write-behind: fixes issues with iobuf length for large writes

Use of an unsigned type in some calculations of size would lead to segmentation faults, if several large adjacent writes came in concurrently. Also, improves buffer allocation code to take the size required into account. Credits for the patch go to Amar. Change-Id: I8a09c52d49909e4ee8e7d4dcfa02ec33ea36a551 BUG: 880948 Signed-off-by: Kaushal M <kaushal@redhat.com> Reviewed-on: http://review.gluster.org/4307 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Anand Avati <avati@redhat.com>
author: Kaushal M <kaushal@redhat.com> 2012-12-13 12:51:03 +0530
committer: Anand Avati <avati@redhat.com> 2012-12-13 14:46:42 -0800
commit: 1b681f27e04ae3b74a461ef2601ac8c48ad6b1c3 (patch)
tree: 7607160f0b0fdc434d8864b6072ee8f860ddfd5f
parent: ef95b1ec0981e5f9859f5308e15ae33608bd6c29 (diff)
1 files changed, 15 insertions, 5 deletions
diff --git a/xlators/performance/write-behind/src/write-behind.c b/xlators/performance/write-behind/src/write-behind.c
index 232e6c0dec0..a256419e867 100644
--- a/xlators/performance/write-behind/src/write-behind.c
+++ b/xlators/performance/write-behind/src/write-behind.c
@@ -123,7 +123,7 @@ typedef struct wb_request {
 
         call_stub_t          *stub;
 
-        size_t                write_size;  /* currently held size
+        ssize_t               write_size;  /* currently held size
 					      (after collapsing) */
 	size_t                orig_size;   /* size which arrived with the request.
 					      This is the size by which we grow
@@ -861,10 +861,20 @@ __wb_collapse_small_writes (wb_request_t *holder, wb_request_t *req)
         struct iobuf  *iobuf  = NULL;
         struct iobref *iobref = NULL;
         int            ret    = -1;
+        ssize_t        required_size = 0;
+        size_t         holder_len = 0;
+        size_t         req_len = 0;
 
         if (!holder->iobref) {
-                /* TODO: check the required size */
-                iobuf = iobuf_get (req->wb_inode->this->ctx->iobuf_pool);
+                holder_len = iov_length (holder->stub->args.writev.vector,
+                                         holder->stub->args.writev.count);
+                req_len = iov_length (req->stub->args.writev.vector,
+                                      req->stub->args.writev.count);
+
+                required_size = max ((THIS->ctx->page_size),
+                                     (holder_len + req_len));
+                iobuf = iobuf_get2 (req->wb_inode->this->ctx->iobuf_pool,
+                                    required_size);
                 if (iobuf == NULL) {
                         goto out;
                 }
@@ -917,13 +927,13 @@ void
 __wb_preprocess_winds (wb_inode_t *wb_inode)
 {
         off_t         offset_expected = 0;
-        size_t        space_left      = 0;
+        ssize_t       space_left      = 0;
 	wb_request_t *req             = NULL;
 	wb_request_t *tmp             = NULL;
 	wb_request_t *holder          = NULL;
 	wb_conf_t    *conf            = NULL;
         int           ret             = 0;
-	size_t        page_size       = 0;
+	ssize_t       page_size       = 0;
 
 	/* With asynchronous IO from a VM guest (as a file), there
 	   can be two sequential writes happening in two regions
diff --git a/xlators/Makefile.am b/xlators/Makefile.am
index 876a31f8e81..ef20cbb64fa 100644
--- a/xlators/Makefile.am
+++ b/xlators/Makefile.am
@@ -1,16 +1,12 @@
-if ENABLE_EXPERIMENTAL
-  EXPERIMENTAL = experimental
-endif
-
 if BUILD_GNFS
   GNFS_DIR = nfs
 endif
 
-DIST_SUBDIRS = cluster storage protocol performance debug features encryption \
-          mount nfs mgmt system playground meta experimental
+DIST_SUBDIRS = cluster storage protocol performance debug features \
+          mount nfs mgmt system playground meta
 
-SUBDIRS = cluster storage protocol performance debug features encryption \
-          mount ${GNFS_DIR} mgmt system playground meta $(EXPERIMENTAL)
+SUBDIRS = cluster storage protocol performance debug features \
+          mount ${GNFS_DIR} mgmt system playground meta
 
 EXTRA_DIST = xlator.sym
 
diff --git a/xlators/cluster/Makefile.am b/xlators/cluster/Makefile.am
index 903fbb39f12..8e067d5ab58 100644
--- a/xlators/cluster/Makefile.am
+++ b/xlators/cluster/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = stripe afr dht ec
+SUBDIRS = afr dht ec
 
 CLEANFILES = 
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index c313294961f..032ab5c8001 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -15,22 +15,20 @@
 #include <stdlib.h>
 #include <signal.h>
 
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-#include "statedump.h"
-#include "events.h"
-#include "upcall-utils.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/hashfn.h>
+#include <glusterfs/list.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/events.h>
+#include <glusterfs/upcall-utils.h>
 
 #include "afr-inode-read.h"
 #include "afr-inode-write.h"
@@ -42,157 +40,796 @@
 #include "afr-messages.h"
 
 int32_t
-afr_quorum_errno (afr_private_t *priv)
+afr_quorum_errno(afr_private_t *priv)
 {
-        return ENOTCONN;
+    return ENOTCONN;
+}
+
+gf_boolean_t
+afr_is_private_directory(afr_private_t *priv, uuid_t pargfid, const char *name,
+                         pid_t pid)
+{
+    if (!__is_root_gfid(pargfid)) {
+        return _gf_false;
+    }
+
+    if (strcmp(name, GF_REPLICATE_TRASH_DIR) == 0) {
+        /*For backward compatibility /.landfill is private*/
+        return _gf_true;
+    }
+
+    if (pid == GF_CLIENT_PID_GSYNCD) {
+        /*geo-rep needs to create/sync private directory on slave because
+         * it appears in changelog*/
+        return _gf_false;
+    }
+
+    if (pid == GF_CLIENT_PID_GLFS_HEAL || pid == GF_CLIENT_PID_SELF_HEALD) {
+        if (strcmp(name, priv->anon_inode_name) == 0) {
+            /* anonymous-inode dir is private*/
+            return _gf_true;
+        }
+    } else {
+        if (strncmp(name, AFR_ANON_DIR_PREFIX, strlen(AFR_ANON_DIR_PREFIX)) ==
+            0) {
+            /* anonymous-inode dir prefix is private for geo-rep to work*/
+            return _gf_true;
+        }
+    }
+
+    return _gf_false;
+}
+
+void
+afr_fill_success_replies(afr_local_t *local, afr_private_t *priv,
+                         unsigned char *replies)
+{
+    int i = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].op_ret == 0) {
+            replies[i] = 1;
+        } else {
+            replies[i] = 0;
+        }
+    }
 }
 
 int
-afr_fav_child_reset_sink_xattrs (void *opaque);
+afr_fav_child_reset_sink_xattrs(void *opaque);
 
 int
-afr_fav_child_reset_sink_xattrs_cbk (int ret, call_frame_t *frame,
-                                     void *opaque);
+afr_fav_child_reset_sink_xattrs_cbk(int ret, call_frame_t *frame, void *opaque);
 
 static void
-afr_discover_done (call_frame_t *frame, xlator_t *this);
+afr_discover_done(call_frame_t *frame, xlator_t *this);
 
-gf_boolean_t
-afr_is_consistent_io_possible (afr_local_t *local, afr_private_t *priv,
-                               int32_t *op_errno)
+int
+afr_dom_lock_acquire_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
 {
-        if (priv->consistent_io && local->call_count != priv->child_count) {
-                gf_msg (THIS->name, GF_LOG_INFO, 0,
-                        AFR_MSG_SUBVOLS_DOWN, "All subvolumes are not up");
-                if (op_errno)
-                        *op_errno = ENOTCONN;
-                return _gf_false;
-        }
-        return _gf_true;
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = (long)cookie;
+
+    local->cont.lk.dom_lock_op_ret[i] = op_ret;
+    local->cont.lk.dom_lock_op_errno[i] = op_errno;
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to acquire %s on %s",
+               uuid_utoa(local->fd->inode->gfid), AFR_LK_HEAL_DOM,
+               priv->children[i]->name);
+    } else {
+        local->cont.lk.dom_locked_nodes[i] = 1;
+    }
+
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
 }
 
-call_frame_t *
-afr_copy_frame (call_frame_t *base)
-{
-	afr_local_t *local = NULL;
-	call_frame_t *frame = NULL;
-	int op_errno = 0;
+int
+afr_dom_lock_acquire(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    int i = 0;
+
+    priv = frame->this->private;
+    local = frame->local;
+    local->cont.lk.dom_locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.locked_nodes),
+        gf_afr_mt_char);
+    if (!local->cont.lk.dom_locked_nodes) {
+        return -ENOMEM;
+    }
+    local->cont.lk.dom_lock_op_ret = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.dom_lock_op_ret),
+        gf_afr_mt_int32_t);
+    if (!local->cont.lk.dom_lock_op_ret) {
+        return -ENOMEM; /* CALLOC'd members are freed in afr_local_cleanup. */
+    }
+    local->cont.lk.dom_lock_op_errno = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.dom_lock_op_errno),
+        gf_afr_mt_int32_t);
+    if (!local->cont.lk.dom_lock_op_errno) {
+        return -ENOMEM; /* CALLOC'd members are freed in afr_local_cleanup. */
+    }
+    flock.l_type = F_WRLCK;
+
+    AFR_ONALL(frame, afr_dom_lock_acquire_cbk, finodelk, AFR_LK_HEAL_DOM,
+              local->fd, F_SETLK, &flock, NULL);
+
+    if (!afr_has_quorum(local->cont.lk.dom_locked_nodes, frame->this, NULL))
+        goto blocking_lock;
+
+    /*If any of the bricks returned EAGAIN, we still need blocking locks.*/
+    if (AFR_COUNT(local->cont.lk.dom_locked_nodes, priv->child_count) !=
+        priv->child_count) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->cont.lk.dom_lock_op_ret[i] == -1 &&
+                local->cont.lk.dom_lock_op_errno[i] == EAGAIN)
+                goto blocking_lock;
+        }
+    }
+
+    return 0;
 
-	frame = copy_frame (base);
-	if (!frame)
-		return NULL;
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local) {
-		AFR_STACK_DESTROY (frame);
-		return NULL;
-	}
+blocking_lock:
+    afr_dom_lock_release(frame);
+    AFR_ONALL(frame, afr_dom_lock_acquire_cbk, finodelk, AFR_LK_HEAL_DOM,
+              local->fd, F_SETLKW, &flock, NULL);
+    if (!afr_has_quorum(local->cont.lk.dom_locked_nodes, frame->this, NULL)) {
+        afr_dom_lock_release(frame);
+        return -afr_quorum_errno(priv);
+    }
 
-	return frame;
+    return 0;
 }
 
-/* Check if an entry or inode could be undergoing a transaction. */
-gf_boolean_t
-afr_is_possibly_under_txn (afr_transaction_type type, afr_local_t *local,
-                           xlator_t *this)
+int
+afr_dom_lock_release_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
 {
-        int i = 0;
-        int tmp = 0;
-	afr_private_t *priv = NULL;
-        GF_UNUSED char *key = NULL;
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = (long)cookie;
 
-        priv = this->private;
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to release %s on %s", local->loc.path,
+               AFR_LK_HEAL_DOM, priv->children[i]->name);
+    }
+    local->cont.lk.dom_locked_nodes[i] = 0;
 
-        if (type == AFR_ENTRY_TRANSACTION)
-                key = GLUSTERFS_PARENT_ENTRYLK;
-        else if (type == AFR_DATA_TRANSACTION)
-                /*FIXME: Use GLUSTERFS_INODELK_DOM_COUNT etc. once
-                 * pl_inodelk_xattr_fill supports separate keys for different
-                 * domains.*/
-                key = GLUSTERFS_INODELK_COUNT;
+    syncbarrier_wake(&local->barrier);
 
-        for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].xdata)
-			continue;
-		if (dict_get_int32 (local->replies[i].xdata, key, &tmp) == 0)
-			if (tmp)
-				return _gf_true;
-	}
+    return 0;
+}
 
-        return _gf_false;
+void
+afr_dom_lock_release(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *locked_on = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+
+    local = frame->local;
+    priv = frame->this->private;
+    locked_on = local->cont.lk.dom_locked_nodes;
+    if (AFR_COUNT(locked_on, priv->child_count) == 0)
+        return;
+    flock.l_type = F_UNLCK;
+
+    AFR_ONLIST(locked_on, frame, afr_dom_lock_release_cbk, finodelk,
+               AFR_LK_HEAL_DOM, local->fd, F_SETLK, &flock, NULL);
+
+    return;
 }
 
 static void
-afr_inode_ctx_destroy (afr_inode_ctx_t *ctx)
+afr_lk_heal_info_cleanup(afr_lk_heal_info_t *info)
 {
-        int i = 0;
-
-        if (!ctx)
-                return;
+    if (!info)
+        return;
+    if (info->xdata_req)
+        dict_unref(info->xdata_req);
+    if (info->fd)
+        fd_unref(info->fd);
+    GF_FREE(info->locked_nodes);
+    GF_FREE(info->child_up_event_gen);
+    GF_FREE(info->child_down_event_gen);
+    GF_FREE(info);
+}
 
-        for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
-                GF_FREE (ctx->pre_op_done[i]);
+static int
+afr_add_lock_to_saved_locks(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+    afr_lk_heal_info_t *info = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = -ENOMEM;
+
+    info = GF_CALLOC(sizeof(*info), 1, gf_afr_mt_lk_heal_info_t);
+    if (!info) {
+        goto cleanup;
+    }
+    INIT_LIST_HEAD(&info->pos);
+    info->fd = fd_ref(local->fd);
+    info->cmd = local->cont.lk.cmd;
+    info->pid = frame->root->pid;
+    info->flock = local->cont.lk.user_flock;
+    info->xdata_req = dict_copy_with_ref(local->xdata_req, NULL);
+    if (!info->xdata_req) {
+        goto cleanup;
+    }
+    info->lk_owner = frame->root->lk_owner;
+    info->locked_nodes = GF_MALLOC(
+        sizeof(*info->locked_nodes) * priv->child_count, gf_afr_mt_char);
+    if (!info->locked_nodes) {
+        goto cleanup;
+    }
+    memcpy(info->locked_nodes, local->cont.lk.locked_nodes,
+           sizeof(*info->locked_nodes) * priv->child_count);
+    info->child_up_event_gen = GF_CALLOC(sizeof(*info->child_up_event_gen),
+                                         priv->child_count, gf_afr_mt_int32_t);
+    if (!info->child_up_event_gen) {
+        goto cleanup;
+    }
+    info->child_down_event_gen = GF_CALLOC(sizeof(*info->child_down_event_gen),
+                                           priv->child_count,
+                                           gf_afr_mt_int32_t);
+    if (!info->child_down_event_gen) {
+        goto cleanup;
+    }
+
+    LOCK(&local->fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(local->fd, this);
+        if (fd_ctx)
+            fd_ctx->lk_heal_info = info;
+    }
+    UNLOCK(&local->fd->lock);
+    if (!fd_ctx) {
+        goto cleanup;
+    }
+
+    LOCK(&priv->lock);
+    {
+        list_add_tail(&info->pos, &priv->saved_locks);
+    }
+    UNLOCK(&priv->lock);
+
+    return 0;
+cleanup:
+    gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_LK_HEAL_DOM,
+           "%s: Failed to add lock to healq",
+           uuid_utoa(local->fd->inode->gfid));
+    if (info) {
+        afr_lk_heal_info_cleanup(info);
+        if (fd_ctx) {
+            LOCK(&local->fd->lock);
+            {
+                fd_ctx->lk_heal_info = NULL;
+            }
+            UNLOCK(&local->fd->lock);
         }
+    }
+    return ret;
+}
 
-        GF_FREE (ctx);
+static int
+afr_remove_lock_from_saved_locks(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    struct gf_flock flock = local->cont.lk.user_flock;
+    afr_lk_heal_info_t *info = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = -EINVAL;
+
+    fd_ctx = afr_fd_ctx_get(local->fd, this);
+    if (!fd_ctx || !fd_ctx->lk_heal_info) {
+        goto out;
+    }
+
+    info = fd_ctx->lk_heal_info;
+    if ((info->flock.l_start != flock.l_start) ||
+        (info->flock.l_whence != flock.l_whence) ||
+        (info->flock.l_len != flock.l_len)) {
+        /*TODO: Compare lkowners too.*/
+        goto out;
+    }
+
+    LOCK(&priv->lock);
+    {
+        list_del(&fd_ctx->lk_heal_info->pos);
+    }
+    UNLOCK(&priv->lock);
+
+    afr_lk_heal_info_cleanup(info);
+    fd_ctx->lk_heal_info = NULL;
+    ret = 0;
+out:
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to remove lock from healq",
+               uuid_utoa(local->fd->inode->gfid));
+    return ret;
 }
 
 int
-__afr_inode_ctx_get (xlator_t *this, inode_t *inode, afr_inode_ctx_t **ctx)
+afr_lock_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                  dict_t *xdata)
 {
-        uint64_t        ctx_int   = 0;
-        int             ret       = -1;
-        int             i         = -1;
-        int             num_locks = -1;
-        afr_inode_ctx_t *ictx     = NULL;
-        afr_lock_t      *lock     = NULL;
-        afr_private_t   *priv     = this->private;
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
 
-        ret = __inode_ctx_get (inode, this, &ctx_int);
-        if (ret == 0) {
-                *ctx = (afr_inode_ctx_t *)ctx_int;
-                return 0;
-        }
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "Failed to heal lock on child %d for %s", i,
+               uuid_utoa(local->fd->inode->gfid));
+    }
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
 
-        ictx = GF_CALLOC (1, sizeof (afr_inode_ctx_t), gf_afr_mt_inode_ctx_t);
-        if (!ictx)
-                goto out;
+int
+afr_getlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
 
-        for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
-                ictx->pre_op_done[i] = GF_CALLOC (sizeof *ictx->pre_op_done[i],
-                                                  priv->child_count,
-                                                  gf_afr_mt_int32_t);
-                if (!ictx->pre_op_done[i]) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "Failed getlk for %s", uuid_utoa(local->fd->inode->gfid));
+    } else {
+        local->cont.lk.getlk_rsp[i] = *lock;
+    }
+
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
+
+static gf_boolean_t
+afr_does_lk_owner_match(call_frame_t *frame, afr_private_t *priv,
+                        afr_lk_heal_info_t *info)
+{
+    int i = 0;
+    afr_local_t *local = frame->local;
+    struct gf_flock flock = {
+        0,
+    };
+    gf_boolean_t ret = _gf_true;
+    char *wind_on = alloca0(priv->child_count);
+    unsigned char *success_replies = alloca0(priv->child_count);
+    local->cont.lk.getlk_rsp = GF_CALLOC(sizeof(*local->cont.lk.getlk_rsp),
+                                         priv->child_count, gf_afr_mt_gf_lock);
+
+    flock = info->flock;
+    for (i = 0; i < priv->child_count; i++) {
+        if (info->locked_nodes[i])
+            wind_on[i] = 1;
+    }
+
+    AFR_ONLIST(wind_on, frame, afr_getlk_cbk, lk, info->fd, F_GETLK, &flock,
+               info->xdata_req);
+
+    afr_fill_success_replies(local, priv, success_replies);
+    if (AFR_COUNT(success_replies, priv->child_count) == 0) {
+        ret = _gf_false;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid || local->replies[i].op_ret != 0)
+            continue;
+        if (local->cont.lk.getlk_rsp[i].l_type == F_UNLCK)
+            continue;
+        /*TODO: Do we really need to compare lkowner if F_UNLCK is true?*/
+        if (!is_same_lkowner(&local->cont.lk.getlk_rsp[i].l_owner,
+                             &info->lk_owner)) {
+            ret = _gf_false;
+            break;
+        }
+    }
+out:
+    afr_local_replies_wipe(local, priv);
+    GF_FREE(local->cont.lk.getlk_rsp);
+    local->cont.lk.getlk_rsp = NULL;
+    return ret;
+}
+
+static void
+afr_mark_fd_bad(fd_t *fd, xlator_t *this)
+{
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    if (!fd)
+        return;
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(fd, this);
+        if (fd_ctx) {
+            fd_ctx->is_fd_bad = _gf_true;
+            fd_ctx->lk_heal_info = NULL;
         }
+    }
+    UNLOCK(&fd->lock);
+}
+
+static void
+afr_add_lock_to_lkhealq(afr_private_t *priv, afr_lk_heal_info_t *info)
+{
+    LOCK(&priv->lock);
+    {
+        list_del(&info->pos);
+        list_add_tail(&info->pos, &priv->lk_healq);
+    }
+    UNLOCK(&priv->lock);
+}
+
+static void
+afr_lock_heal_do(call_frame_t *frame, afr_private_t *priv,
+                 afr_lk_heal_info_t *info)
+{
+    int i = 0;
+    int op_errno = 0;
+    int32_t *current_event_gen = NULL;
+    afr_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    char *wind_on = alloca0(priv->child_count);
+    gf_boolean_t retry = _gf_true;
+
+    frame->root->pid = info->pid;
+    lk_owner_copy(&frame->root->lk_owner, &info->lk_owner);
+
+    op_errno = -afr_dom_lock_acquire(frame);
+    if ((op_errno != 0)) {
+        goto release;
+    }
+
+    if (!afr_does_lk_owner_match(frame, priv, info)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_LK_HEAL_DOM,
+               "Ignoring lock heal for %s since lk-onwers mismatch. "
+               "Lock possibly pre-empted by another client.",
+               uuid_utoa(info->fd->inode->gfid));
+        goto release;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (info->locked_nodes[i])
+            continue;
+        wind_on[i] = 1;
+    }
+
+    current_event_gen = alloca(priv->child_count);
+    memcpy(current_event_gen, info->child_up_event_gen,
+           priv->child_count * sizeof *current_event_gen);
+    AFR_ONLIST(wind_on, frame, afr_lock_heal_cbk, lk, info->fd, info->cmd,
+               &info->flock, info->xdata_req);
+
+    LOCK(&priv->lock);
+    {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!wind_on[i])
+                continue;
+            if ((!local->replies[i].valid) || (local->replies[i].op_ret != 0)) {
+                continue;
+            }
 
-        num_locks = sizeof(ictx->lock)/sizeof(afr_lock_t);
-        for (i = 0; i < num_locks; i++) {
-                lock = &ictx->lock[i];
-                INIT_LIST_HEAD (&lock->post_op);
-                INIT_LIST_HEAD (&lock->frozen);
-                INIT_LIST_HEAD (&lock->waiting);
-                INIT_LIST_HEAD (&lock->owners);
+            if ((current_event_gen[i] == info->child_up_event_gen[i]) &&
+                (current_event_gen[i] > info->child_down_event_gen[i])) {
+                info->locked_nodes[i] = 1;
+                retry = _gf_false;
+                list_del_init(&info->pos);
+                list_add_tail(&info->pos, &priv->saved_locks);
+            } else {
+                /*We received subsequent child up/down events while heal was in
+                 * progress; don't mark child as healed. Attempt again on the
+                 * new child up*/
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_LK_HEAL_DOM,
+                       "Event gen mismatch: skipped healing lock on child %d "
+                       "for %s.",
+                       i, uuid_utoa(info->fd->inode->gfid));
+            }
         }
+    }
+    UNLOCK(&priv->lock);
 
-        ctx_int = (uint64_t)ictx;
-        ret = __inode_ctx_set (inode, this, &ctx_int);
-        if (ret) {
-                goto out;
+release:
+    afr_dom_lock_release(frame);
+    if (retry)
+        afr_add_lock_to_lkhealq(priv, info);
+    return;
+}
+
+static int
+afr_lock_heal_done(int ret, call_frame_t *frame, void *opaque)
+{
+    STACK_DESTROY(frame->root);
+    return 0;
+}
+
+static int
+afr_lock_heal(void *opaque)
+{
+    call_frame_t *frame = (call_frame_t *)opaque;
+    call_frame_t *iter_frame = NULL;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
+    struct list_head healq = {
+        0,
+    };
+    int ret = 0;
+
+    iter_frame = afr_copy_frame(frame);
+    if (!iter_frame) {
+        return ENOMEM;
+    }
+
+    INIT_LIST_HEAD(&healq);
+    LOCK(&priv->lock);
+    {
+        list_splice_init(&priv->lk_healq, &healq);
+    }
+    UNLOCK(&priv->lock);
+
+    list_for_each_entry_safe(info, tmp, &healq, pos)
+    {
+        GF_ASSERT((AFR_COUNT(info->locked_nodes, priv->child_count) <
+                   priv->child_count));
+        ((afr_local_t *)(iter_frame->local))->fd = fd_ref(info->fd);
+        afr_lock_heal_do(iter_frame, priv, info);
+        AFR_STACK_RESET(iter_frame);
+        if (iter_frame->local == NULL) {
+            ret = ENOTCONN;
+            gf_msg(frame->this->name, GF_LOG_ERROR, ENOTCONN,
+                   AFR_MSG_LK_HEAL_DOM,
+                   "Aborting processing of lk_healq."
+                   "Healing will be reattempted on next child up for locks "
+                   "that are still in quorum.");
+            LOCK(&priv->lock);
+            {
+                list_add_tail(&healq, &priv->lk_healq);
+            }
+            UNLOCK(&priv->lock);
+            break;
         }
+    }
 
-        ictx->spb_choice = -1;
-        ictx->read_subvol = 0;
-        ictx->write_subvol = 0;
-        ictx->lock_count = 0;
-        ret = 0;
-        *ctx = ictx;
+    AFR_STACK_DESTROY(iter_frame);
+    return ret;
+}
+
+static int
+__afr_lock_heal_synctask(xlator_t *this, afr_private_t *priv, int child)
+{
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
+
+    if (priv->shd.iamshd)
+        return 0;
+
+    list_for_each_entry_safe(info, tmp, &priv->saved_locks, pos)
+    {
+        info->child_up_event_gen[child] = priv->event_generation;
+        list_del_init(&info->pos);
+        list_add_tail(&info->pos, &priv->lk_healq);
+    }
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        return -1;
+
+    ret = synctask_new(this->ctx->env, afr_lock_heal, afr_lock_heal_done, frame,
+                       frame);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_LK_HEAL_DOM,
+               "Failed to launch lock heal synctask");
+
+    return ret;
+}
+
+static int
+__afr_mark_pending_lk_heal(xlator_t *this, afr_private_t *priv, int child)
+{
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
+
+    if (priv->shd.iamshd)
+        return 0;
+    list_for_each_entry_safe(info, tmp, &priv->saved_locks, pos)
+    {
+        info->child_down_event_gen[child] = priv->event_generation;
+        if (info->locked_nodes[child] == 1)
+            info->locked_nodes[child] = 0;
+        if (!afr_has_quorum(info->locked_nodes, this, NULL)) {
+            /* Since the lock was lost on quorum no. of nodes, we should
+             * not attempt to heal it anymore. Some other client could have
+             * acquired the lock, modified data and released it and this
+             * client wouldn't know about it if we heal it.*/
+            afr_mark_fd_bad(info->fd, this);
+            list_del(&info->pos);
+            afr_lk_heal_info_cleanup(info);
+            /* We're not winding an unlock on the node where the lock is still
+             * present because when fencing logic switches over to the new
+             * client (since we marked the fd bad), it should preempt any
+             * existing lock. */
+        }
+    }
+    return 0;
+}
+
+gf_boolean_t
+afr_is_consistent_io_possible(afr_local_t *local, afr_private_t *priv,
+                              int32_t *op_errno)
+{
+    if (priv->consistent_io && local->call_count != priv->child_count) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOLS_DOWN,
+               "All subvolumes are not up");
+        if (op_errno)
+            *op_errno = ENOTCONN;
+        return _gf_false;
+    }
+    return _gf_true;
+}
+
+gf_boolean_t
+afr_is_lock_mode_mandatory(dict_t *xdata)
+{
+    int ret = 0;
+    uint32_t lk_mode = GF_LK_ADVISORY;
+
+    ret = dict_get_uint32(xdata, GF_LOCK_MODE, &lk_mode);
+    if (!ret && lk_mode == GF_LK_MANDATORY)
+        return _gf_true;
+
+    return _gf_false;
+}
+
+call_frame_t *
+afr_copy_frame(call_frame_t *base)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    int op_errno = 0;
+
+    frame = copy_frame(base);
+    if (!frame)
+        return NULL;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local) {
+        AFR_STACK_DESTROY(frame);
+        return NULL;
+    }
+
+    return frame;
+}
+
+/* Check if an entry or inode could be undergoing a transaction. */
+gf_boolean_t
+afr_is_possibly_under_txn(afr_transaction_type type, afr_local_t *local,
+                          xlator_t *this)
+{
+    int i = 0;
+    int tmp = 0;
+    afr_private_t *priv = NULL;
+    GF_UNUSED char *key = NULL;
+    int keylen = 0;
+
+    priv = this->private;
+
+    if (type == AFR_ENTRY_TRANSACTION) {
+        key = GLUSTERFS_PARENT_ENTRYLK;
+        keylen = SLEN(GLUSTERFS_PARENT_ENTRYLK);
+    } else if (type == AFR_DATA_TRANSACTION) {
+        /*FIXME: Use GLUSTERFS_INODELK_DOM_COUNT etc. once
+         * pl_inodelk_xattr_fill supports separate keys for different
+         * domains.*/
+        key = GLUSTERFS_INODELK_COUNT;
+        keylen = SLEN(GLUSTERFS_INODELK_COUNT);
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].xdata)
+            continue;
+        if (dict_get_int32n(local->replies[i].xdata, key, keylen, &tmp) == 0)
+            if (tmp)
+                return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+static void
+afr_inode_ctx_destroy(afr_inode_ctx_t *ctx)
+{
+    int i = 0;
+
+    if (!ctx)
+        return;
+
+    for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+        GF_FREE(ctx->pre_op_done[i]);
+    }
+
+    GF_FREE(ctx);
+}
+
+int
+__afr_inode_ctx_get(xlator_t *this, inode_t *inode, afr_inode_ctx_t **ctx)
+{
+    uint64_t ctx_int = 0;
+    int ret = -1;
+    int i = -1;
+    int num_locks = -1;
+    afr_inode_ctx_t *ictx = NULL;
+    afr_lock_t *lock = NULL;
+    afr_private_t *priv = this->private;
+
+    ret = __inode_ctx_get(inode, this, &ctx_int);
+    if (ret == 0) {
+        *ctx = (afr_inode_ctx_t *)(uintptr_t)ctx_int;
+        return 0;
+    }
+
+    ictx = GF_CALLOC(1, sizeof(afr_inode_ctx_t), gf_afr_mt_inode_ctx_t);
+    if (!ictx)
+        goto out;
+
+    for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+        ictx->pre_op_done[i] = GF_CALLOC(sizeof *ictx->pre_op_done[i],
+                                         priv->child_count, gf_afr_mt_int32_t);
+        if (!ictx->pre_op_done[i]) {
+            ret = -ENOMEM;
+            goto out;
+        }
+    }
+
+    num_locks = sizeof(ictx->lock) / sizeof(afr_lock_t);
+    for (i = 0; i < num_locks; i++) {
+        lock = &ictx->lock[i];
+        INIT_LIST_HEAD(&lock->post_op);
+        INIT_LIST_HEAD(&lock->frozen);
+        INIT_LIST_HEAD(&lock->waiting);
+        INIT_LIST_HEAD(&lock->owners);
+    }
+
+    ctx_int = (uint64_t)(uintptr_t)ictx;
+    ret = __inode_ctx_set(inode, this, &ctx_int);
+    if (ret) {
+        goto out;
+    }
+
+    ictx->spb_choice = -1;
+    ictx->read_subvol = 0;
+    ictx->write_subvol = 0;
+    ictx->lock_count = 0;
+    ret = 0;
+    *ctx = ictx;
 out:
-        if (ret) {
-                afr_inode_ctx_destroy (ictx);
-        }
-        return ret;
+    if (ret) {
+        afr_inode_ctx_destroy(ictx);
+    }
+    return ret;
 }
 
 /*
@@ -226,1694 +863,1681 @@ out:
  */
 
 int
-__afr_set_in_flight_sb_status (xlator_t *this, afr_local_t *local,
-                               inode_t *inode)
-{
-        int                 i               = 0;
-        int                 txn_type        = 0;
-        int                 count           = 0;
-        int                 index           = -1;
-        uint16_t            datamap_old     = 0;
-        uint16_t            metadatamap_old = 0;
-        uint16_t            datamap         = 0;
-        uint16_t            metadatamap     = 0;
-        uint16_t            tmp_map         = 0;
-        uint16_t            mask            = 0;
-        uint32_t            event           = 0;
-        uint64_t            val             = 0;
-        afr_private_t      *priv            = NULL;
-
-        priv = this->private;
-        txn_type = local->transaction.type;
-
-        if (txn_type == AFR_DATA_TRANSACTION)
-                val = local->inode_ctx->write_subvol;
-        else
-                val = local->inode_ctx->read_subvol;
-
-        metadatamap_old = metadatamap = (val & 0x000000000000ffff);
-        datamap_old = datamap = (val & 0x00000000ffff0000) >> 16;
-        event = (val & 0xffffffff00000000) >> 32;
-
-        if (txn_type == AFR_DATA_TRANSACTION)
-                tmp_map = datamap;
-        else if (txn_type == AFR_METADATA_TRANSACTION)
-                tmp_map = metadatamap;
-
-        count = gf_bits_count (tmp_map);
-
-        if (count == 1)
-                index = gf_bits_index (tmp_map);
-
-        for (i = 0; i < priv->child_count; i++) {
-                mask = 0;
-                if (!local->transaction.failed_subvols[i])
-                        continue;
-
-                mask = 1 << i;
-                if (txn_type == AFR_METADATA_TRANSACTION)
-                        metadatamap &= ~mask;
-                else if (txn_type == AFR_DATA_TRANSACTION)
-                        datamap &= ~mask;
-        }
-
-        switch (txn_type) {
+__afr_set_in_flight_sb_status(xlator_t *this, afr_local_t *local,
+                              inode_t *inode)
+{
+    int i = 0;
+    int txn_type = 0;
+    int count = 0;
+    int index = -1;
+    uint16_t datamap_old = 0;
+    uint16_t metadatamap_old = 0;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint16_t tmp_map = 0;
+    uint16_t mask = 0;
+    uint32_t event = 0;
+    uint64_t val = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    txn_type = local->transaction.type;
+
+    if (txn_type == AFR_DATA_TRANSACTION)
+        val = local->inode_ctx->write_subvol;
+    else
+        val = local->inode_ctx->read_subvol;
+
+    metadatamap_old = metadatamap = (val & 0x000000000000ffff);
+    datamap_old = datamap = (val & 0x00000000ffff0000) >> 16;
+    event = (val & 0xffffffff00000000) >> 32;
+
+    if (txn_type == AFR_DATA_TRANSACTION)
+        tmp_map = datamap;
+    else if (txn_type == AFR_METADATA_TRANSACTION)
+        tmp_map = metadatamap;
+
+    count = gf_bits_count(tmp_map);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.failed_subvols[i])
+            continue;
+
+        mask = 1 << i;
+        if (txn_type == AFR_METADATA_TRANSACTION)
+            metadatamap &= ~mask;
+        else if (txn_type == AFR_DATA_TRANSACTION)
+            datamap &= ~mask;
+    }
+
+    switch (txn_type) {
         case AFR_METADATA_TRANSACTION:
-                if ((metadatamap_old != 0) && (metadatamap == 0) &&
-                    (count == 1)) {
-                        local->transaction.in_flight_sb_errno =
-                                                local->replies[index].op_errno;
-                        local->transaction.in_flight_sb = _gf_true;
-                        metadatamap |= (1 << index);
-                }
-                if (metadatamap_old != metadatamap) {
-                        event = 0;
-                }
-                break;
+            if ((metadatamap_old != 0) && (metadatamap == 0) && (count == 1)) {
+                index = gf_bits_index(tmp_map);
+                local->transaction.in_flight_sb_errno = local->replies[index]
+                                                            .op_errno;
+                local->transaction.in_flight_sb = _gf_true;
+                metadatamap |= (1 << index);
+            }
+            if (metadatamap_old != metadatamap) {
+                __afr_inode_need_refresh_set(inode, this);
+            }
+            break;
 
         case AFR_DATA_TRANSACTION:
-                if ((datamap_old != 0) && (datamap == 0) && (count == 1)) {
-                        local->transaction.in_flight_sb_errno =
-                                                local->replies[index].op_errno;
-                        local->transaction.in_flight_sb = _gf_true;
-                        datamap |= (1 << index);
-                }
-                if (datamap_old != datamap)
-                        event = 0;
-                break;
+            if ((datamap_old != 0) && (datamap == 0) && (count == 1)) {
+                index = gf_bits_index(tmp_map);
+                local->transaction.in_flight_sb_errno = local->replies[index]
+                                                            .op_errno;
+                local->transaction.in_flight_sb = _gf_true;
+                datamap |= (1 << index);
+            }
+            if (datamap_old != datamap)
+                __afr_inode_need_refresh_set(inode, this);
+            break;
 
         default:
-                break;
-        }
+            break;
+    }
 
-        val = ((uint64_t) metadatamap) |
-                (((uint64_t) datamap) << 16) |
-                (((uint64_t) event) << 32);
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
 
-        if (txn_type == AFR_DATA_TRANSACTION)
-                local->inode_ctx->write_subvol = val;
-        local->inode_ctx->read_subvol = val;
+    if (txn_type == AFR_DATA_TRANSACTION)
+        local->inode_ctx->write_subvol = val;
+    local->inode_ctx->read_subvol = val;
 
-        return 0;
+    return 0;
 }
 
 gf_boolean_t
-afr_is_symmetric_error (call_frame_t *frame, xlator_t *this)
+afr_is_symmetric_error(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int op_errno = 0;
-        int i_errno = 0;
-        gf_boolean_t matching_errors = _gf_true;
-        int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int op_errno = 0;
+    int i_errno = 0;
+    gf_boolean_t matching_errors = _gf_true;
+    int i = 0;
 
-        priv = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->replies[i].valid)
-                        continue;
-                if (local->replies[i].op_ret != -1) {
-                        /* Operation succeeded on at least one subvol,
-                           so it is not a failed-everywhere situation.
-                        */
-                        matching_errors = _gf_false;
-                        break;
-                }
-                i_errno = local->replies[i].op_errno;
-
-                if (i_errno == ENOTCONN) {
-                        /* ENOTCONN is not a symmetric error. We do not
-                           know if the operation was performed on the
-                           backend or not.
-                        */
-                        matching_errors = _gf_false;
-                        break;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret != -1) {
+            /* Operation succeeded on at least one subvol,
+               so it is not a failed-everywhere situation.
+            */
+            matching_errors = _gf_false;
+            break;
+        }
+        i_errno = local->replies[i].op_errno;
 
-                if (!op_errno) {
-                        op_errno = i_errno;
-                } else if (op_errno != i_errno) {
-                        /* Mismatching op_errno's */
-                        matching_errors = _gf_false;
-                        break;
-                }
+        if (i_errno == ENOTCONN) {
+            /* ENOTCONN is not a symmetric error. We do not
+               know if the operation was performed on the
+               backend or not.
+            */
+            matching_errors = _gf_false;
+            break;
         }
 
-        return matching_errors;
+        if (!op_errno) {
+            op_errno = i_errno;
+        } else if (op_errno != i_errno) {
+            /* Mismatching op_errno's */
+            matching_errors = _gf_false;
+            break;
+        }
+    }
+
+    return matching_errors;
 }
 
 int
-afr_set_in_flight_sb_status (xlator_t *this, call_frame_t *frame,
-                             inode_t *inode)
+afr_set_in_flight_sb_status(xlator_t *this, call_frame_t *frame, inode_t *inode)
 {
-        int           ret    = -1;
-        afr_private_t *priv  = NULL;
-        afr_local_t   *local = NULL;
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        priv = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        /* If this transaction saw no failures, then exit. */
-        if (AFR_COUNT (local->transaction.failed_subvols,
-                       priv->child_count) == 0)
-                return 0;
+    /* If this transaction saw no failures, then exit. */
+    if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count) == 0)
+        return 0;
 
-        if (afr_is_symmetric_error (frame, this))
-                return 0;
+    if (afr_is_symmetric_error(frame, this))
+        return 0;
 
-        LOCK (&inode->lock);
-        {
-                ret = __afr_set_in_flight_sb_status (this, local, inode);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_set_in_flight_sb_status(this, local, inode);
+    }
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
 int
-__afr_inode_read_subvol_get_small (inode_t *inode, xlator_t *this,
-				   unsigned char *data, unsigned char *metadata,
-				   int *event_p)
-{
-	afr_private_t *priv = NULL;
-	int ret = -1;
-	uint16_t datamap = 0;
-	uint16_t metadatamap = 0;
-	uint32_t event = 0;
-	uint64_t val = 0;
-	int i = 0;
-        afr_inode_ctx_t *ctx = NULL;
-
-	priv = this->private;
-
-	ret = __afr_inode_ctx_get (this, inode, &ctx);
-	if (ret < 0)
-		return ret;
+__afr_inode_read_subvol_get_small(inode_t *inode, xlator_t *this,
+                                  unsigned char *data, unsigned char *metadata,
+                                  int *event_p)
+{
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint32_t event = 0;
+    uint64_t val = 0;
+    int i = 0;
+    afr_inode_ctx_t *ctx = NULL;
+
+    priv = this->private;
+
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret < 0)
+        return ret;
 
-        val = ctx->read_subvol;
+    val = ctx->read_subvol;
 
-	metadatamap = (val & 0x000000000000ffff);
-	datamap =     (val & 0x00000000ffff0000) >> 16;
-	event =       (val & 0xffffffff00000000) >> 32;
+    metadatamap = (val & 0x000000000000ffff);
+    datamap = (val & 0x00000000ffff0000) >> 16;
+    event = (val & 0xffffffff00000000) >> 32;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (metadata)
-			metadata[i] = (metadatamap >> i) & 1;
-		if (data)
-			data[i] = (datamap >> i) & 1;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (metadata)
+            metadata[i] = (metadatamap >> i) & 1;
+        if (data)
+            data[i] = (datamap >> i) & 1;
+    }
 
-	if (event_p)
-		*event_p = event;
-	return ret;
+    if (event_p)
+        *event_p = event;
+    return ret;
 }
 
-
 int
-__afr_inode_read_subvol_set_small (inode_t *inode, xlator_t *this,
-				   unsigned char *data, unsigned char *metadata,
-				   int event)
+__afr_inode_read_subvol_set_small(inode_t *inode, xlator_t *this,
+                                  unsigned char *data, unsigned char *metadata,
+                                  int event)
 {
-	afr_private_t *priv = NULL;
-	uint16_t datamap = 0;
-	uint16_t metadatamap = 0;
-	uint64_t val = 0;
-	int i = 0;
-        int ret = -1;
-        afr_inode_ctx_t *ctx = NULL;
+    afr_private_t *priv = NULL;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint64_t val = 0;
+    int i = 0;
+    int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
 
-	priv = this->private;
+    priv = this->private;
 
-        ret = __afr_inode_ctx_get (this, inode, &ctx);
-        if (ret)
-                goto out;
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret)
+        goto out;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (data[i])
-			datamap |= (1 << i);
-		if (metadata[i])
-			metadatamap |= (1 << i);
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (data[i])
+            datamap |= (1 << i);
+        if (metadata[i])
+            metadatamap |= (1 << i);
+    }
 
-	val = ((uint64_t) metadatamap) |
-		(((uint64_t) datamap) << 16) |
-		(((uint64_t) event) << 32);
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
 
-        ctx->read_subvol = val;
+    ctx->read_subvol = val;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-__afr_inode_event_gen_reset_small (inode_t *inode, xlator_t *this)
+__afr_inode_read_subvol_get(inode_t *inode, xlator_t *this, unsigned char *data,
+                            unsigned char *metadata, int *event_p)
 {
-	int               ret         = -1;
-	uint16_t          datamap     = 0;
-	uint16_t          metadatamap = 0;
-	uint32_t          event       = 0;
-	uint64_t          val         = 0;
-        afr_inode_ctx_t  *ctx         = NULL;
-
-	ret = __afr_inode_ctx_get (this, inode, &ctx);
-        if (ret)
-                return ret;
-
-        val = ctx->read_subvol;
-
-	metadatamap = (val & 0x000000000000ffff) >> 0;
-	datamap =     (val & 0x00000000ffff0000) >> 16;
-	event = 0;
+    afr_private_t *priv = NULL;
+    int ret = -1;
 
-	val = ((uint64_t) metadatamap) |
-		(((uint64_t) datamap) << 16) |
-		(((uint64_t) event) << 32);
+    priv = this->private;
 
-        ctx->read_subvol = val;
+    if (priv->child_count <= 16)
+        ret = __afr_inode_read_subvol_get_small(inode, this, data, metadata,
+                                                event_p);
+    else
+        /* TBD: allocate structure with array and read from it */
+        ret = -1;
 
-        return ret;
+    return ret;
 }
 
-
 int
-__afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
-			     unsigned char *data, unsigned char *metadata,
-			     int *event_p)
+__afr_inode_split_brain_choice_get(inode_t *inode, xlator_t *this,
+                                   int *spb_choice)
 {
-	afr_private_t *priv = NULL;
-	int ret = -1;
-
-	priv = this->private;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-	if (priv->child_count <= 16)
-		ret = __afr_inode_read_subvol_get_small (inode, this, data,
-							 metadata, event_p);
-	else
-		/* TBD: allocate structure with array and read from it */
-		ret = -1;
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret < 0)
+        return ret;
 
-	return ret;
+    *spb_choice = ctx->spb_choice;
+    return 0;
 }
 
 int
-__afr_inode_split_brain_choice_get (inode_t *inode, xlator_t *this,
-			            int *spb_choice)
+__afr_inode_read_subvol_set(inode_t *inode, xlator_t *this, unsigned char *data,
+                            unsigned char *metadata, int event)
 {
-        afr_inode_ctx_t *ctx = NULL;
-        int ret = -1;
+    afr_private_t *priv = NULL;
+    int ret = -1;
 
-        ret = __afr_inode_ctx_get (this, inode, &ctx);
-        if (ret < 0)
-                return ret;
+    priv = this->private;
 
-        *spb_choice = ctx->spb_choice;
-        return 0;
+    if (priv->child_count <= 16)
+        ret = __afr_inode_read_subvol_set_small(inode, this, data, metadata,
+                                                event);
+    else
+        ret = -1;
+
+    return ret;
 }
 
 int
-__afr_inode_read_subvol_set (inode_t *inode, xlator_t *this, unsigned char *data,
-			     unsigned char *metadata, int event)
+__afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this,
+                                   int spb_choice)
 {
-	afr_private_t *priv = NULL;
-	int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-	priv = this->private;
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret)
+        goto out;
 
-	if (priv->child_count <= 16)
-		ret = __afr_inode_read_subvol_set_small (inode, this, data,
-							 metadata, event);
-	else
-		ret = -1;
+    ctx->spb_choice = spb_choice;
 
-	return ret;
+    ret = 0;
+out:
+    return ret;
 }
 
 int
-__afr_inode_split_brain_choice_set (inode_t *inode, xlator_t *this,
-                                    int spb_choice)
+afr_inode_read_subvol_get(inode_t *inode, xlator_t *this, unsigned char *data,
+                          unsigned char *metadata, int *event_p)
 {
-        afr_inode_ctx_t *ctx = NULL;
-	int ret = -1;
+    int ret = -1;
 
-	ret = __afr_inode_ctx_get (this, inode, &ctx);
-        if (ret)
-                goto out;
-
-        ctx->spb_choice = spb_choice;
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        ret = 0;
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_read_subvol_get(inode, this, data, metadata, event_p);
+    }
+    UNLOCK(&inode->lock);
 out:
-        return ret;
+    return ret;
 }
 
 int
-__afr_inode_event_gen_reset (inode_t *inode, xlator_t *this)
+afr_inode_get_readable(call_frame_t *frame, inode_t *inode, xlator_t *this,
+                       unsigned char *readable, int *event_p, int type)
 {
-	afr_private_t *priv = NULL;
-	int ret = -1;
-
-	priv = this->private;
-
-	if (priv->child_count <= 16)
-		ret = __afr_inode_event_gen_reset_small (inode, this);
-	else
-		ret = -1;
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+    unsigned char *data = alloca0(priv->child_count);
+    unsigned char *metadata = alloca0(priv->child_count);
+    int data_count = 0;
+    int metadata_count = 0;
+    int event_generation = 0;
+    int ret = 0;
+
+    ret = afr_inode_read_subvol_get(inode, this, data, metadata,
+                                    &event_generation);
+    if (ret == -1)
+        return -EIO;
+
+    data_count = AFR_COUNT(data, priv->child_count);
+    metadata_count = AFR_COUNT(metadata, priv->child_count);
+
+    if (inode->ia_type == IA_IFDIR) {
+        /* For directories, allow even if it is in data split-brain. */
+        if (type == AFR_METADATA_TRANSACTION || local->op == GF_FOP_STAT ||
+            local->op == GF_FOP_FSTAT) {
+            if (!metadata_count)
+                return -EIO;
+        }
+    } else {
+        /* For files, abort in case of data/metadata split-brain. */
+        if (!data_count || !metadata_count) {
+            return -EIO;
+        }
+    }
 
-	return ret;
+    if (type == AFR_METADATA_TRANSACTION && readable)
+        memcpy(readable, metadata, priv->child_count * sizeof *metadata);
+    if (type == AFR_DATA_TRANSACTION && readable) {
+        if (!data_count)
+            memcpy(readable, local->child_up,
+                   priv->child_count * sizeof *readable);
+        else
+            memcpy(readable, data, priv->child_count * sizeof *data);
+    }
+    if (event_p)
+        *event_p = event_generation;
+    return 0;
 }
 
-
-int
-afr_inode_read_subvol_get (inode_t *inode, xlator_t *this, unsigned char *data,
-			   unsigned char *metadata, int *event_p)
+static int
+afr_inode_split_brain_choice_get(inode_t *inode, xlator_t *this,
+                                 int *spb_choice)
 {
-	int ret = -1;
-
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    int ret = -1;
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK(&inode->lock);
-	{
-		ret = __afr_inode_read_subvol_get (inode, this, data,
-						   metadata, event_p);
-	}
-	UNLOCK(&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_split_brain_choice_get(inode, this, spb_choice);
+    }
+    UNLOCK(&inode->lock);
 out:
-        return ret;
+    return ret;
 }
 
+/*
+ * frame is used to get the favourite policy. Since
+ * afr_inode_split_brain_choice_get was called with afr_open, it is possible to
+ * have a frame with out local->replies. So in that case, frame is passed as
+ * null, hence this function will handle the frame NULL case.
+ */
 int
-afr_inode_get_readable (call_frame_t *frame, inode_t *inode, xlator_t *this,
-                       unsigned char *readable, int *event_p, int type)
+afr_split_brain_read_subvol_get(inode_t *inode, xlator_t *this,
+                                call_frame_t *frame, int *spb_subvol)
 {
+    int ret = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        afr_private_t *priv = this->private;
-        afr_local_t *local = frame->local;
-        unsigned char *data = alloca0 (priv->child_count);
-        unsigned char *metadata = alloca0 (priv->child_count);
-        int data_count = 0;
-        int metadata_count = 0;
-        int event_generation = 0;
-        int ret = 0;
-
-        ret = afr_inode_read_subvol_get (inode, this, data, metadata,
-                                         &event_generation);
-        if (ret == -1)
-                return -EIO;
+    GF_VALIDATE_OR_GOTO("afr", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, spb_subvol, out);
 
-        data_count = AFR_COUNT (data, priv->child_count);
-        metadata_count = AFR_COUNT (metadata, priv->child_count);
+    priv = this->private;
 
-        if (inode->ia_type == IA_IFDIR) {
-                /* For directories, allow even if it is in data split-brain. */
-                if (type == AFR_METADATA_TRANSACTION ||
-                    local->op == GF_FOP_STAT || local->op == GF_FOP_FSTAT) {
-                        if (!metadata_count)
-                                return -EIO;
-                }
-        } else {
-                /* For files, abort in case of data/metadata split-brain. */
-                if (!data_count || !metadata_count) {
-                        return -EIO;
-                }
-        }
-
-        if (type == AFR_METADATA_TRANSACTION && readable)
-                memcpy (readable, metadata, priv->child_count * sizeof *metadata);
-        if (type == AFR_DATA_TRANSACTION && readable) {
-                if (!data_count)
-                        memcpy (readable, local->child_up,
-                                priv->child_count * sizeof *readable);
-                else
-                        memcpy (readable, data, priv->child_count * sizeof *data);
+    ret = afr_inode_split_brain_choice_get(inode, this, spb_subvol);
+    if (*spb_subvol < 0 && priv->fav_child_policy && frame && frame->local) {
+        local = frame->local;
+        *spb_subvol = afr_sh_get_fav_by_policy(this, local->replies, inode,
+                                               NULL);
+        if (*spb_subvol >= 0) {
+            ret = 0;
         }
-        if (event_p)
-                *event_p = event_generation;
-        return 0;
-}
+    }
 
-int
-afr_inode_split_brain_choice_get (inode_t *inode, xlator_t *this,
-                                  int *spb_choice)
-{
-	int ret = -1;
-
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-	LOCK(&inode->lock);
-	{
-		ret = __afr_inode_split_brain_choice_get (inode, this,
-                                                          spb_choice);
-	}
-	UNLOCK(&inode->lock);
 out:
-        return ret;
+    return ret;
 }
-
-
 int
-afr_inode_read_subvol_set (inode_t *inode, xlator_t *this, unsigned char *data,
-			   unsigned char *metadata, int event)
+afr_inode_read_subvol_set(inode_t *inode, xlator_t *this, unsigned char *data,
+                          unsigned char *metadata, int event)
 {
-	int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK(&inode->lock);
-	{
-		ret = __afr_inode_read_subvol_set (inode, this, data, metadata,
-						   event);
-	}
-	UNLOCK(&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_read_subvol_set(inode, this, data, metadata, event);
+    }
+    UNLOCK(&inode->lock);
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-afr_inode_split_brain_choice_set (inode_t *inode, xlator_t *this,
-                                  int spb_choice)
+afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this, int spb_choice)
 {
-	int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-	LOCK(&inode->lock);
-	{
-		ret = __afr_inode_split_brain_choice_set (inode, this,
-                                                          spb_choice);
-	}
-	UNLOCK(&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_split_brain_choice_set(inode, this, spb_choice);
+    }
+    UNLOCK(&inode->lock);
 out:
-	return ret;
+    return ret;
 }
 
-
 /* The caller of this should perform afr_inode_refresh, if this function
  * returns _gf_true
  */
 gf_boolean_t
-afr_is_inode_refresh_reqd (inode_t *inode, xlator_t *this,
-                           int event_gen1, int event_gen2)
+afr_is_inode_refresh_reqd(inode_t *inode, xlator_t *this, int event_gen1,
+                          int event_gen2)
 {
-        gf_boolean_t     need_refresh = _gf_false;
-        afr_inode_ctx_t  *ctx         = NULL;
-        int              ret          = -1;
+    gf_boolean_t need_refresh = _gf_false;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK(&inode->lock);
-        {
-                ret = __afr_inode_ctx_get (this, inode, &ctx);
-                if (ret)
-                        goto unlock;
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
+        if (ret)
+            goto unlock;
 
-                need_refresh = ctx->need_refresh;
-                /* Hoping that the caller will do inode_refresh followed by
-                 * this, hence setting the need_refresh to false */
-                ctx->need_refresh = _gf_false;
-        }
+        need_refresh = ctx->need_refresh;
+        /* Hoping that the caller will do inode_refresh followed by
+         * this, hence setting the need_refresh to false */
+        ctx->need_refresh = _gf_false;
+    }
 unlock:
-        UNLOCK(&inode->lock);
+    UNLOCK(&inode->lock);
 
-        if (event_gen1 != event_gen2)
-                need_refresh = _gf_true;
+    if (event_gen1 != event_gen2)
+        need_refresh = _gf_true;
 out:
-        return need_refresh;
+    return need_refresh;
 }
 
-
-static int
-afr_inode_need_refresh_set (inode_t *inode, xlator_t *this)
+int
+__afr_inode_need_refresh_set(inode_t *inode, xlator_t *this)
 {
-        int               ret         = -1;
-        afr_inode_ctx_t  *ctx         = NULL;
-
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
 
-        LOCK(&inode->lock);
-        {
-                ret = __afr_inode_ctx_get (this, inode, &ctx);
-                if (ret)
-                        goto unlock;
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret == 0) {
+        ctx->need_refresh = _gf_true;
+    }
 
-                ctx->need_refresh = _gf_true;
-        }
-unlock:
-        UNLOCK(&inode->lock);
-out:
-        return ret;
+    return ret;
 }
 
 int
-afr_inode_event_gen_reset (inode_t *inode, xlator_t *this)
+afr_inode_need_refresh_set(inode_t *inode, xlator_t *this)
 {
-	int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK(&inode->lock);
-        {
-                ret = __afr_inode_event_gen_reset (inode, this);
-        }
-        UNLOCK(&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_need_refresh_set(inode, this);
+    }
+    UNLOCK(&inode->lock);
 out:
-	return ret;
+    return ret;
 }
 
 int
-afr_spb_choice_timeout_cancel (xlator_t *this, inode_t *inode)
+afr_spb_choice_timeout_cancel(xlator_t *this, inode_t *inode)
 {
-        afr_inode_ctx_t *ctx    = NULL;
-        int              ret    = -1;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        if (!inode)
-                return ret;
+    if (!inode)
+        return ret;
 
-        LOCK(&inode->lock);
-        {
-                ret = __afr_inode_ctx_get (this, inode, &ctx);
-                if (ret < 0 || !ctx) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
-                                "Failed to cancel split-brain choice timer.");
-                        goto out;
-                }
-                ctx->spb_choice = -1;
-                if (ctx->timer) {
-                        gf_timer_call_cancel (this->ctx, ctx->timer);
-                        ctx->timer = NULL;
-                }
-                ret = 0;
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
+        if (ret < 0 || !ctx) {
+            UNLOCK(&inode->lock);
+            gf_msg(this->name, GF_LOG_WARNING, 0,
+                   AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+                   "Failed to cancel split-brain choice timer.");
+            goto out;
+        }
+        ctx->spb_choice = -1;
+        if (ctx->timer) {
+            gf_timer_call_cancel(this->ctx, ctx->timer);
+            ctx->timer = NULL;
         }
+        ret = 0;
+    }
+    UNLOCK(&inode->lock);
 out:
-        UNLOCK(&inode->lock);
-        return ret;
+    return ret;
 }
 
 void
-afr_set_split_brain_choice_cbk (void *data)
+afr_set_split_brain_choice_cbk(void *data)
 {
-        inode_t      *inode     = data;
-        xlator_t     *this      = THIS;
+    inode_t *inode = data;
+    xlator_t *this = THIS;
 
-        afr_spb_choice_timeout_cancel (this, inode);
-        inode_invalidate (inode);
-        inode_unref (inode);
-        return;
+    afr_spb_choice_timeout_cancel(this, inode);
+    inode_invalidate(inode);
+    inode_unref(inode);
+    return;
 }
 
-
 int
-afr_set_split_brain_choice (int ret, call_frame_t *frame, void *opaque)
-{
-        int                 op_errno         = ENOMEM;
-        afr_private_t      *priv             = NULL;
-        afr_inode_ctx_t    *ctx              = NULL;
-        inode_t            *inode            = NULL;
-        loc_t              *loc              = NULL;
-        xlator_t           *this             = NULL;
-        afr_spbc_timeout_t *data             = opaque;
-        struct              timespec delta   = {0, };
-        gf_boolean_t        timer_set        = _gf_false;
-        gf_boolean_t        timer_cancelled  = _gf_false;
-        gf_boolean_t        timer_reset      = _gf_false;
-        gf_boolean_t        need_invalidate  = _gf_true;
-        int                 old_spb_choice   = -1;
-
-        frame = data->frame;
-        loc = data->loc;
-        this = frame->this;
-        priv = this->private;
-
+afr_set_split_brain_choice(int ret, call_frame_t *frame, void *opaque)
+{
+    int op_errno = ENOMEM;
+    afr_private_t *priv = NULL;
+    afr_inode_ctx_t *ctx = NULL;
+    inode_t *inode = NULL;
+    loc_t *loc = NULL;
+    xlator_t *this = NULL;
+    afr_spbc_timeout_t *data = opaque;
+    struct timespec delta = {
+        0,
+    };
+    gf_boolean_t timer_set = _gf_false;
+    gf_boolean_t timer_cancelled = _gf_false;
+    gf_boolean_t timer_reset = _gf_false;
+    int old_spb_choice = -1;
+
+    frame = data->frame;
+    loc = data->loc;
+    this = frame->this;
+    priv = this->private;
+
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    delta.tv_sec = priv->spb_choice_timeout;
+    delta.tv_nsec = 0;
+
+    if (!loc->inode) {
+        ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    if (!(data->d_spb || data->m_spb)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+               "Cannot set "
+               "replica.split-brain-choice on %s. File is"
+               " not in data/metadata split-brain.",
+               uuid_utoa(loc->gfid));
+        ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /*
+     * we're ref'ing the inode before LOCK like it is done elsewhere in the
+     * code. If we ref after LOCK, coverity complains of possible deadlocks.
+     */
+    inode = inode_ref(loc->inode);
+
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
         if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-
-        delta.tv_sec = priv->spb_choice_timeout;
-        delta.tv_nsec = 0;
-
-        if (!loc->inode) {
-                ret = -1;
-                op_errno = EINVAL;
-                goto out;
+            UNLOCK(&inode->lock);
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+                   "Failed to get inode_ctx for %s", loc->name);
+            goto post_unlock;
         }
 
-        if (!(data->d_spb || data->m_spb)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR, "Cannot set "
-                        "replica.split-brain-choice on %s. File is"
-                        " not in data/metadata split-brain.",
-                        uuid_utoa (loc->gfid));
-                ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+        old_spb_choice = ctx->spb_choice;
+        ctx->spb_choice = data->spb_child_index;
 
-        /*
-         * we're ref'ing the inode before LOCK like it is done elsewhere in the
-         * code. If we ref after LOCK, coverity complains of possible deadlocks.
+        /* Possible changes in spb-choice :
+         *         valid to -1    : cancel timer and unref
+         *         valid to valid : cancel timer and inject new one
+         *         -1    to -1    : unref and do not do anything
+         *         -1 to valid    : inject timer
          */
-        inode = inode_ref (loc->inode);
 
-        LOCK(&inode->lock);
-        {
-                ret = __afr_inode_ctx_get (this, inode, &ctx);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
-                                "Failed to get inode_ctx for %s", loc->name);
-                        goto unlock;
+        /* ctx->timer is NULL iff previous value of
+         * ctx->spb_choice is -1
+         */
+        if (ctx->timer) {
+            if (ctx->spb_choice == -1) {
+                if (!gf_timer_call_cancel(this->ctx, ctx->timer)) {
+                    ctx->timer = NULL;
+                    timer_cancelled = _gf_true;
                 }
-
-                old_spb_choice = ctx->spb_choice;
-                ctx->spb_choice = data->spb_child_index;
-
-                /* Possible changes in spb-choice :
-                 *         valid to -1    : cancel timer and unref
-                 *         valid to valid : cancel timer and inject new one
-                 *         -1    to -1    : unref and do not do anything
-                 *         -1 to valid    : inject timer
-                 */
-
-                /* ctx->timer is NULL iff previous value of
-                 * ctx->spb_choice is -1
+                /* If timer cancel failed here it means that the
+                 *  previous cbk will be executed which will set
+                 *  spb_choice to -1. So we can consider the
+                 *  'valid to -1' case to be a success
+                 *  (i.e. ret = 0) and goto unlock.
                  */
-                if (ctx->timer) {
-                        if (ctx->spb_choice == -1) {
-                                if (!gf_timer_call_cancel (this->ctx,
-                                                            ctx->timer)) {
-                                        ctx->timer = NULL;
-                                        timer_cancelled = _gf_true;
-                                }
-                                /* If timer cancel failed here it means that the
-                                *  previous cbk will be executed which will set
-                                *  spb_choice to -1. So we can consider the
-                                *  'valid to -1' case to be a sucess
-                                *  (i.e. ret = 0) and goto unlock.
-                                */
-                                goto unlock;
-                        }
-                        goto reset_timer;
-                } else {
-                        if (ctx->spb_choice == -1)
-                                goto unlock;
-                        goto set_timer;
-                }
-
-reset_timer:
-                ret = gf_timer_call_cancel (this->ctx, ctx->timer);
-                if (ret != 0) {
-                        /* We need to bail out now instead of launching a new
-                         * timer. Otherwise the cbk of the previous timer event
-                         * will cancel the new ctx->timer.
-                         */
-                        ctx->spb_choice = old_spb_choice;
-                        ret = -1;
-                        op_errno = EAGAIN;
-                        goto unlock;
-                }
-                ctx->timer = NULL;
-                timer_reset = _gf_true;
-
-set_timer:
-                ctx->timer = gf_timer_call_after (this->ctx, delta,
-                                                  afr_set_split_brain_choice_cbk,
-                                                  inode);
-                if (!ctx->timer) {
-                        ctx->spb_choice = old_spb_choice;
-                        ret = -1;
-                        op_errno = ENOMEM;
-                }
-                if (!timer_reset && ctx->timer)
-                        timer_set = _gf_true;
-                if (timer_reset && !ctx->timer)
-                        timer_cancelled = _gf_true;
-                need_invalidate = _gf_false;
+                goto unlock;
+            }
+            goto reset_timer;
+        } else {
+            if (ctx->spb_choice == -1)
+                goto unlock;
+            goto set_timer;
         }
+
+    reset_timer:
+        ret = gf_timer_call_cancel(this->ctx, ctx->timer);
+        if (ret != 0) {
+            /* We need to bail out now instead of launching a new
+             * timer. Otherwise the cbk of the previous timer event
+             * will cancel the new ctx->timer.
+             */
+            ctx->spb_choice = old_spb_choice;
+            ret = -1;
+            op_errno = EAGAIN;
+            goto unlock;
+        }
+        ctx->timer = NULL;
+        timer_reset = _gf_true;
+
+    set_timer:
+        ctx->timer = gf_timer_call_after(this->ctx, delta,
+                                         afr_set_split_brain_choice_cbk, inode);
+        if (!ctx->timer) {
+            ctx->spb_choice = old_spb_choice;
+            ret = -1;
+            op_errno = ENOMEM;
+        }
+        if (!timer_reset && ctx->timer)
+            timer_set = _gf_true;
+        if (timer_reset && !ctx->timer)
+            timer_cancelled = _gf_true;
+    }
 unlock:
-        UNLOCK(&inode->lock);
-        if (!timer_set)
-                inode_unref (inode);
-        if (timer_cancelled)
-                inode_unref (inode);
-        /*
-         * We need to invalidate the inode to prevent the kernel from serving
-         * reads from an older cached value despite a change in spb_choice to
-         * a new value.
-         */
-        if (need_invalidate)
-                inode_invalidate (inode);
+    UNLOCK(&inode->lock);
+post_unlock:
+    if (!timer_set)
+        inode_unref(inode);
+    if (timer_cancelled)
+        inode_unref(inode);
+    /*
+     * We need to invalidate the inode to prevent the kernel from serving
+     * reads from an older cached value despite a change in spb_choice to
+     * a new value.
+     */
+    inode_invalidate(inode);
 out:
-        GF_FREE (data);
-        AFR_STACK_UNWIND (setxattr, frame, ret, op_errno, NULL);
-        return 0;
+    GF_FREE(data);
+    AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    return 0;
 }
 
 int
-afr_accused_fill (xlator_t *this, dict_t *xdata, unsigned char *accused,
-		  afr_transaction_type type)
+afr_accused_fill(xlator_t *this, dict_t *xdata, unsigned char *accused,
+                 afr_transaction_type type)
 {
-	afr_private_t *priv = NULL;
-	int i = 0;
-	int idx = afr_index_for_transaction_type (type);
-	void *pending_raw = NULL;
-	int pending[3];
-	int ret = 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int idx = afr_index_for_transaction_type(type);
+    void *pending_raw = NULL;
+    int pending[3];
+    int ret = 0;
 
-	priv = this->private;
+    priv = this->private;
 
-	for (i = 0; i < priv->child_count; i++) {
-		ret = dict_get_ptr (xdata, priv->pending_key[i],
-				    &pending_raw);
-		if (ret) /* no pending flags */
-			continue;
-		memcpy (pending, pending_raw, sizeof(pending));
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_get_ptr(xdata, priv->pending_key[i], &pending_raw);
+        if (ret) /* no pending flags */
+            continue;
+        memcpy(pending, pending_raw, sizeof(pending));
 
-		if (ntoh32 (pending[idx]))
-			accused[i] = 1;
-	}
+        if (ntoh32(pending[idx]))
+            accused[i] = 1;
+    }
 
-	return 0;
+    return 0;
 }
 
 int
-afr_accuse_smallfiles (xlator_t *this, struct afr_reply *replies,
-		       unsigned char *data_accused)
+afr_accuse_smallfiles(xlator_t *this, struct afr_reply *replies,
+                      unsigned char *data_accused)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-	uint64_t maxsize = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint64_t maxsize = 0;
 
-	priv = this->private;
+    priv = this->private;
 
-	for (i = 0; i < priv->child_count; i++) {
-                if (replies[i].valid && replies[i].xdata &&
-                    dict_get (replies[i].xdata, GLUSTERFS_BAD_INODE))
-                        continue;
-		if (data_accused[i])
-			continue;
-		if (replies[i].poststat.ia_size > maxsize)
-			maxsize = replies[i].poststat.ia_size;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].xdata &&
+            dict_get_sizen(replies[i].xdata, GLUSTERFS_BAD_INODE))
+            continue;
+        if (data_accused[i])
+            continue;
+        if (replies[i].poststat.ia_size > maxsize)
+            maxsize = replies[i].poststat.ia_size;
+    }
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (data_accused[i])
-			continue;
-                if (AFR_IS_ARBITER_BRICK(priv, i))
-                        continue;
-		if (replies[i].poststat.ia_size < maxsize)
-			data_accused[i] = 1;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_accused[i])
+            continue;
+        if (AFR_IS_ARBITER_BRICK(priv, i))
+            continue;
+        if (replies[i].poststat.ia_size < maxsize)
+            data_accused[i] = 1;
+    }
 
-	return 0;
+    return 0;
 }
 
 int
-afr_readables_fill (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                    unsigned char *data_accused,
-                    unsigned char *metadata_accused,
-                    unsigned char *data_readable,
-                    unsigned char *metadata_readable,
-                    struct afr_reply *replies)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        dict_t *xdata = NULL;
-        int i = 0;
-        int ret = 0;
-        ia_type_t ia_type = IA_INVAL;
-
-        local = frame->local;
-        priv = this->private;
+afr_readables_fill(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                   unsigned char *data_accused, unsigned char *metadata_accused,
+                   unsigned char *data_readable,
+                   unsigned char *metadata_readable, struct afr_reply *replies)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
+    int ret = 0;
+    ia_type_t ia_type = IA_INVAL;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        data_readable[i] = 1;
+        metadata_readable[i] = 1;
+    }
+    if (AFR_IS_ARBITER_BRICK(priv, ARBITER_BRICK_INDEX)) {
+        data_readable[ARBITER_BRICK_INDEX] = 0;
+        metadata_readable[ARBITER_BRICK_INDEX] = 0;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies) { /* Lookup */
+            if (!replies[i].valid || replies[i].op_ret == -1 ||
+                (replies[i].xdata &&
+                 dict_get_sizen(replies[i].xdata, GLUSTERFS_BAD_INODE))) {
+                data_readable[i] = 0;
+                metadata_readable[i] = 0;
+                continue;
+            }
 
-        for (i = 0; i < priv->child_count; i++) {
-                data_readable[i] = 1;
-                metadata_readable[i] = 1;
-        }
-        if (AFR_IS_ARBITER_BRICK (priv, ARBITER_BRICK_INDEX)) {
-                data_readable[ARBITER_BRICK_INDEX] =  0;
-                metadata_readable[ARBITER_BRICK_INDEX] = 0;
+            xdata = replies[i].xdata;
+            ia_type = replies[i].poststat.ia_type;
+        } else { /* pre-op xattrop */
+            xdata = local->transaction.changelog_xdata[i];
+            ia_type = inode->ia_type;
         }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (replies) {/* Lookup */
-                        if (!replies[i].valid || replies[i].op_ret == -1 ||
-                            (replies[i].xdata && dict_get (replies[i].xdata,
-                                                        GLUSTERFS_BAD_INODE))) {
-                                data_readable[i] = 0;
-                                metadata_readable[i] = 0;
-                                continue;
-                        }
-
-                        xdata = replies[i].xdata;
-                        ia_type = replies[i].poststat.ia_type;
-                } else {/* pre-op xattrop */
-                        xdata = local->transaction.changelog_xdata[i];
-                        ia_type = inode->ia_type;
-                }
+        if (!xdata)
+            continue; /* mkdir_cbk sends NULL xdata_rsp. */
+        afr_accused_fill(this, xdata, data_accused,
+                         (ia_type == IA_IFDIR) ? AFR_ENTRY_TRANSACTION
+                                               : AFR_DATA_TRANSACTION);
 
-                afr_accused_fill (this, xdata, data_accused,
-                                  (ia_type == IA_IFDIR) ?
-                                  AFR_ENTRY_TRANSACTION : AFR_DATA_TRANSACTION);
+        afr_accused_fill(this, xdata, metadata_accused,
+                         AFR_METADATA_TRANSACTION);
+    }
 
-                afr_accused_fill (this, xdata,
-                                  metadata_accused, AFR_METADATA_TRANSACTION);
-        }
+    if (replies && ia_type != IA_INVAL && ia_type != IA_IFDIR &&
+        /* We want to accuse small files only when we know for
+         * sure that there is no IO happening. Otherwise, the
+         * ia_sizes obtained in post-refresh replies may
+         * mismatch due to a race between inode-refresh and
+         * ongoing writes, causing spurious heal launches*/
+        !afr_is_possibly_under_txn(AFR_DATA_TRANSACTION, local, this)) {
+        afr_accuse_smallfiles(this, replies, data_accused);
+    }
 
-        if (replies && ia_type != IA_INVAL && ia_type != IA_IFDIR &&
-            /* We want to accuse small files only when we know for
-             * sure that there is no IO happening. Otherwise, the
-             * ia_sizes obtained in post-refresh replies may
-             * mismatch due to a race between inode-refresh and
-             * ongoing writes, causing spurious heal launches*/
-            !afr_is_possibly_under_txn (AFR_DATA_TRANSACTION, local, this)) {
-                afr_accuse_smallfiles (this, replies, data_accused);
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_accused[i]) {
+            data_readable[i] = 0;
+            ret = 1;
         }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (data_accused[i]) {
-                        data_readable[i] = 0;
-                        ret = 1;
-                }
-                if (metadata_accused[i]) {
-                        metadata_readable[i] = 0;
-                        ret = 1;
-                }
+        if (metadata_accused[i]) {
+            metadata_readable[i] = 0;
+            ret = 1;
         }
-        return ret;
+    }
+    return ret;
 }
 
 int
-afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                       gf_boolean_t *start_heal)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	struct afr_reply *replies = NULL;
-	int event_generation = 0;
-	int i = 0;
-	unsigned char *data_accused = NULL;
-	unsigned char *metadata_accused = NULL;
-	unsigned char *data_readable = NULL;
-	unsigned char *metadata_readable = NULL;
-	int ret = 0;
-
-	local = frame->local;
-	priv = this->private;
-	replies = local->replies;
-	event_generation = local->event_generation;
-
-	data_accused = alloca0 (priv->child_count);
-	data_readable = alloca0 (priv->child_count);
-	metadata_accused = alloca0 (priv->child_count);
-	metadata_readable = alloca0 (priv->child_count);
-
-        ret = afr_readables_fill (frame, this, inode, data_accused,
-                                  metadata_accused, data_readable,
-                                  metadata_readable, replies);
-
-	for (i = 0; i < priv->child_count; i++) {
-                if (start_heal && priv->child_up[i] &&
-                    (data_accused[i] || metadata_accused[i])) {
-                        *start_heal = _gf_true;
-                        break;
-                }
-        }
-	afr_inode_read_subvol_set (inode, this, data_readable,
-				   metadata_readable, event_generation);
-	return ret;
+afr_replies_interpret(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                      gf_boolean_t *start_heal)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int event_generation = 0;
+    int i = 0;
+    unsigned char *data_accused = NULL;
+    unsigned char *metadata_accused = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    priv = this->private;
+    replies = local->replies;
+    event_generation = local->event_generation;
+
+    data_accused = alloca0(priv->child_count);
+    data_readable = alloca0(priv->child_count);
+    metadata_accused = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+
+    ret = afr_readables_fill(frame, this, inode, data_accused, metadata_accused,
+                             data_readable, metadata_readable, replies);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (start_heal && priv->child_up[i] &&
+            (data_accused[i] || metadata_accused[i])) {
+            *start_heal = _gf_true;
+            break;
+        }
+    }
+    afr_inode_read_subvol_set(inode, this, data_readable, metadata_readable,
+                              event_generation);
+    return ret;
 }
 
 int
-afr_refresh_selfheal_done (int ret, call_frame_t *heal, void *opaque)
+afr_refresh_selfheal_done(int ret, call_frame_t *heal, void *opaque)
 {
-	if (heal)
-                AFR_STACK_DESTROY (heal);
-	return 0;
+    if (heal)
+        AFR_STACK_DESTROY(heal);
+    return 0;
 }
 
 int
-afr_inode_refresh_err (call_frame_t *frame, xlator_t *this)
+afr_inode_refresh_err(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-	int err = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int err = 0;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->replies[i].valid && !local->replies[i].op_ret) {
-			err = 0;
-			goto ret;
-		}
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && !local->replies[i].op_ret) {
+            err = 0;
+            goto ret;
+        }
+    }
 
-	err = afr_final_errno (local, priv);
+    err = afr_final_errno(local, priv);
 ret:
-	return -err;
+    return err;
 }
 
 gf_boolean_t
-afr_selfheal_enabled (xlator_t *this)
+afr_selfheal_enabled(const xlator_t *this)
 {
-	afr_private_t *priv = NULL;
-	gf_boolean_t data = _gf_false;
-        int ret = 0;
-
-	priv = this->private;
+    const afr_private_t *priv = this->private;
 
-	ret = gf_string2boolean (priv->data_self_heal, &data);
-        GF_ASSERT (!ret);
-
-	return data || priv->metadata_self_heal || priv->entry_self_heal;
+    return priv->data_self_heal || priv->metadata_self_heal ||
+           priv->entry_self_heal;
 }
 
-
 int
-afr_txn_refresh_done (call_frame_t *frame, xlator_t *this, int err)
+afr_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
 {
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *heal_local = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    int event_generation = 0;
+    int read_subvol = -1;
+    int ret = 0;
 
-        call_frame_t *heal_frame = NULL;
-        afr_local_t *heal_local = NULL;
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        inode_t *inode = NULL;
-        int event_generation = 0;
-        int read_subvol = -1;
-        int op_errno = ENOMEM;
-        int ret = 0;
-
-        local = frame->local;
-        inode = local->inode;
-        priv = this->private;
+    local = frame->local;
+    inode = local->inode;
+    priv = this->private;
 
-        if (err)
-                goto refresh_done;
+    if (err)
+        goto refresh_done;
 
-        if (local->op == GF_FOP_LOOKUP)
-                goto refresh_done;
+    if (local->op == GF_FOP_LOOKUP)
+        goto refresh_done;
 
-        ret = afr_inode_get_readable (frame, inode, this, local->readable,
-                                      &event_generation,
-                                      local->transaction.type);
+    ret = afr_inode_get_readable(frame, inode, this, local->readable,
+                                 &event_generation, local->transaction.type);
 
-        if (ret == -EIO || (local->is_read_txn && !event_generation)) {
-                /* No readable subvolume even after refresh ==> splitbrain.*/
-                if (!priv->fav_child_policy) {
-                        err = -EIO;
-                        goto refresh_done;
-                }
-                read_subvol = afr_sh_get_fav_by_policy (this, local->replies,
-                                                        inode, NULL);
-                if (read_subvol == -1) {
-                        err = -EIO;
-                        goto refresh_done;
-                }
+    if (ret == -EIO) {
+        /* No readable subvolume even after refresh ==> splitbrain.*/
+        if (!priv->fav_child_policy) {
+            err = EIO;
+            goto refresh_done;
+        }
+        read_subvol = afr_sh_get_fav_by_policy(this, local->replies, inode,
+                                               NULL);
+        if (read_subvol == -1) {
+            err = EIO;
+            goto refresh_done;
+        }
 
-                heal_frame = copy_frame (frame);
-                if (!heal_frame) {
-                        err = -EIO;
-                        goto refresh_done;
-                }
-                heal_frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
-                heal_local = AFR_FRAME_INIT (heal_frame, op_errno);
-                if (!heal_local) {
-                        err = -EIO;
-                        AFR_STACK_DESTROY (heal_frame);
-                        goto refresh_done;
-                }
-                heal_local->xdata_req = dict_new();
-                if (!heal_local->xdata_req) {
-                        err = -EIO;
-                        AFR_STACK_DESTROY (heal_frame);
-                        goto refresh_done;
-                }
-                heal_local->heal_frame = frame;
-                ret = synctask_new (this->ctx->env,
-                                    afr_fav_child_reset_sink_xattrs,
-                                    afr_fav_child_reset_sink_xattrs_cbk,
-                                    heal_frame,
-                                    heal_frame);
-                return 0;
+        heal_frame = afr_frame_create(this, NULL);
+        if (!heal_frame) {
+            err = EIO;
+            goto refresh_done;
         }
+        heal_local = heal_frame->local;
+        heal_local->xdata_req = dict_new();
+        if (!heal_local->xdata_req) {
+            err = EIO;
+            AFR_STACK_DESTROY(heal_frame);
+            goto refresh_done;
+        }
+        heal_local->heal_frame = frame;
+        ret = synctask_new(this->ctx->env, afr_fav_child_reset_sink_xattrs,
+                           afr_fav_child_reset_sink_xattrs_cbk, heal_frame,
+                           heal_frame);
+        return 0;
+    }
 
 refresh_done:
-        afr_local_replies_wipe (local, this->private);
-        local->refreshfn (frame, this, err);
+    afr_local_replies_wipe(local, this->private);
+    local->refreshfn(frame, this, err);
 
-        return 0;
+    return 0;
 }
 
 int
-afr_inode_refresh_done (call_frame_t *frame, xlator_t *this, int error)
-{
-	call_frame_t *heal_frame = NULL;
-	afr_local_t *local = NULL;
-        gf_boolean_t start_heal = _gf_false;
-        afr_local_t *heal_local = NULL;
-        int op_errno = ENOMEM;
-	int ret = 0;
-	int err = 0;
-
-	if (error != 0) {
-		err = error;
-		goto refresh_done;
-	}
-
-	local = frame->local;
-
-	ret = afr_replies_interpret (frame, this, local->refreshinode,
-                                     &start_heal);
-
-	err = afr_inode_refresh_err (frame, this);
-
-	if (ret && afr_selfheal_enabled (this) && start_heal) {
-                heal_frame = copy_frame (frame);
-                if (!heal_frame)
-                        goto refresh_done;
-                heal_frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
-                heal_local = AFR_FRAME_INIT (heal_frame, op_errno);
-                if (!heal_local) {
-                        AFR_STACK_DESTROY (heal_frame);
-                        goto refresh_done;
-                }
-                heal_local->refreshinode = inode_ref (local->refreshinode);
-                heal_local->heal_frame = heal_frame;
-                if (!afr_throttled_selfheal (heal_frame, this)) {
-                        AFR_STACK_DESTROY (heal_frame);
-                        goto refresh_done;
-                }
+afr_inode_refresh_done(call_frame_t *frame, xlator_t *this, int error)
+{
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t start_heal = _gf_false;
+    afr_local_t *heal_local = NULL;
+    unsigned char *success_replies = NULL;
+    int ret = 0;
+
+    if (error != 0) {
+        goto refresh_done;
+    }
+
+    local = frame->local;
+    priv = this->private;
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
+
+    if (priv->thin_arbiter_count && local->is_read_txn &&
+        AFR_COUNT(success_replies, priv->child_count) != priv->child_count) {
+        /* We need to query the good bricks and/or thin-arbiter.*/
+        if (success_replies[0]) {
+            local->read_txn_query_child = AFR_CHILD_ZERO;
+        } else if (success_replies[1]) {
+            local->read_txn_query_child = AFR_CHILD_ONE;
+        }
+        error = EINVAL;
+        goto refresh_done;
+    }
+
+    if (!afr_has_quorum(success_replies, this, frame)) {
+        error = afr_final_errno(frame->local, this->private);
+        if (!error)
+            error = afr_quorum_errno(priv);
+        goto refresh_done;
+    }
+
+    ret = afr_replies_interpret(frame, this, local->refreshinode, &start_heal);
+
+    if (ret && afr_selfheal_enabled(this) && start_heal) {
+        heal_frame = afr_frame_create(this, NULL);
+        if (!heal_frame)
+            goto refresh_done;
+        heal_local = heal_frame->local;
+        heal_local->refreshinode = inode_ref(local->refreshinode);
+        heal_local->heal_frame = heal_frame;
+        if (!afr_throttled_selfheal(heal_frame, this)) {
+            AFR_STACK_DESTROY(heal_frame);
+            goto refresh_done;
         }
+    }
 
 refresh_done:
-        afr_txn_refresh_done (frame, this, err);
+    afr_txn_refresh_done(frame, this, error);
 
-	return 0;
+    return 0;
 }
 
 void
-afr_inode_refresh_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int op_ret, int op_errno, struct iatt *buf,
-                              dict_t *xdata, struct iatt *par)
-{
-        afr_local_t *local = NULL;
-        int call_child = (long) cookie;
-        int8_t need_heal = 1;
-        int call_count = 0;
-        GF_UNUSED int ret = 0;
+afr_inode_refresh_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, struct iatt *buf,
+                             dict_t *xdata, struct iatt *par)
+{
+    afr_local_t *local = NULL;
+    int call_child = (long)cookie;
+    int8_t need_heal = 1;
+    int call_count = 0;
+    int ret = 0;
+
+    local = frame->local;
+    local->replies[call_child].valid = 1;
+    local->replies[call_child].op_ret = op_ret;
+    local->replies[call_child].op_errno = op_errno;
+    if (op_ret != -1) {
+        local->replies[call_child].poststat = *buf;
+        if (par)
+            local->replies[call_child].postparent = *par;
+        if (xdata)
+            local->replies[call_child].xdata = dict_ref(xdata);
+    }
 
-        local = frame->local;
-        local->replies[call_child].valid = 1;
-        local->replies[call_child].op_ret = op_ret;
-        local->replies[call_child].op_errno = op_errno;
-	if (op_ret != -1) {
-		local->replies[call_child].poststat = *buf;
-		if (par)
-                        local->replies[call_child].postparent = *par;
-                if (xdata)
-		        local->replies[call_child].xdata = dict_ref (xdata);
-	}
-        if (xdata) {
-                ret = dict_get_int8 (xdata, "link-count", &need_heal);
-                local->replies[call_child].need_heal = need_heal;
-        } else {
-                local->replies[call_child].need_heal = need_heal;
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        if (ret) {
+            gf_msg_debug(this->name, -ret, "Unable to get link count");
         }
+    }
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                afr_set_need_heal (this, local);
-		afr_inode_refresh_done (frame, this, 0);
+    local->replies[call_child].need_heal = need_heal;
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        ret = afr_inode_refresh_err(frame, this);
+        if (ret) {
+            gf_msg_debug(this->name, ret, "afr_inode_refresh_err failed");
         }
-
+        afr_inode_refresh_done(frame, this, ret);
+    }
 }
 
 int
-afr_inode_refresh_subvol_with_lookup_cbk (call_frame_t *frame, void *cookie,
-                                          xlator_t *this, int op_ret,
-                                          int op_errno, inode_t *inode,
-                                          struct iatt *buf, dict_t *xdata,
-                                          struct iatt *par)
+afr_inode_refresh_subvol_with_lookup_cbk(call_frame_t *frame, void *cookie,
+                                         xlator_t *this, int op_ret,
+                                         int op_errno, inode_t *inode,
+                                         struct iatt *buf, dict_t *xdata,
+                                         struct iatt *par)
 {
-        afr_inode_refresh_subvol_cbk (frame, cookie, this, op_ret, op_errno,
-                                      buf, xdata, par);
-	return 0;
+    afr_inode_refresh_subvol_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                                 xdata, par);
+    return 0;
 }
 
-
 int
-afr_inode_refresh_subvol_with_lookup (call_frame_t *frame, xlator_t *this,
-                                      int i, inode_t *inode, uuid_t gfid,
-                                      dict_t *xdata)
+afr_inode_refresh_subvol_with_lookup(call_frame_t *frame, xlator_t *this, int i,
+                                     inode_t *inode, uuid_t gfid, dict_t *xdata)
 {
-	loc_t loc = {0, };
-	afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    afr_private_t *priv = NULL;
 
-	priv = this->private;
+    priv = this->private;
 
-	loc.inode = inode;
-        if (gf_uuid_is_null (inode->gfid) && gfid) {
-                /* To handle setattr/setxattr on yet to be linked inode from
-                 * dht */
-                gf_uuid_copy (loc.gfid, gfid);
-        } else {
-                gf_uuid_copy (loc.gfid, inode->gfid);
-        }
+    loc.inode = inode;
+    if (gf_uuid_is_null(inode->gfid) && gfid) {
+        /* To handle setattr/setxattr on yet to be linked inode from
+         * dht */
+        gf_uuid_copy(loc.gfid, gfid);
+    } else {
+        gf_uuid_copy(loc.gfid, inode->gfid);
+    }
 
-	STACK_WIND_COOKIE (frame, afr_inode_refresh_subvol_with_lookup_cbk,
-			   (void *) (long) i, priv->children[i],
-			   priv->children[i]->fops->lookup, &loc, xdata);
-	return 0;
+    STACK_WIND_COOKIE(frame, afr_inode_refresh_subvol_with_lookup_cbk,
+                      (void *)(long)i, priv->children[i],
+                      priv->children[i]->fops->lookup, &loc, xdata);
+    return 0;
 }
 
 int
-afr_inode_refresh_subvol_with_fstat_cbk (call_frame_t *frame,
-                                         void *cookie, xlator_t *this,
-                                         int32_t op_ret, int32_t op_errno,
-                                         struct iatt *buf, dict_t *xdata)
+afr_inode_refresh_subvol_with_fstat_cbk(call_frame_t *frame, void *cookie,
+                                        xlator_t *this, int32_t op_ret,
+                                        int32_t op_errno, struct iatt *buf,
+                                        dict_t *xdata)
 {
-        afr_inode_refresh_subvol_cbk (frame, cookie, this, op_ret, op_errno,
-                                      buf, xdata, NULL);
-        return 0;
+    afr_inode_refresh_subvol_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                                 xdata, NULL);
+    return 0;
 }
 
 int
-afr_inode_refresh_subvol_with_fstat (call_frame_t *frame, xlator_t *this, int i,
-			             dict_t *xdata)
+afr_inode_refresh_subvol_with_fstat(call_frame_t *frame, xlator_t *this, int i,
+                                    dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        priv = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        STACK_WIND_COOKIE (frame, afr_inode_refresh_subvol_with_fstat_cbk,
-                           (void *) (long) i, priv->children[i],
-                           priv->children[i]->fops->fstat, local->fd, xdata);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_inode_refresh_subvol_with_fstat_cbk,
+                      (void *)(long)i, priv->children[i],
+                      priv->children[i]->fops->fstat, local->fd, xdata);
+    return 0;
 }
 
 int
-afr_inode_refresh_do (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int call_count = 0;
-	int i = 0;
-        int ret = 0;
-	dict_t *xdata = NULL;
-        afr_fd_ctx_t  *fd_ctx = NULL;
-        unsigned char *wind_subvols = NULL;
-
-	priv = this->private;
-	local = frame->local;
-        wind_subvols = alloca0 (priv->child_count);
-
-        afr_local_replies_wipe (local, priv);
-
-        if (local->fd) {
-                fd_ctx = afr_fd_ctx_get (local->fd, this);
-                if (!fd_ctx) {
-                        afr_inode_refresh_done (frame, this, EINVAL);
-                        return 0;
-                }
-        }
+afr_inode_refresh_do(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int i = 0;
+    int ret = 0;
+    dict_t *xdata = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    unsigned char *wind_subvols = NULL;
 
-	xdata = dict_new ();
-	if (!xdata) {
-		afr_inode_refresh_done (frame, this, ENOMEM);
-		return 0;
-	}
+    priv = this->private;
+    local = frame->local;
+    wind_subvols = alloca0(priv->child_count);
 
-	ret = afr_xattr_req_prepare (this, xdata);
-	if (ret != 0) {
-		dict_unref (xdata);
-		afr_inode_refresh_done (frame, this, -ret);
-		return 0;
-	}
+    afr_local_replies_wipe(local, priv);
 
-        ret = dict_set_str (xdata, "link-count", GF_XATTROP_INDEX_COUNT);
-        if (ret) {
-                gf_msg_debug (this->name, -ret,
-                              "Unable to set link-count in dict ");
+    if (local->fd) {
+        fd_ctx = afr_fd_ctx_get(local->fd, this);
+        if (!fd_ctx) {
+            afr_inode_refresh_done(frame, this, EINVAL);
+            return 0;
         }
+    }
 
-        ret = dict_set_str (xdata, GLUSTERFS_INODELK_DOM_COUNT, this->name);
-        if (ret) {
-                gf_msg_debug (this->name, -ret,
-                              "Unable to set inodelk-dom-count in dict ");
+    xdata = dict_new();
+    if (!xdata) {
+        afr_inode_refresh_done(frame, this, ENOMEM);
+        return 0;
+    }
 
-        }
+    ret = afr_xattr_req_prepare(this, xdata);
+    if (ret != 0) {
+        dict_unref(xdata);
+        afr_inode_refresh_done(frame, this, -ret);
+        return 0;
+    }
 
-        if (local->fd) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] &&
-                            fd_ctx->opened_on[i] == AFR_FD_OPENED)
-                                wind_subvols[i] = 1;
-                }
-        } else {
-                memcpy (wind_subvols, local->child_up,
-                        sizeof (*local->child_up) * priv->child_count);
-        }
+    ret = dict_set_sizen_str_sizen(xdata, "link-count", GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
 
-	local->call_count = AFR_COUNT (wind_subvols, priv->child_count);
+    ret = dict_set_str_sizen(xdata, GLUSTERFS_INODELK_DOM_COUNT, this->name);
+    if (ret) {
+        gf_msg_debug(this->name, -ret,
+                     "Unable to set inodelk-dom-count in dict ");
+    }
 
-	call_count = local->call_count;
-        if (!call_count) {
-                dict_unref (xdata);
-		if (local->fd && AFR_COUNT(local->child_up, priv->child_count))
-	                afr_inode_refresh_done (frame, this, EBADFD);
-		else
-	                afr_inode_refresh_done (frame, this, ENOTCONN);
-                return 0;
+    if (local->fd) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->child_up[i] && fd_ctx->opened_on[i] == AFR_FD_OPENED)
+                wind_subvols[i] = 1;
         }
-	for (i = 0; i < priv->child_count; i++) {
-		if (!wind_subvols[i])
-			continue;
+    } else {
+        memcpy(wind_subvols, local->child_up,
+               sizeof(*local->child_up) * priv->child_count);
+    }
 
-                if (local->fd)
-                        afr_inode_refresh_subvol_with_fstat (frame, this, i,
-                                                             xdata);
-                else
-                        afr_inode_refresh_subvol_with_lookup (frame, this, i,
-                                                    local->refreshinode,
-                                                    local->refreshgfid, xdata);
+    local->call_count = AFR_COUNT(wind_subvols, priv->child_count);
 
-		if (!--call_count)
-			break;
-	}
+    call_count = local->call_count;
+    if (!call_count) {
+        dict_unref(xdata);
+        if (local->fd && AFR_COUNT(local->child_up, priv->child_count))
+            afr_inode_refresh_done(frame, this, EBADFD);
+        else
+            afr_inode_refresh_done(frame, this, ENOTCONN);
+        return 0;
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!wind_subvols[i])
+            continue;
 
-	dict_unref (xdata);
+        if (local->fd)
+            afr_inode_refresh_subvol_with_fstat(frame, this, i, xdata);
+        else
+            afr_inode_refresh_subvol_with_lookup(
+                frame, this, i, local->refreshinode, local->refreshgfid, xdata);
 
-	return 0;
-}
+        if (!--call_count)
+            break;
+    }
+
+    dict_unref(xdata);
 
+    return 0;
+}
 
 int
-afr_inode_refresh (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                   uuid_t gfid, afr_inode_refresh_cbk_t refreshfn)
+afr_inode_refresh(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                  uuid_t gfid, afr_inode_refresh_cbk_t refreshfn)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	local->refreshfn = refreshfn;
+    local->refreshfn = refreshfn;
 
-	if (local->refreshinode) {
-		inode_unref (local->refreshinode);
-		local->refreshinode = NULL;
-	}
+    if (local->refreshinode) {
+        inode_unref(local->refreshinode);
+        local->refreshinode = NULL;
+    }
 
-	local->refreshinode = inode_ref (inode);
+    local->refreshinode = inode_ref(inode);
 
-        if (gfid)
-                gf_uuid_copy (local->refreshgfid, gfid);
-        else
-                gf_uuid_clear (local->refreshgfid);
+    if (gfid)
+        gf_uuid_copy(local->refreshgfid, gfid);
+    else
+        gf_uuid_clear(local->refreshgfid);
 
-	afr_inode_refresh_do (frame, this);
+    afr_inode_refresh_do(frame, this);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req)
+afr_xattr_req_prepare(xlator_t *this, dict_t *xattr_req)
 {
-        int             i           = 0;
-        afr_private_t   *priv       = NULL;
-        int             ret         = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int ret = 0;
 
-        priv   = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_set_uint64 (xattr_req, priv->pending_key[i],
-                                       AFR_NUM_CHANGE_LOGS * sizeof(int));
-                if (ret < 0)
-                        gf_msg (this->name, GF_LOG_WARNING,
-                                -ret, AFR_MSG_DICT_SET_FAILED,
-                                "Unable to set dict value for %s",
-                                priv->pending_key[i]);
-                /* 3 = data+metadata+entry */
-        }
-        ret = dict_set_uint64 (xattr_req, AFR_DIRTY,
-			       AFR_NUM_CHANGE_LOGS * sizeof(int));
-        if (ret) {
-                gf_msg_debug (this->name, -ret, "failed to set dirty "
-                              "query flag");
-        }
-
-        ret = dict_set_int32 (xattr_req, "list-xattr", 1);
-        if (ret) {
-                gf_msg_debug (this->name, -ret,
-                              "Unable to set list-xattr in dict ");
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_uint64(xattr_req, priv->pending_key[i],
+                              AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Unable to set dict value for %s", priv->pending_key[i]);
+        /* 3 = data+metadata+entry */
+    }
+    ret = dict_set_uint64(xattr_req, AFR_DIRTY,
+                          AFR_NUM_CHANGE_LOGS * sizeof(int));
+    if (ret) {
+        gf_msg_debug(this->name, -ret,
+                     "failed to set dirty "
+                     "query flag");
+    }
+
+    ret = dict_set_int32_sizen(xattr_req, "list-xattr", 1);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set list-xattr in dict ");
+    }
+
+    return ret;
+}
 
-	return ret;
+int
+afr_lookup_xattr_req_prepare(afr_local_t *local, xlator_t *this,
+                             dict_t *xattr_req, loc_t *loc)
+{
+    int ret = -ENOMEM;
+
+    if (!local->xattr_req)
+        local->xattr_req = dict_new();
+
+    if (!local->xattr_req)
+        goto out;
+
+    if (xattr_req && (xattr_req != local->xattr_req))
+        dict_copy(xattr_req, local->xattr_req);
+
+    ret = afr_xattr_req_prepare(this, local->xattr_req);
+
+    ret = dict_set_uint64(local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_INODELK_COUNT);
+    }
+    ret = dict_set_uint64(local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_ENTRYLK_COUNT);
+    }
+
+    ret = dict_set_uint32(local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_PARENT_ENTRYLK);
+    }
+
+    ret = dict_set_sizen_str_sizen(local->xattr_req, "link-count",
+                                   GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
+
+    ret = 0;
+out:
+    return ret;
 }
 
 int
-afr_lookup_xattr_req_prepare (afr_local_t *local, xlator_t *this,
-                              dict_t *xattr_req, loc_t *loc)
+afr_least_pending_reads_child(afr_private_t *priv, unsigned char *readable)
 {
-        int     ret = -ENOMEM;
+    int i = 0;
+    int child = -1;
+    int64_t read_iter = -1;
+    int64_t pending_read = -1;
 
-        if (!local->xattr_req)
-                local->xattr_req = dict_new ();
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i])
+            continue;
+        read_iter = GF_ATOMIC_GET(priv->pending_reads[i]);
+        if (child == -1 || read_iter < pending_read) {
+            pending_read = read_iter;
+            child = i;
+        }
+    }
 
-        if (!local->xattr_req)
-                goto out;
+    return child;
+}
 
-        if (xattr_req && (xattr_req != local->xattr_req))
-                dict_copy (xattr_req, local->xattr_req);
+static int32_t
+afr_least_latency_child(afr_private_t *priv, unsigned char *readable)
+{
+    int32_t i = 0;
+    int child = -1;
 
-        ret = afr_xattr_req_prepare (this, local->xattr_req);
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i] ||
+            priv->child_latency[i] < 0)
+            continue;
 
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_INODELK_COUNT);
-        }
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_ENTRYLK_COUNT);
+        if (child == -1 ||
+            priv->child_latency[i] < priv->child_latency[child]) {
+            child = i;
         }
+    }
+    return child;
+}
 
-        ret = dict_set_uint32 (local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_PARENT_ENTRYLK);
-        }
+static int32_t
+afr_least_latency_times_pending_reads_child(afr_private_t *priv,
+                                            unsigned char *readable)
+{
+    int32_t i = 0;
+    int child = -1;
+    int64_t pending_read = 0;
+    int64_t latency = -1;
+    int64_t least_latency = -1;
 
-        ret = dict_set_str (local->xattr_req, "link-count",
-                            GF_XATTROP_INDEX_COUNT);
-        if (ret) {
-                gf_msg_debug (this->name, -ret,
-                              "Unable to set link-count in dict ");
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i] ||
+            priv->child_latency[i] < 0)
+            continue;
 
-        ret = 0;
-out:
-        return ret;
-}
+        pending_read = GF_ATOMIC_GET(priv->pending_reads[i]);
+        latency = (pending_read + 1) * priv->child_latency[i];
 
-int
-afr_least_pending_reads_child (afr_private_t *priv)
-{
-        int i = 0;
-        int child = 0;
-        int64_t read_iter = -1;
-        int64_t pending_read = -1;
-
-        pending_read = GF_ATOMIC_GET (priv->pending_reads[0]);
-        for (i = 1; i < priv->child_count; i++) {
-                if (AFR_IS_ARBITER_BRICK(priv, i))
-                        continue;
-                read_iter =  GF_ATOMIC_GET(priv->pending_reads[i]);
-                if (read_iter < pending_read) {
-                        pending_read = read_iter;
-                        child = i;
-                }
+        if (child == -1 || latency < least_latency) {
+            least_latency = latency;
+            child = i;
         }
-
-        return child;
+    }
+    return child;
 }
 
 int
-afr_hash_child (afr_read_subvol_args_t *args, afr_private_t *priv)
-{
-        uuid_t gfid_copy = {0,};
-        pid_t pid;
-        int child = -1;
-
-        switch (priv->hash_mode) {
-        case 0:
-                break;
-        case 1:
-                gf_uuid_copy (gfid_copy, args->gfid);
-                child = SuperFastHash((char *)gfid_copy,
-                                      sizeof(gfid_copy)) % priv->child_count;
-                break;
-        case 2:
-                if (args->ia_type != IA_IFDIR) {
-                        /*
-                         * Why getpid?  Because it's one of the cheapest calls
-                         * available - faster than gethostname etc. - and
-                         * returns a constant-length value that's sure to be
-                         * shorter than a UUID. It's still very unlikely to be
-                         * the same across clients, so it still provides good
-                         * mixing.  We're not trying for perfection here. All we
-                         * need is a low probability that multiple clients
-                         * won't converge on the same subvolume.
-                         */
-                        pid = getpid();
-                        memcpy (gfid_copy, &pid, sizeof(pid));
-                }
-                child = SuperFastHash((char *)gfid_copy,
-                                      sizeof(gfid_copy)) % priv->child_count;
-                break;
-        case 3:
-                child = afr_least_pending_reads_child (priv);
-                break;
-        }
-
-        return child;
+afr_hash_child(afr_read_subvol_args_t *args, afr_private_t *priv,
+               unsigned char *readable)
+{
+    uuid_t gfid_copy = {
+        0,
+    };
+    pid_t pid;
+    int child = -1;
+
+    switch (priv->hash_mode) {
+        case AFR_READ_POLICY_FIRST_UP:
+            break;
+        case AFR_READ_POLICY_GFID_HASH:
+            gf_uuid_copy(gfid_copy, args->gfid);
+            child = SuperFastHash((char *)gfid_copy, sizeof(gfid_copy)) %
+                    priv->child_count;
+            break;
+        case AFR_READ_POLICY_GFID_PID_HASH:
+            if (args->ia_type != IA_IFDIR) {
+                /*
+                 * Why getpid?  Because it's one of the cheapest calls
+                 * available - faster than gethostname etc. - and
+                 * returns a constant-length value that's sure to be
+                 * shorter than a UUID. It's still very unlikely to be
+                 * the same across clients, so it still provides good
+                 * mixing.  We're not trying for perfection here. All we
+                 * need is a low probability that multiple clients
+                 * won't converge on the same subvolume.
+                 */
+                gf_uuid_copy(gfid_copy, args->gfid);
+                pid = getpid();
+                *(pid_t *)gfid_copy ^= pid;
+            }
+            child = SuperFastHash((char *)gfid_copy, sizeof(gfid_copy)) %
+                    priv->child_count;
+            break;
+        case AFR_READ_POLICY_LESS_LOAD:
+            child = afr_least_pending_reads_child(priv, readable);
+            break;
+        case AFR_READ_POLICY_LEAST_LATENCY:
+            child = afr_least_latency_child(priv, readable);
+            break;
+        case AFR_READ_POLICY_LOAD_LATENCY_HYBRID:
+            child = afr_least_latency_times_pending_reads_child(priv, readable);
+            break;
+    }
+
+    return child;
 }
 
 int
-afr_read_subvol_select_by_policy (inode_t *inode, xlator_t *this,
-				  unsigned char *readable,
-                                  afr_read_subvol_args_t *args)
+afr_read_subvol_select_by_policy(inode_t *inode, xlator_t *this,
+                                 unsigned char *readable,
+                                 afr_read_subvol_args_t *args)
 {
-	int             i           = 0;
-	int             read_subvol = -1;
-	afr_private_t  *priv        = NULL;
-        afr_read_subvol_args_t local_args = {0,};
+    int i = 0;
+    int read_subvol = -1;
+    afr_private_t *priv = NULL;
+    afr_read_subvol_args_t local_args = {
+        0,
+    };
 
-	priv = this->private;
+    priv = this->private;
 
-	/* first preference - explicitly specified or local subvolume */
-	if (priv->read_child >= 0 && readable[priv->read_child])
-                return priv->read_child;
+    /* first preference - explicitly specified or local subvolume */
+    if (priv->read_child >= 0 && readable[priv->read_child])
+        return priv->read_child;
 
-        if (inode_is_linked (inode)) {
-                gf_uuid_copy (local_args.gfid, inode->gfid);
-                local_args.ia_type = inode->ia_type;
-        } else if (args) {
-                local_args = *args;
-        }
+    if (inode_is_linked(inode)) {
+        gf_uuid_copy(local_args.gfid, inode->gfid);
+        local_args.ia_type = inode->ia_type;
+    } else if (args) {
+        local_args = *args;
+    }
 
-	/* second preference - use hashed mode */
-        read_subvol = afr_hash_child (&local_args, priv);
-	if (read_subvol >= 0 && readable[read_subvol])
-                return read_subvol;
+    /* second preference - use hashed mode */
+    read_subvol = afr_hash_child(&local_args, priv, readable);
+    if (read_subvol >= 0 && readable[read_subvol])
+        return read_subvol;
 
-	for (i = 0; i < priv->child_count; i++) {
-                if (readable[i])
-                return i;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (readable[i])
+            return i;
+    }
 
-        /* no readable subvolumes, either split brain or all subvols down */
+    /* no readable subvolumes, either split brain or all subvols down */
 
-        return -1;
+    return -1;
 }
 
-
 int
-afr_inode_read_subvol_type_get (inode_t *inode, xlator_t *this,
-				unsigned char *readable, int *event_p,
-				int type)
+afr_inode_read_subvol_type_get(inode_t *inode, xlator_t *this,
+                               unsigned char *readable, int *event_p, int type)
 {
-	int ret = -1;
+    int ret = -1;
 
-	if (type == AFR_METADATA_TRANSACTION)
-		ret = afr_inode_read_subvol_get (inode, this, 0, readable,
-						 event_p);
-	else
-		ret = afr_inode_read_subvol_get (inode, this, readable, 0,
-						 event_p);
-	return ret;
+    if (type == AFR_METADATA_TRANSACTION)
+        ret = afr_inode_read_subvol_get(inode, this, 0, readable, event_p);
+    else
+        ret = afr_inode_read_subvol_get(inode, this, readable, 0, event_p);
+    return ret;
 }
 
 void
-afr_readables_intersect_get (inode_t *inode, xlator_t *this, int *event,
-                             unsigned char *intersection)
+afr_readables_intersect_get(inode_t *inode, xlator_t *this, int *event,
+                            unsigned char *intersection)
 {
-	afr_private_t *priv = NULL;
-	unsigned char *data_readable = NULL;
-	unsigned char *metadata_readable = NULL;
-	unsigned char *intersect = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    unsigned char *intersect = NULL;
 
-	priv = this->private;
-	data_readable = alloca0 (priv->child_count);
-	metadata_readable = alloca0 (priv->child_count);
-	intersect = alloca0 (priv->child_count);
+    priv = this->private;
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+    intersect = alloca0(priv->child_count);
 
-	afr_inode_read_subvol_get (inode, this, data_readable,
-                                   metadata_readable, event);
+    afr_inode_read_subvol_get(inode, this, data_readable, metadata_readable,
+                              event);
 
-	AFR_INTERSECT (intersect, data_readable, metadata_readable,
-		       priv->child_count);
-        if (intersection)
-                memcpy (intersection, intersect,
-                        sizeof (*intersection) * priv->child_count);
+    AFR_INTERSECT(intersect, data_readable, metadata_readable,
+                  priv->child_count);
+    if (intersection)
+        memcpy(intersection, intersect,
+               sizeof(*intersection) * priv->child_count);
 }
 
 int
-afr_read_subvol_get (inode_t *inode, xlator_t *this, int *subvol_p,
-                     unsigned char *readables,
-		     int *event_p, afr_transaction_type type,
-                     afr_read_subvol_args_t *args)
+afr_read_subvol_get(inode_t *inode, xlator_t *this, int *subvol_p,
+                    unsigned char *readables, int *event_p,
+                    afr_transaction_type type, afr_read_subvol_args_t *args)
 {
-	afr_private_t *priv = NULL;
-	unsigned char *readable = NULL;
-	unsigned char *intersection = NULL;
-	int subvol = -1;
-	int event = 0;
+    afr_private_t *priv = NULL;
+    unsigned char *readable = NULL;
+    unsigned char *intersection = NULL;
+    int subvol = -1;
+    int event = 0;
 
-	priv = this->private;
+    priv = this->private;
 
-	readable = alloca0 (priv->child_count);
-	intersection = alloca0 (priv->child_count);
+    readable = alloca0(priv->child_count);
+    intersection = alloca0(priv->child_count);
 
-	afr_inode_read_subvol_type_get (inode, this, readable, &event, type);
+    afr_inode_read_subvol_type_get(inode, this, readable, &event, type);
 
-        afr_readables_intersect_get (inode, this, &event, intersection);
+    afr_readables_intersect_get(inode, this, &event, intersection);
 
-	if (AFR_COUNT (intersection, priv->child_count) > 0)
-		subvol = afr_read_subvol_select_by_policy (inode, this,
-							   intersection, args);
-	else
-		subvol = afr_read_subvol_select_by_policy (inode, this,
-							   readable, args);
-	if (subvol_p)
-		*subvol_p = subvol;
-	if (event_p)
-		*event_p = event;
-        if (readables)
-                memcpy (readables, readable,
-                        sizeof (*readables) * priv->child_count);
-	return subvol;
+    if (AFR_COUNT(intersection, priv->child_count) > 0)
+        subvol = afr_read_subvol_select_by_policy(inode, this, intersection,
+                                                  args);
+    else
+        subvol = afr_read_subvol_select_by_policy(inode, this, readable, args);
+    if (subvol_p)
+        *subvol_p = subvol;
+    if (event_p)
+        *event_p = event;
+    if (readables)
+        memcpy(readables, readable, sizeof(*readables) * priv->child_count);
+    return subvol;
 }
 
-
 void
-afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
+afr_local_transaction_cleanup(afr_local_t *local, xlator_t *this)
 {
-        afr_private_t *priv    = NULL;
-        int           i        = 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-        priv = this->private;
+    priv = this->private;
 
-        afr_matrix_cleanup (local->pending, priv->child_count);
+    afr_matrix_cleanup(local->pending, priv->child_count);
 
-        GF_FREE (local->internal_lock.locked_nodes);
+    GF_FREE(local->internal_lock.lower_locked_nodes);
 
-        GF_FREE (local->internal_lock.lower_locked_nodes);
+    afr_lockees_cleanup(&local->internal_lock);
 
-        afr_entry_lockee_cleanup (&local->internal_lock);
+    GF_FREE(local->transaction.pre_op);
 
-        GF_FREE (local->transaction.pre_op);
-
-        GF_FREE (local->transaction.pre_op_sources);
-        if (local->transaction.changelog_xdata) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!local->transaction.changelog_xdata[i])
-                                continue;
-                        dict_unref (local->transaction.changelog_xdata[i]);
-                }
-                GF_FREE (local->transaction.changelog_xdata);
+    GF_FREE(local->transaction.pre_op_sources);
+    if (local->transaction.changelog_xdata) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!local->transaction.changelog_xdata[i])
+                continue;
+            dict_unref(local->transaction.changelog_xdata[i]);
         }
+        GF_FREE(local->transaction.changelog_xdata);
+    }
 
-        GF_FREE (local->transaction.failed_subvols);
+    GF_FREE(local->transaction.failed_subvols);
 
-        GF_FREE (local->transaction.basename);
-        GF_FREE (local->transaction.new_basename);
-
-        loc_wipe (&local->transaction.parent_loc);
-        loc_wipe (&local->transaction.new_parent_loc);
+    GF_FREE(local->transaction.basename);
+    GF_FREE(local->transaction.new_basename);
 
+    loc_wipe(&local->transaction.parent_loc);
+    loc_wipe(&local->transaction.new_parent_loc);
 }
 
 void
-afr_reply_wipe (struct afr_reply *reply)
+afr_reply_wipe(struct afr_reply *reply)
 {
-        if (reply->xdata) {
-                dict_unref (reply->xdata);
-                reply->xdata = NULL;
-        }
+    if (reply->xdata) {
+        dict_unref(reply->xdata);
+        reply->xdata = NULL;
+    }
 
-        if (reply->xattr) {
-                dict_unref (reply->xattr);
-                reply->xattr = NULL;
-        }
+    if (reply->xattr) {
+        dict_unref(reply->xattr);
+        reply->xattr = NULL;
+    }
 }
 
 void
-afr_replies_wipe (struct afr_reply *replies, int count)
+afr_replies_wipe(struct afr_reply *replies, int count)
 {
-        int i = 0;
+    int i = 0;
 
-        for (i = 0; i < count; i++) {
-                afr_reply_wipe (&replies[i]);
-        }
+    for (i = 0; i < count; i++) {
+        afr_reply_wipe(&replies[i]);
+    }
 }
 
 void
-afr_local_replies_wipe (afr_local_t *local, afr_private_t *priv)
+afr_local_replies_wipe(afr_local_t *local, afr_private_t *priv)
 {
+    if (!local->replies)
+        return;
 
-	if (!local->replies)
-		return;
-
-        afr_replies_wipe (local->replies, priv->child_count);
+    afr_replies_wipe(local->replies, priv->child_count);
 
-	memset (local->replies, 0, sizeof(*local->replies) * priv->child_count);
+    memset(local->replies, 0, sizeof(*local->replies) * priv->child_count);
 }
 
 static gf_boolean_t
-afr_fop_lock_is_unlock (call_frame_t *frame)
+afr_fop_lock_is_unlock(call_frame_t *frame)
 {
-        afr_local_t *local = frame->local;
-        switch (local->op) {
+    afr_local_t *local = frame->local;
+    switch (local->op) {
         case GF_FOP_INODELK:
         case GF_FOP_FINODELK:
-                if ((F_UNLCK == local->cont.inodelk.in_flock.l_type) &&
-                   (local->cont.inodelk.in_cmd == F_SETLKW ||
-                    local->cont.inodelk.in_cmd == F_SETLK))
-                        return _gf_true;
-                break;
+            if ((F_UNLCK == local->cont.inodelk.in_flock.l_type) &&
+                (local->cont.inodelk.in_cmd == F_SETLKW ||
+                 local->cont.inodelk.in_cmd == F_SETLK))
+                return _gf_true;
+            break;
         case GF_FOP_ENTRYLK:
         case GF_FOP_FENTRYLK:
-                if (ENTRYLK_UNLOCK == local->cont.entrylk.in_cmd)
-                        return _gf_true;
-                break;
+            if (ENTRYLK_UNLOCK == local->cont.entrylk.in_cmd)
+                return _gf_true;
+            break;
         default:
-                return _gf_false;
-        }
-        return _gf_false;
+            return _gf_false;
+    }
+    return _gf_false;
 }
 
 static gf_boolean_t
-afr_lk_is_unlock (int32_t cmd, struct gf_flock *flock)
+afr_lk_is_unlock(int32_t cmd, struct gf_flock *flock)
 {
-        switch (cmd) {
+    switch (cmd) {
         case F_RESLK_UNLCK:
-                return _gf_true;
-                break;
+            return _gf_true;
+            break;
 
 #if F_SETLKW != F_SETLKW64
         case F_SETLKW64:
@@ -1924,575 +2548,573 @@ afr_lk_is_unlock (int32_t cmd, struct gf_flock *flock)
         case F_SETLK64:
 #endif
         case F_SETLK:
-                if (F_UNLCK == flock->l_type)
-                        return _gf_true;
-                break;
+            if (F_UNLCK == flock->l_type)
+                return _gf_true;
+            break;
         default:
-                return _gf_false;
-        }
-        return _gf_false;
+            return _gf_false;
+    }
+    return _gf_false;
 }
 
 void
-afr_handle_inconsistent_fop (call_frame_t *frame, int32_t *op_ret,
-                             int32_t *op_errno)
+afr_handle_inconsistent_fop(call_frame_t *frame, int32_t *op_ret,
+                            int32_t *op_errno)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t   *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        if (!frame || !frame->this || !frame->local || !frame->this->private)
-                return;
+    if (!frame || !frame->this || !frame->local || !frame->this->private)
+        return;
 
-        if (*op_ret < 0)
-                return;
+    if (*op_ret < 0)
+        return;
 
-        /* Failing inodelk/entrylk/lk here is not a good idea because we
-         * need to cleanup the locks on the other bricks if we choose to fail
-         * the fop here. The brick may go down just after unwind happens as well
-         * so anyways the fop will fail when the next fop is sent so leaving
-         * it like this for now.*/
-        local = frame->local;
-        switch (local->op) {
+    /* Failing inodelk/entrylk/lk here is not a good idea because we
+     * need to cleanup the locks on the other bricks if we choose to fail
+     * the fop here. The brick may go down just after unwind happens as well
+     * so anyways the fop will fail when the next fop is sent so leaving
+     * it like this for now.*/
+    local = frame->local;
+    switch (local->op) {
         case GF_FOP_LOOKUP:
         case GF_FOP_INODELK:
         case GF_FOP_FINODELK:
         case GF_FOP_ENTRYLK:
         case GF_FOP_FENTRYLK:
         case GF_FOP_LK:
-                return;
+            return;
         default:
-                break;
-        }
+            break;
+    }
 
-        priv = frame->this->private;
-        if (!priv->consistent_io)
-                return;
+    priv = frame->this->private;
+    if (!priv->consistent_io)
+        return;
 
-        if (local->event_generation &&
-            (local->event_generation != priv->event_generation))
-                goto inconsistent;
+    if (local->event_generation &&
+        (local->event_generation != priv->event_generation))
+        goto inconsistent;
 
-        return;
+    return;
 inconsistent:
-        *op_ret = -1;
-        *op_errno = ENOTCONN;
+    *op_ret = -1;
+    *op_errno = ENOTCONN;
 }
 
 void
-afr_local_cleanup (afr_local_t *local, xlator_t *this)
+afr_local_cleanup(afr_local_t *local, xlator_t *this)
 {
-        afr_private_t * priv = NULL;
-
-        if (!local)
-                return;
+    afr_private_t *priv = NULL;
 
-	syncbarrier_destroy (&local->barrier);
-
-        afr_local_transaction_cleanup (local, this);
-
-        priv = this->private;
+    if (!local)
+        return;
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->newloc);
+    syncbarrier_destroy(&local->barrier);
 
-        if (local->fd)
-                fd_unref (local->fd);
+    afr_local_transaction_cleanup(local, this);
 
-        if (local->xattr_req)
-                dict_unref (local->xattr_req);
+    priv = this->private;
 
-        if (local->xattr_rsp)
-                dict_unref (local->xattr_rsp);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->newloc);
 
-        if (local->dict)
-                dict_unref (local->dict);
+    if (local->fd)
+        fd_unref(local->fd);
 
-        afr_local_replies_wipe (local, priv);
-	GF_FREE(local->replies);
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
 
-        GF_FREE (local->child_up);
+    if (local->xattr_rsp)
+        dict_unref(local->xattr_rsp);
 
-        GF_FREE (local->read_attempted);
+    if (local->dict)
+        dict_unref(local->dict);
 
-        GF_FREE (local->readable);
-        GF_FREE (local->readable2);
+    afr_local_replies_wipe(local, priv);
+    GF_FREE(local->replies);
 
-	if (local->inode)
-		inode_unref (local->inode);
+    GF_FREE(local->child_up);
 
-	if (local->parent)
-		inode_unref (local->parent);
+    GF_FREE(local->read_attempted);
+
+    GF_FREE(local->readable);
+    GF_FREE(local->readable2);
+
+    if (local->inode)
+        inode_unref(local->inode);
 
-	if (local->parent2)
-		inode_unref (local->parent2);
+    if (local->parent)
+        inode_unref(local->parent);
 
-	if (local->refreshinode)
-		inode_unref (local->refreshinode);
+    if (local->parent2)
+        inode_unref(local->parent2);
 
-        { /* getxattr */
-                GF_FREE (local->cont.getxattr.name);
-        }
+    if (local->refreshinode)
+        inode_unref(local->refreshinode);
 
-        { /* lk */
-                GF_FREE (local->cont.lk.locked_nodes);
-        }
+    { /* getxattr */
+        GF_FREE(local->cont.getxattr.name);
+    }
 
-        { /* create */
-                if (local->cont.create.fd)
-                        fd_unref (local->cont.create.fd);
-                if (local->cont.create.params)
-                        dict_unref (local->cont.create.params);
-        }
+    { /* lk */
+        GF_FREE(local->cont.lk.locked_nodes);
+        GF_FREE(local->cont.lk.dom_locked_nodes);
+        GF_FREE(local->cont.lk.dom_lock_op_ret);
+        GF_FREE(local->cont.lk.dom_lock_op_errno);
+    }
 
-        { /* mknod */
-                if (local->cont.mknod.params)
-                        dict_unref (local->cont.mknod.params);
-        }
+    { /* create */
+        if (local->cont.create.fd)
+            fd_unref(local->cont.create.fd);
+        if (local->cont.create.params)
+            dict_unref(local->cont.create.params);
+    }
 
-        { /* mkdir */
-                if (local->cont.mkdir.params)
-                        dict_unref (local->cont.mkdir.params);
-        }
-
-        { /* symlink */
-                if (local->cont.symlink.params)
-                        dict_unref (local->cont.symlink.params);
-        }
-
-        { /* writev */
-                GF_FREE (local->cont.writev.vector);
-                if (local->cont.writev.iobref)
-                        iobref_unref (local->cont.writev.iobref);
-        }
-
-        { /* setxattr */
-                if (local->cont.setxattr.dict)
-                        dict_unref (local->cont.setxattr.dict);
-        }
-
-        { /* fsetxattr */
-                if (local->cont.fsetxattr.dict)
-                        dict_unref (local->cont.fsetxattr.dict);
-        }
-
-        { /* removexattr */
-                GF_FREE (local->cont.removexattr.name);
-        }
-        { /* xattrop */
-                if (local->cont.xattrop.xattr)
-                        dict_unref (local->cont.xattrop.xattr);
-        }
-        { /* symlink */
-                GF_FREE (local->cont.symlink.linkpath);
-        }
+    { /* mknod */
+        if (local->cont.mknod.params)
+            dict_unref(local->cont.mknod.params);
+    }
 
-        { /* opendir */
-                GF_FREE (local->cont.opendir.checksum);
-        }
+    { /* mkdir */
+        if (local->cont.mkdir.params)
+            dict_unref(local->cont.mkdir.params);
+    }
 
-        { /* open */
-                if (local->cont.open.fd)
-                        fd_unref (local->cont.open.fd);
-        }
+    { /* symlink */
+        if (local->cont.symlink.params)
+            dict_unref(local->cont.symlink.params);
+    }
 
-        { /* readdirp */
-                if (local->cont.readdir.dict)
-                        dict_unref (local->cont.readdir.dict);
-        }
+    { /* writev */
+        GF_FREE(local->cont.writev.vector);
+        if (local->cont.writev.iobref)
+            iobref_unref(local->cont.writev.iobref);
+    }
 
-        { /* inodelk */
-                GF_FREE (local->cont.inodelk.volume);
-                if (local->cont.inodelk.xdata)
-                        dict_unref (local->cont.inodelk.xdata);
-        }
+    { /* setxattr */
+        if (local->cont.setxattr.dict)
+            dict_unref(local->cont.setxattr.dict);
+    }
 
-        { /* entrylk */
-                GF_FREE (local->cont.entrylk.volume);
-                GF_FREE (local->cont.entrylk.basename);
-                if (local->cont.entrylk.xdata)
-                        dict_unref (local->cont.entrylk.xdata);
-        }
+    { /* fsetxattr */
+        if (local->cont.fsetxattr.dict)
+            dict_unref(local->cont.fsetxattr.dict);
+    }
 
-        if (local->xdata_req)
-                dict_unref (local->xdata_req);
+    { /* removexattr */
+        GF_FREE(local->cont.removexattr.name);
+    }
+    { /* xattrop */
+        if (local->cont.xattrop.xattr)
+            dict_unref(local->cont.xattrop.xattr);
+    }
+    { /* symlink */
+        GF_FREE(local->cont.symlink.linkpath);
+    }
 
-        if (local->xdata_rsp)
-                dict_unref (local->xdata_rsp);
+    { /* opendir */
+        GF_FREE(local->cont.opendir.checksum);
+    }
+
+    { /* open */
+        if (local->cont.open.fd)
+            fd_unref(local->cont.open.fd);
+    }
+
+    { /* readdirp */
+        if (local->cont.readdir.dict)
+            dict_unref(local->cont.readdir.dict);
+    }
+
+    { /* inodelk */
+        GF_FREE(local->cont.inodelk.volume);
+        if (local->cont.inodelk.xdata)
+            dict_unref(local->cont.inodelk.xdata);
+    }
+
+    { /* entrylk */
+        GF_FREE(local->cont.entrylk.volume);
+        GF_FREE(local->cont.entrylk.basename);
+        if (local->cont.entrylk.xdata)
+            dict_unref(local->cont.entrylk.xdata);
+    }
+
+    if (local->xdata_req)
+        dict_unref(local->xdata_req);
+
+    if (local->xdata_rsp)
+        dict_unref(local->xdata_rsp);
 }
 
-
 int
-afr_frame_return (call_frame_t *frame)
+afr_frame_return(call_frame_t *frame)
 {
-        afr_local_t *local = NULL;
-        int          call_count = 0;
+    afr_local_t *local = NULL;
+    int call_count = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                call_count = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
+    LOCK(&frame->lock);
+    {
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        return call_count;
+    return call_count;
 }
 
-static char *afr_ignore_xattrs[] = {
-        GF_SELINUX_XATTR_KEY,
-        QUOTA_SIZE_KEY,
-        NULL
-};
+static char *afr_ignore_xattrs[] = {GF_SELINUX_XATTR_KEY, QUOTA_SIZE_KEY, NULL};
 
 gf_boolean_t
-afr_is_xattr_ignorable (char *key)
+afr_is_xattr_ignorable(char *key)
 {
-        int i = 0;
+    int i = 0;
 
-        if (!strncmp (key, AFR_XATTR_PREFIX, strlen(AFR_XATTR_PREFIX)))
-                return _gf_true;
-        for (i = 0; afr_ignore_xattrs[i]; i++) {
-                if (!strcmp (key, afr_ignore_xattrs[i]))
-                       return _gf_true;
-        }
-        return _gf_false;
+    if (!strncmp(key, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX)))
+        return _gf_true;
+    for (i = 0; afr_ignore_xattrs[i]; i++) {
+        if (!strcmp(key, afr_ignore_xattrs[i]))
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
 static gf_boolean_t
-afr_xattr_match_needed (dict_t *this, char *key1, data_t *value1, void *data)
+afr_xattr_match_needed(dict_t *this, char *key1, data_t *value1, void *data)
 {
-        /* Ignore all non-disk (i.e. virtual) xattrs right away. */
-        if (!gf_is_valid_xattr_namespace (key1))
-                return _gf_false;
+    /* Ignore all non-disk (i.e. virtual) xattrs right away. */
+    if (!gf_is_valid_xattr_namespace(key1))
+        return _gf_false;
 
-        /* Ignore on-disk xattrs that AFR doesn't need to heal. */
-        if (!afr_is_xattr_ignorable (key1))
-                return _gf_true;
+    /* Ignore on-disk xattrs that AFR doesn't need to heal. */
+    if (!afr_is_xattr_ignorable(key1))
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 gf_boolean_t
-afr_xattrs_are_equal (dict_t *dict1, dict_t *dict2)
+afr_xattrs_are_equal(dict_t *dict1, dict_t *dict2)
 {
-        return are_dicts_equal (dict1, dict2, afr_xattr_match_needed, NULL);
+    return are_dicts_equal(dict1, dict2, afr_xattr_match_needed, NULL);
 }
 
 static int
-afr_get_parent_read_subvol (xlator_t *this, inode_t *parent,
-                            struct afr_reply *replies, unsigned char *readable)
+afr_get_parent_read_subvol(xlator_t *this, inode_t *parent,
+                           struct afr_reply *replies, unsigned char *readable)
 {
-        int             i                    = 0;
-        int             par_read_subvol      = -1;
-        int             par_read_subvol_iter = -1;
-        afr_private_t  *priv                 = NULL;
+    int i = 0;
+    int par_read_subvol = -1;
+    int par_read_subvol_iter = -1;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (parent)
-                par_read_subvol = afr_data_subvol_get (parent, this, NULL, NULL,
-                                                       NULL, NULL);
+    if (parent)
+        par_read_subvol = afr_data_subvol_get(parent, this, NULL, NULL, NULL,
+                                              NULL);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid)
-                        continue;
-
-                if (replies[i].op_ret < 0)
-                        continue;
-
-                if (par_read_subvol_iter == -1) {
-                        par_read_subvol_iter = i;
-                        continue;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                if ((par_read_subvol_iter != par_read_subvol) && readable[i])
-                        par_read_subvol_iter = i;
+        if (replies[i].op_ret < 0)
+            continue;
 
-                if (i == par_read_subvol)
-                        par_read_subvol_iter = i;
+        if (par_read_subvol_iter == -1) {
+            par_read_subvol_iter = i;
+            continue;
         }
-        /* At the end of the for-loop, the only reason why @par_read_subvol_iter
-         * could be -1 is when this LOOKUP has failed on all sub-volumes.
-         * So it is okay to send an arbitrary subvolume (0 in this case)
-         * as parent read subvol.
-         */
-        if (par_read_subvol_iter == -1)
-                par_read_subvol_iter = 0;
 
-        return par_read_subvol_iter;
+        if ((par_read_subvol_iter != par_read_subvol) && readable[i])
+            par_read_subvol_iter = i;
+
+        if (i == par_read_subvol)
+            par_read_subvol_iter = i;
+    }
+    /* At the end of the for-loop, the only reason why @par_read_subvol_iter
+     * could be -1 is when this LOOKUP has failed on all sub-volumes.
+     * So it is okay to send an arbitrary subvolume (0 in this case)
+     * as parent read subvol.
+     */
+    if (par_read_subvol_iter == -1)
+        par_read_subvol_iter = 0;
 
+    return par_read_subvol_iter;
 }
 
 int
-afr_read_subvol_decide (inode_t *inode, xlator_t *this,
-                        afr_read_subvol_args_t *args, unsigned char *readable)
+afr_read_subvol_decide(inode_t *inode, xlator_t *this,
+                       afr_read_subvol_args_t *args, unsigned char *readable)
 {
-        int event = 0;
-        afr_private_t *priv = NULL;
-        unsigned char *intersection = NULL;
+    int event = 0;
+    afr_private_t *priv = NULL;
+    unsigned char *intersection = NULL;
 
-        priv = this->private;
-        intersection = alloca0 (priv->child_count);
+    priv = this->private;
+    intersection = alloca0(priv->child_count);
 
-        afr_readables_intersect_get (inode, this, &event, intersection);
+    afr_readables_intersect_get(inode, this, &event, intersection);
 
-        if (AFR_COUNT (intersection, priv->child_count) <= 0) {
-                /* TODO: If we have one brick with valid data_readable and
-                 * another with metadata_readable, try to send an iatt with
-                 * valid bits from both.*/
-                return -1;
-        }
+    if (AFR_COUNT(intersection, priv->child_count) <= 0) {
+        /* TODO: If we have one brick with valid data_readable and
+         * another with metadata_readable, try to send an iatt with
+         * valid bits from both.*/
+        return -1;
+    }
 
-        memcpy (readable, intersection, sizeof (*readable) * priv->child_count);
+    memcpy(readable, intersection, sizeof(*readable) * priv->child_count);
 
-        return afr_read_subvol_select_by_policy (inode, this, intersection,
-                                                 args);
+    return afr_read_subvol_select_by_policy(inode, this, intersection, args);
 }
 
 static inline int
-afr_first_up_child (call_frame_t *frame, xlator_t *this)
+afr_first_up_child(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t       *priv  = NULL;
-        afr_local_t         *local = NULL;
-        int                  i     = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++)
-                if (local->replies[i].valid &&
-                    local->replies[i].op_ret == 0)
-                        return i;
-        return -1;
+    for (i = 0; i < priv->child_count; i++)
+        if (local->replies[i].valid && local->replies[i].op_ret == 0)
+            return i;
+    return -1;
 }
 
 static void
-afr_attempt_readsubvol_set (call_frame_t *frame, xlator_t *this,
-                            unsigned char *success_replies,
-                            unsigned char *data_readable, int *read_subvol)
+afr_attempt_readsubvol_set(call_frame_t *frame, xlator_t *this,
+                           unsigned char *success_replies,
+                           unsigned char *data_readable, int *read_subvol)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int spb_choice = -1;
-        int child_count = -1;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int spb_subvol = -1;
+    int child_count = -1;
 
-        if (*read_subvol != -1)
-                return;
+    if (*read_subvol != -1)
+        return;
 
-        priv = this->private;
-        local = frame->local;
-        child_count = priv->child_count;
-
-        afr_inode_split_brain_choice_get (local->inode, this,
-                                          &spb_choice);
-        if ((spb_choice >= 0) &&
-            (AFR_COUNT(success_replies, child_count) == child_count)) {
-                *read_subvol = spb_choice;
-        } else if (!priv->quorum_count) {
-                *read_subvol = afr_first_up_child (frame, this);
-        } else if (priv->quorum_count &&
-                   afr_has_quorum (data_readable, this)) {
-                /* read_subvol is guaranteed to be valid if we hit this path. */
-                *read_subvol = afr_first_up_child (frame, this);
-        } else {
-               /* If quorum is enabled and we do not have a
-                  readable yet, it means all good copies are down.
-               */
-               local->op_ret = -1;
-               local->op_errno = ENOTCONN;
-               gf_msg (this->name, GF_LOG_WARNING, 0,
-                       AFR_MSG_READ_SUBVOL_ERROR, "no read "
-                       "subvols for %s", local->loc.path);
-        }
-        if (*read_subvol >= 0)
-                dict_del (local->replies[*read_subvol].xdata, GF_CONTENT_KEY);
+    priv = this->private;
+    local = frame->local;
+    child_count = priv->child_count;
+
+    afr_split_brain_read_subvol_get(local->inode, this, frame, &spb_subvol);
+    if ((spb_subvol >= 0) &&
+        (AFR_COUNT(success_replies, child_count) == child_count)) {
+        *read_subvol = spb_subvol;
+    } else if (!priv->quorum_count ||
+               frame->root->pid == GF_CLIENT_PID_GLFS_HEAL) {
+        *read_subvol = afr_first_up_child(frame, this);
+    } else if (priv->quorum_count &&
+               afr_has_quorum(data_readable, this, NULL)) {
+        /* read_subvol is guaranteed to be valid if we hit this path. */
+        *read_subvol = afr_first_up_child(frame, this);
+    } else {
+        /* If quorum is enabled and we do not have a
+           readable yet, it means all good copies are down.
+        */
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_READ_SUBVOL_ERROR,
+               "no read "
+               "subvols for %s",
+               local->loc.path);
+    }
+    if (*read_subvol >= 0)
+        dict_del_sizen(local->replies[*read_subvol].xdata, GF_CONTENT_KEY);
 }
 
 static void
-afr_lookup_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t       *priv  = NULL;
-        afr_local_t         *local = NULL;
-	int                 i = -1;
-	int                 op_errno = 0;
-	int                 read_subvol = 0;
-        int                 par_read_subvol = 0;
-        int                 ret         = -1;
-	unsigned char      *readable = NULL;
-        unsigned char      *success_replies = NULL;
-	int                 event = 0;
-	struct afr_reply   *replies = NULL;
-	uuid_t              read_gfid = {0, };
-	gf_boolean_t        locked_entry = _gf_false;
-	gf_boolean_t        can_interpret = _gf_true;
-        inode_t            *parent = NULL;
-        ia_type_t           ia_type = IA_INVAL;
-        afr_read_subvol_args_t args = {0,};
-        char               *gfid_heal_msg = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-	replies = local->replies;
-        parent = local->loc.parent;
-
-	locked_entry = afr_is_possibly_under_txn (AFR_ENTRY_TRANSACTION, local,
-                                                  this);
-
-	readable = alloca0 (priv->child_count);
-	success_replies = alloca0 (priv->child_count);
-
-	afr_inode_read_subvol_get (parent, this, readable, NULL, &event);
-        par_read_subvol = afr_get_parent_read_subvol (this, parent, replies,
-                                                      readable);
-
-	/* First, check if we have a gfid-change from somewhere,
-	   If so, propagate that so that a fresh lookup can be
-	   issued
-	*/
-        if (local->cont.lookup.needs_fresh_lookup) {
-                local->op_ret = -1;
-                local->op_errno = ESTALE;
-                goto error;
-        }
-
-	op_errno = afr_final_errno (frame->local, this->private);
-	local->op_errno = op_errno;
-
-	read_subvol = -1;
-        for (i = 0; i < priv->child_count; i++)
-                if (replies[i].valid && replies[i].op_ret == 0)
-                        success_replies[i] = 1;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-
-		if (locked_entry && replies[i].op_ret == -1 &&
-		    replies[i].op_errno == ENOENT) {
-			/* Second, check entry is still
-			   "underway" in creation */
-                        local->op_ret = -1;
-                        local->op_errno = ENOENT;
-                        goto error;
-		}
-
-		if (replies[i].op_ret == -1)
-			continue;
-
-		if (read_subvol == -1 || !readable[read_subvol]) {
-			read_subvol = i;
-			gf_uuid_copy (read_gfid, replies[i].poststat.ia_gfid);
-                        ia_type = replies[i].poststat.ia_type;
-			local->op_ret = 0;
-		}
-	}
+afr_lookup_done(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = -1;
+    int op_errno = 0;
+    int read_subvol = 0;
+    int par_read_subvol = 0;
+    int ret = -1;
+    unsigned char *readable = NULL;
+    unsigned char *success_replies = NULL;
+    int event = 0;
+    struct afr_reply *replies = NULL;
+    uuid_t read_gfid = {
+        0,
+    };
+    gf_boolean_t locked_entry = _gf_false;
+    gf_boolean_t in_flight_create = _gf_false;
+    gf_boolean_t can_interpret = _gf_true;
+    inode_t *parent = NULL;
+    ia_type_t ia_type = IA_INVAL;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+    char *gfid_heal_msg = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    replies = local->replies;
+    parent = local->loc.parent;
+
+    locked_entry = afr_is_possibly_under_txn(AFR_ENTRY_TRANSACTION, local,
+                                             this);
+
+    readable = alloca0(priv->child_count);
+    success_replies = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(parent, this, readable, NULL, &event);
+    par_read_subvol = afr_get_parent_read_subvol(this, parent, replies,
+                                                 readable);
+
+    /* First, check if we have a gfid-change from somewhere,
+       If so, propagate that so that a fresh lookup can be
+       issued
+    */
+    if (local->cont.lookup.needs_fresh_lookup) {
+        local->op_ret = -1;
+        local->op_errno = ESTALE;
+        goto error;
+    }
 
-        if (read_subvol == -1)
-                goto error;
-	/* We now have a read_subvol, which is readable[] (if there
-	   were any). Next we look for GFID mismatches. We don't
-	   consider a GFID mismatch as an error if read_subvol is
-	   readable[] but the mismatching GFID subvol is not.
-	*/
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid || replies[i].op_ret == -1) {
-			if (priv->child_up[i])
-				can_interpret = _gf_false;
-			continue;
-		}
-
-		if (!gf_uuid_compare (replies[i].poststat.ia_gfid, read_gfid))
-                        continue;
-
-		can_interpret = _gf_false;
-
-		if (locked_entry)
-			continue;
-
-		/* Now GFIDs mismatch. It's OK as long as this subvol
-		   is not readable[] but read_subvol is */
-		if (readable[read_subvol] && !readable[i])
-			continue;
-
-                /* If we were called from glfsheal and there is still a gfid
-                 * mismatch, succeed the lookup and let glfsheal print the
-                 * response via gfid-heal-msg.*/
-                if (!dict_get_str (local->xattr_req, "gfid-heal-msg",
-                                   &gfid_heal_msg))
-                        goto cant_interpret;
-
-		/* LOG ERROR */
-		local->op_ret = -1;
-		local->op_errno = EIO;
-		goto error;
-	}
-
-	/* Forth, for the finalized GFID, pick the best subvolume
-	   to return stats from.
-	*/
-        read_subvol = -1;
-        memset (readable, 0, sizeof (*readable) * priv->child_count);
-	if (can_interpret) {
-                if (!afr_has_quorum (success_replies, this))
-                        goto cant_interpret;
-		/* It is safe to call afr_replies_interpret() because we have
-		   a response from all the UP subvolumes and all of them resolved
-		   to the same GFID
-		*/
-                gf_uuid_copy (args.gfid, read_gfid);
-                args.ia_type = ia_type;
-		ret = afr_replies_interpret (frame, this, local->inode, NULL);
-                read_subvol = afr_read_subvol_decide (local->inode, this, &args,
-                                                      readable);
-                if (read_subvol == -1)
-                        goto cant_interpret;
-                if (ret) {
-			afr_inode_event_gen_reset (local->inode, this);
-                        dict_del (local->replies[read_subvol].xdata,
-                                  GF_CONTENT_KEY);
-                }
-	} else {
-	cant_interpret:
-                afr_attempt_readsubvol_set (frame, this, success_replies,
-                                            readable, &read_subvol);
-                if (read_subvol == -1) {
-                        goto error;
-                }
-	}
+    op_errno = afr_final_errno(frame->local, this->private);
+    local->op_errno = op_errno;
 
-	afr_handle_quota_size (frame, this);
+    read_subvol = -1;
+    afr_fill_success_replies(local, priv, success_replies);
 
-        afr_set_need_heal (this, local);
-        if (AFR_IS_ARBITER_BRICK (priv, read_subvol) && local->op_ret == 0) {
-                local->op_ret = -1;
-                local->op_errno = ENOTCONN;
-                gf_msg_debug(this->name, 0, "Arbiter cannot be a read subvol "
-                             "for %s", local->loc.path);
-                goto error;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret == -1) {
+            if (locked_entry && replies[i].op_errno == ENOENT) {
+                in_flight_create = _gf_true;
+            }
+            continue;
         }
 
-        ret = dict_get_str (local->xattr_req, "gfid-heal-msg", &gfid_heal_msg);
-        if (!ret) {
-                ret = dict_set_str (local->replies[read_subvol].xdata,
-                                    "gfid-heal-msg", gfid_heal_msg);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_DICT_SET_FAILED,
-                                "Error setting gfid-heal-msg dict");
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                }
+        if (read_subvol == -1 || !readable[read_subvol]) {
+            read_subvol = i;
+            gf_uuid_copy(read_gfid, replies[i].poststat.ia_gfid);
+            ia_type = replies[i].poststat.ia_type;
+            local->op_ret = 0;
         }
+    }
 
-	AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-			  local->inode, &local->replies[read_subvol].poststat,
-			  local->replies[read_subvol].xdata,
-			  &local->replies[par_read_subvol].postparent);
-        return;
+    if (in_flight_create && !afr_has_quorum(success_replies, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = ENOENT;
+        goto error;
+    }
+
+    if (read_subvol == -1)
+        goto error;
+    /* We now have a read_subvol, which is readable[] (if there
+       were any). Next we look for GFID mismatches. We don't
+       consider a GFID mismatch as an error if read_subvol is
+       readable[] but the mismatching GFID subvol is not.
+    */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1) {
+            continue;
+        }
+
+        if (!gf_uuid_compare(replies[i].poststat.ia_gfid, read_gfid))
+            continue;
+
+        can_interpret = _gf_false;
+
+        if (locked_entry)
+            continue;
+
+        /* Now GFIDs mismatch. It's OK as long as this subvol
+           is not readable[] but read_subvol is */
+        if (readable[read_subvol] && !readable[i])
+            continue;
+
+        /* If we were called from glfsheal and there is still a gfid
+         * mismatch, succeed the lookup and let glfsheal print the
+         * response via gfid-heal-msg.*/
+        if (!dict_get_str_sizen(local->xattr_req, "gfid-heal-msg",
+                                &gfid_heal_msg))
+            goto cant_interpret;
+
+        /* LOG ERROR */
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        goto error;
+    }
+
+    /* Forth, for the finalized GFID, pick the best subvolume
+       to return stats from.
+    */
+    read_subvol = -1;
+    memset(readable, 0, sizeof(*readable) * priv->child_count);
+    if (can_interpret) {
+        if (!afr_has_quorum(success_replies, this, NULL))
+            goto cant_interpret;
+        /* It is safe to call afr_replies_interpret() because we have
+           a response from all the UP subvolumes and all of them resolved
+           to the same GFID
+        */
+        gf_uuid_copy(args.gfid, read_gfid);
+        args.ia_type = ia_type;
+        ret = afr_replies_interpret(frame, this, local->inode, NULL);
+        read_subvol = afr_read_subvol_decide(local->inode, this, &args,
+                                             readable);
+        if (read_subvol == -1)
+            goto cant_interpret;
+        if (ret) {
+            afr_inode_need_refresh_set(local->inode, this);
+            dict_del_sizen(local->replies[read_subvol].xdata, GF_CONTENT_KEY);
+        }
+    } else {
+    cant_interpret:
+        afr_attempt_readsubvol_set(frame, this, success_replies, readable,
+                                   &read_subvol);
+        if (read_subvol == -1) {
+            goto error;
+        }
+    }
+
+    afr_handle_quota_size(frame, this);
+
+    afr_set_need_heal(this, local);
+    if (AFR_IS_ARBITER_BRICK(priv, read_subvol) && local->op_ret == 0) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg_debug(this->name, 0,
+                     "Arbiter cannot be a read subvol "
+                     "for %s",
+                     local->loc.path);
+        goto error;
+    }
+
+    ret = dict_get_str_sizen(local->xattr_req, "gfid-heal-msg", &gfid_heal_msg);
+    if (!ret) {
+        ret = dict_set_str_sizen(local->replies[read_subvol].xdata,
+                                 "gfid-heal-msg", gfid_heal_msg);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting gfid-heal-msg dict");
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+        }
+    }
+
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->replies[read_subvol].poststat,
+                     local->replies[read_subvol].xdata,
+                     &local->replies[par_read_subvol].postparent);
+    return;
 
 error:
-        AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno, NULL,
-                          NULL, NULL, NULL);
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno, NULL, NULL,
+                     NULL, NULL);
 }
 
 /*
@@ -2500,751 +3122,845 @@ error:
  * others in that they must be given higher priority while
  * returning to the user.
  *
- * The hierarchy is ENODATA > ENOENT > ESTALE > others
+ * The hierarchy is ENODATA > ENOENT > ESTALE > ENOSPC others
  */
 
 int
-afr_higher_errno (int32_t old_errno, int32_t new_errno)
+afr_higher_errno(int32_t old_errno, int32_t new_errno)
 {
-	if (old_errno == ENODATA || new_errno == ENODATA)
-		return ENODATA;
-        if (old_errno == ENOENT || new_errno == ENOENT)
-                return ENOENT;
-	if (old_errno == ESTALE || new_errno == ESTALE)
-		return ESTALE;
+    if (old_errno == ENODATA || new_errno == ENODATA)
+        return ENODATA;
+    if (old_errno == ENOENT || new_errno == ENOENT)
+        return ENOENT;
+    if (old_errno == ESTALE || new_errno == ESTALE)
+        return ESTALE;
+    if (old_errno == ENOSPC || new_errno == ENOSPC)
+        return ENOSPC;
 
-	return new_errno;
+    return new_errno;
 }
 
-
 int
-afr_final_errno (afr_local_t *local, afr_private_t *priv)
+afr_final_errno(afr_local_t *local, afr_private_t *priv)
 {
-	int i = 0;
-	int op_errno = 0;
-	int tmp_errno = 0;
+    int i = 0;
+    int op_errno = 0;
+    int tmp_errno = 0;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].valid)
-			continue;
-		if (local->replies[i].op_ret >= 0)
-			continue;
-		tmp_errno = local->replies[i].op_errno;
-		op_errno = afr_higher_errno (op_errno, tmp_errno);
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret >= 0)
+            continue;
+        tmp_errno = local->replies[i].op_errno;
+        op_errno = afr_higher_errno(op_errno, tmp_errno);
+    }
 
-	return op_errno;
+    return op_errno;
 }
 
 static int32_t
-afr_local_discovery_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			 int32_t op_ret, int32_t op_errno, dict_t *dict,
-			 dict_t *xdata)
-{
-        int              ret            = 0;
-        char            *pathinfo       = NULL;
-        gf_boolean_t     is_local        = _gf_false;
-        afr_private_t   *priv           = NULL;
-        int32_t          child_index    = -1;
-
-        if (op_ret != 0) {
-                goto out;
-        }
-
-	priv = this->private;
-	child_index = (int32_t)(long)cookie;
-
-        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
-        if (ret != 0) {
-                goto out;
-        }
-
-        ret = glusterfs_is_local_pathinfo (pathinfo, &is_local);
-        if (ret) {
-                goto out;
-        }
-
-        /*
-         * Note that one local subvolume will override another here.  The only
-         * way to avoid that would be to retain extra information about whether
-         * the previous read_child is local, and it's just not worth it.  Even
-         * the slowest local subvolume is far preferable to a remote one.
-         */
-        if (is_local) {
-                priv->local[child_index] = 1;
-                /* Don't set arbiter as read child. */
-                if (AFR_IS_ARBITER_BRICK(priv, child_index))
-                        goto out;
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        AFR_MSG_LOCAL_CHILD, "selecting local read_child %s",
-                        priv->children[child_index]->name);
-
-                priv->read_child = child_index;
-        }
+afr_local_discovery_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *dict,
+                        dict_t *xdata)
+{
+    int ret = 0;
+    char *pathinfo = NULL;
+    gf_boolean_t is_local = _gf_false;
+    afr_private_t *priv = NULL;
+    int32_t child_index = -1;
+
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    priv = this->private;
+    child_index = (int32_t)(long)cookie;
+
+    ret = dict_get_str_sizen(dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
+    if (ret != 0) {
+        goto out;
+    }
+
+    ret = glusterfs_is_local_pathinfo(pathinfo, &is_local);
+    if (ret) {
+        goto out;
+    }
+
+    /*
+     * Note that one local subvolume will override another here.  The only
+     * way to avoid that would be to retain extra information about whether
+     * the previous read_child is local, and it's just not worth it.  Even
+     * the slowest local subvolume is far preferable to a remote one.
+     */
+    if (is_local) {
+        priv->local[child_index] = 1;
+        /* Don't set arbiter as read child. */
+        if (AFR_IS_ARBITER_BRICK(priv, child_index))
+            goto out;
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_LOCAL_CHILD,
+               "selecting local read_child %s",
+               priv->children[child_index]->name);
+
+        priv->read_child = child_index;
+    }
 out:
-        STACK_DESTROY(frame->root);
-        return 0;
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
 static void
-afr_attempt_local_discovery (xlator_t *this, int32_t child_index)
+afr_attempt_local_discovery(xlator_t *this, int32_t child_index)
 {
-        call_frame_t    *newframe = NULL;
-        loc_t            tmploc = {0,};
-        afr_private_t   *priv = this->private;
+    call_frame_t *newframe = NULL;
+    loc_t tmploc = {
+        0,
+    };
+    afr_private_t *priv = this->private;
 
-        newframe = create_frame(this,this->ctx->pool);
-        if (!newframe) {
-                return;
-        }
+    newframe = create_frame(this, this->ctx->pool);
+    if (!newframe) {
+        return;
+    }
 
-        tmploc.gfid[sizeof(tmploc.gfid)-1] = 1;
-        STACK_WIND_COOKIE (newframe, afr_local_discovery_cbk,
-                           (void *)(long)child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->getxattr,
-                           &tmploc, GF_XATTR_PATHINFO_KEY, NULL);
+    tmploc.gfid[sizeof(tmploc.gfid) - 1] = 1;
+    STACK_WIND_COOKIE(newframe, afr_local_discovery_cbk,
+                      (void *)(long)child_index, priv->children[child_index],
+                      priv->children[child_index]->fops->getxattr, &tmploc,
+                      GF_XATTR_PATHINFO_KEY, NULL);
 }
 
 int
-afr_lookup_sh_metadata_wrap (void *opaque)
-{
-        call_frame_t *frame       = opaque;
-        afr_local_t  *local       = NULL;
-        xlator_t     *this        = NULL;
-        inode_t      *inode       = NULL;
-        afr_private_t *priv       = NULL;
-        struct afr_reply *replies = NULL;
-        int i= 0, first = -1;
-        int ret = -1;
-        dict_t *dict = NULL;
+afr_lookup_sh_metadata_wrap(void *opaque)
+{
+    call_frame_t *frame = opaque;
+    afr_local_t *local = NULL;
+    xlator_t *this = NULL;
+    inode_t *inode = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0, first = -1;
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    priv = this->private;
+    replies = local->replies;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        first = i;
+        break;
+    }
+    if (first == -1)
+        goto out;
+
+    if (afr_selfheal_metadata_by_stbuf(this, &replies[first].poststat))
+        goto out;
+
+    afr_local_replies_wipe(local, this->private);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+    if (local->xattr_req) {
+        dict_copy(local->xattr_req, dict);
+    }
+
+    ret = dict_set_sizen_str_sizen(dict, "link-count", GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
+
+    if (loc_is_nameless(&local->loc)) {
+        ret = afr_selfheal_unlocked_discover_on(frame, local->inode,
+                                                local->loc.gfid, local->replies,
+                                                local->child_up, dict);
+    } else {
+        inode = afr_selfheal_unlocked_lookup_on(frame, local->loc.parent,
+                                                local->loc.name, local->replies,
+                                                local->child_up, dict);
+    }
+    if (inode)
+        inode_unref(inode);
+out:
+    if (loc_is_nameless(&local->loc))
+        afr_discover_done(frame, this);
+    else
+        afr_lookup_done(frame, this);
 
-        local = frame->local;
-        this  = frame->this;
-        priv  = this->private;
-        replies = local->replies;
-
-        for (i =0; i < priv->child_count; i++) {
-                if(!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-                first = i;
-                break;
-        }
-        if (first == -1)
-                goto out;
+    if (dict)
+        dict_unref(dict);
 
-        if (afr_selfheal_metadata_by_stbuf (this, &replies[first].poststat))
-                goto out;
+    return 0;
+}
 
-        afr_local_replies_wipe (local, this->private);
+gf_boolean_t
+afr_is_pending_set(xlator_t *this, dict_t *xdata, int type)
+{
+    int idx = -1;
+    afr_private_t *priv = NULL;
+    void *pending_raw = NULL;
+    int *pending_int = NULL;
+    int i = 0;
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-        ret = dict_set_str (dict, "link-count", GF_XATTROP_INDEX_COUNT);
-        if (ret) {
-                gf_msg_debug (this->name, -ret,
-                              "Unable to set link-count in dict ");
+    priv = this->private;
+    idx = afr_index_for_transaction_type(type);
+
+    if (dict_get_ptr(xdata, AFR_DIRTY, &pending_raw) == 0) {
+        if (pending_raw) {
+            pending_int = pending_raw;
+
+            if (ntoh32(pending_int[idx]))
+                return _gf_true;
         }
+    }
 
-        if (loc_is_nameless (&local->loc)) {
-                ret = afr_selfheal_unlocked_discover_on (frame, local->inode,
-                                                         local->loc.gfid,
-                                                         local->replies,
-                                                         local->child_up);
-        } else {
-                inode = afr_selfheal_unlocked_lookup_on (frame,
-                                                        local->loc.parent,
-                                                        local->loc.name,
-                                                        local->replies,
-                                                        local->child_up, dict);
-        }
-        if (inode)
-                inode_unref (inode);
-out:
-        if (loc_is_nameless (&local->loc))
-                afr_discover_done (frame, this);
-        else
-                afr_lookup_done (frame, this);
+    for (i = 0; i < priv->child_count; i++) {
+        if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
+            continue;
+        if (!pending_raw)
+            continue;
+        pending_int = pending_raw;
 
-        if (dict)
-                dict_unref (dict);
+        if (ntoh32(pending_int[idx]))
+            return _gf_true;
+    }
 
-        return 0;
+    return _gf_false;
 }
 
 static gf_boolean_t
 afr_can_start_metadata_self_heal(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        struct afr_reply *replies = NULL;
-        int i = 0, first = -1;
-        gf_boolean_t start = _gf_false;
-        struct iatt stbuf = {0, };
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0, first = -1;
+    gf_boolean_t start = _gf_false;
+    struct iatt stbuf = {
+        0,
+    };
 
-        local = frame->local;
-        replies = local->replies;
-        priv = this->private;
+    local = frame->local;
+    replies = local->replies;
+    priv = this->private;
 
-        if (!priv->metadata_self_heal)
-                return _gf_false;
+    if (!priv->metadata_self_heal)
+        return _gf_false;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if(!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-                if (first == -1) {
-                        first = i;
-                        stbuf = replies[i].poststat;
-                        continue;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (first == -1) {
+            first = i;
+            stbuf = replies[i].poststat;
+            continue;
+        }
 
-                if (gf_uuid_compare (stbuf.ia_gfid, replies[i].poststat.ia_gfid)) {
-                        start = _gf_false;
-                        break;
-                }
-                if (!IA_EQUAL (stbuf, replies[i].poststat, type)) {
-                        start = _gf_false;
-                        break;
-                }
+        if (afr_is_pending_set(this, replies[i].xdata,
+                               AFR_METADATA_TRANSACTION)) {
+            /* Let shd do the heal so that lookup is not blocked
+             * on getting metadata lock/doing the heal */
+            start = _gf_false;
+            break;
+        }
 
-                /*Check if iattrs need heal*/
-                if ((!IA_EQUAL (stbuf, replies[i].poststat, uid)) ||
-                    (!IA_EQUAL (stbuf, replies[i].poststat, gid)) ||
-                    (!IA_EQUAL (stbuf, replies[i].poststat, prot))) {
-                        start = _gf_true;
-                        continue;
-                }
+        if (gf_uuid_compare(stbuf.ia_gfid, replies[i].poststat.ia_gfid)) {
+            start = _gf_false;
+            break;
+        }
+        if (!IA_EQUAL(stbuf, replies[i].poststat, type)) {
+            start = _gf_false;
+            break;
+        }
 
-                /*Check if xattrs need heal*/
-                if (!afr_xattrs_are_equal (replies[first].xdata,
-                                           replies[i].xdata))
-                        start = _gf_true;
+        /*Check if iattrs need heal*/
+        if ((!IA_EQUAL(stbuf, replies[i].poststat, uid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, gid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, prot))) {
+            start = _gf_true;
+            continue;
         }
 
-        return start;
+        /*Check if xattrs need heal*/
+        if (!afr_xattrs_are_equal(replies[first].xdata, replies[i].xdata))
+            start = _gf_true;
+    }
+
+    return start;
 }
 
 int
-afr_lookup_metadata_heal_check (call_frame_t *frame, xlator_t *this)
+afr_lookup_metadata_heal_check(call_frame_t *frame, xlator_t *this)
 
 {
-        call_frame_t *heal = NULL;
-        afr_local_t *local = NULL;
-        int ret            = 0;
+    call_frame_t *heal = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
 
-        local = frame->local;
-        if (!afr_can_start_metadata_self_heal (frame, this))
-                goto out;
+    local = frame->local;
+    if (!afr_can_start_metadata_self_heal(frame, this))
+        goto out;
 
-        heal = afr_frame_create (this, &ret);
-        if (!heal) {
-                ret = -ret;
-                goto out;
-        }
+    heal = afr_frame_create(this, &ret);
+    if (!heal) {
+        ret = -ret;
+        goto out;
+    }
 
-        ret = synctask_new (this->ctx->env, afr_lookup_sh_metadata_wrap,
-                            afr_refresh_selfheal_done, heal, frame);
-        if (ret)
-                goto out;
-        return ret;
+    ret = synctask_new(this->ctx->env, afr_lookup_sh_metadata_wrap,
+                       afr_refresh_selfheal_done, heal, frame);
+    if (ret)
+        goto out;
+    return ret;
 out:
-        if (loc_is_nameless (&local->loc))
-                afr_discover_done (frame, this);
-        else
-                afr_lookup_done (frame, this);
-        if (heal)
-                AFR_STACK_DESTROY (heal);
-        return ret;
+    if (loc_is_nameless(&local->loc))
+        afr_discover_done(frame, this);
+    else
+        afr_lookup_done(frame, this);
+    if (heal)
+        AFR_STACK_DESTROY(heal);
+    return ret;
 }
 
 int
-afr_lookup_selfheal_wrap (void *opaque)
+afr_lookup_selfheal_wrap(void *opaque)
 {
-        int ret = 0;
-	call_frame_t *frame = opaque;
-	afr_local_t *local = NULL;
-	xlator_t *this = NULL;
-	inode_t *inode = NULL;
-        uuid_t pargfid = {0,};
+    int ret = 0;
+    call_frame_t *frame = opaque;
+    afr_local_t *local = NULL;
+    xlator_t *this = NULL;
+    inode_t *inode = NULL;
+    uuid_t pargfid = {
+        0,
+    };
 
-	local = frame->local;
-	this = frame->this;
-        loc_pargfid (&local->loc, pargfid);
+    local = frame->local;
+    this = frame->this;
+    loc_pargfid(&local->loc, pargfid);
 
-	ret = afr_selfheal_name (frame->this, pargfid, local->loc.name,
-                                 &local->cont.lookup.gfid_req, local->xattr_req);
-        if (ret == -EIO)
-                goto unwind;
+    ret = afr_selfheal_name(frame->this, pargfid, local->loc.name,
+                            &local->cont.lookup.gfid_req, local->xattr_req);
+    if (ret == -EIO)
+        goto unwind;
 
-        afr_local_replies_wipe (local, this->private);
+    afr_local_replies_wipe(local, this->private);
 
-	inode = afr_selfheal_unlocked_lookup_on (frame, local->loc.parent,
-						 local->loc.name, local->replies,
-						 local->child_up, NULL);
-	if (inode)
-		inode_unref (inode);
+    inode = afr_selfheal_unlocked_lookup_on(frame, local->loc.parent,
+                                            local->loc.name, local->replies,
+                                            local->child_up, local->xattr_req);
+    if (inode)
+        inode_unref(inode);
 
-        afr_lookup_metadata_heal_check(frame, this);
-        return 0;
+    afr_lookup_metadata_heal_check(frame, this);
+    return 0;
 
 unwind:
-	AFR_STACK_UNWIND (lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-afr_lookup_entry_heal (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	call_frame_t *heal = NULL;
-	int i = 0, first = -1;
-	gf_boolean_t need_heal = _gf_false;
-	struct afr_reply *replies = NULL;
-	int ret = 0;
-
-	local = frame->local;
-	replies = local->replies;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-
-                if ((replies[i].op_ret == -1) &&
-                    (replies[i].op_errno == ENODATA))
-                        need_heal = _gf_true;
-
-		if (first == -1) {
-			first = i;
-			continue;
-		}
-
-		if (replies[i].op_ret != replies[first].op_ret) {
-			need_heal = _gf_true;
-			break;
-		}
-
-		if (gf_uuid_compare (replies[i].poststat.ia_gfid,
-				  replies[first].poststat.ia_gfid)) {
-			need_heal = _gf_true;
-			break;
-		}
-	}
-
-	if (need_heal) {
-		heal = afr_frame_create (this, NULL);
-		if (!heal)
-                        goto metadata_heal;
-
-		ret = synctask_new (this->ctx->env, afr_lookup_selfheal_wrap,
-				    afr_refresh_selfheal_done, heal, frame);
-		if (ret) {
-                        AFR_STACK_DESTROY (heal);
-			goto metadata_heal;
-                }
-                return ret;
-	}
-metadata_heal:
-        ret = afr_lookup_metadata_heal_check (frame, this);
+afr_lookup_entry_heal(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    call_frame_t *heal = NULL;
+    int i = 0, first = -1;
+    gf_boolean_t name_state_mismatch = _gf_false;
+    struct afr_reply *replies = NULL;
+    int ret = 0;
+    unsigned char *par_readables = NULL;
+    unsigned char *success = NULL;
+    int32_t op_errno = 0;
+    uuid_t gfid = {0};
+
+    local = frame->local;
+    replies = local->replies;
+    priv = this->private;
+    par_readables = alloca0(priv->child_count);
+    success = alloca0(priv->child_count);
+
+    ret = afr_inode_read_subvol_get(local->loc.parent, this, par_readables,
+                                    NULL, NULL);
+    if (ret < 0 || AFR_COUNT(par_readables, priv->child_count) == 0) {
+        /* In this case set par_readables to all 1 so that name_heal
+         * need checks at the end of this function will flag missing
+         * entry when name state mismatches*/
+        memset(par_readables, 1, priv->child_count);
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret == 0) {
+            if (gf_uuid_is_null(gfid)) {
+                gf_uuid_copy(gfid, replies[i].poststat.ia_gfid);
+            }
+            success[i] = 1;
+        } else {
+            if ((replies[i].op_errno != ENOTCONN) &&
+                (replies[i].op_errno != ENOENT) &&
+                (replies[i].op_errno != ESTALE)) {
+                op_errno = replies[i].op_errno;
+            }
+        }
 
-	return ret;
-}
+        /*gfid is missing, needs heal*/
+        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA)) {
+            goto name_heal;
+        }
 
+        if (first == -1) {
+            first = i;
+            continue;
+        }
 
-int
-afr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
-		dict_t *xdata, struct iatt *postparent)
-{
-        afr_local_t *   local = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        GF_UNUSED int   ret             = 0;
-	int8_t need_heal                = 1;
-
-	child_index = (long) cookie;
-
-	local = frame->local;
-
-	local->replies[child_index].valid = 1;
-	local->replies[child_index].op_ret = op_ret;
-	local->replies[child_index].op_errno = op_errno;
-        /*
-         * On revalidate lookup if the gfid-changed, afr should unwind the fop
-         * with ESTALE so that a fresh lookup will be sent by the top xlator.
-         * So remember it.
-         */
-        if (xdata && dict_get (xdata, "gfid-changed"))
-                local->cont.lookup.needs_fresh_lookup = _gf_true;
+        if (replies[i].op_ret != replies[first].op_ret) {
+            name_state_mismatch = _gf_true;
+        }
 
-        if (xdata) {
-                ret = dict_get_int8 (xdata, "link-count", &need_heal);
-                local->replies[child_index].need_heal = need_heal;
-        } else {
-                local->replies[child_index].need_heal = need_heal;
+        if (replies[i].op_ret == 0) {
+            /* Rename after this lookup may succeed if we don't do
+             * a name-heal and the destination may not have pending xattrs
+             * to indicate which name is good and which is bad so always do
+             * this heal*/
+            if (gf_uuid_compare(replies[i].poststat.ia_gfid, gfid)) {
+                goto name_heal;
+            }
         }
-	if (op_ret != -1) {
-		local->replies[child_index].poststat = *buf;
-		local->replies[child_index].postparent = *postparent;
-		if (xdata)
-			local->replies[child_index].xdata = dict_ref (xdata);
-	}
+    }
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                afr_set_need_heal (this, local);
-		afr_lookup_entry_heal (frame, this);
+    if (name_state_mismatch) {
+        if (!priv->quorum_count)
+            goto name_heal;
+        if (!afr_has_quorum(success, this, NULL))
+            goto name_heal;
+        if (op_errno)
+            goto name_heal;
+        for (i = 0; i < priv->child_count; i++) {
+            if (!replies[i].valid)
+                continue;
+            if (par_readables[i] && replies[i].op_ret < 0 &&
+                replies[i].op_errno != ENOTCONN) {
+                goto name_heal;
+            }
         }
+    }
+
+    goto metadata_heal;
 
-	return 0;
+name_heal:
+    heal = afr_frame_create(this, NULL);
+    if (!heal)
+        goto metadata_heal;
+
+    ret = synctask_new(this->ctx->env, afr_lookup_selfheal_wrap,
+                       afr_refresh_selfheal_done, heal, frame);
+    if (ret) {
+        AFR_STACK_DESTROY(heal);
+        goto metadata_heal;
+    }
+    return ret;
+
+metadata_heal:
+    ret = afr_lookup_metadata_heal_check(frame, this);
+
+    return ret;
+}
+
+int
+afr_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+               struct iatt *postparent)
+{
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = -1;
+    GF_UNUSED int ret = 0;
+    int8_t need_heal = 1;
+
+    child_index = (long)cookie;
+
+    local = frame->local;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    /*
+     * On revalidate lookup if the gfid-changed, afr should unwind the fop
+     * with ESTALE so that a fresh lookup will be sent by the top xlator.
+     * So remember it.
+     */
+    if (xdata && dict_get_sizen(xdata, "gfid-changed"))
+        local->cont.lookup.needs_fresh_lookup = _gf_true;
+
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        local->replies[child_index].need_heal = need_heal;
+    } else {
+        local->replies[child_index].need_heal = need_heal;
+    }
+    if (op_ret != -1) {
+        local->replies[child_index].poststat = *buf;
+        local->replies[child_index].postparent = *postparent;
+        if (xdata)
+            local->replies[child_index].xdata = dict_ref(xdata);
+    }
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        afr_lookup_entry_heal(frame, this);
+    }
+
+    return 0;
 }
 
 static void
-afr_discover_unwind (call_frame_t *frame, xlator_t *this)
+afr_discover_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t       *priv  = NULL;
-        afr_local_t         *local = NULL;
-	int                 i = -1;
-	int                 op_errno = 0;
-	int                 read_subvol = -1;
-        unsigned char *data_readable = NULL;
-        unsigned char *success_replies = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int read_subvol = -1;
+    int ret = 0;
+    unsigned char *data_readable = NULL;
+    unsigned char *success_replies = NULL;
 
-        priv  = this->private;
-        local = frame->local;
-        data_readable = alloca0 (priv->child_count);
-        success_replies = alloca0 (priv->child_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].valid)
-			continue;
-		if (local->replies[i].op_ret == 0) {
-                        success_replies[i] = 1;
-			local->op_ret = 0;
-                }
-	}
+    priv = this->private;
+    local = frame->local;
+    data_readable = alloca0(priv->child_count);
+    success_replies = alloca0(priv->child_count);
 
-	op_errno = afr_final_errno (frame->local, this->private);
+    afr_fill_success_replies(local, priv, success_replies);
+    if (AFR_COUNT(success_replies, priv->child_count) > 0)
+        local->op_ret = 0;
 
-        if (local->op_ret < 0) {
-                AFR_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL,
-                                  NULL, NULL);
-                return;
-	}
+    if (local->op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(frame->local, this->private);
+        goto error;
+    }
 
-        if (!afr_has_quorum (success_replies, this))
-                goto unwind;
+    if (!afr_has_quorum(success_replies, this, frame))
+        goto unwind;
 
-	afr_replies_interpret (frame, this, local->inode, NULL);
+    ret = afr_replies_interpret(frame, this, local->inode, NULL);
+    if (ret) {
+        afr_inode_need_refresh_set(local->inode, this);
+    }
 
-	read_subvol = afr_read_subvol_decide (local->inode, this, NULL,
-                                              data_readable);
+    read_subvol = afr_read_subvol_decide(local->inode, this, NULL,
+                                         data_readable);
 
 unwind:
-        afr_attempt_readsubvol_set (frame, this, success_replies, data_readable,
-                                    &read_subvol);
-	if (read_subvol == -1) {
-                AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                                  NULL, NULL, NULL, NULL);
-                return;
-        }
+    afr_attempt_readsubvol_set(frame, this, success_replies, data_readable,
+                               &read_subvol);
+    if (read_subvol == -1)
+        goto error;
 
-        if (AFR_IS_ARBITER_BRICK (priv, read_subvol) && local->op_ret == 0) {
-                local->op_ret = -1;
-                local->op_errno = ENOTCONN;
-	        gf_msg_debug (this->name, 0, "Arbiter cannot be a read subvol "
-                              "for %s", local->loc.path);
-        }
+    if (AFR_IS_ARBITER_BRICK(priv, read_subvol) && local->op_ret == 0) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg_debug(this->name, 0,
+                     "Arbiter cannot be a read subvol "
+                     "for %s",
+                     local->loc.path);
+    }
+
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->replies[read_subvol].poststat,
+                     local->replies[read_subvol].xdata,
+                     &local->replies[read_subvol].postparent);
+    return;
 
-	AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-			  local->inode, &local->replies[read_subvol].poststat,
-			  local->replies[read_subvol].xdata,
-			  &local->replies[read_subvol].postparent);
+error:
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno, NULL, NULL,
+                     NULL, NULL);
 }
 
 static int
-afr_ta_id_file_check (void *opaque)
-{
-        afr_private_t *priv = NULL;
-        xlator_t *this = NULL;
-        loc_t loc = {0, };
-        struct iatt  stbuf = {0,};
-        dict_t *dict = NULL;
-        uuid_t  gfid = {0,};
-        fd_t *fd = NULL;
-        int ret = 0;
-
-        this = opaque;
-        priv = this->private;
-
-        ret = afr_fill_ta_loc (this, &loc);
-        if (ret)
-                goto out;
-
-        ret = syncop_lookup (priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
-                             &stbuf, 0, 0, 0);
-        if (ret == 0) {
-                goto out;
-        } else if (ret == -ENOENT) {
-                fd = fd_create (loc.inode, getpid());
-                if (!fd)
-                        goto out;
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-                gf_uuid_generate (gfid);
-                ret = dict_set_gfuuid (dict, "gfid-req", gfid, true);
-                ret = syncop_create (priv->children[THIN_ARBITER_BRICK_INDEX],
-                                     &loc, O_RDWR, 0664, fd, &stbuf, dict,
-                                     NULL);
-        }
+afr_ta_id_file_check(void *opaque)
+{
+    afr_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    dict_t *dict = NULL;
+    uuid_t gfid = {
+        0,
+    };
+    fd_t *fd = NULL;
+    int ret = 0;
+
+    this = opaque;
+    priv = this->private;
+
+    ret = afr_fill_ta_loc(this, &loc, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        goto out;
+    }
+
+    ret = syncop_lookup(priv->children[THIN_ARBITER_BRICK_INDEX], &loc, &stbuf,
+                        0, 0, 0);
+    if (ret == 0) {
+        goto out;
+    } else if (ret == -ENOENT) {
+        fd = fd_create(loc.inode, getpid());
+        if (!fd)
+            goto out;
+        dict = dict_new();
+        if (!dict)
+            goto out;
+        gf_uuid_generate(gfid);
+        ret = dict_set_gfuuid(dict, "gfid-req", gfid, true);
+        ret = syncop_create(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
+                            O_RDWR, 0664, fd, &stbuf, dict, NULL);
+    }
 
 out:
-        if (ret == 0) {
-                gf_uuid_copy (priv->ta_gfid, stbuf.ia_gfid);
-        } else {
-                gf_msg (this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
-                        "Failed to lookup/create thin-arbiter id file.");
-        }
-        if (dict)
-                dict_unref (dict);
-        if (fd)
-                fd_unref (fd);
-        loc_wipe (&loc);
+    if (ret == 0) {
+        gf_uuid_copy(priv->ta_gfid, stbuf.ia_gfid);
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to lookup/create thin-arbiter id file.");
+    }
+    if (dict)
+        dict_unref(dict);
+    if (fd)
+        fd_unref(fd);
+    loc_wipe(&loc);
 
-        return 0;
+    return 0;
 }
 
 static int
-afr_ta_id_file_check_cbk (int ret, call_frame_t *ta_frame, void *opaque)
+afr_ta_id_file_check_cbk(int ret, call_frame_t *ta_frame, void *opaque)
 {
-        return 0;
+    return 0;
 }
 
 static void
-afr_discover_done (call_frame_t *frame, xlator_t *this)
+afr_discover_done(call_frame_t *frame, xlator_t *this)
 {
-        int ret = 0;
-        afr_private_t *priv = NULL;
+    int ret = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
-        if (!priv->thin_arbiter_count)
-                goto unwind;
-        if (!gf_uuid_is_null(priv->ta_gfid))
-                goto unwind;
+    priv = this->private;
+    if (!priv->thin_arbiter_count)
+        goto unwind;
+    if (!gf_uuid_is_null(priv->ta_gfid))
+        goto unwind;
 
-        ret = synctask_new (this->ctx->env, afr_ta_id_file_check,
-                            afr_ta_id_file_check_cbk, NULL, this);
-        if (ret)
-                goto unwind;
+    ret = synctask_new(this->ctx->env, afr_ta_id_file_check,
+                       afr_ta_id_file_check_cbk, NULL, this);
+    if (ret)
+        goto unwind;
 unwind:
-        afr_discover_unwind (frame, this);
+    afr_discover_unwind(frame, this);
 }
 
 int
-afr_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
-		  dict_t *xdata, struct iatt *postparent)
-{
-        afr_local_t *   local = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        GF_UNUSED int ret               = 0;
-	int8_t need_heal                = 1;
-
-	child_index = (long) cookie;
-
-	local = frame->local;
-
-	local->replies[child_index].valid = 1;
-	local->replies[child_index].op_ret = op_ret;
-	local->replies[child_index].op_errno = op_errno;
-	if (op_ret != -1) {
-		local->replies[child_index].poststat = *buf;
-		local->replies[child_index].postparent = *postparent;
-		if (xdata)
-			local->replies[child_index].xdata = dict_ref (xdata);
-	}
-
-        if (local->do_discovery && (op_ret == 0))
-                afr_attempt_local_discovery (this, child_index);
-
-        if (xdata) {
-                ret = dict_get_int8 (xdata, "link-count", &need_heal);
-                local->replies[child_index].need_heal = need_heal;
-        } else {
-                local->replies[child_index].need_heal = need_heal;
-        }
-
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                afr_set_need_heal (this, local);
-                afr_lookup_metadata_heal_check (frame, this);
-        }
+afr_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+                 struct iatt *postparent)
+{
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = -1;
+    GF_UNUSED int ret = 0;
+    int8_t need_heal = 1;
+
+    child_index = (long)cookie;
+
+    local = frame->local;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (op_ret != -1) {
+        local->replies[child_index].poststat = *buf;
+        local->replies[child_index].postparent = *postparent;
+        if (xdata)
+            local->replies[child_index].xdata = dict_ref(xdata);
+    }
+
+    if (local->do_discovery && (op_ret == 0))
+        afr_attempt_local_discovery(this, child_index);
+
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        local->replies[child_index].need_heal = need_heal;
+    } else {
+        local->replies[child_index].need_heal = need_heal;
+    }
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        afr_lookup_metadata_heal_check(frame, this);
+    }
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_discover_do (call_frame_t *frame, xlator_t *this, int err)
-{
-	int ret = 0;
-	int i = 0;
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int call_count = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (err) {
-		local->op_errno = -err;
-		goto out;
-	}
-
-	call_count = local->call_count = AFR_COUNT (local->child_up,
-						    priv->child_count);
-
-        ret = afr_lookup_xattr_req_prepare (local, this, local->xattr_req,
-					    &local->loc);
-        if (ret) {
-                local->op_errno = -ret;
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_discover_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->lookup,
-                                           &local->loc, local->xattr_req);
-                        if (!--call_count)
-                                break;
-                }
+afr_discover_do(call_frame_t *frame, xlator_t *this, int err)
+{
+    int ret = 0;
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (err) {
+        local->op_errno = err;
+        goto out;
+    }
+
+    call_count = local->call_count = AFR_COUNT(local->child_up,
+                                               priv->child_count);
+
+    ret = afr_lookup_xattr_req_prepare(local, this, local->xattr_req,
+                                       &local->loc);
+    if (ret) {
+        local->op_errno = -ret;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(
+                frame, afr_discover_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->lookup, &local->loc, local->xattr_req);
+            if (!--call_count)
+                break;
         }
+    }
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
-	return 0;
+    AFR_STACK_UNWIND(lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+    return 0;
 }
 
-
 int
-afr_discover (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+afr_discover(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-	int op_errno = ENOMEM;
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	int event = 0;
+    int op_errno = ENOMEM;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int event = 0;
 
-	priv = this->private;
+    priv = this->private;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        if (!local->call_count) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
+    if (!local->call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
 
-	if (__is_root_gfid (loc->inode->gfid)) {
-		if (!this->itable)
-			this->itable = loc->inode->table;
-		if (!priv->root_inode)
-			priv->root_inode = inode_ref (loc->inode);
+    if (__is_root_gfid(loc->inode->gfid)) {
+        if (!priv->root_inode)
+            priv->root_inode = inode_ref(loc->inode);
 
-		if (priv->choose_local && !priv->did_discovery) {
-			/* Logic to detect which subvolumes of AFR are
-			   local, in order to prefer them for reads
-			*/
-			local->do_discovery = _gf_true;
-                        priv->did_discovery = _gf_true;
-                }
-	}
+        if (priv->choose_local && !priv->did_discovery) {
+            /* Logic to detect which subvolumes of AFR are
+               local, in order to prefer them for reads
+            */
+            local->do_discovery = _gf_true;
+            priv->did_discovery = _gf_true;
+        }
+    }
 
-        local->op = GF_FOP_LOOKUP;
+    local->op = GF_FOP_LOOKUP;
 
-        loc_copy (&local->loc, loc);
+    loc_copy(&local->loc, loc);
 
-	local->inode = inode_ref (loc->inode);
+    local->inode = inode_ref(loc->inode);
 
-	if (xattr_req)
-		/* If xattr_req was null, afr_lookup_xattr_req_prepare() will
-		   allocate one for us */
-		local->xattr_req = dict_ref (xattr_req);
+    if (xattr_req) {
+        /* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+           allocate one for us */
+        local->xattr_req = dict_copy_with_ref(xattr_req, NULL);
+        if (!local->xattr_req) {
+            op_errno = ENOMEM;
+            goto out;
+        }
+    }
 
-	if (gf_uuid_is_null (loc->inode->gfid)) {
-		afr_discover_do (frame, this, 0);
-		return 0;
-	}
+    if (gf_uuid_is_null(loc->inode->gfid)) {
+        afr_discover_do(frame, this, 0);
+        return 0;
+    }
 
-	afr_read_subvol_get (loc->inode, this, NULL, NULL, &event,
-			     AFR_DATA_TRANSACTION, NULL);
+    afr_read_subvol_get(loc->inode, this, NULL, NULL, &event,
+                        AFR_DATA_TRANSACTION, NULL);
 
-	if (afr_is_inode_refresh_reqd (loc->inode, this, event,
-                                       local->event_generation))
-		afr_inode_refresh (frame, this, loc->inode, NULL,
-                                   afr_discover_do);
-	else
-		afr_discover_do (frame, this, 0);
+    afr_discover_do(frame, this, 0);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
-	return 0;
+    AFR_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int
-afr_lookup_do (call_frame_t *frame, xlator_t *this, int err)
-{
-	int ret = 0;
-	int i = 0;
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int call_count = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (err < 0) {
-		local->op_errno = -err;
-		ret = -1;
-		goto out;
-	}
-
-	call_count = local->call_count = AFR_COUNT (local->child_up,
-						    priv->child_count);
-
-        ret = afr_lookup_xattr_req_prepare (local, this, local->xattr_req,
-					    &local->loc);
-        if (ret) {
-                local->op_errno = -ret;
-		ret = -1;
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_lookup_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->lookup,
-                                           &local->loc, local->xattr_req);
-                        if (!--call_count)
-                                break;
-                }
+afr_lookup_do(call_frame_t *frame, xlator_t *this, int err)
+{
+    int ret = 0;
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (err < 0) {
+        local->op_errno = err;
+        goto out;
+    }
+
+    call_count = local->call_count = AFR_COUNT(local->child_up,
+                                               priv->child_count);
+
+    ret = afr_lookup_xattr_req_prepare(local, this, local->xattr_req,
+                                       &local->loc);
+    if (ret) {
+        local->op_errno = -ret;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(
+                frame, afr_lookup_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->lookup, &local->loc, local->xattr_req);
+            if (!--call_count)
+                break;
         }
-	return 0;
+    }
+    return 0;
 out:
-	AFR_STACK_UNWIND (lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
-	return 0;
+    AFR_STACK_UNWIND(lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+    return 0;
 }
 
 /*
@@ -3284,1618 +4000,1722 @@ out:
  */
 
 int
-afr_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
-{
-        afr_local_t   *local = NULL;
-        int32_t        op_errno = 0;
-	int            event = 0;
-        int            ret = 0;
-
-        if (loc_is_nameless (loc)) {
-                if (xattr_req)
-                        dict_del (xattr_req, "gfid-req");
-		afr_discover (frame, this, loc, xattr_req);
-		return 0;
-	}
-
-	if (__is_root_gfid (loc->parent->gfid)) {
-		if (!strcmp (loc->name, GF_REPLICATE_TRASH_DIR)) {
-			op_errno = EPERM;
-			goto out;
-		}
-	}
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-        if (!local->call_count) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
+afr_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    int event = 0;
+    int ret = 0;
+
+    if (loc_is_nameless(loc)) {
+        if (xattr_req)
+            dict_del_sizen(xattr_req, "gfid-req");
+        afr_discover(frame, this, loc, xattr_req);
+        return 0;
+    }
 
-        local->op = GF_FOP_LOOKUP;
+    if (afr_is_private_directory(this->private, loc->parent->gfid, loc->name,
+                                 frame->root->pid)) {
+        op_errno = EPERM;
+        goto out;
+    }
 
-        loc_copy (&local->loc, loc);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->inode = inode_ref (loc->inode);
+    if (!local->call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
 
-	if (xattr_req) {
-		/* If xattr_req was null, afr_lookup_xattr_req_prepare() will
-		   allocate one for us */
-		local->xattr_req = dict_copy_with_ref (xattr_req, NULL);
-		if (!local->xattr_req) {
-		        op_errno = ENOMEM;
-		        goto out;
-                }
-                ret = dict_get_gfuuid (local->xattr_req, "gfid-req",
-                                       &local->cont.lookup.gfid_req);
-                if (ret == 0) {
-                        dict_del (local->xattr_req, "gfid-req");
-                }
+    local->op = GF_FOP_LOOKUP;
+
+    loc_copy(&local->loc, loc);
+
+    local->inode = inode_ref(loc->inode);
+
+    if (xattr_req) {
+        /* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+           allocate one for us */
+        local->xattr_req = dict_copy_with_ref(xattr_req, NULL);
+        if (!local->xattr_req) {
+            op_errno = ENOMEM;
+            goto out;
         }
+        ret = dict_get_gfuuid(local->xattr_req, "gfid-req",
+                              &local->cont.lookup.gfid_req);
+        if (ret == 0) {
+            dict_del_sizen(local->xattr_req, "gfid-req");
+        }
+    }
 
-	afr_read_subvol_get (loc->parent, this, NULL, NULL, &event,
-			     AFR_DATA_TRANSACTION, NULL);
+    afr_read_subvol_get(loc->parent, this, NULL, NULL, &event,
+                        AFR_DATA_TRANSACTION, NULL);
 
-	if (afr_is_inode_refresh_reqd (loc->inode, this, event,
-                                       local->event_generation))
-		afr_inode_refresh (frame, this, loc->parent, NULL,
-                                   afr_lookup_do);
-	else
-		afr_lookup_do (frame, this, 0);
+    afr_lookup_do(frame, this, 0);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    AFR_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 void
-_afr_cleanup_fd_ctx (afr_fd_ctx_t *fd_ctx)
+_afr_cleanup_fd_ctx(xlator_t *this, afr_fd_ctx_t *fd_ctx)
 {
-        GF_FREE (fd_ctx->opened_on);
-        GF_FREE (fd_ctx);
-        return;
+    afr_private_t *priv = this->private;
+
+    if (fd_ctx->lk_heal_info) {
+        LOCK(&priv->lock);
+        {
+            list_del(&fd_ctx->lk_heal_info->pos);
+        }
+        afr_lk_heal_info_cleanup(fd_ctx->lk_heal_info);
+        fd_ctx->lk_heal_info = NULL;
+    }
+    GF_FREE(fd_ctx->opened_on);
+    GF_FREE(fd_ctx);
+    return;
 }
 
 int
-afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
+afr_cleanup_fd_ctx(xlator_t *this, fd_t *fd)
 {
-        uint64_t        ctx = 0;
-        afr_fd_ctx_t    *fd_ctx = NULL;
-        int             ret = 0;
+    uint64_t ctx = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = 0;
 
-        ret = fd_ctx_get (fd, this, &ctx);
-        if (ret < 0)
-                goto out;
+    ret = fd_ctx_get(fd, this, &ctx);
+    if (ret < 0)
+        goto out;
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+    fd_ctx = (afr_fd_ctx_t *)(long)ctx;
 
-        if (fd_ctx) {
-                _afr_cleanup_fd_ctx (fd_ctx);
-        }
+    if (fd_ctx) {
+        _afr_cleanup_fd_ctx(this, fd_ctx);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 int
-afr_release (xlator_t *this, fd_t *fd)
+afr_release(xlator_t *this, fd_t *fd)
 {
-        afr_cleanup_fd_ctx (this, fd);
+    afr_cleanup_fd_ctx(this, fd);
 
-        return 0;
+    return 0;
 }
 
 afr_fd_ctx_t *
-__afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+__afr_fd_ctx_get(fd_t *fd, xlator_t *this)
 {
-        uint64_t       ctx = 0;
-        int            ret = 0;
-        afr_fd_ctx_t  *fd_ctx = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-        ret = __fd_ctx_get (fd, this, &ctx);
+    ret = __fd_ctx_get(fd, this, &ctx);
 
-        if (ret < 0) {
-                ret = __afr_fd_ctx_set (this, fd);
-                if (ret < 0)
-                        goto out;
+    if (ret < 0) {
+        ret = __afr_fd_ctx_set(this, fd);
+        if (ret < 0)
+            goto out;
 
-                ret = __fd_ctx_get (fd, this, &ctx);
-                if (ret < 0)
-                        goto out;
-        }
+        ret = __fd_ctx_get(fd, this, &ctx);
+        if (ret < 0)
+            goto out;
+    }
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+    fd_ctx = (afr_fd_ctx_t *)(long)ctx;
 out:
-        return fd_ctx;
+    return fd_ctx;
 }
 
-
 afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+afr_fd_ctx_get(fd_t *fd, xlator_t *this)
 {
-        afr_fd_ctx_t  *fd_ctx = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-        LOCK(&fd->lock);
-        {
-                fd_ctx = __afr_fd_ctx_get (fd, this);
-        }
-        UNLOCK(&fd->lock);
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(fd, this);
+    }
+    UNLOCK(&fd->lock);
 
-        return fd_ctx;
+    return fd_ctx;
 }
 
-
 int
-__afr_fd_ctx_set (xlator_t *this, fd_t *fd)
+__afr_fd_ctx_set(xlator_t *this, fd_t *fd)
 {
-        afr_private_t * priv   = NULL;
-        int             ret    = -1;
-        uint64_t        ctx    = 0;
-        afr_fd_ctx_t *  fd_ctx = NULL;
-	int             i = 0;
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    uint64_t ctx = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int i = 0;
 
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (fd, out);
+    VALIDATE_OR_GOTO(this->private, out);
+    VALIDATE_OR_GOTO(fd, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        ret = __fd_ctx_get (fd, this, &ctx);
+    ret = __fd_ctx_get(fd, this, &ctx);
 
-        if (ret == 0)
-                goto out;
+    if (ret == 0)
+        goto out;
 
-        fd_ctx = GF_CALLOC (1, sizeof (afr_fd_ctx_t),
-                            gf_afr_mt_afr_fd_ctx_t);
-        if (!fd_ctx) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    fd_ctx = GF_CALLOC(1, sizeof(afr_fd_ctx_t), gf_afr_mt_afr_fd_ctx_t);
+    if (!fd_ctx) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        fd_ctx->opened_on = GF_CALLOC (sizeof (*fd_ctx->opened_on),
-                                       priv->child_count,
-                                       gf_afr_mt_int32_t);
-        if (!fd_ctx->opened_on) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    fd_ctx->opened_on = GF_CALLOC(sizeof(*fd_ctx->opened_on), priv->child_count,
+                                  gf_afr_mt_int32_t);
+    if (!fd_ctx->opened_on) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (fd_is_anonymous (fd))
-			fd_ctx->opened_on[i] = AFR_FD_OPENED;
-		else
-			fd_ctx->opened_on[i] = AFR_FD_NOT_OPENED;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (fd_is_anonymous(fd))
+            fd_ctx->opened_on[i] = AFR_FD_OPENED;
+        else
+            fd_ctx->opened_on[i] = AFR_FD_NOT_OPENED;
+    }
 
-	fd_ctx->readdir_subvol = -1;
+    fd_ctx->readdir_subvol = -1;
+    fd_ctx->lk_heal_info = NULL;
 
-        ret = __fd_ctx_set (fd, this, (uint64_t)(long) fd_ctx);
-        if (ret)
-                gf_msg_debug (this->name, 0,
-                              "failed to set fd ctx (%p)", fd);
+    ret = __fd_ctx_set(fd, this, (uint64_t)(long)fd_ctx);
+    if (ret)
+        gf_msg_debug(this->name, 0, "failed to set fd ctx (%p)", fd);
 out:
-        if (ret && fd_ctx)
-                _afr_cleanup_fd_ctx (fd_ctx);
-        return ret;
+    if (ret && fd_ctx)
+        _afr_cleanup_fd_ctx(this, fd_ctx);
+    return ret;
 }
 
-
 /* {{{ flush */
 
 int
-afr_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
+    afr_local_t *local = NULL;
+    int call_count = -1;
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-			local->op_ret = op_ret;
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-		} else {
-			local->op_errno = op_errno;
-		}
+    LOCK(&frame->lock);
+    {
+        if (op_ret != -1) {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        } else {
+            local->op_errno = op_errno;
         }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-	if (call_count == 0)
-		AFR_STACK_UNWIND (flush, frame, local->op_ret,
-				  local->op_errno, local->xdata_rsp);
+    if (call_count == 0)
+        AFR_STACK_UNWIND(flush, frame, local->op_ret, local->op_errno,
+                         local->xdata_rsp);
 
-        return 0;
+    return 0;
 }
 
 static int
-afr_flush_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
-{
-        int           i      = 0;
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int call_count       = -1;
-
-        priv = this->private;
-        local = frame->local;
-        call_count = local->call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_flush_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->flush,
-                                           local->fd, xdata);
-                        if (!--call_count)
-                                break;
-
-                }
+afr_flush_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = -1;
+
+    priv = this->private;
+    local = frame->local;
+    call_count = local->call_count;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, afr_flush_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->flush,
+                              local->fd, xdata);
+            if (!--call_count)
+                break;
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-afr_local_t*
-afr_wakeup_same_fd_delayed_op (xlator_t *this, afr_lock_t *lock, fd_t *fd)
+afr_local_t *
+afr_wakeup_same_fd_delayed_op(xlator_t *this, afr_lock_t *lock, fd_t *fd)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-        if (lock->delay_timer) {
-                local = list_entry(lock->post_op.next, afr_local_t,
-                                   transaction.owner_list);
-                if (fd == local->fd) {
-                        if (gf_timer_call_cancel (this->ctx,
-                                                  lock->delay_timer)) {
-                                local = NULL;
-                        } else {
-                                lock->delay_timer = NULL;
-                        }
-                } else {
-                        local = NULL;
-                }
+    if (lock->delay_timer) {
+        local = list_entry(lock->post_op.next, afr_local_t,
+                           transaction.owner_list);
+        if (fd == local->fd) {
+            if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
+                local = NULL;
+            } else {
+                lock->delay_timer = NULL;
+            }
+        } else {
+            local = NULL;
         }
+    }
 
-        return local;
+    return local;
 }
 
 void
-afr_delayed_changelog_wake_resume (xlator_t *this, inode_t *inode,
-                                   call_stub_t *stub)
-{
-        afr_inode_ctx_t *ctx = NULL;
-        afr_lock_t      *lock = NULL;
-        afr_local_t     *metadata_local = NULL;
-        afr_local_t     *data_local = NULL;
-        LOCK (&inode->lock);
-        {
-                (void)__afr_inode_ctx_get (this, inode, &ctx);
-                lock = &ctx->lock[AFR_DATA_TRANSACTION];
-                data_local = afr_wakeup_same_fd_delayed_op (this, lock,
-                                                            stub->args.fd);
-                lock = &ctx->lock[AFR_METADATA_TRANSACTION];
-                metadata_local = afr_wakeup_same_fd_delayed_op (this, lock,
-                                                                stub->args.fd);
-        }
-        UNLOCK (&inode->lock);
-
-        if (data_local) {
-                data_local->transaction.resume_stub = stub;
-        } else if (metadata_local) {
-                metadata_local->transaction.resume_stub = stub;
-        } else {
-                call_resume (stub);
-        }
-        if (data_local) {
-                afr_delayed_changelog_wake_up_cbk (data_local);
-        }
-        if (metadata_local) {
-                afr_delayed_changelog_wake_up_cbk (metadata_local);
-        }
+afr_delayed_changelog_wake_resume(xlator_t *this, inode_t *inode,
+                                  call_stub_t *stub)
+{
+    afr_inode_ctx_t *ctx = NULL;
+    afr_lock_t *lock = NULL;
+    afr_local_t *metadata_local = NULL;
+    afr_local_t *data_local = NULL;
+    LOCK(&inode->lock);
+    {
+        (void)__afr_inode_ctx_get(this, inode, &ctx);
+        lock = &ctx->lock[AFR_DATA_TRANSACTION];
+        data_local = afr_wakeup_same_fd_delayed_op(this, lock, stub->args.fd);
+        lock = &ctx->lock[AFR_METADATA_TRANSACTION];
+        metadata_local = afr_wakeup_same_fd_delayed_op(this, lock,
+                                                       stub->args.fd);
+    }
+    UNLOCK(&inode->lock);
+
+    if (data_local) {
+        data_local->transaction.resume_stub = stub;
+    } else if (metadata_local) {
+        metadata_local->transaction.resume_stub = stub;
+    } else {
+        call_resume(stub);
+    }
+    if (data_local) {
+        afr_delayed_changelog_wake_up_cbk(data_local);
+    }
+    if (metadata_local) {
+        afr_delayed_changelog_wake_up_cbk(metadata_local);
+    }
 }
 
 int
-afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+afr_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *local   = NULL;
-        call_stub_t *stub    = NULL;
-        int         op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = ENOMEM;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->op = GF_FOP_FLUSH;
-	if (!afr_is_consistent_io_possible (local, this->private, &op_errno))
-		goto out;
+    local->op = GF_FOP_FLUSH;
+    if (!afr_is_consistent_io_possible(local, this->private, &op_errno))
+        goto out;
 
-	local->fd = fd_ref(fd);
+    local->fd = fd_ref(fd);
 
-        stub = fop_flush_stub (frame, afr_flush_wrapper, fd, xdata);
-        if (!stub)
-                goto out;
+    stub = fop_flush_stub(frame, afr_flush_wrapper, fd, xdata);
+    if (!stub)
+        goto out;
 
-        afr_delayed_changelog_wake_resume (this, fd->inode, stub);
+    afr_delayed_changelog_wake_resume(this, fd->inode, stub);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
-        return 0;
+    AFR_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+    return 0;
 }
 
 int
-afr_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
+    afr_local_t *local = NULL;
+    int call_count = -1;
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-		} else {
-			local->op_errno = op_errno;
-		}
+    LOCK(&frame->lock);
+    {
+        if (op_ret == 0) {
+            local->op_ret = 0;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        } else {
+            local->op_errno = op_errno;
         }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-				  local->op_errno, local->xdata_rsp);
+    if (call_count == 0)
+        AFR_STACK_UNWIND(fsyncdir, frame, local->op_ret, local->op_errno,
+                         local->xdata_rsp);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
-	      dict_t *xdata)
-{
-	afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = ENOMEM;
-
-	priv = this->private;
-
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-        local->op = GF_FOP_FSYNCDIR;
-	if (!afr_is_consistent_io_possible (local, priv, &op_errno))
-		goto out;
-
-        call_count = local->call_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_fsyncdir_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->fsyncdir,
-                                    fd, datasync, xdata);
-                        if (!--call_count)
-                                break;
-                }
+afr_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+             dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int32_t call_count = 0;
+    int32_t op_errno = ENOMEM;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_FSYNCDIR;
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
+
+    call_count = local->call_count;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND(frame, afr_fsyncdir_cbk, priv->children[i],
+                       priv->children[i]->fops->fsyncdir, fd, datasync, xdata);
+            if (!--call_count)
+                break;
         }
+    }
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
+    AFR_STACK_UNWIND(fsyncdir, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* }}} */
 
 static int
-afr_serialized_lock_wind (call_frame_t *frame, xlator_t *this);
+afr_serialized_lock_wind(call_frame_t *frame, xlator_t *this);
 
 static gf_boolean_t
-afr_is_conflicting_lock_present (int32_t op_ret, int32_t op_errno)
+afr_is_conflicting_lock_present(int32_t op_ret, int32_t op_errno)
 {
-        if (op_ret == -1 && op_errno == EAGAIN)
-                return _gf_true;
-        return _gf_false;
+    if (op_ret == -1 && op_errno == EAGAIN)
+        return _gf_true;
+    return _gf_false;
 }
 
 static void
-afr_fop_lock_unwind (call_frame_t *frame, glusterfs_fop_t op, int32_t op_ret,
-                     int32_t op_errno, dict_t *xdata)
+afr_fop_lock_unwind(call_frame_t *frame, glusterfs_fop_t op, int32_t op_ret,
+                    int32_t op_errno, dict_t *xdata)
 {
-        switch (op) {
+    switch (op) {
         case GF_FOP_INODELK:
-                AFR_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
-                break;
+            AFR_STACK_UNWIND(inodelk, frame, op_ret, op_errno, xdata);
+            break;
         case GF_FOP_FINODELK:
-                AFR_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
-                break;
+            AFR_STACK_UNWIND(finodelk, frame, op_ret, op_errno, xdata);
+            break;
         case GF_FOP_ENTRYLK:
-                AFR_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
-                break;
+            AFR_STACK_UNWIND(entrylk, frame, op_ret, op_errno, xdata);
+            break;
         case GF_FOP_FENTRYLK:
-                AFR_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, xdata);
-                break;
+            AFR_STACK_UNWIND(fentrylk, frame, op_ret, op_errno, xdata);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 }
 
 static void
-afr_fop_lock_wind (call_frame_t *frame, xlator_t *this, int child_index,
-                   int32_t (*lock_cbk) (call_frame_t *, void *, xlator_t *,
-                                        int32_t, int32_t, dict_t *))
+afr_fop_lock_wind(call_frame_t *frame, xlator_t *this, int child_index,
+                  int32_t (*lock_cbk)(call_frame_t *, void *, xlator_t *,
+                                      int32_t, int32_t, dict_t *))
 {
-        afr_local_t *local = frame->local;
-        afr_private_t *priv = this->private;
-        int i = child_index;
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = child_index;
 
-        switch (local->op) {
+    switch (local->op) {
         case GF_FOP_INODELK:
-                STACK_WIND_COOKIE (frame, lock_cbk, (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->inodelk,
-                                   (const char *)local->cont.inodelk.volume,
-                                   &local->loc, local->cont.inodelk.cmd,
-                                   &local->cont.inodelk.flock,
-                                   local->cont.inodelk.xdata);
-                break;
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->inodelk,
+                (const char *)local->cont.inodelk.volume, &local->loc,
+                local->cont.inodelk.cmd, &local->cont.inodelk.flock,
+                local->cont.inodelk.xdata);
+            break;
         case GF_FOP_FINODELK:
-                STACK_WIND_COOKIE (frame, lock_cbk, (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->finodelk,
-                                   (const char *)local->cont.inodelk.volume,
-                                   local->fd, local->cont.inodelk.cmd,
-                                   &local->cont.inodelk.flock,
-                                   local->cont.inodelk.xdata);
-                break;
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->finodelk,
+                (const char *)local->cont.inodelk.volume, local->fd,
+                local->cont.inodelk.cmd, &local->cont.inodelk.flock,
+                local->cont.inodelk.xdata);
+            break;
         case GF_FOP_ENTRYLK:
-                STACK_WIND_COOKIE (frame, lock_cbk, (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->entrylk,
-                                   local->cont.entrylk.volume, &local->loc,
-                                   local->cont.entrylk.basename,
-                                   local->cont.entrylk.cmd,
-                                   local->cont.entrylk.type,
-                                   local->cont.entrylk.xdata);
-                break;
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->entrylk, local->cont.entrylk.volume,
+                &local->loc, local->cont.entrylk.basename,
+                local->cont.entrylk.cmd, local->cont.entrylk.type,
+                local->cont.entrylk.xdata);
+            break;
         case GF_FOP_FENTRYLK:
-                STACK_WIND_COOKIE (frame, lock_cbk, (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->fentrylk,
-                                   local->cont.entrylk.volume, local->fd,
-                                   local->cont.entrylk.basename,
-                                   local->cont.entrylk.cmd,
-                                   local->cont.entrylk.type,
-                                   local->cont.entrylk.xdata);
-                break;
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->fentrylk, local->cont.entrylk.volume,
+                local->fd, local->cont.entrylk.basename,
+                local->cont.entrylk.cmd, local->cont.entrylk.type,
+                local->cont.entrylk.xdata);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 }
 
 void
-afr_fop_lock_proceed (call_frame_t *frame)
+afr_fop_lock_proceed(call_frame_t *frame)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = frame->this->private;
-
-        if (local->fop_lock_state != AFR_FOP_LOCK_PARALLEL) {
-                afr_fop_lock_unwind (frame, local->op, local->op_ret,
-                                     local->op_errno, local->xdata_rsp);
-                return;
-        }
-        /* At least one child is up */
-        /*
-         * Non-blocking locks also need to be serialized.  Otherwise there is
-         * a chance that both the mounts which issued same non-blocking inodelk
-         * may endup not acquiring the lock on any-brick.
-         * Ex: Mount1 and Mount2
-         * request for full length lock on file f1.  Mount1 afr may acquire the
-         * partial lock on brick-1 and may not acquire the lock on brick-2
-         * because Mount2 already got the lock on brick-2, vice versa.  Since
-         * both the mounts only got partial locks, afr treats them as failure in
-         * gaining the locks and unwinds with EAGAIN errno.
-         */
-        local->op_ret = -1;
-        local->op_errno = EUCLEAN;
-        local->fop_lock_state = AFR_FOP_LOCK_SERIAL;
-        afr_local_replies_wipe (local, priv);
-        if (local->xdata_rsp)
-                dict_unref (local->xdata_rsp);
-        local->xdata_rsp = NULL;
-        switch (local->op) {
+    local = frame->local;
+    priv = frame->this->private;
+
+    if (local->fop_lock_state != AFR_FOP_LOCK_PARALLEL) {
+        afr_fop_lock_unwind(frame, local->op, local->op_ret, local->op_errno,
+                            local->xdata_rsp);
+        return;
+    }
+    /* At least one child is up */
+    /*
+     * Non-blocking locks also need to be serialized.  Otherwise there is
+     * a chance that both the mounts which issued same non-blocking inodelk
+     * may endup not acquiring the lock on any-brick.
+     * Ex: Mount1 and Mount2
+     * request for full length lock on file f1.  Mount1 afr may acquire the
+     * partial lock on brick-1 and may not acquire the lock on brick-2
+     * because Mount2 already got the lock on brick-2, vice versa.  Since
+     * both the mounts only got partial locks, afr treats them as failure in
+     * gaining the locks and unwinds with EAGAIN errno.
+     */
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+    local->fop_lock_state = AFR_FOP_LOCK_SERIAL;
+    afr_local_replies_wipe(local, priv);
+    if (local->xdata_rsp)
+        dict_unref(local->xdata_rsp);
+    local->xdata_rsp = NULL;
+    switch (local->op) {
         case GF_FOP_INODELK:
         case GF_FOP_FINODELK:
-                local->cont.inodelk.cmd = local->cont.inodelk.in_cmd;
-                local->cont.inodelk.flock = local->cont.inodelk.in_flock;
-                if (local->cont.inodelk.xdata)
-                        dict_unref (local->cont.inodelk.xdata);
-                local->cont.inodelk.xdata = NULL;
-                if (local->xdata_req)
-                        local->cont.inodelk.xdata = dict_ref (local->xdata_req);
-                break;
+            local->cont.inodelk.cmd = local->cont.inodelk.in_cmd;
+            local->cont.inodelk.flock = local->cont.inodelk.in_flock;
+            if (local->cont.inodelk.xdata)
+                dict_unref(local->cont.inodelk.xdata);
+            local->cont.inodelk.xdata = NULL;
+            if (local->xdata_req)
+                local->cont.inodelk.xdata = dict_ref(local->xdata_req);
+            break;
         case GF_FOP_ENTRYLK:
         case GF_FOP_FENTRYLK:
-                local->cont.entrylk.cmd = local->cont.entrylk.in_cmd;
-                if (local->cont.entrylk.xdata)
-                        dict_unref (local->cont.entrylk.xdata);
-                local->cont.entrylk.xdata = NULL;
-                if (local->xdata_req)
-                        local->cont.entrylk.xdata = dict_ref (local->xdata_req);
-                break;
+            local->cont.entrylk.cmd = local->cont.entrylk.in_cmd;
+            if (local->cont.entrylk.xdata)
+                dict_unref(local->cont.entrylk.xdata);
+            local->cont.entrylk.xdata = NULL;
+            if (local->xdata_req)
+                local->cont.entrylk.xdata = dict_ref(local->xdata_req);
+            break;
         default:
-                break;
-        }
-        afr_serialized_lock_wind (frame, frame->this);
+            break;
+    }
+    afr_serialized_lock_wind(frame, frame->this);
 }
 
 static int32_t
-afr_unlock_partial_lock_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret,
-                             int32_t op_errno, dict_t *xdata)
+afr_unlock_partial_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int child_index = (long)cookie;
-        uuid_t  gfid = {0};
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = -1;
+    int child_index = (long)cookie;
+    uuid_t gfid = {0};
 
-        local = frame->local;
-        priv = this->private;
-
-        if (op_ret < 0 && op_errno != ENOTCONN) {
-                if (local->fd)
-                        gf_uuid_copy (gfid, local->fd->inode->gfid);
-                else
-                        loc_gfid (&local->loc, gfid);
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        AFR_MSG_UNLOCK_FAIL,
-                        "%s: Failed to unlock %s on %s "
-                        "with lk_owner: %s", uuid_utoa (gfid),
-                        gf_fop_list[local->op],
-                        priv->children[child_index]->name,
-                        lkowner_utoa (&frame->root->lk_owner));
-        }
-
-        call_count = afr_frame_return (frame);
-        if (call_count == 0)
-                afr_fop_lock_proceed (frame);
+    local = frame->local;
+    priv = this->private;
 
-        return 0;
+    if (op_ret < 0 && op_errno != ENOTCONN) {
+        if (local->fd)
+            gf_uuid_copy(gfid, local->fd->inode->gfid);
+        else
+            loc_gfid(&local->loc, gfid);
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "%s: Failed to unlock %s on %s "
+               "with lk_owner: %s",
+               uuid_utoa(gfid), gf_fop_list[local->op],
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        afr_fop_lock_proceed(frame);
+
+    return 0;
 }
 
 static int32_t
-afr_unlock_locks_and_proceed (call_frame_t *frame, xlator_t *this,
+afr_unlock_locks_and_proceed(call_frame_t *frame, xlator_t *this,
                              int call_count)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        if (call_count == 0) {
-                afr_fop_lock_proceed (frame);
-                goto out;
-        }
+    if (call_count == 0) {
+        afr_fop_lock_proceed(frame);
+        goto out;
+    }
 
-        local = frame->local;
-        priv = this->private;
-        local->call_count = call_count;
-        switch (local->op) {
+    local = frame->local;
+    priv = this->private;
+    local->call_count = call_count;
+    switch (local->op) {
         case GF_FOP_INODELK:
         case GF_FOP_FINODELK:
-                local->cont.inodelk.flock.l_type = F_UNLCK;
-                local->cont.inodelk.cmd = F_SETLK;
-                if (local->cont.inodelk.xdata)
-                        dict_unref (local->cont.inodelk.xdata);
-                local->cont.inodelk.xdata = NULL;
-                break;
+            local->cont.inodelk.flock.l_type = F_UNLCK;
+            local->cont.inodelk.cmd = F_SETLK;
+            if (local->cont.inodelk.xdata)
+                dict_unref(local->cont.inodelk.xdata);
+            local->cont.inodelk.xdata = NULL;
+            break;
         case GF_FOP_ENTRYLK:
         case GF_FOP_FENTRYLK:
-                local->cont.entrylk.cmd = ENTRYLK_UNLOCK;
-                if (local->cont.entrylk.xdata)
-                        dict_unref (local->cont.entrylk.xdata);
-                local->cont.entrylk.xdata = NULL;
-                break;
+            local->cont.entrylk.cmd = ENTRYLK_UNLOCK;
+            if (local->cont.entrylk.xdata)
+                dict_unref(local->cont.entrylk.xdata);
+            local->cont.entrylk.xdata = NULL;
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->replies[i].valid)
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
 
-                if (local->replies[i].op_ret == -1)
-                        continue;
+        if (local->replies[i].op_ret == -1)
+            continue;
 
-                afr_fop_lock_wind (frame, this, i, afr_unlock_partial_lock_cbk);
+        afr_fop_lock_wind(frame, this, i, afr_unlock_partial_lock_cbk);
 
-                if (!--call_count)
-                        break;
-        }
+        if (!--call_count)
+            break;
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-afr_fop_lock_done (call_frame_t *frame, xlator_t *this)
+afr_fop_lock_done(call_frame_t *frame, xlator_t *this)
 {
-        int i = 0;
-        int lock_count = 0;
-        unsigned char *success = NULL;
-
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv = this->private;
-        success = alloca0(priv->child_count);
+    int i = 0;
+    int lock_count = 0;
+    unsigned char *success = NULL;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->replies[i].valid)
-                        continue;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-                if (local->replies[i].op_ret == 0) {
-                        lock_count++;
-                        success[i] = 1;
-                }
+    local = frame->local;
+    priv = this->private;
+    success = alloca0(priv->child_count);
 
-                if (local->op_ret == -1 && local->op_errno == EAGAIN)
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
 
-                if ((local->replies[i].op_ret == -1) &&
-                    (local->replies[i].op_errno == EAGAIN)) {
-                        local->op_ret = -1;
-                        local->op_errno = EAGAIN;
-                        continue;
-                }
+        if (local->replies[i].op_ret == 0) {
+            lock_count++;
+            success[i] = 1;
+        }
 
-                if (local->replies[i].op_ret == 0)
-                        local->op_ret = 0;
+        if (local->op_ret == -1 && local->op_errno == EAGAIN)
+            continue;
 
-                local->op_errno = local->replies[i].op_errno;
+        if ((local->replies[i].op_ret == -1) &&
+            (local->replies[i].op_errno == EAGAIN)) {
+            local->op_ret = -1;
+            local->op_errno = EAGAIN;
+            continue;
         }
 
-        if (afr_fop_lock_is_unlock (frame))
-                goto unwind;
+        if (local->replies[i].op_ret == 0)
+            local->op_ret = 0;
 
-        if (afr_is_conflicting_lock_present (local->op_ret, local->op_errno)) {
-                afr_unlock_locks_and_proceed (frame, this, lock_count);
-        } else if (priv->quorum_count && !afr_has_quorum (success, this)) {
-                local->fop_lock_state = AFR_FOP_LOCK_QUORUM_FAILED;
-                local->op_ret = -1;
-                local->op_errno = afr_final_errno (local, priv);
-                if (local->op_errno == 0)
-                        local->op_errno = afr_quorum_errno (priv);
-                afr_unlock_locks_and_proceed (frame, this, lock_count);
-        } else {
-                goto unwind;
-        }
+        local->op_errno = local->replies[i].op_errno;
+    }
 
-        return 0;
+    if (afr_fop_lock_is_unlock(frame))
+        goto unwind;
+
+    if (afr_is_conflicting_lock_present(local->op_ret, local->op_errno)) {
+        afr_unlock_locks_and_proceed(frame, this, lock_count);
+    } else if (priv->quorum_count && !afr_has_quorum(success, this, NULL)) {
+        local->fop_lock_state = AFR_FOP_LOCK_QUORUM_FAILED;
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
+        if (local->op_errno == 0)
+            local->op_errno = afr_quorum_errno(priv);
+        afr_unlock_locks_and_proceed(frame, this, lock_count);
+    } else {
+        goto unwind;
+    }
+
+    return 0;
 unwind:
-        afr_fop_lock_unwind (frame, local->op, local->op_ret,
-                             local->op_errno, local->xdata_rsp);
-        return 0;
+    afr_fop_lock_unwind(frame, local->op, local->op_ret, local->op_errno,
+                        local->xdata_rsp);
+    return 0;
 }
 
 static int
-afr_common_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_common_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int child_index = (long)cookie;
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->replies[child_index].valid = 1;
-        local->replies[child_index].op_ret = op_ret;
-        local->replies[child_index].op_errno = op_errno;
-        if (op_ret == 0 && xdata) {
-                local->replies[child_index].xdata = dict_ref (xdata);
-                LOCK (&frame->lock);
-                {
-                        if (!local->xdata_rsp)
-                                local->xdata_rsp = dict_ref (xdata);
-                }
-                UNLOCK (&frame->lock);
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (op_ret == 0 && xdata) {
+        local->replies[child_index].xdata = dict_ref(xdata);
+        LOCK(&frame->lock);
+        {
+            if (!local->xdata_rsp)
+                local->xdata_rsp = dict_ref(xdata);
         }
-        return 0;
+        UNLOCK(&frame->lock);
+    }
+    return 0;
 }
 
 static int32_t
-afr_serialized_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_serialized_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int child_index = (long)cookie;
-        int next_child  = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = (long)cookie;
+    int next_child = 0;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        afr_common_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
 
-        for (next_child = child_index + 1; next_child < priv->child_count;
-             next_child++) {
-                if (local->child_up[next_child])
-                        break;
-        }
+    for (next_child = child_index + 1; next_child < priv->child_count;
+         next_child++) {
+        if (local->child_up[next_child])
+            break;
+    }
 
-        if (afr_is_conflicting_lock_present (op_ret, op_errno) ||
-            (next_child == priv->child_count)) {
-                afr_fop_lock_done (frame, this);
-        } else {
-                afr_fop_lock_wind (frame, this, next_child,
-                                   afr_serialized_lock_cbk);
-        }
+    if (afr_is_conflicting_lock_present(op_ret, op_errno) ||
+        (next_child == priv->child_count)) {
+        afr_fop_lock_done(frame, this);
+    } else {
+        afr_fop_lock_wind(frame, this, next_child, afr_serialized_lock_cbk);
+    }
 
-        return 0;
+    return 0;
 }
 
 static int
-afr_serialized_lock_wind (call_frame_t *frame, xlator_t *this)
+afr_serialized_lock_wind(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-        priv = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        afr_fop_lock_wind (frame, this, i,
-                                           afr_serialized_lock_cbk);
-                        break;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            afr_fop_lock_wind(frame, this, i, afr_serialized_lock_cbk);
+            break;
         }
-        return 0;
+    }
+    return 0;
 }
 
 static int32_t
-afr_parallel_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_parallel_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        int     call_count = 0;
+    int call_count = 0;
 
-        afr_common_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0)
-                afr_fop_lock_done (frame, this);
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        afr_fop_lock_done(frame, this);
 
-        return 0;
+    return 0;
 }
 
 static int
-afr_parallel_lock_wind (call_frame_t *frame, xlator_t *this)
+afr_parallel_lock_wind(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int         call_count = 0;
-        int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    int i = 0;
 
-        priv = this->private;
-        local = frame->local;
-        call_count = local->call_count;
+    priv = this->private;
+    local = frame->local;
+    call_count = local->call_count;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->child_up[i])
-                        continue;
-                afr_fop_lock_wind (frame, this, i, afr_parallel_lock_cbk);
-                if (!--call_count)
-                        break;
-        }
-        return 0;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
+        afr_fop_lock_wind(frame, this, i, afr_parallel_lock_cbk);
+        if (!--call_count)
+            break;
+    }
+    return 0;
 }
 
 static int
-afr_fop_handle_lock (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = frame->local;
-        int op_errno = 0;
-
-        if (!afr_fop_lock_is_unlock (frame)) {
-                if (!afr_is_consistent_io_possible (local, this->private,
-                                                &op_errno))
-                        goto out;
-
-                switch (local->op) {
-                case GF_FOP_INODELK:
-                case GF_FOP_FINODELK:
-                        local->cont.inodelk.cmd = F_SETLK;
-                        break;
-                case GF_FOP_ENTRYLK:
-                case GF_FOP_FENTRYLK:
-                        local->cont.entrylk.cmd = ENTRYLK_LOCK_NB;
-                        break;
-                default:
-                        break;
-                }
+afr_fop_handle_lock(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = frame->local;
+    int op_errno = 0;
+
+    if (!afr_fop_lock_is_unlock(frame)) {
+        if (!afr_is_consistent_io_possible(local, this->private, &op_errno))
+            goto out;
+
+        switch (local->op) {
+            case GF_FOP_INODELK:
+            case GF_FOP_FINODELK:
+                local->cont.inodelk.cmd = F_SETLK;
+                break;
+            case GF_FOP_ENTRYLK:
+            case GF_FOP_FENTRYLK:
+                local->cont.entrylk.cmd = ENTRYLK_LOCK_NB;
+                break;
+            default:
+                break;
         }
+    }
 
-        if (local->xdata_req) {
-                switch (local->op) {
-                case GF_FOP_INODELK:
-                case GF_FOP_FINODELK:
-                        local->cont.inodelk.xdata = dict_ref (local->xdata_req);
-                        break;
-                case GF_FOP_ENTRYLK:
-                case GF_FOP_FENTRYLK:
-                        local->cont.entrylk.xdata = dict_ref (local->xdata_req);
-                        break;
-                default:
-                        break;
-                }
+    if (local->xdata_req) {
+        switch (local->op) {
+            case GF_FOP_INODELK:
+            case GF_FOP_FINODELK:
+                local->cont.inodelk.xdata = dict_ref(local->xdata_req);
+                break;
+            case GF_FOP_ENTRYLK:
+            case GF_FOP_FENTRYLK:
+                local->cont.entrylk.xdata = dict_ref(local->xdata_req);
+                break;
+            default:
+                break;
         }
+    }
 
-        local->fop_lock_state = AFR_FOP_LOCK_PARALLEL;
-        afr_parallel_lock_wind (frame, this);
+    local->fop_lock_state = AFR_FOP_LOCK_PARALLEL;
+    afr_parallel_lock_wind(frame, this);
 out:
-        return -op_errno;
+    return -op_errno;
 }
 
 static int32_t
-afr_handle_inodelk (call_frame_t *frame, glusterfs_fop_t fop,
-                    const char *volume, loc_t *loc, fd_t *fd, int32_t cmd,
-                    struct gf_flock *flock, dict_t *xdata)
-{
-        afr_local_t *local  = NULL;
-        int32_t op_errno = ENOMEM;
-
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local)
-                goto out;
-
-        local->op = fop;
-        if (loc)
-                loc_copy (&local->loc, loc);
-        if (fd)
-                local->fd = fd_ref (fd);
-
-        local->cont.inodelk.volume = gf_strdup (volume);
-        if (!local->cont.inodelk.volume) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        local->cont.inodelk.in_cmd = cmd;
-        local->cont.inodelk.cmd = cmd;
-        local->cont.inodelk.in_flock = *flock;
-        local->cont.inodelk.flock = *flock;
-        if (xdata)
-                local->xdata_req = dict_ref (xdata);
-
-        op_errno = -afr_fop_handle_lock (frame, frame->this);
-        if (op_errno)
-                goto out;
-        return 0;
+afr_handle_inodelk(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
+                   const char *volume, loc_t *loc, fd_t *fd, int32_t cmd,
+                   struct gf_flock *flock, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = fop;
+    if (loc)
+        loc_copy(&local->loc, loc);
+    if (fd && (flock->l_type != F_UNLCK)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        local->fd = fd_ref(fd);
+    }
+
+    local->cont.inodelk.volume = gf_strdup(volume);
+    if (!local->cont.inodelk.volume) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->cont.inodelk.in_cmd = cmd;
+    local->cont.inodelk.cmd = cmd;
+    local->cont.inodelk.in_flock = *flock;
+    local->cont.inodelk.flock = *flock;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    op_errno = -afr_fop_handle_lock(frame, frame->this);
+    if (op_errno)
+        goto out;
+    return 0;
 out:
-        afr_fop_lock_unwind (frame, fop, -1, op_errno, NULL);
+    afr_fop_lock_unwind(frame, fop, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-afr_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd,
-             struct gf_flock *flock, dict_t *xdata)
+afr_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        afr_handle_inodelk (frame, GF_FOP_INODELK, volume, loc, NULL, cmd,
-                            flock, xdata);
-        return 0;
+    afr_handle_inodelk(frame, this, GF_FOP_INODELK, volume, loc, NULL, cmd,
+                       flock, xdata);
+    return 0;
 }
 
 int32_t
-afr_finodelk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
-	      int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+afr_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        afr_handle_inodelk (frame, GF_FOP_FINODELK, volume, NULL, fd, cmd,
-                            flock, xdata);
-        return 0;
+    afr_handle_inodelk(frame, this, GF_FOP_FINODELK, volume, NULL, fd, cmd,
+                       flock, xdata);
+    return 0;
 }
 
 static int
-afr_handle_entrylk (call_frame_t *frame, glusterfs_fop_t fop,
-                    const char *volume, loc_t *loc, fd_t *fd,
-                    const char *basename, entrylk_cmd cmd,
-                    entrylk_type type, dict_t *xdata)
+afr_handle_entrylk(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
+                   const char *volume, loc_t *loc, fd_t *fd,
+                   const char *basename, entrylk_cmd cmd, entrylk_type type,
+                   dict_t *xdata)
 {
-        afr_local_t *local  = NULL;
-        int32_t op_errno = ENOMEM;
-
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local)
-                goto out;
-
-        local->op = fop;
-        if (loc)
-                loc_copy (&local->loc, loc);
-        if (fd)
-                local->fd = fd_ref (fd);
-        local->cont.entrylk.cmd = cmd;
-        local->cont.entrylk.in_cmd = cmd;
-        local->cont.entrylk.type = type;
-        local->cont.entrylk.volume = gf_strdup (volume);
-        local->cont.entrylk.basename = gf_strdup (basename);
-        if (!local->cont.entrylk.volume || !local->cont.entrylk.basename) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-        if (xdata)
-                local->xdata_req = dict_ref (xdata);
-        op_errno = -afr_fop_handle_lock (frame, frame->this);
-        if (op_errno)
-                goto out;
-
-        return 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = fop;
+    if (loc)
+        loc_copy(&local->loc, loc);
+    if (fd && (cmd != ENTRYLK_UNLOCK)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        local->fd = fd_ref(fd);
+    }
+    local->cont.entrylk.cmd = cmd;
+    local->cont.entrylk.in_cmd = cmd;
+    local->cont.entrylk.type = type;
+    local->cont.entrylk.volume = gf_strdup(volume);
+    local->cont.entrylk.basename = gf_strdup(basename);
+    if (!local->cont.entrylk.volume || !local->cont.entrylk.basename) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+    op_errno = -afr_fop_handle_lock(frame, frame->this);
+    if (op_errno)
+        goto out;
+
+    return 0;
 out:
-        afr_fop_lock_unwind (frame, fop, -1, op_errno, NULL);
-        return 0;
+    afr_fop_lock_unwind(frame, fop, -1, op_errno, NULL);
+    return 0;
 }
 
 int
-afr_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-	     loc_t *loc, const char *basename, entrylk_cmd cmd,
-	     entrylk_type type, dict_t *xdata)
+afr_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
 {
-        afr_handle_entrylk (frame, GF_FOP_ENTRYLK, volume, loc, NULL, basename,
-                            cmd, type, xdata);
-        return 0;
+    afr_handle_entrylk(frame, this, GF_FOP_ENTRYLK, volume, loc, NULL, basename,
+                       cmd, type, xdata);
+    return 0;
 }
 
 int
-afr_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
-              const char *basename, entrylk_cmd cmd, entrylk_type type,
-	      dict_t *xdata)
+afr_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata)
 {
-        afr_handle_entrylk (frame, GF_FOP_FENTRYLK, volume, NULL, fd, basename,
-                            cmd, type, xdata);
-        return 0;
+    afr_handle_entrylk(frame, this, GF_FOP_FENTRYLK, volume, NULL, fd, basename,
+                       cmd, type, xdata);
+    return 0;
 }
 
-
 int
-afr_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-		int op_errno, struct statvfs *statvfs, dict_t *xdata)
+afr_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = 0;
-	struct statvfs *buf = NULL;
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    struct statvfs *buf = NULL;
 
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
-
-                if (op_ret != 0) {
-                        local->op_errno = op_errno;
-			goto unlock;
-		}
-
-		local->op_ret = op_ret;
-
-		buf = &local->cont.statfs.buf;
-		if (local->cont.statfs.buf_set) {
-			if (statvfs->f_bavail < buf->f_bavail) {
-				*buf = *statvfs;
-				if (xdata) {
-					if (local->xdata_rsp)
-						dict_unref (local->xdata_rsp);
-					local->xdata_rsp = dict_ref (xdata);
-				}
-			}
-		} else {
-			*buf = *statvfs;
-			local->cont.statfs.buf_set = 1;
-			if (xdata)
-				local->xdata_rsp = dict_ref (xdata);
-		}
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret != 0) {
+            local->op_errno = op_errno;
+            goto unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        call_count = afr_frame_return (frame);
+        local->op_ret = op_ret;
+
+        buf = &local->cont.statfs.buf;
+        if (local->cont.statfs.buf_set) {
+            if (statvfs->f_bavail < buf->f_bavail) {
+                *buf = *statvfs;
+                if (xdata) {
+                    if (local->xdata_rsp)
+                        dict_unref(local->xdata_rsp);
+                    local->xdata_rsp = dict_ref(xdata);
+                }
+            }
+        } else {
+            *buf = *statvfs;
+            local->cont.statfs.buf_set = 1;
+            if (xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        }
+    }
+unlock:
+    call_count = --local->call_count;
+    UNLOCK(&frame->lock);
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->cont.statfs.buf, local->xdata_rsp);
+    if (call_count == 0)
+        AFR_STACK_UNWIND(statfs, frame, local->op_ret, local->op_errno,
+                         &local->cont.statfs.buf, local->xdata_rsp);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-        afr_local_t   *  local       = NULL;
-	afr_private_t   *priv        = NULL;
-        int              i           = 0;
-        int              call_count = 0;
-        int32_t          op_errno    = ENOMEM;
-
-	priv = this->private;
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-        local->op = GF_FOP_STATFS;
-	if (!afr_is_consistent_io_possible (local, priv, &op_errno))
-		goto out;
-
-        if (priv->arbiter_count == 1 && local->child_up[ARBITER_BRICK_INDEX])
-                local->call_count--;
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        if (AFR_IS_ARBITER_BRICK(priv, i))
-                                continue;
-                        STACK_WIND (frame, afr_statfs_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->statfs,
-                                    loc, xdata);
-                        if (!--call_count)
-                                break;
-                }
+afr_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
+    int32_t op_errno = ENOMEM;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_STATFS;
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
+
+    if (priv->arbiter_count == 1 && local->child_up[ARBITER_BRICK_INDEX])
+        local->call_count--;
+    call_count = local->call_count;
+    if (!call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            if (AFR_IS_ARBITER_BRICK(priv, i))
+                continue;
+            STACK_WIND(frame, afr_statfs_cbk, priv->children[i],
+                       priv->children[i]->fops->statfs, loc, xdata);
+            if (!--call_count)
+                break;
         }
+    }
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-                   dict_t *xdata)
+afr_lk_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                  dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = this->private;
-        int call_count = -1;
-        int child_index = (long)cookie;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
+    int call_count = -1;
+    int child_index = (long)cookie;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        AFR_MSG_UNLOCK_FAIL,
-                        "gfid=%s: unlock failed on subvolume %s "
-                        "with lock owner %s",
-                        uuid_utoa (local->fd->inode->gfid),
-                        priv->children[child_index]->name,
-                        lkowner_utoa (&frame->root->lk_owner));
-        }
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "gfid=%s: unlock failed on subvolume %s "
+               "with lock owner %s",
+               uuid_utoa(local->fd->inode->gfid),
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0)
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  NULL, local->xdata_rsp);
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno, NULL,
+                         local->xdata_rsp);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_lk_unlock (call_frame_t *frame, xlator_t *this)
+afr_lk_unlock(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   * local = NULL;
-        afr_private_t * priv  = NULL;
-        int i = 0;
-        int call_count = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        call_count = afr_locked_nodes_count (local->cont.lk.locked_nodes,
-                                             priv->child_count);
+    call_count = afr_locked_nodes_count(local->cont.lk.locked_nodes,
+                                        priv->child_count);
 
-        if (call_count == 0) {
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  NULL, local->xdata_rsp);
-                return 0;
-        }
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno, NULL,
+                         local->xdata_rsp);
+        return 0;
+    }
 
-        local->call_count = call_count;
+    local->call_count = call_count;
 
-        local->cont.lk.user_flock.l_type = F_UNLCK;
+    local->cont.lk.user_flock.l_type = F_UNLCK;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->cont.lk.locked_nodes[i]) {
-                        STACK_WIND_COOKIE (frame, afr_lk_unlock_cbk,
-                                    (void *) (long) i, priv->children[i],
-                                    priv->children[i]->fops->lk,
-                                    local->fd, F_SETLK,
-                                    &local->cont.lk.user_flock, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->cont.lk.locked_nodes[i]) {
+            STACK_WIND_COOKIE(frame, afr_lk_unlock_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->lk,
+                              local->fd, F_SETLK, &local->cont.lk.user_flock,
+                              NULL);
+
+            if (!--call_count)
+                break;
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-            dict_t *xdata)
+afr_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+           int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int child_index = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = -1;
 
+    local = frame->local;
+    priv = this->private;
 
-        local = frame->local;
-        priv  = this->private;
+    child_index = (long)cookie;
 
-        child_index = (long) cookie;
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret < 0 && op_errno == EAGAIN) {
+        local->op_ret = -1;
+        local->op_errno = EAGAIN;
 
-        afr_common_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-        if (op_ret < 0 && op_errno == EAGAIN) {
-                local->op_ret   = -1;
-                local->op_errno = EAGAIN;
+        afr_lk_unlock(frame, this);
+        return 0;
+    }
+
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lk.locked_nodes[child_index] = 1;
+        local->cont.lk.ret_flock = *lock;
+    }
+
+    child_index++;
+
+    if (child_index < priv->child_count) {
+        STACK_WIND_COOKIE(frame, afr_lk_cbk, (void *)(long)child_index,
+                          priv->children[child_index],
+                          priv->children[child_index]->fops->lk, local->fd,
+                          local->cont.lk.cmd, &local->cont.lk.user_flock,
+                          local->xdata_req);
+    } else if (priv->quorum_count &&
+               !afr_has_quorum(local->cont.lk.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
 
-                afr_lk_unlock (frame, this);
-                return 0;
-        }
+        afr_lk_unlock(frame, this);
+    } else {
+        if (local->op_ret < 0)
+            local->op_errno = afr_final_errno(local, priv);
 
-        if (op_ret == 0) {
-                local->op_ret        = 0;
-                local->op_errno      = 0;
-                local->cont.lk.locked_nodes[child_index] = 1;
-                local->cont.lk.ret_flock = *lock;
-        }
-
-        child_index++;
-
-        if (child_index < priv->child_count) {
-                STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) child_index,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->lk,
-                                   local->fd, local->cont.lk.cmd,
-                                   &local->cont.lk.user_flock,
-                                   local->xdata_req);
-        } else if (priv->quorum_count &&
-                   !afr_has_quorum (local->cont.lk.locked_nodes, this)) {
-                local->op_ret   = -1;
-                local->op_errno = afr_final_errno (local, priv);
-
-                afr_lk_unlock (frame, this);
-        } else {
-                if (local->op_ret < 0)
-                        local->op_errno = afr_final_errno (local, priv);
-
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock, local->xdata_rsp);
-        }
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno,
+                         &local->cont.lk.ret_flock, local->xdata_rsp);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-afr_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+afr_lk_transaction_cbk(int ret, call_frame_t *frame, void *opaque)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int i = 0;
-        int32_t op_errno = ENOMEM;
+    return 0;
+}
 
-        priv = this->private;
+int
+afr_lk_txn_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                    dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int child_index = -1;
+
+    local = frame->local;
+    child_index = (long)cookie;
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lk.locked_nodes[child_index] = 1;
+        local->cont.lk.ret_flock = *lock;
+    }
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local)
-                goto out;
+int
+afr_lk_txn_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int child_index = (long)cookie;
 
-        local->op = GF_FOP_LK;
-        if (!afr_lk_is_unlock (cmd, flock) &&
-            !afr_is_consistent_io_possible (local, priv, &op_errno))
-                goto out;
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "gfid=%s: unlock failed on subvolume %s "
+               "with lock owner %s",
+               uuid_utoa(local->fd->inode->gfid),
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
+    return 0;
+}
+int
+afr_lk_transaction(void *opaque)
+{
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    char *wind_on = NULL;
+    int op_errno = 0;
+    int i = 0;
+    int ret = 0;
+
+    frame = (call_frame_t *)opaque;
+    local = frame->local;
+    this = frame->this;
+    priv = this->private;
+    wind_on = alloca0(priv->child_count);
+
+    if (priv->arbiter_count || priv->child_count != 3) {
+        op_errno = ENOTSUP;
+        gf_msg(frame->this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Lock healing supported only for replica 3 volumes.",
+               uuid_utoa(local->fd->inode->gfid));
+        goto err;
+    }
+
+    op_errno = -afr_dom_lock_acquire(frame);  // Released during
+                                              // AFR_STACK_UNWIND
+    if (op_errno != 0) {
+        goto err;
+    }
+    if (priv->quorum_count &&
+        !afr_has_quorum(local->cont.lk.dom_locked_nodes, this, NULL)) {
+        op_errno = afr_final_errno(local, priv);
+        goto err;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->child_up[i] && local->cont.lk.dom_locked_nodes[i])
+            wind_on[i] = 1;
+    }
+    AFR_ONLIST(wind_on, frame, afr_lk_txn_wind_cbk, lk, local->fd,
+               local->cont.lk.cmd, &local->cont.lk.user_flock,
+               local->xdata_req);
+
+    if (priv->quorum_count &&
+        !afr_has_quorum(local->cont.lk.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
+        goto unlock;
+    } else {
+        if (local->cont.lk.user_flock.l_type == F_UNLCK)
+            ret = afr_remove_lock_from_saved_locks(local, this);
+        else
+            ret = afr_add_lock_to_saved_locks(frame, this);
+        if (ret) {
+            local->op_ret = -1;
+            local->op_errno = -ret;
+            goto unlock;
+        }
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno,
+                         &local->cont.lk.ret_flock, local->xdata_rsp);
+    }
 
-        local->cont.lk.locked_nodes = GF_CALLOC (priv->child_count,
-                                                 sizeof (*local->cont.lk.locked_nodes),
-                                                 gf_afr_mt_char);
+    return 0;
 
-        if (!local->cont.lk.locked_nodes) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+unlock:
+    local->cont.lk.user_flock.l_type = F_UNLCK;
+    AFR_ONLIST(local->cont.lk.locked_nodes, frame, afr_lk_txn_unlock_cbk, lk,
+               local->fd, F_SETLK, &local->cont.lk.user_flock, NULL);
+err:
+    AFR_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
+    return -1;
+}
 
-        local->fd            = fd_ref (fd);
-        local->cont.lk.cmd   = cmd;
-        local->cont.lk.user_flock = *flock;
-        local->cont.lk.ret_flock = *flock;
-        if (xdata)
-                local->xdata_req = dict_ref (xdata);
+int
+afr_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+       struct gf_flock *flock, dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+    int i = 0;
+    int32_t op_errno = ENOMEM;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_LK;
+    if (!afr_lk_is_unlock(cmd, flock)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+            goto out;
+    }
+
+    local->cont.lk.locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.locked_nodes),
+        gf_afr_mt_char);
+
+    if (!local->cont.lk.locked_nodes) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->fd = fd_ref(fd);
+    local->cont.lk.cmd = cmd;
+    local->cont.lk.user_flock = *flock;
+    local->cont.lk.ret_flock = *flock;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    if (afr_is_lock_mode_mandatory(xdata)) {
+        ret = synctask_new(this->ctx->env, afr_lk_transaction,
+                           afr_lk_transaction_cbk, frame, frame);
+        if (ret) {
+            op_errno = ENOMEM;
+            goto out;
+        }
+        return 0;
+    }
 
-        STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) 0,
-                           priv->children[i],
-                           priv->children[i]->fops->lk,
-                           fd, cmd, flock, local->xdata_req);
+    STACK_WIND_COOKIE(frame, afr_lk_cbk, (void *)(long)0, priv->children[i],
+                      priv->children[i]->fops->lk, fd, cmd, flock,
+                      local->xdata_req);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-afr_lease_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_lease *lease,
-                      dict_t *xdata)
+afr_lease_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct gf_lease *lease,
+                     dict_t *xdata)
 {
-        afr_local_t  *local      = NULL;
-        int           call_count = -1;
+    afr_local_t *local = NULL;
+    int call_count = -1;
 
-        local = frame->local;
-        call_count = afr_frame_return (frame);
+    local = frame->local;
+    call_count = afr_frame_return(frame);
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (lease, frame, local->op_ret, local->op_errno,
-                                  lease, xdata);
+    if (call_count == 0)
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno, lease,
+                         xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-afr_lease_unlock (call_frame_t *frame, xlator_t *this)
+afr_lease_unlock(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local      = NULL;
-        afr_private_t *priv       = NULL;
-        int            i          = 0;
-        int            call_count = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        call_count = afr_locked_nodes_count (local->cont.lease.locked_nodes,
-                                             priv->child_count);
+    call_count = afr_locked_nodes_count(local->cont.lease.locked_nodes,
+                                        priv->child_count);
 
-        if (call_count == 0) {
-                AFR_STACK_UNWIND (lease, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lease.ret_lease, NULL);
-                return 0;
-        }
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno,
+                         &local->cont.lease.ret_lease, NULL);
+        return 0;
+    }
 
-        local->call_count = call_count;
+    local->call_count = call_count;
 
-        local->cont.lease.user_lease.cmd = GF_UNLK_LEASE;
+    local->cont.lease.user_lease.cmd = GF_UNLK_LEASE;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->cont.lease.locked_nodes[i]) {
-                        STACK_WIND (frame, afr_lease_unlock_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->lease,
-                                    &local->loc, &local->cont.lease.user_lease, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->cont.lease.locked_nodes[i]) {
+            STACK_WIND(frame, afr_lease_unlock_cbk, priv->children[i],
+                       priv->children[i]->fops->lease, &local->loc,
+                       &local->cont.lease.user_lease, NULL);
+
+            if (!--call_count)
+                break;
         }
+    }
 
-        return 0;
+    return 0;
 }
 
 int32_t
-afr_lease_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct gf_lease *lease,
-               dict_t *xdata)
+afr_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct gf_lease *lease, dict_t *xdata)
 {
-        afr_local_t   *local       = NULL;
-        afr_private_t *priv        = NULL;
-        int            child_index = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = -1;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        child_index = (long) cookie;
+    child_index = (long)cookie;
 
-        afr_common_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-        if (op_ret < 0 && op_errno == EAGAIN) {
-                local->op_ret   = -1;
-                local->op_errno = EAGAIN;
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret < 0 && op_errno == EAGAIN) {
+        local->op_ret = -1;
+        local->op_errno = EAGAIN;
 
-                afr_lease_unlock (frame, this);
-                return 0;
-        }
+        afr_lease_unlock(frame, this);
+        return 0;
+    }
 
-        if (op_ret == 0) {
-                local->op_ret        = 0;
-                local->op_errno      = 0;
-                local->cont.lease.locked_nodes[child_index] = 1;
-                local->cont.lease.ret_lease = *lease;
-        }
-
-        child_index++;
-        if (child_index < priv->child_count) {
-                STACK_WIND_COOKIE (frame, afr_lease_cbk, (void *) (long) child_index,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->lease,
-                                   &local->loc, &local->cont.lease.user_lease, xdata);
-        } else if  (priv->quorum_count &&
-                   !afr_has_quorum (local->cont.lk.locked_nodes, this)) {
-                local->op_ret   = -1;
-                local->op_errno = afr_final_errno (local, priv);
-
-                afr_lease_unlock (frame, this);
-        } else {
-                if (local->op_ret < 0)
-                        local->op_errno = afr_final_errno (local, priv);
-                AFR_STACK_UNWIND (lease, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lease.ret_lease, NULL);
-        }
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lease.locked_nodes[child_index] = 1;
+        local->cont.lease.ret_lease = *lease;
+    }
+
+    child_index++;
+    if (child_index < priv->child_count) {
+        STACK_WIND_COOKIE(frame, afr_lease_cbk, (void *)(long)child_index,
+                          priv->children[child_index],
+                          priv->children[child_index]->fops->lease, &local->loc,
+                          &local->cont.lease.user_lease, xdata);
+    } else if (priv->quorum_count &&
+               !afr_has_quorum(local->cont.lease.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
 
-        return 0;
+        afr_lease_unlock(frame, this);
+    } else {
+        if (local->op_ret < 0)
+            local->op_errno = afr_final_errno(local, priv);
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno,
+                         &local->cont.lease.ret_lease, NULL);
+    }
+
+    return 0;
 }
 
 int
-afr_lease (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, struct gf_lease *lease, dict_t *xdata)
+afr_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata)
 {
-        afr_private_t *priv     = NULL;
-        afr_local_t   *local    = NULL;
-        int32_t        op_errno = ENOMEM;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
 
-        priv = this->private;
+    priv = this->private;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local)
-                goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->op = GF_FOP_LEASE;
-        local->cont.lease.locked_nodes = GF_CALLOC (priv->child_count,
-                                                    sizeof (*local->cont.lease.locked_nodes),
-                                                    gf_afr_mt_char);
+    local->op = GF_FOP_LEASE;
+    local->cont.lease.locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lease.locked_nodes),
+        gf_afr_mt_char);
 
-        if (!local->cont.lease.locked_nodes) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    if (!local->cont.lease.locked_nodes) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-        loc_copy (&local->loc, loc);
-        local->cont.lease.user_lease = *lease;
-        local->cont.lease.ret_lease = *lease;
+    loc_copy(&local->loc, loc);
+    local->cont.lease.user_lease = *lease;
+    local->cont.lease.ret_lease = *lease;
 
-        STACK_WIND_COOKIE (frame, afr_lease_cbk, (void *) (long) 0,
-                           priv->children[0],
-                           priv->children[0]->fops->lease,
-                           loc, lease, xdata);
+    STACK_WIND_COOKIE(frame, afr_lease_cbk, (void *)(long)0, priv->children[0],
+                      priv->children[0]->fops->lease, loc, lease, xdata);
 
-        return 0;
+    return 0;
 out:
-        AFR_STACK_UNWIND (lease, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(lease, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-afr_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_local_t   *local          = NULL;
-        int            child_index    = (long)cookie;
-        int            call_count     = 0;
-        gf_boolean_t   failed         = _gf_false;
-        gf_boolean_t   succeded       = _gf_false;
-        int            i              = 0;
-        afr_private_t *priv           = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        local->replies[child_index].valid = 1;
-        local->replies[child_index].op_ret = op_ret;
-        local->replies[child_index].op_errno = op_errno;
-        if (xdata)
-                local->replies[child_index].xdata = dict_ref (xdata);
-
-        call_count = afr_frame_return (frame);
-        if (call_count)
-                goto out;
-        /* If any of the subvolumes failed with other than ENOTCONN
-         * return error else return success unless all the subvolumes
-         * failed.
-         * TODO: In case of failure, we need to unregister the xattrs
-         * from the other subvolumes where it succeded (once upcall
-         * fixes the Bz-1371622)*/
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->replies[i].valid)
-                        continue;
-                if (local->replies[i].op_ret < 0 &&
-                    local->replies[i].op_errno != ENOTCONN) {
-                        local->op_ret = local->replies[i].op_ret;
-                        local->op_errno = local->replies[i].op_errno;
-                        if (local->xdata_rsp)
-                                dict_unref (local->xdata_rsp);
-                        local->xdata_rsp = NULL;
-                        if (local->replies[i].xdata) {
-                                local->xdata_rsp =
-                                        dict_ref (local->replies[i].xdata);
-                        }
-                        failed = _gf_true;
-                        break;
-                }
-                if (local->replies[i].op_ret == 0) {
-                        succeded = _gf_true;
-                        local->op_ret = 0;
-                        local->op_errno = 0;
-                        if (!local->xdata_rsp && local->replies[i].xdata) {
-                                local->xdata_rsp =
-                                        dict_ref (local->replies[i].xdata);
-                        }
-                }
+afr_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = 0;
+    gf_boolean_t failed = _gf_false;
+    gf_boolean_t succeeded = _gf_false;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
+
+    call_count = afr_frame_return(frame);
+    if (call_count)
+        goto out;
+    /* If any of the subvolumes failed with other than ENOTCONN
+     * return error else return success unless all the subvolumes
+     * failed.
+     * TODO: In case of failure, we need to unregister the xattrs
+     * from the other subvolumes where it succeeded (once upcall
+     * fixes the Bz-1371622)*/
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0 &&
+            local->replies[i].op_errno != ENOTCONN) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+            if (local->xdata_rsp)
+                dict_unref(local->xdata_rsp);
+            local->xdata_rsp = NULL;
+            if (local->replies[i].xdata) {
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+            failed = _gf_true;
+            break;
+        }
+        if (local->replies[i].op_ret == 0) {
+            succeeded = _gf_true;
+            local->op_ret = 0;
+            local->op_errno = 0;
+            if (!local->xdata_rsp && local->replies[i].xdata) {
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
         }
+    }
 
-        if (!succeded && !failed) {
-                local->op_ret = -1;
-                local->op_errno = ENOTCONN;
-        }
+    if (!succeeded && !failed) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+    }
 
-        AFR_STACK_UNWIND (ipc, frame, local->op_ret, local->op_errno,
-                          local->xdata_rsp);
+    AFR_STACK_UNWIND(ipc, frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
 
 out:
-        return 0;
+    return 0;
 }
 
 int
-afr_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+afr_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        afr_local_t    *local    = NULL;
-        int32_t         op_errno = -1;
-        afr_private_t  *priv     = NULL;
-        int             i        = 0;
-        int             call_cnt = -1;
+    afr_local_t *local = NULL;
+    int32_t op_errno = -1;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_cnt = -1;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
 
-        if (op != GF_IPC_TARGET_UPCALL)
-                goto wind_default;
+    if (op != GF_IPC_TARGET_UPCALL)
+        goto wind_default;
 
-        VALIDATE_OR_GOTO (this->private, err);
-        priv = this->private;
+    VALIDATE_OR_GOTO(this->private, err);
+    priv = this->private;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local)
-                goto err;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto err;
 
-        call_cnt = local->call_count;
+    call_cnt = local->call_count;
 
-        if (xdata) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (dict_set_int8 (xdata, priv->pending_key[i], 0) < 0)
-                                goto err;
-                }
+    if (xdata) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (dict_set_int8(xdata, priv->pending_key[i], 0) < 0)
+                goto err;
         }
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->child_up[i])
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
 
-                STACK_WIND_COOKIE (frame, afr_ipc_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->ipc,
-                                   op, xdata);
-                if (!--call_cnt)
-                        break;
-        }
-        return 0;
+        STACK_WIND_COOKIE(frame, afr_ipc_cbk, (void *)(long)i,
+                          priv->children[i], priv->children[i]->fops->ipc, op,
+                          xdata);
+        if (!--call_cnt)
+            break;
+    }
+    return 0;
 
 err:
-        if (op_errno == -1)
-                op_errno = errno;
-        AFR_STACK_UNWIND (ipc, frame, -1, op_errno, NULL);
+    if (op_errno == -1)
+        op_errno = errno;
+    AFR_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 
 wind_default:
-        STACK_WIND (frame, default_ipc_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ipc, op, xdata);
-        return 0;
+    STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
 }
 
 int
-afr_forget (xlator_t *this, inode_t *inode)
+afr_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t        ctx_int = 0;
-        afr_inode_ctx_t *ctx    = NULL;
+    uint64_t ctx_int = 0;
+    afr_inode_ctx_t *ctx = NULL;
 
-        afr_spb_choice_timeout_cancel (this, inode);
-        inode_ctx_del (inode, this, &ctx_int);
-        if (!ctx_int)
-                return 0;
-
-        ctx = (afr_inode_ctx_t *)ctx_int;
-        afr_inode_ctx_destroy (ctx);
+    afr_spb_choice_timeout_cancel(this, inode);
+    inode_ctx_del(inode, this, &ctx_int);
+    if (!ctx_int)
         return 0;
+
+    ctx = (afr_inode_ctx_t *)(uintptr_t)ctx_int;
+    afr_inode_ctx_destroy(ctx);
+    return 0;
 }
 
 int
-afr_priv_dump (xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        char  key_prefix[GF_DUMP_MAX_BUF_LEN];
-        char  key[GF_DUMP_MAX_BUF_LEN];
-        int   i = 0;
-
-
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-        snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
-        gf_proc_dump_add_section(key_prefix);
-        gf_proc_dump_write("child_count", "%u", priv->child_count);
-        for (i = 0; i < priv->child_count; i++) {
-                sprintf (key, "child_up[%d]", i);
-                gf_proc_dump_write(key, "%d", priv->child_up[i]);
-                sprintf (key, "pending_key[%d]", i);
-                gf_proc_dump_write(key, "%s", priv->pending_key[i]);
-                sprintf (key, "pending_reads[%d]", i);
-                gf_proc_dump_write(key, "%"PRId64, GF_ATOMIC_GET(priv->pending_reads[i]));
-                sprintf (key, "child_latency[%d]", i);
-                gf_proc_dump_write(key, "%"PRId64, priv->child_latency[i]);
-        }
-        gf_proc_dump_write("data_self_heal", "%s", priv->data_self_heal);
-        gf_proc_dump_write("metadata_self_heal", "%d", priv->metadata_self_heal);
-        gf_proc_dump_write("entry_self_heal", "%d", priv->entry_self_heal);
-        gf_proc_dump_write("read_child", "%d", priv->read_child);
-        gf_proc_dump_write("wait_count", "%u", priv->wait_count);
-        gf_proc_dump_write("heal-wait-queue-length", "%d",
-                           priv->heal_wait_qlen);
-        gf_proc_dump_write("heal-waiters", "%d", priv->heal_waiters);
-        gf_proc_dump_write("background-self-heal-count", "%d",
-                           priv->background_self_heal_count);
-        gf_proc_dump_write("healers", "%d", priv->healers);
-        gf_proc_dump_write("read-hash-mode", "%d", priv->hash_mode);
-        if (priv->quorum_count == AFR_QUORUM_AUTO) {
-                gf_proc_dump_write ("quorum-type", "auto");
-        } else if (priv->quorum_count == 0) {
-                gf_proc_dump_write ("quorum-type", "none");
-        } else {
-                gf_proc_dump_write("quorum-type", "fixed");
-                gf_proc_dump_write("quorum-count", "%d", priv->quorum_count);
-        }
-
-        return 0;
+afr_priv_dump(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+    snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
+    gf_proc_dump_add_section("%s", key_prefix);
+    gf_proc_dump_write("child_count", "%u", priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        sprintf(key, "child_up[%d]", i);
+        gf_proc_dump_write(key, "%d", priv->child_up[i]);
+        sprintf(key, "pending_key[%d]", i);
+        gf_proc_dump_write(key, "%s", priv->pending_key[i]);
+        sprintf(key, "pending_reads[%d]", i);
+        gf_proc_dump_write(key, "%" PRId64,
+                           GF_ATOMIC_GET(priv->pending_reads[i]));
+        sprintf(key, "child_latency[%d]", i);
+        gf_proc_dump_write(key, "%" PRId64, priv->child_latency[i]);
+        sprintf(key, "halo_child_up[%d]", i);
+        gf_proc_dump_write(key, "%d", priv->halo_child_up[i]);
+    }
+    gf_proc_dump_write("data_self_heal", "%d", priv->data_self_heal);
+    gf_proc_dump_write("metadata_self_heal", "%d", priv->metadata_self_heal);
+    gf_proc_dump_write("entry_self_heal", "%d", priv->entry_self_heal);
+    gf_proc_dump_write("read_child", "%d", priv->read_child);
+    gf_proc_dump_write("wait_count", "%u", priv->wait_count);
+    gf_proc_dump_write("heal-wait-queue-length", "%d", priv->heal_wait_qlen);
+    gf_proc_dump_write("heal-waiters", "%d", priv->heal_waiters);
+    gf_proc_dump_write("background-self-heal-count", "%d",
+                       priv->background_self_heal_count);
+    gf_proc_dump_write("healers", "%d", priv->healers);
+    gf_proc_dump_write("read-hash-mode", "%d", priv->hash_mode);
+    gf_proc_dump_write("use-anonymous-inode", "%d", priv->use_anon_inode);
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        gf_proc_dump_write("quorum-type", "auto");
+    } else if (priv->quorum_count == 0) {
+        gf_proc_dump_write("quorum-type", "none");
+    } else {
+        gf_proc_dump_write("quorum-type", "fixed");
+        gf_proc_dump_write("quorum-count", "%d", priv->quorum_count);
+    }
+    gf_proc_dump_write("up", "%u", afr_has_quorum(priv->child_up, this, NULL));
+    if (priv->thin_arbiter_count) {
+        gf_proc_dump_write("ta_child_up", "%d", priv->ta_child_up);
+        gf_proc_dump_write("ta_bad_child_index", "%d",
+                           priv->ta_bad_child_index);
+        gf_proc_dump_write("ta_notify_dom_lock_offset", "%" PRId64,
+                           priv->ta_notify_dom_lock_offset);
+    }
+
+    return 0;
 }
 
-
 /**
  * find_child_index - find the child's index in the array of subvolumes
  * @this: AFR
@@ -4903,1820 +5723,2156 @@ afr_priv_dump (xlator_t *this)
  */
 
 static int
-find_child_index (xlator_t *this, xlator_t *child)
+afr_find_child_index(xlator_t *this, xlator_t *child)
 {
-        afr_private_t *priv = NULL;
-        int i = -1;
+    afr_private_t *priv = NULL;
+    int child_count = -1;
+    int i = -1;
 
-        priv = this->private;
+    priv = this->private;
+    child_count = priv->child_count;
+    if (priv->thin_arbiter_count) {
+        child_count++;
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if ((xlator_t *) child == priv->children[i])
-                        break;
-        }
+    for (i = 0; i < child_count; i++) {
+        if ((xlator_t *)child == priv->children[i])
+            break;
+    }
 
-        return i;
+    return i;
 }
 
-static int
-__afr_get_up_children_count (afr_private_t *priv)
+int
+__afr_get_up_children_count(afr_private_t *priv)
 {
-        int             up_children         = 0;
-        int             i = 0;
+    int up_children = 0;
+    int i = 0;
 
-        for (i = 0; i < priv->child_count; i++)
-                if (priv->child_up[i] == 1)
-                        up_children++;
+    for (i = 0; i < priv->child_count; i++)
+        if (priv->child_up[i] == 1)
+            up_children++;
 
-        return up_children;
+    return up_children;
 }
 
-glusterfs_event_t
-__afr_transform_event_from_state (afr_private_t *priv)
+static int
+__get_heard_from_all_status(xlator_t *this)
 {
-        int i = 0;
-        int up_children = 0;
-
-        if (AFR_COUNT (priv->last_event, priv->child_count) ==
-                       priv->child_count)
-                /* have_heard_from_all. Let afr_notify() do the propagation. */
-                return GF_EVENT_MAXVAL;
+    afr_private_t *priv = this->private;
+    int i;
 
-        up_children = __afr_get_up_children_count (priv);
-        /* Treat the children with pending notification, as having sent a
-         * GF_EVENT_CHILD_DOWN. i.e. set the event as GF_EVENT_SOME_DESCENDENT_DOWN,
-         * as done in afr_notify() */
-        for (i = 0; i < priv->child_count; i++) {
-                if (priv->last_event[i])
-                        continue;
-                priv->last_event[i] = GF_EVENT_SOME_DESCENDENT_DOWN;
-                priv->child_up[i] = 0;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!priv->last_event[i]) {
+            return 0;
         }
+    }
+    if (priv->thin_arbiter_count && !priv->ta_child_up) {
+        return 0;
+    }
+    return 1;
+}
 
-        if (up_children)
-                /* We received at least one child up */
-                return GF_EVENT_CHILD_UP;
-        else
-                return GF_EVENT_CHILD_DOWN;
+glusterfs_event_t
+__afr_transform_event_from_state(xlator_t *this)
+{
+    int i = 0;
+    int up_children = 0;
+    afr_private_t *priv = this->private;
 
+    if (__get_heard_from_all_status(this))
+        /* have_heard_from_all. Let afr_notify() do the propagation. */
         return GF_EVENT_MAXVAL;
+
+    up_children = __afr_get_up_children_count(priv);
+    /* Treat the children with pending notification, as having sent a
+     * GF_EVENT_CHILD_DOWN. i.e. set the event as GF_EVENT_SOME_DESCENDENT_DOWN,
+     * as done in afr_notify() */
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->last_event[i])
+            continue;
+        priv->last_event[i] = GF_EVENT_SOME_DESCENDENT_DOWN;
+        priv->child_up[i] = 0;
+    }
+
+    if (up_children)
+        /* We received at least one child up */
+        return GF_EVENT_CHILD_UP;
+    else
+        return GF_EVENT_CHILD_DOWN;
+
+    return GF_EVENT_MAXVAL;
 }
 
 static void
-afr_notify_cbk (void *data)
-{
-        xlator_t *this = data;
-        afr_private_t *priv = this->private;
-        glusterfs_event_t event = GF_EVENT_MAXVAL;
-        gf_boolean_t propagate = _gf_false;
-
-        LOCK (&priv->lock);
-        {
-                if (!priv->timer) {
-                        /*
-                         * Either child_up/child_down is already sent to parent.
-                         * This is a spurious wake up.
-                         */
-                        goto unlock;
-                }
-                priv->timer = NULL;
-                event = __afr_transform_event_from_state (priv);
-                if (event != GF_EVENT_MAXVAL)
-                        propagate = _gf_true;
-        }
+afr_notify_cbk(void *data)
+{
+    xlator_t *this = data;
+    afr_private_t *priv = this->private;
+    glusterfs_event_t event = GF_EVENT_MAXVAL;
+    gf_boolean_t propagate = _gf_false;
+
+    LOCK(&priv->lock);
+    {
+        if (!priv->timer) {
+            /*
+             * Either child_up/child_down is already sent to parent.
+             * This is a spurious wake up.
+             */
+            goto unlock;
+        }
+        priv->timer = NULL;
+        event = __afr_transform_event_from_state(this);
+        if (event != GF_EVENT_MAXVAL)
+            propagate = _gf_true;
+    }
 unlock:
-        UNLOCK (&priv->lock);
-        if (propagate)
-                default_notify (this, event, NULL);
+    UNLOCK(&priv->lock);
+    if (propagate)
+        default_notify(this, event, NULL);
 }
 
 static void
-__afr_launch_notify_timer (xlator_t *this, afr_private_t *priv)
+__afr_launch_notify_timer(xlator_t *this, afr_private_t *priv)
 {
+    struct timespec delay = {
+        0,
+    };
 
-        struct timespec delay = {0, };
-
-        gf_msg_debug (this->name, 0, "Initiating child-down timer");
-        delay.tv_sec = 10;
-        delay.tv_nsec = 0;
-        priv->timer = gf_timer_call_after (this->ctx, delay,
-                                           afr_notify_cbk, this);
-        if (priv->timer == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, AFR_MSG_TIMER_CREATE_FAIL,
-                        "Cannot create timer for delayed initialization");
-        }
-}
-
-int
-__get_heard_from_all_status (xlator_t *this)
-{
-        afr_private_t *priv          = this->private;
-        int           heard_from_all = 1;
-        int           i              = 0;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!priv->last_event[i]) {
-                        heard_from_all = 0;
-                        break;
-                }
-        }
-        return heard_from_all;
+    gf_msg_debug(this->name, 0, "Initiating child-down timer");
+    delay.tv_sec = 10;
+    delay.tv_nsec = 0;
+    priv->timer = gf_timer_call_after(this->ctx, delay, afr_notify_cbk, this);
+    if (priv->timer == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_TIMER_CREATE_FAIL,
+               "Cannot create timer for delayed initialization");
+    }
 }
 
 static int
-find_best_down_child (xlator_t *this)
+find_best_down_child(xlator_t *this)
 {
-        afr_private_t   *priv               = NULL;
-        int             i                   = -1;
-        int32_t         best_child          = -1;
-        int64_t         best_latency        = INT64_MAX;
+    afr_private_t *priv = NULL;
+    int i = -1;
+    int32_t best_child = -1;
+    int64_t best_latency = INT64_MAX;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (priv->child_up[i] &&
-                    priv->child_latency[i] >= 0 &&
-                    priv->child_latency[i] < best_latency) {
-                        best_child = i;
-                        best_latency = priv->child_latency[i];
-                }
-        }
-        if (best_child >= 0) {
-                gf_msg_debug (this->name, 0, "Found best down child (%d) "
-                              "@ %ld ms latency", best_child, best_latency);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!priv->child_up[i] && priv->child_latency[i] >= 0 &&
+            priv->child_latency[i] < best_latency) {
+            best_child = i;
+            best_latency = priv->child_latency[i];
         }
-        return best_child;
+    }
+    if (best_child >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "Found best down child (%d) @ %" PRId64 " ms latency",
+                     best_child, best_latency);
+    }
+    return best_child;
 }
 
 int
-find_worst_up_child (xlator_t *this)
+find_worst_up_child(xlator_t *this)
 {
-        afr_private_t   *priv               = NULL;
-        int             i                   = -1;
-        int32_t         worst_child         = -1;
-        int64_t         worst_latency       = INT64_MIN;
+    afr_private_t *priv = NULL;
+    int i = -1;
+    int32_t worst_child = -1;
+    int64_t worst_latency = INT64_MIN;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (priv->child_up[i] &&
-                    priv->child_latency[i] >= 0 &&
-                    priv->child_latency[i] > worst_latency) {
-                        worst_child = i;
-                        worst_latency = priv->child_latency[i];
-                }
-        }
-        if (worst_child >= 0) {
-                gf_msg_debug (this->name, 0, "Found worst up child (%d)"
-                              " @ %ld ms latency", worst_child, worst_latency);
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->child_up[i] && priv->child_latency[i] >= 0 &&
+            priv->child_latency[i] > worst_latency) {
+            worst_child = i;
+            worst_latency = priv->child_latency[i];
         }
-        return worst_child;
+    }
+    if (worst_child >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "Found worst up child (%d) @ %" PRId64 " ms latency",
+                     worst_child, worst_latency);
+    }
+    return worst_child;
 }
 
 void
-__afr_handle_ping_event (xlator_t *this, xlator_t *child_xlator,
-                const int idx, int64_t halo_max_latency_msec, int32_t *event,
-                int64_t child_latency_msec)
+__afr_handle_ping_event(xlator_t *this, xlator_t *child_xlator, const int idx,
+                        int64_t halo_max_latency_msec, int32_t *event,
+                        int64_t child_latency_msec)
 {
-        afr_private_t   *priv               = NULL;
-        int             up_children         = 0;
+    afr_private_t *priv = NULL;
+    int up_children = 0;
 
-        priv = this->private;
+    priv = this->private;
 
-        priv->child_latency[idx] = child_latency_msec;
-        gf_msg_debug (child_xlator->name, 0, "Client ping @ %ld ms",
-                      child_latency_msec);
-        if (priv->shd.iamshd)
-                return;
-
-        up_children = __afr_get_up_children_count (priv);
-
-        if (child_latency_msec > halo_max_latency_msec &&
-            priv->child_up[idx] == 1 &&
-            up_children > priv->halo_min_replicas) {
-                if ((up_children - 1) <
-                    priv->halo_min_replicas) {
-                        gf_log (child_xlator->name, GF_LOG_INFO,
-                               "Overriding halo threshold, "
-                               "min replicas: %d",
-                               priv->halo_min_replicas);
-                } else {
-                        gf_log (child_xlator->name, GF_LOG_INFO,
-                                "Child latency (%ld ms) "
-                                "exceeds halo threshold (%ld), "
-                                "marking child down.",
-                                child_latency_msec,
-                                halo_max_latency_msec);
-                        *event = GF_EVENT_CHILD_DOWN;
-                }
-        } else if (child_latency_msec < halo_max_latency_msec &&
-                   priv->child_up[idx] == 0) {
-                if (up_children < priv->halo_max_replicas) {
-                        gf_log (child_xlator->name, GF_LOG_INFO,
-                                "Child latency (%ld ms) "
-                                "below halo threshold (%ld), "
-                                "marking child up.",
-                                child_latency_msec,
-                                halo_max_latency_msec);
-                        *event = GF_EVENT_CHILD_UP;
-                } else {
-                        gf_log (child_xlator->name, GF_LOG_INFO,
-                            "Not marking child %d up, "
-                            "max replicas (%d) reached.", idx,
-                            priv->halo_max_replicas);
-                }
+    priv->child_latency[idx] = child_latency_msec;
+    gf_msg_debug(child_xlator->name, 0, "Client ping @ %" PRId64 " ms",
+                 child_latency_msec);
+    if (priv->shd.iamshd)
+        return;
+
+    up_children = __afr_get_up_children_count(priv);
+
+    if (child_latency_msec > halo_max_latency_msec &&
+        priv->child_up[idx] == 1 && up_children > priv->halo_min_replicas) {
+        if ((up_children - 1) < priv->halo_min_replicas) {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Overriding halo threshold, "
+                   "min replicas: %d",
+                   priv->halo_min_replicas);
+        } else {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Child latency (%" PRId64
+                   " ms) "
+                   "exceeds halo threshold (%" PRId64
+                   "), "
+                   "marking child down.",
+                   child_latency_msec, halo_max_latency_msec);
+            if (priv->halo_child_up[idx]) {
+                *event = GF_EVENT_CHILD_DOWN;
+            }
         }
+    } else if (child_latency_msec < halo_max_latency_msec &&
+               priv->child_up[idx] == 0) {
+        if (up_children < priv->halo_max_replicas) {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Child latency (%" PRId64
+                   " ms) "
+                   "below halo threshold (%" PRId64
+                   "), "
+                   "marking child up.",
+                   child_latency_msec, halo_max_latency_msec);
+            if (priv->halo_child_up[idx]) {
+                *event = GF_EVENT_CHILD_UP;
+            }
+        } else {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Not marking child %d up, "
+                   "max replicas (%d) reached.",
+                   idx, priv->halo_max_replicas);
+        }
+    }
 }
 
 static int64_t
-afr_get_halo_latency (xlator_t *this)
+afr_get_halo_latency(xlator_t *this)
 {
-        afr_private_t *priv           = NULL;
-        int64_t halo_max_latency_msec = 0;
+    afr_private_t *priv = NULL;
+    int64_t halo_max_latency_msec = 0;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->shd.iamshd) {
-                halo_max_latency_msec = priv->shd.halo_max_latency_msec;
-        } else if (priv->nfsd.iamnfsd) {
-                halo_max_latency_msec =
-                        priv->nfsd.halo_max_latency_msec;
-        } else {
-                halo_max_latency_msec = priv->halo_max_latency_msec;
-        }
-        gf_msg_debug (this->name, 0, "Using halo latency %ld",
-                halo_max_latency_msec);
-        return halo_max_latency_msec;
+    if (priv->shd.iamshd) {
+        halo_max_latency_msec = priv->shd.halo_max_latency_msec;
+    } else if (priv->nfsd.iamnfsd) {
+        halo_max_latency_msec = priv->nfsd.halo_max_latency_msec;
+    } else {
+        halo_max_latency_msec = priv->halo_max_latency_msec;
+    }
+    gf_msg_debug(this->name, 0, "Using halo latency %" PRId64,
+                 halo_max_latency_msec);
+    return halo_max_latency_msec;
 }
 
 void
-__afr_handle_child_up_event (xlator_t *this, xlator_t *child_xlator,
-                const int idx, int64_t child_latency_msec,
-                int32_t *event, int32_t *call_psh, int32_t *up_child)
-{
-        afr_private_t   *priv               = NULL;
-        int             up_children         = 0;
-        int             worst_up_child      = -1;
-        int64_t         halo_max_latency_msec = afr_get_halo_latency (this);
-
-        priv = this->private;
-
-        /*
-         * This only really counts if the child was never up
-         * (value = -1) or had been down (value = 0).  See
-         * comment at GF_EVENT_CHILD_DOWN for a more detailed
-         * explanation.
-         */
-        if (priv->child_up[idx] != 1) {
-                priv->event_generation++;
-        }
-        priv->child_up[idx] = 1;
-
-        *call_psh = 1;
-        *up_child = idx;
-        up_children = __afr_get_up_children_count (priv);
-        /*
-         * If this is an _actual_ CHILD_UP event, we
-         * want to set the child_latency to MAX to indicate
-         * the child needs ping data to be available before doing child-up
-         */
-        if (child_latency_msec < 0 && priv->halo_enabled) {
-                /*set to INT64_MAX-1 so that it is found for best_down_child*/
-                priv->child_latency[idx] = AFR_HALO_MAX_LATENCY;
-        }
-
-        /*
-         * Handle the edge case where we exceed
-         * halo_min_replicas and we've got a child which is
-         * marked up as it was helping to satisfy the
-         * halo_min_replicas even though it's latency exceeds
-         * halo_max_latency_msec.
-         */
-        if (up_children > priv->halo_min_replicas) {
-                worst_up_child = find_worst_up_child (this);
-                if (worst_up_child >= 0 &&
-                    priv->child_latency[worst_up_child] >
-                    halo_max_latency_msec) {
-                        gf_msg_debug (this->name, 0, "Marking child %d down, "
-                                "doesn't meet halo threshold (%ld), and > "
-                                "halo_min_replicas (%d)",
-                                worst_up_child, halo_max_latency_msec,
-                                priv->halo_min_replicas);
-                        priv->child_up[worst_up_child] = 0;
-                        up_children--;
-                }
-        }
-
-        if (up_children > priv->halo_max_replicas &&
-            !priv->shd.iamshd) {
-                worst_up_child = find_worst_up_child (this);
-                if (worst_up_child < 0) {
-                        worst_up_child = idx;
-                }
-                priv->child_up[worst_up_child] = 0;
-                up_children--;
-                gf_msg_debug (this->name, 0, "Marking child %d down, "
-                        "up_children (%d) > halo_max_replicas (%d)",
-                        worst_up_child, up_children, priv->halo_max_replicas);
-        }
-
-        if (up_children == 1) {
-                gf_msg (this->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOL_UP,
-                        "Subvolume '%s' came back up; "
-                        "going online.",
-                        child_xlator->name);
-                gf_event (EVENT_AFR_SUBVOL_UP, "subvol=%s", this->name);
-        } else {
-                *event = GF_EVENT_SOME_DESCENDENT_UP;
-        }
+__afr_handle_child_up_event(xlator_t *this, xlator_t *child_xlator,
+                            const int idx, int64_t child_latency_msec,
+                            int32_t *event, int32_t *call_psh,
+                            int32_t *up_child)
+{
+    afr_private_t *priv = NULL;
+    int up_children = 0;
+    int worst_up_child = -1;
+    int64_t halo_max_latency_msec = afr_get_halo_latency(this);
+
+    priv = this->private;
+
+    /*
+     * This only really counts if the child was never up
+     * (value = -1) or had been down (value = 0).  See
+     * comment at GF_EVENT_CHILD_DOWN for a more detailed
+     * explanation.
+     */
+    if (priv->child_up[idx] != 1) {
+        priv->event_generation++;
+    }
+    priv->child_up[idx] = 1;
+
+    *call_psh = 1;
+    *up_child = idx;
+    up_children = __afr_get_up_children_count(priv);
+    /*
+     * If this is an _actual_ CHILD_UP event, we
+     * want to set the child_latency to MAX to indicate
+     * the child needs ping data to be available before doing child-up
+     */
+    if (!priv->halo_enabled)
+        goto out;
+
+    if (child_latency_msec < 0) {
+        /*set to INT64_MAX-1 so that it is found for best_down_child*/
+        priv->halo_child_up[idx] = 1;
+        if (priv->child_latency[idx] < 0) {
+            priv->child_latency[idx] = AFR_HALO_MAX_LATENCY;
+        }
+    }
+
+    /*
+     * Handle the edge case where we exceed
+     * halo_min_replicas and we've got a child which is
+     * marked up as it was helping to satisfy the
+     * halo_min_replicas even though it's latency exceeds
+     * halo_max_latency_msec.
+     */
+    if (up_children > priv->halo_min_replicas) {
+        worst_up_child = find_worst_up_child(this);
+        if (worst_up_child >= 0 &&
+            priv->child_latency[worst_up_child] > halo_max_latency_msec) {
+            gf_msg_debug(this->name, 0,
+                         "Marking child %d down, "
+                         "doesn't meet halo threshold (%" PRId64
+                         "), and > "
+                         "halo_min_replicas (%d)",
+                         worst_up_child, halo_max_latency_msec,
+                         priv->halo_min_replicas);
+            priv->child_up[worst_up_child] = 0;
+            up_children--;
+        }
+    }
+
+    if (up_children > priv->halo_max_replicas && !priv->shd.iamshd) {
+        worst_up_child = find_worst_up_child(this);
+        if (worst_up_child < 0) {
+            worst_up_child = idx;
+        }
+        priv->child_up[worst_up_child] = 0;
+        up_children--;
+        gf_msg_debug(this->name, 0,
+                     "Marking child %d down, "
+                     "up_children (%d) > halo_max_replicas (%d)",
+                     worst_up_child, up_children, priv->halo_max_replicas);
+    }
+out:
+    if (up_children == 1) {
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOL_UP,
+               "Subvolume '%s' came back up; "
+               "going online.",
+               child_xlator->name);
+        gf_event(EVENT_AFR_SUBVOL_UP, "client-pid=%d; subvol=%s",
+                 this->ctx->cmd_args.client_pid, this->name);
+    } else {
+        *event = GF_EVENT_SOME_DESCENDENT_UP;
+    }
 
-        priv->last_event[idx] = *event;
+    priv->last_event[idx] = *event;
 }
 
 void
-__afr_handle_child_down_event (xlator_t *this, xlator_t *child_xlator,
-                int idx, int64_t child_latency_msec, int32_t *event,
-                int32_t *call_psh, int32_t *up_child)
-{
-        afr_private_t   *priv               = NULL;
-        int             i                   = 0;
-        int             up_children         = 0;
-        int             down_children       = 0;
-        int             best_down_child     = -1;
-
-        priv = this->private;
-
-        /*
-         * If a brick is down when we start, we'll get a
-         * CHILD_DOWN to indicate its initial state.  There
-         * was never a CHILD_UP in this case, so if we
-         * increment "down_count" the difference between than
-         * and "up_count" will no longer be the number of
-         * children that are currently up.  This has serious
-         * implications e.g. for quorum enforcement, so we
-         * don't increment these values unless the event
-         * represents an actual state transition between "up"
-         * (value = 1) and anything else.
-         */
-        if (priv->child_up[idx] == 1) {
-                priv->event_generation++;
-        }
+__afr_handle_child_down_event(xlator_t *this, xlator_t *child_xlator, int idx,
+                              int64_t child_latency_msec, int32_t *event,
+                              int32_t *call_psh, int32_t *up_child)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int up_children = 0;
+    int down_children = 0;
+    int best_down_child = -1;
+
+    priv = this->private;
+
+    /*
+     * If a brick is down when we start, we'll get a
+     * CHILD_DOWN to indicate its initial state.  There
+     * was never a CHILD_UP in this case, so if we
+     * increment "down_count" the difference between than
+     * and "up_count" will no longer be the number of
+     * children that are currently up.  This has serious
+     * implications e.g. for quorum enforcement, so we
+     * don't increment these values unless the event
+     * represents an actual state transition between "up"
+     * (value = 1) and anything else.
+     */
+    if (priv->child_up[idx] == 1) {
+        priv->event_generation++;
+    }
+
+    /*
+     * If this is an _actual_ CHILD_DOWN event, we
+     * want to set the child_latency to < 0 to indicate
+     * the child is really disconnected.
+     */
+    if (child_latency_msec < 0) {
+        priv->child_latency[idx] = child_latency_msec;
+        priv->halo_child_up[idx] = 0;
+    }
+    priv->child_up[idx] = 0;
+
+    up_children = __afr_get_up_children_count(priv);
+    /*
+     * Handle the edge case where we need to find the
+     * next best child (to mark up) as marking this child
+     * down would cause us to fall below halo_min_replicas.
+     * We will also force the SHD to heal this child _now_
+     * as we want it to be up to date if we are going to
+     * begin using it synchronously.
+     */
+    if (priv->halo_enabled && up_children < priv->halo_min_replicas) {
+        best_down_child = find_best_down_child(this);
+        if (best_down_child >= 0) {
+            gf_msg_debug(this->name, 0,
+                         "Swapping out child %d for "
+                         "child %d to satisfy halo_min_replicas (%d).",
+                         idx, best_down_child, priv->halo_min_replicas);
+            priv->child_up[best_down_child] = 1;
+            *call_psh = 1;
+            *up_child = best_down_child;
+        }
+    }
+    for (i = 0; i < priv->child_count; i++)
+        if (priv->child_up[i] == 0)
+            down_children++;
+    if (down_children == priv->child_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SUBVOLS_DOWN,
+               "All subvolumes are down. Going "
+               "offline until at least one of them "
+               "comes back up.");
+        gf_event(EVENT_AFR_SUBVOLS_DOWN, "client-pid=%d; subvol=%s",
+                 this->ctx->cmd_args.client_pid, this->name);
+    } else {
+        *event = GF_EVENT_SOME_DESCENDENT_DOWN;
+    }
+    priv->last_event[idx] = *event;
+}
 
-        /*
-         * If this is an _actual_ CHILD_DOWN event, we
-         * want to set the child_latency to < 0 to indicate
-         * the child is really disconnected.
-         */
-        if (child_latency_msec < 0) {
-                priv->child_latency[idx] = child_latency_msec;
-        }
-        priv->child_up[idx] = 0;
+void
+afr_ta_lock_release_synctask(xlator_t *this)
+{
+    call_frame_t *ta_frame = NULL;
+    int ret = 0;
 
-        up_children = __afr_get_up_children_count (priv);
-        /*
-         * Handle the edge case where we need to find the
-         * next best child (to mark up) as marking this child
-         * down would cause us to fall below halo_min_replicas.
-         * We will also force the SHD to heal this child _now_
-         * as we want it to be up to date if we are going to
-         * begin using it synchronously.
-         */
-        if (up_children < priv->halo_min_replicas) {
-                best_down_child = find_best_down_child (this);
-                if (best_down_child >= 0) {
-                        gf_msg_debug (this->name, 0,
-                                "Swapping out child %d for "
-                                "child %d to satisfy halo_min_replicas (%d).",
-                                idx, best_down_child, priv->halo_min_replicas);
-                        priv->child_up[best_down_child] = 1;
-                        *call_psh = 1;
-                        *up_child = best_down_child;
-                }
-        }
+    ta_frame = afr_ta_frame_create(this);
+    if (!ta_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to create ta_frame");
+        return;
+    }
 
-        for (i = 0; i < priv->child_count; i++)
-                if (priv->child_up[i] == 0)
-                        down_children++;
-        if (down_children == priv->child_count) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, AFR_MSG_SUBVOLS_DOWN,
-                        "All subvolumes are down. Going "
-                        "offline until atleast one of them "
-                        "comes back up.");
-                gf_event (EVENT_AFR_SUBVOLS_DOWN, "subvol=%s", this->name);
-        } else {
-                *event = GF_EVENT_SOME_DESCENDENT_DOWN;
-        }
-        priv->last_event[idx] = *event;
+    ret = synctask_new(this->ctx->env, afr_release_notify_lock_for_ta,
+                       afr_ta_lock_release_done, ta_frame, this);
+    if (ret) {
+        STACK_DESTROY(ta_frame->root);
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to release "
+               "AFR_TA_DOM_NOTIFY lock.");
+    }
 }
 
-int32_t
-afr_notify (xlator_t *this, int32_t event,
-            void *data, void *data2)
-{
-        afr_private_t   *priv               = NULL;
-        xlator_t        *child_xlator       = NULL;
-        int             i                   = -1;
-        int             propagate           = 0;
-        int             had_heard_from_all  = 0;
-        int             have_heard_from_all = 0;
-        int             idx                 = -1;
-        int             ret                 = -1;
-        int             call_psh            = 0;
-        int             up_child            = -1;
-        dict_t          *input              = NULL;
-        dict_t          *output             = NULL;
-        gf_boolean_t    had_quorum          = _gf_false;
-        gf_boolean_t    has_quorum          = _gf_false;
-        struct gf_upcall *up_data           = NULL;
-        struct gf_upcall_cache_invalidation *up_ci = NULL;
-        inode_table_t  *itable              = NULL;
-        inode_t        *inode               = NULL;
-        int64_t         halo_max_latency_msec = 0;
-        int64_t         child_latency_msec   = -1;
-
-        child_xlator = (xlator_t *)data;
-
-        priv = this->private;
-
-        if (!priv)
-                return 0;
-
-        /*
-         * We need to reset this in case children come up in "staggered"
-         * fashion, so that we discover a late-arriving local subvolume.  Note
-         * that we could end up issuing N lookups to the first subvolume, and
-         * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
-         */
-        priv->did_discovery = _gf_false;
+static void
+afr_handle_inodelk_contention(xlator_t *this, struct gf_upcall *upcall)
+{
+    struct gf_upcall_inodelk_contention *lc = NULL;
+    unsigned int inmem_count = 0;
+    unsigned int onwire_count = 0;
+    afr_private_t *priv = this->private;
 
+    lc = upcall->data;
 
-        /* parent xlators dont need to know about every child_up, child_down
-         * because of afr ha. If all subvolumes go down, child_down has
-         * to be triggered. In that state when 1 subvolume comes up child_up
-         * needs to be triggered. dht optimizes revalidate lookup by sending
-         * it only to one of its subvolumes. When child up/down happens
-         * for afr's subvolumes dht should be notified by child_modified. The
-         * subsequent revalidate lookup happens on all the dht's subvolumes
-         * which triggers afr self-heals if any.
-         */
-        idx = find_child_index (this, child_xlator);
-        if (idx < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_CHILD_UP,
-                        "Received child_up from invalid subvolume");
-                goto out;
-        }
+    if (strcmp(lc->domain, AFR_TA_DOM_NOTIFY) != 0)
+        return;
 
-        had_quorum = priv->quorum_count && afr_has_quorum (priv->child_up,
-                                                           this);
-        if (priv->halo_enabled) {
-                halo_max_latency_msec = afr_get_halo_latency (this);
-
-                if (event == GF_EVENT_CHILD_PING) {
-                        /* Calculates the child latency and sets event
-                         */
-                        child_latency_msec = (int64_t)(uintptr_t)data2;
-                        LOCK (&priv->lock);
-                        {
-                                __afr_handle_ping_event (this, child_xlator,
-                                        idx, halo_max_latency_msec, &event,
-                                        child_latency_msec);
-                        }
-                        UNLOCK (&priv->lock);
-                }
-        }
+    if (priv->shd.iamshd) {
+        /* shd should ignore AFR_TA_DOM_NOTIFY release requests. */
+        return;
+    }
+    LOCK(&priv->lock);
+    {
+        if (priv->release_ta_notify_dom_lock == _gf_true) {
+            /* Ignore multiple release requests from shds.*/
+            UNLOCK(&priv->lock);
+            return;
+        }
+        priv->release_ta_notify_dom_lock = _gf_true;
+        inmem_count = priv->ta_in_mem_txn_count;
+        onwire_count = priv->ta_on_wire_txn_count;
+    }
+    UNLOCK(&priv->lock);
+    if (inmem_count || onwire_count)
+        /* lock release will happen in txn code path after
+         * in-memory or on-wire txns are over.*/
+        return;
 
-        if (event == GF_EVENT_CHILD_PING) {
-                /* This is the only xlator that handles PING, no reason to
-                 * propagate.
-                 */
-                goto out;
+    afr_ta_lock_release_synctask(this);
+}
+
+static void
+afr_handle_upcall_event(xlator_t *this, struct gf_upcall *upcall)
+{
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+    afr_private_t *priv = this->private;
+    inode_t *inode = NULL;
+    inode_table_t *itable = NULL;
+    int i = 0;
+
+    switch (upcall->event_type) {
+        case GF_UPCALL_INODELK_CONTENTION:
+            afr_handle_inodelk_contention(this, upcall);
+            break;
+        case GF_UPCALL_CACHE_INVALIDATION:
+            up_ci = (struct gf_upcall_cache_invalidation *)upcall->data;
+
+            /* Since md-cache will be aggressively filtering
+             * lookups, the stale read issue will be more
+             * pronounced. Hence when a pending xattr is set notify
+             * all the md-cache clients to invalidate the existing
+             * stat cache and send the lookup next time */
+            if (!up_ci->dict)
+                break;
+            for (i = 0; i < priv->child_count; i++) {
+                if (!dict_get(up_ci->dict, priv->pending_key[i]))
+                    continue;
+                up_ci->flags |= UP_INVAL_ATTR;
+                itable = ((xlator_t *)this->graph->top)->itable;
+                /*Internal processes may not have itable for
+                 *top xlator*/
+                if (itable)
+                    inode = inode_find(itable, upcall->gfid);
+                if (inode)
+                    afr_inode_need_refresh_set(inode, this);
+                break;
             }
+            break;
+        default:
+            break;
+    }
+}
 
-        if (event == GF_EVENT_TRANSLATOR_OP) {
-                LOCK (&priv->lock);
-                {
-                        had_heard_from_all = __get_heard_from_all_status (this);
-                }
-                UNLOCK (&priv->lock);
-
-                if (!had_heard_from_all) {
-                        ret = -1;
-                } else {
-                        input = data;
-                        output = data2;
-                        ret = afr_xl_op (this, input, output);
-                }
-                goto out;
+int32_t
+afr_notify(xlator_t *this, int32_t event, void *data, void *data2)
+{
+    afr_private_t *priv = NULL;
+    xlator_t *child_xlator = NULL;
+    int i = -1;
+    int propagate = 0;
+    int had_heard_from_all = 0;
+    int have_heard_from_all = 0;
+    int idx = -1;
+    int ret = -1;
+    int call_psh = 0;
+    int up_child = -1;
+    dict_t *input = NULL;
+    dict_t *output = NULL;
+    gf_boolean_t had_quorum = _gf_false;
+    gf_boolean_t has_quorum = _gf_false;
+    int64_t halo_max_latency_msec = 0;
+    int64_t child_latency_msec = -1;
+
+    child_xlator = (xlator_t *)data;
+
+    priv = this->private;
+
+    if (!priv)
+        return 0;
+
+    /*
+     * We need to reset this in case children come up in "staggered"
+     * fashion, so that we discover a late-arriving local subvolume.  Note
+     * that we could end up issuing N lookups to the first subvolume, and
+     * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
+     */
+    priv->did_discovery = _gf_false;
+
+    /* parent xlators don't need to know about every child_up, child_down
+     * because of afr ha. If all subvolumes go down, child_down has
+     * to be triggered. In that state when 1 subvolume comes up child_up
+     * needs to be triggered. dht optimizes revalidate lookup by sending
+     * it only to one of its subvolumes. When child up/down happens
+     * for afr's subvolumes dht should be notified by child_modified. The
+     * subsequent revalidate lookup happens on all the dht's subvolumes
+     * which triggers afr self-heals if any.
+     */
+    idx = afr_find_child_index(this, child_xlator);
+    if (idx < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_CHILD_UP,
+               "Received child_up from invalid subvolume");
+        goto out;
+    }
+
+    had_quorum = priv->quorum_count &&
+                 afr_has_quorum(priv->child_up, this, NULL);
+    if (event == GF_EVENT_CHILD_PING) {
+        child_latency_msec = (int64_t)(uintptr_t)data2;
+        if (priv->halo_enabled) {
+            halo_max_latency_msec = afr_get_halo_latency(this);
+
+            /* Calculates the child latency and sets event
+             */
+            LOCK(&priv->lock);
+            {
+                __afr_handle_ping_event(this, child_xlator, idx,
+                                        halo_max_latency_msec, &event,
+                                        child_latency_msec);
+            }
+            UNLOCK(&priv->lock);
+        } else {
+            LOCK(&priv->lock);
+            {
+                priv->child_latency[idx] = child_latency_msec;
+            }
+            UNLOCK(&priv->lock);
         }
+    }
+
+    if (event == GF_EVENT_CHILD_PING) {
+        /* This is the only xlator that handles PING, no reason to
+         * propagate.
+         */
+        goto out;
+    }
 
-        LOCK (&priv->lock);
+    if (event == GF_EVENT_TRANSLATOR_OP) {
+        LOCK(&priv->lock);
         {
-                had_heard_from_all = __get_heard_from_all_status (this);
-                switch (event) {
-                case GF_EVENT_PARENT_UP:
-                        __afr_launch_notify_timer (this, priv);
-                        propagate = 1;
-                        break;
-                case GF_EVENT_CHILD_UP:
-                        __afr_handle_child_up_event (this, child_xlator,
-                                idx, child_latency_msec, &event, &call_psh,
-                                &up_child);
-                        break;
-
-                case GF_EVENT_CHILD_DOWN:
-                        __afr_handle_child_down_event (this, child_xlator, idx,
-                                child_latency_msec, &event, &call_psh,
-                                &up_child);
-                        break;
-
-                case GF_EVENT_CHILD_CONNECTING:
-                        priv->last_event[idx] = event;
-
-                        break;
-
-                case GF_EVENT_SOME_DESCENDENT_DOWN:
-                        priv->last_event[idx] = event;
-                        break;
-                case GF_EVENT_UPCALL:
-                        up_data = (struct gf_upcall *)data;
-                        if (up_data->event_type != GF_UPCALL_CACHE_INVALIDATION)
-                                break;
-                        up_ci = (struct gf_upcall_cache_invalidation *)up_data->data;
-
-                        /* Since md-cache will be aggressively filtering
-                         * lookups, the stale read issue will be more
-                         * pronounced. Hence when a pending xattr is set notify
-                         * all the md-cache clients to invalidate the existing
-                         * stat cache and send the lookup next time */
-                        if (!up_ci->dict)
-                                break;
-                        for (i = 0; i < priv->child_count; i++) {
-                                if (dict_get (up_ci->dict, priv->pending_key[i])) {
-                                         up_ci->flags |= UP_INVAL_ATTR;
-                                         itable = ((xlator_t *)this->graph->top)->itable;
-                                         /*Internal processes may not have itable for top xlator*/
-                                         if (itable)
-                                                 inode = inode_find (itable, up_data->gfid);
-                                         if (inode)
-                                                 afr_inode_need_refresh_set (inode, this);
-
-                                         break;
-                                }
-                        }
-                        break;
-                default:
-                        propagate = 1;
-                        break;
-                }
-                have_heard_from_all = __get_heard_from_all_status (this);
-                if (!had_heard_from_all && have_heard_from_all) {
-                        if (priv->timer) {
-                                gf_timer_call_cancel (this->ctx, priv->timer);
-                                priv->timer = NULL;
-                        }
-                        /* This is the first event which completes aggregation
-                           of events from all subvolumes. If at least one subvol
-                           had come up, propagate CHILD_UP, but only this time
-                        */
-                        event = GF_EVENT_CHILD_DOWN;
-                        for (i = 0; i < priv->child_count; i++) {
-                                if (priv->last_event[i] == GF_EVENT_CHILD_UP) {
-                                        event = GF_EVENT_CHILD_UP;
-                                        break;
-                                }
-
-                                if (priv->last_event[i] ==
-                                                GF_EVENT_CHILD_CONNECTING) {
-                                        event = GF_EVENT_CHILD_CONNECTING;
-                               /* continue to check other events for CHILD_UP */
-                                }
-                        }
-                }
+            had_heard_from_all = __get_heard_from_all_status(this);
         }
-        UNLOCK (&priv->lock);
+        UNLOCK(&priv->lock);
 
-        if (priv->quorum_count) {
-                has_quorum = afr_has_quorum (priv->child_up, this);
-                if (!had_quorum && has_quorum) {
-                        gf_msg (this->name, GF_LOG_INFO, 0, AFR_MSG_QUORUM_MET,
-                                "Client-quorum is met");
-                        gf_event  (EVENT_AFR_QUORUM_MET,
-                                   "subvol=%s", this->name);
+        if (!had_heard_from_all) {
+            ret = -1;
+        } else {
+            input = data;
+            output = data2;
+            ret = afr_xl_op(this, input, output);
+        }
+        goto out;
+    }
+
+    if (event == GF_EVENT_UPCALL) {
+        afr_handle_upcall_event(this, data);
+    }
+
+    LOCK(&priv->lock);
+    {
+        had_heard_from_all = __get_heard_from_all_status(this);
+        switch (event) {
+            case GF_EVENT_PARENT_UP:
+                __afr_launch_notify_timer(this, priv);
+                propagate = 1;
+                break;
+            case GF_EVENT_CHILD_UP:
+                if (priv->thin_arbiter_count &&
+                    (idx == AFR_CHILD_THIN_ARBITER)) {
+                    priv->ta_child_up = 1;
+                    priv->ta_event_gen++;
+                    break;
                 }
-                if (had_quorum && !has_quorum) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_QUORUM_FAIL,
-                                "Client-quorum is not met");
-                        gf_event  (EVENT_AFR_QUORUM_FAIL, "subvol=%s",
-                                   this->name);
+                __afr_handle_child_up_event(this, child_xlator, idx,
+                                            child_latency_msec, &event,
+                                            &call_psh, &up_child);
+                __afr_lock_heal_synctask(this, priv, idx);
+                break;
+
+            case GF_EVENT_CHILD_DOWN:
+                if (priv->thin_arbiter_count &&
+                    (idx == AFR_CHILD_THIN_ARBITER)) {
+                    priv->ta_child_up = 0;
+                    priv->ta_event_gen++;
+                    afr_ta_locked_priv_invalidate(priv);
+                    break;
                 }
-        }
+                __afr_handle_child_down_event(this, child_xlator, idx,
+                                              child_latency_msec, &event,
+                                              &call_psh, &up_child);
+                __afr_mark_pending_lk_heal(this, priv, idx);
+                break;
+
+            case GF_EVENT_CHILD_CONNECTING:
+                priv->last_event[idx] = event;
+
+                break;
 
-        /* if all subvols have reported status, no need to hide anything
-           or wait for anything else. Just propagate blindly */
-        if (have_heard_from_all)
+            case GF_EVENT_SOME_DESCENDENT_DOWN:
+                priv->last_event[idx] = event;
+                break;
+            default:
                 propagate = 1;
+                break;
+        }
+        have_heard_from_all = __get_heard_from_all_status(this);
+        if (!had_heard_from_all && have_heard_from_all) {
+            if (priv->timer) {
+                gf_timer_call_cancel(this->ctx, priv->timer);
+                priv->timer = NULL;
+            }
+            /* This is the first event which completes aggregation
+               of events from all subvolumes. If at least one subvol
+               had come up, propagate CHILD_UP, but only this time
+            */
+            event = GF_EVENT_CHILD_DOWN;
+            for (i = 0; i < priv->child_count; i++) {
+                if (priv->last_event[i] == GF_EVENT_CHILD_UP) {
+                    event = GF_EVENT_CHILD_UP;
+                    break;
+                }
 
-        ret = 0;
-        if (propagate)
-                ret = default_notify (this, event, data);
-
-        if ((!had_heard_from_all) || call_psh) {
-                /* Launch self-heal on all local subvolumes if:
-                 * a) We have_heard_from_all for the first time
-                 * b) Already heard from everyone, but we now got a child-up
-                 *    event.
-                 */
-                if (have_heard_from_all && priv->shd.iamshd) {
-                        for (i = 0; i < priv->child_count; i++)
-                                if (priv->child_up[i])
-                                        afr_selfheal_childup (this, i);
+                if (priv->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
+                    event = GF_EVENT_CHILD_CONNECTING;
+                    /* continue to check other events for CHILD_UP */
                 }
+            }
         }
+    }
+    UNLOCK(&priv->lock);
+
+    if (priv->quorum_count) {
+        has_quorum = afr_has_quorum(priv->child_up, this, NULL);
+        if (!had_quorum && has_quorum) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_QUORUM_MET,
+                   "Client-quorum is met");
+            gf_event(EVENT_AFR_QUORUM_MET, "client-pid=%d; subvol=%s",
+                     this->ctx->cmd_args.client_pid, this->name);
+        }
+        if (had_quorum && !has_quorum) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
+                   "Client-quorum is not met");
+            gf_event(EVENT_AFR_QUORUM_FAIL, "client-pid=%d; subvol=%s",
+                     this->ctx->cmd_args.client_pid, this->name);
+        }
+    }
+
+    /* if all subvols have reported status, no need to hide anything
+       or wait for anything else. Just propagate blindly */
+    if (have_heard_from_all)
+        propagate = 1;
+
+    ret = 0;
+    if (propagate)
+        ret = default_notify(this, event, data);
+
+    if ((!had_heard_from_all) || call_psh) {
+        /* Launch self-heal on all local subvolumes if:
+         * a) We have_heard_from_all for the first time
+         * b) Already heard from everyone, but we now got a child-up
+         *    event.
+         */
+        if (have_heard_from_all) {
+            afr_selfheal_childup(this, priv);
+        }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
+afr_local_init(afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
 {
-        int __ret = -1;
-        local->op_ret = -1;
-        local->op_errno = EUCLEAN;
+    int __ret = -1;
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
 
-	__ret = syncbarrier_init (&local->barrier);
-        if (__ret) {
-                if (op_errno)
-                        *op_errno = __ret;
-                goto out;
-        }
-
-        local->child_up = GF_CALLOC (priv->child_count,
-                                     sizeof (*local->child_up),
-                                     gf_afr_mt_char);
-        if (!local->child_up) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
-        }
-
-        memcpy (local->child_up, priv->child_up,
-                sizeof (*local->child_up) * priv->child_count);
-        local->call_count = AFR_COUNT (local->child_up, priv->child_count);
-        if (local->call_count == 0) {
-                gf_msg (THIS->name, GF_LOG_INFO, 0,
-                        AFR_MSG_SUBVOLS_DOWN, "no subvolumes up");
-                if (op_errno)
-                        *op_errno = ENOTCONN;
-                goto out;
-        }
-
-	local->event_generation = priv->event_generation;
+    __ret = syncbarrier_init(&local->barrier);
+    if (__ret) {
+        if (op_errno)
+            *op_errno = __ret;
+        goto out;
+    }
 
-	local->read_attempted = GF_CALLOC (priv->child_count, sizeof (char),
-					   gf_afr_mt_char);
-	if (!local->read_attempted) {
-		if (op_errno)
-			*op_errno = ENOMEM;
-		goto out;
-	}
+    local->child_up = GF_MALLOC(priv->child_count * sizeof(*local->child_up),
+                                gf_afr_mt_char);
+    if (!local->child_up) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    memcpy(local->child_up, priv->child_up,
+           sizeof(*local->child_up) * priv->child_count);
+    local->call_count = AFR_COUNT(local->child_up, priv->child_count);
+    if (local->call_count == 0) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOLS_DOWN,
+               "no subvolumes up");
+        if (op_errno)
+            *op_errno = ENOTCONN;
+        goto out;
+    }
 
-	local->readable = GF_CALLOC (priv->child_count, sizeof (char),
-				     gf_afr_mt_char);
-	if (!local->readable) {
-		if (op_errno)
-			*op_errno = ENOMEM;
-		goto out;
-	}
+    local->event_generation = priv->event_generation;
 
-        local->readable2 = GF_CALLOC (priv->child_count, sizeof (char),
+    local->read_attempted = GF_CALLOC(priv->child_count, sizeof(char),
                                       gf_afr_mt_char);
-        if (!local->readable2) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
-        }
+    if (!local->read_attempted) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
 
-        local->read_subvol = -1;
+    local->readable = GF_CALLOC(priv->child_count, sizeof(char),
+                                gf_afr_mt_char);
+    if (!local->readable) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
 
-	local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
-				   gf_afr_mt_reply_t);
-	if (!local->replies) {
-		if (op_errno)
-			*op_errno = ENOMEM;
-		goto out;
-	}
+    local->readable2 = GF_CALLOC(priv->child_count, sizeof(char),
+                                 gf_afr_mt_char);
+    if (!local->readable2) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
 
-        local->need_full_crawl = _gf_false;
+    local->read_subvol = -1;
 
-        INIT_LIST_HEAD (&local->healer);
-	return 0;
+    local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
+                               gf_afr_mt_reply_t);
+    if (!local->replies) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->need_full_crawl = _gf_false;
+    if (priv->thin_arbiter_count) {
+        local->ta_child_up = priv->ta_child_up;
+        local->ta_failed_subvol = AFR_CHILD_UNKNOWN;
+        local->read_txn_query_child = AFR_CHILD_UNKNOWN;
+        local->ta_event_gen = priv->ta_event_gen;
+        local->fop_state = TA_SUCCESS;
+    }
+    local->is_new_entry = _gf_false;
+
+    INIT_LIST_HEAD(&local->healer);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int
-afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count)
+afr_internal_lock_init(afr_internal_lock_t *lk, size_t child_count)
 {
-        int             ret = -ENOMEM;
+    int ret = -ENOMEM;
 
-        lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
-                                      child_count, gf_afr_mt_char);
-        if (NULL == lk->locked_nodes)
-                goto out;
-
-        lk->lower_locked_nodes = GF_CALLOC (sizeof (*lk->lower_locked_nodes),
-                                            child_count, gf_afr_mt_char);
-        if (NULL == lk->lower_locked_nodes)
-                goto out;
+    lk->lower_locked_nodes = GF_CALLOC(sizeof(*lk->lower_locked_nodes),
+                                       child_count, gf_afr_mt_char);
+    if (NULL == lk->lower_locked_nodes)
+        goto out;
 
-        lk->lock_op_ret   = -1;
-        lk->lock_op_errno = EUCLEAN;
+    lk->lock_op_ret = -1;
+    lk->lock_op_errno = EUCLEAN;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 void
-afr_matrix_cleanup (int32_t **matrix, unsigned int m)
+afr_matrix_cleanup(int32_t **matrix, unsigned int m)
 {
-        int             i         = 0;
+    int i = 0;
 
-        if (!matrix)
-                goto out;
-        for (i = 0; i < m; i++) {
-                GF_FREE (matrix[i]);
-        }
+    if (!matrix)
+        goto out;
+    for (i = 0; i < m; i++) {
+        GF_FREE(matrix[i]);
+    }
 
-        GF_FREE (matrix);
+    GF_FREE(matrix);
 out:
-        return;
+    return;
 }
 
-int32_t**
-afr_matrix_create (unsigned int m, unsigned int n)
+int32_t **
+afr_matrix_create(unsigned int m, unsigned int n)
 {
-        int32_t         **matrix = NULL;
-        int             i       = 0;
+    int32_t **matrix = NULL;
+    int i = 0;
 
-        matrix = GF_CALLOC (sizeof (*matrix), m, gf_afr_mt_int32_t);
-        if (!matrix)
-                goto out;
+    matrix = GF_CALLOC(sizeof(*matrix), m, gf_afr_mt_int32_t);
+    if (!matrix)
+        goto out;
 
-        for (i = 0; i < m; i++) {
-                matrix[i] = GF_CALLOC (sizeof (*matrix[i]), n,
-                                       gf_afr_mt_int32_t);
-                if (!matrix[i])
-                        goto out;
-        }
-        return matrix;
+    for (i = 0; i < m; i++) {
+        matrix[i] = GF_CALLOC(sizeof(*matrix[i]), n, gf_afr_mt_int32_t);
+        if (!matrix[i])
+            goto out;
+    }
+    return matrix;
 out:
-        afr_matrix_cleanup (matrix, m);
-        return NULL;
+    afr_matrix_cleanup(matrix, m);
+    return NULL;
 }
 
 int
-afr_transaction_local_init (afr_local_t *local, xlator_t *this)
-{
-        int            ret = -ENOMEM;
-        afr_private_t *priv = NULL;
-
-        priv = this->private;
-        ret = afr_internal_lock_init (&local->internal_lock, priv->child_count);
-        if (ret < 0)
-                goto out;
-
-        ret = -ENOMEM;
-	local->pre_op_compat = priv->pre_op_compat;
-
-        local->transaction.pre_op = GF_CALLOC (sizeof (*local->transaction.pre_op),
-                                               priv->child_count,
-                                               gf_afr_mt_char);
-        if (!local->transaction.pre_op)
-                goto out;
-
-        local->transaction.changelog_xdata =
-                GF_CALLOC (sizeof (*local->transaction.changelog_xdata),
-                           priv->child_count, gf_afr_mt_dict_t);
-        if (!local->transaction.changelog_xdata)
-                goto out;
-
-        if (priv->arbiter_count == 1) {
-                local->transaction.pre_op_sources =
-                        GF_CALLOC (sizeof (*local->transaction.pre_op_sources),
-                                   priv->child_count, gf_afr_mt_char);
-                if (!local->transaction.pre_op_sources)
-                        goto out;
-        }
-
-        local->transaction.failed_subvols = GF_CALLOC (sizeof (*local->transaction.failed_subvols),
-						       priv->child_count,
-						       gf_afr_mt_char);
-        if (!local->transaction.failed_subvols)
-                goto out;
-
-        local->pending = afr_matrix_create (priv->child_count,
-                                            AFR_NUM_CHANGE_LOGS);
-        if (!local->pending)
-                goto out;
-
-        ret = 0;
-        INIT_LIST_HEAD (&local->transaction.wait_list);
-        INIT_LIST_HEAD (&local->transaction.owner_list);
+afr_transaction_local_init(afr_local_t *local, xlator_t *this)
+{
+    int ret = -ENOMEM;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    INIT_LIST_HEAD(&local->transaction.wait_list);
+    INIT_LIST_HEAD(&local->transaction.owner_list);
+    INIT_LIST_HEAD(&local->ta_waitq);
+    INIT_LIST_HEAD(&local->ta_onwireq);
+    ret = afr_internal_lock_init(&local->internal_lock, priv->child_count);
+    if (ret < 0)
+        goto out;
+
+    ret = -ENOMEM;
+    local->pre_op_compat = priv->pre_op_compat;
+
+    local->transaction.pre_op = GF_CALLOC(sizeof(*local->transaction.pre_op),
+                                          priv->child_count, gf_afr_mt_char);
+    if (!local->transaction.pre_op)
+        goto out;
+
+    local->transaction.changelog_xdata = GF_CALLOC(
+        sizeof(*local->transaction.changelog_xdata), priv->child_count,
+        gf_afr_mt_dict_t);
+    if (!local->transaction.changelog_xdata)
+        goto out;
+
+    if (priv->arbiter_count == 1) {
+        local->transaction.pre_op_sources = GF_CALLOC(
+            sizeof(*local->transaction.pre_op_sources), priv->child_count,
+            gf_afr_mt_char);
+        if (!local->transaction.pre_op_sources)
+            goto out;
+    }
+
+    local->transaction.failed_subvols = GF_CALLOC(
+        sizeof(*local->transaction.failed_subvols), priv->child_count,
+        gf_afr_mt_char);
+    if (!local->transaction.failed_subvols)
+        goto out;
+
+    local->pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!local->pending)
+        goto out;
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 void
-afr_set_low_priority (call_frame_t *frame)
+afr_set_low_priority(call_frame_t *frame)
 {
-        frame->root->pid = LOW_PRIO_PROC_PID;
+    frame->root->pid = LOW_PRIO_PROC_PID;
 }
 
-
 void
-afr_priv_destroy (afr_private_t *priv)
-{
-        int i = 0;
-        int child_count = -1;
-
-        if (!priv)
-                goto out;
-        GF_FREE (priv->last_event);
-
-        child_count = priv->child_count;
-        if (priv->thin_arbiter_count) {
-                child_count++;
-        }
-        if (priv->pending_key) {
-                for (i = 0; i < child_count; i++)
-                        GF_FREE (priv->pending_key[i]);
-        }
-
-        GF_FREE (priv->pending_reads);
-        GF_FREE (priv->local);
-        GF_FREE (priv->pending_key);
-        GF_FREE (priv->children);
-        GF_FREE (priv->child_up);
-        GF_FREE (priv->child_latency);
-        LOCK_DESTROY (&priv->lock);
-
-        GF_FREE (priv);
+afr_priv_destroy(afr_private_t *priv)
+{
+    int i = 0;
+    int child_count = -1;
+
+    if (!priv)
+        goto out;
+
+    GF_FREE(priv->sh_domain);
+    GF_FREE(priv->last_event);
+
+    child_count = priv->child_count;
+    if (priv->thin_arbiter_count) {
+        child_count++;
+    }
+    if (priv->pending_key) {
+        for (i = 0; i < child_count; i++)
+            GF_FREE(priv->pending_key[i]);
+    }
+
+    GF_FREE(priv->pending_reads);
+    GF_FREE(priv->local);
+    GF_FREE(priv->pending_key);
+    GF_FREE(priv->children);
+    GF_FREE(priv->anon_inode);
+    GF_FREE(priv->child_up);
+    GF_FREE(priv->halo_child_up);
+    GF_FREE(priv->child_latency);
+    LOCK_DESTROY(&priv->lock);
+
+    GF_FREE(priv);
 out:
-        return;
+    return;
 }
 
-int**
-afr_mark_pending_changelog (afr_private_t *priv, unsigned char *pending,
-                            dict_t *xattr, ia_type_t iat)
+int **
+afr_mark_pending_changelog(afr_private_t *priv, unsigned char *pending,
+                           dict_t *xattr, ia_type_t iat)
 {
-       int i = 0;
-       int **changelog = NULL;
-       int idx = -1;
-       int m_idx = 0;
-       int d_idx = 0;
-       int ret = 0;
+    int i = 0;
+    int **changelog = NULL;
+    int idx = -1;
+    int m_idx = 0;
+    int d_idx = 0;
+    int ret = 0;
 
-       m_idx = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
-       d_idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
+    m_idx = afr_index_for_transaction_type(AFR_METADATA_TRANSACTION);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
 
-       idx = afr_index_from_ia_type (iat);
+    idx = afr_index_from_ia_type(iat);
 
-       changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS);
-       if (!changelog)
-                goto out;
+    changelog = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!changelog)
+        goto out;
 
-       for (i = 0; i < priv->child_count; i++) {
-               if (!pending[i])
-                       continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!pending[i])
+            continue;
 
-               changelog[i][m_idx] = hton32(1);
-               if (idx != -1)
-                       changelog[i][idx] = hton32(1);
-                /* If the newentry marking is on a newly created directory,
-                 * then mark it with the full-heal indicator.
-                 */
-                if ((IA_ISDIR (iat)) && (priv->esh_granular))
-                        changelog[i][d_idx] = hton32(1);
-       }
-       ret = afr_set_pending_dict (priv, xattr, changelog);
-       if (ret < 0) {
-               afr_matrix_cleanup (changelog, priv->child_count);
-               return NULL;
-       }
+        changelog[i][m_idx] = hton32(1);
+        if (idx != -1)
+            changelog[i][idx] = hton32(1);
+        /* If the newentry marking is on a newly created directory,
+         * then mark it with the full-heal indicator.
+         */
+        if ((IA_ISDIR(iat)) && (priv->esh_granular))
+            changelog[i][d_idx] = hton32(1);
+    }
+    ret = afr_set_pending_dict(priv, xattr, changelog);
+    if (ret < 0) {
+        afr_matrix_cleanup(changelog, priv->child_count);
+        return NULL;
+    }
 out:
-       return changelog;
+    return changelog;
 }
 
-gf_boolean_t
-afr_decide_heal_info (afr_private_t *priv, unsigned char *sources, int source)
+static dict_t *
+afr_set_heal_info(char *status)
 {
-        int sources_count = 0;
+    dict_t *dict = NULL;
+    int ret = -1;
 
-        if (source < 0)
-                goto out;
-
-        sources_count = AFR_COUNT (sources, priv->child_count);
-        if (sources_count == priv->child_count)
-                return _gf_false;
+    dict = dict_new();
+    if (!dict) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_sizen(dict, "heal-info", status);
+    if (ret)
+        gf_msg("", GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Failed to set heal-info key to "
+               "%s",
+               status);
 out:
-        return _gf_true;
-}
+    /* Any error other than EINVAL, dict_set_dynstr frees status */
+    if (ret == -ENOMEM || ret == -EINVAL) {
+        GF_FREE(status);
+    }
 
-int
-afr_selfheal_locked_metadata_inspect (call_frame_t *frame, xlator_t *this,
-                                      inode_t *inode, gf_boolean_t *msh,
-                                      gf_boolean_t *pending)
-{
-        int ret = -1;
-        unsigned char *locked_on = NULL;
-        unsigned char *sources = NULL;
-        unsigned char *sinks = NULL;
-        unsigned char *healed_sinks = NULL;
-        unsigned char *undid_pending = NULL;
-        struct afr_reply *locked_replies = NULL;
-
-        afr_private_t *priv = this->private;
-
-        locked_on = alloca0 (priv->child_count);
-        sources = alloca0 (priv->child_count);
-        sinks = alloca0 (priv->child_count);
-        healed_sinks = alloca0 (priv->child_count);
-        undid_pending = alloca0 (priv->child_count);
-
-        locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-        ret = afr_selfheal_inodelk (frame, this, inode, this->name,
-                                    LLONG_MAX - 1, 0, locked_on);
-        {
-                if (ret == 0) {
-                        /* Not a single lock */
-                        ret = -afr_final_errno (frame->local, priv);
-                        if (ret == 0)
-                                ret = -ENOTCONN;/* all invalid responses */
-                        goto out;
-                }
-                ret = __afr_selfheal_metadata_prepare (frame, this, inode,
-                                                       locked_on, sources,
-                                                       sinks, healed_sinks,
-                                                       undid_pending,
-                                                       locked_replies,
-                                                       pending);
-                *msh = afr_decide_heal_info (priv, sources, ret);
-        }
-        afr_selfheal_uninodelk (frame, this, inode, this->name,
-                                LLONG_MAX - 1, 0, locked_on);
-out:
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-        return ret;
+    if (ret && dict) {
+        dict_unref(dict);
+        dict = NULL;
+    }
+    return dict;
 }
 
-int
-afr_selfheal_locked_data_inspect (call_frame_t *frame, xlator_t *this,
-                                  fd_t *fd, gf_boolean_t *dsh,
-                                  gf_boolean_t *pflag)
-{
-        int ret = -1;
-        unsigned char *data_lock = NULL;
-        unsigned char *sources = NULL;
-        unsigned char *sinks = NULL;
-        unsigned char *healed_sinks = NULL;
-        unsigned char *undid_pending = NULL;
-        afr_private_t   *priv = NULL;
-        struct afr_reply *locked_replies = NULL;
-        inode_t *inode = fd->inode;
-
-        priv = this->private;
-        data_lock = alloca0 (priv->child_count);
-        sources = alloca0 (priv->child_count);
-        sinks = alloca0 (priv->child_count);
-        healed_sinks = alloca0 (priv->child_count);
-        undid_pending = alloca0 (priv->child_count);
-
-        locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-        ret = afr_selfheal_inodelk (frame, this, inode, this->name,
-                                    0, 0, data_lock);
-        {
-                if (ret == 0) {
-                        ret = -afr_final_errno (frame->local, priv);
-                        if (ret == 0)
-                                ret = -ENOTCONN; /* all invalid responses */
-                        goto out;
-                }
-                ret = __afr_selfheal_data_prepare (frame, this, inode,
-                                                   data_lock, sources, sinks,
-                                                   healed_sinks, undid_pending,
-                                                   locked_replies, pflag);
-                *dsh = afr_decide_heal_info (priv, sources, ret);
-        }
-        afr_selfheal_uninodelk (frame, this, inode, this->name, 0, 0,
-                                data_lock);
-out:
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-        return ret;
-}
+static gf_boolean_t
+afr_is_dirty_count_non_unary_for_txn(xlator_t *this, struct afr_reply *replies,
+                                     afr_transaction_type type)
+{
+    afr_private_t *priv = this->private;
+    int *dirty = alloca0(priv->child_count * sizeof(int));
+    int i = 0;
 
-int
-afr_selfheal_locked_entry_inspect (call_frame_t *frame, xlator_t *this,
-                                   inode_t *inode,
-                                   gf_boolean_t *esh, gf_boolean_t *pflag)
-{
-        int ret = -1;
-        int source = -1;
-        afr_private_t   *priv = NULL;
-        unsigned char *locked_on = NULL;
-        unsigned char *data_lock = NULL;
-        unsigned char *sources = NULL;
-        unsigned char *sinks = NULL;
-        unsigned char *healed_sinks = NULL;
-        struct afr_reply *locked_replies = NULL;
-        gf_boolean_t granular_locks = _gf_false;
-
-        priv = this->private;
-	if (strcmp ("granular", priv->locking_scheme) == 0)
-	        granular_locks = _gf_true;
-        locked_on = alloca0 (priv->child_count);
-        data_lock = alloca0 (priv->child_count);
-        sources = alloca0 (priv->child_count);
-        sinks = alloca0 (priv->child_count);
-        healed_sinks = alloca0 (priv->child_count);
-
-        locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-        if (!granular_locks) {
-                ret = afr_selfheal_tryentrylk (frame, this, inode,
-                                              priv->sh_domain, NULL, locked_on);
-        }
-        {
-                if (!granular_locks && ret == 0) {
-                        ret = -afr_final_errno (frame->local, priv);
-                        if (ret == 0)
-                                ret = -ENOTCONN;/* all invalid responses */
-                        goto out;
-                }
+    afr_selfheal_extract_xattr(this, replies, type, dirty, NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (dirty[i] > 1)
+            return _gf_true;
+    }
 
-                ret = afr_selfheal_entrylk (frame, this, inode, this->name,
-                                            NULL, data_lock);
-                {
-                        if (ret == 0) {
-                                ret = -afr_final_errno (frame->local, priv);
-                                if (ret == 0)
-                                        ret = -ENOTCONN;
-                                /* all invalid responses */
-                                goto unlock;
-                        }
-                        ret = __afr_selfheal_entry_prepare (frame, this, inode,
-                                                            data_lock, sources,
-                                                            sinks, healed_sinks,
-                                                            locked_replies,
-                                                            &source, pflag);
-                        if ((ret == 0) && source < 0)
-                                ret = -EIO;
-                        *esh = afr_decide_heal_info (priv, sources, ret);
-                }
-                afr_selfheal_unentrylk (frame, this, inode, this->name, NULL,
-                                        data_lock, NULL);
-        }
-unlock:
-        if (!granular_locks)
-                afr_selfheal_unentrylk (frame, this, inode, priv->sh_domain,
-                                        NULL, locked_on, NULL);
-out:
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-        return ret;
+    return _gf_false;
 }
 
-int
-afr_selfheal_locked_inspect (call_frame_t *frame, xlator_t *this, uuid_t gfid,
-                             inode_t **inode,
-                             gf_boolean_t *entry_selfheal,
-                             gf_boolean_t *data_selfheal,
-                             gf_boolean_t *metadata_selfheal,
-                             gf_boolean_t *pending)
-
-{
-        int ret             = -1;
-        fd_t *fd            = NULL;
-        gf_boolean_t    dsh = _gf_false;
-        gf_boolean_t    msh = _gf_false;
-        gf_boolean_t    esh = _gf_false;
-
-        ret = afr_selfheal_unlocked_inspect (frame, this, gfid, inode,
-                                             &dsh, &msh, &esh);
-        if (ret)
-                goto out;
-
-        /* For every heal type hold locks and check if it indeed needs heal */
-
+static gf_boolean_t
+afr_is_dirty_count_non_unary(xlator_t *this, struct afr_reply *replies,
+                             ia_type_t ia_type)
+{
+    gf_boolean_t data_chk = _gf_false;
+    gf_boolean_t mdata_chk = _gf_false;
+    gf_boolean_t entry_chk = _gf_false;
+
+    switch (ia_type) {
+        case IA_IFDIR:
+            mdata_chk = _gf_true;
+            entry_chk = _gf_true;
+            break;
+        case IA_IFREG:
+            mdata_chk = _gf_true;
+            data_chk = _gf_true;
+            break;
+        default:
+            /*IA_IFBLK, IA_IFCHR, IA_IFLNK, IA_IFIFO, IA_IFSOCK*/
+            mdata_chk = _gf_true;
+            break;
+    }
 
-        /* Heal-info does an open() on the file being examined so that the
-         * current eager-lock holding client, if present, at some point sees
-         * open-fd count being > 1 and releases the eager-lock so that heal-info
-         * doesn't remain blocked forever until IO completes.
-         */
-        if ((*inode)->ia_type == IA_IFREG) {
-                ret = afr_selfheal_data_open (this, *inode, &fd);
-                if (ret < 0) {
-                        gf_msg_debug (this->name, -ret, "%s: Failed to open",
-                                      uuid_utoa ((*inode)->gfid));
-                        goto out;
-                }
-        }
+    if (data_chk && afr_is_dirty_count_non_unary_for_txn(
+                        this, replies, AFR_DATA_TRANSACTION)) {
+        return _gf_true;
+    } else if (mdata_chk && afr_is_dirty_count_non_unary_for_txn(
+                                this, replies, AFR_METADATA_TRANSACTION)) {
+        return _gf_true;
+    } else if (entry_chk && afr_is_dirty_count_non_unary_for_txn(
+                                this, replies, AFR_ENTRY_TRANSACTION)) {
+        return _gf_true;
+    }
 
-        if (msh) {
-                ret = afr_selfheal_locked_metadata_inspect (frame, this,
-                                                            *inode, &msh,
-                                                            pending);
-                if (ret == -EIO)
-                        goto out;
-        }
+    return _gf_false;
+}
 
-        if (dsh) {
-                ret = afr_selfheal_locked_data_inspect (frame, this, fd,
-                                                        &dsh, pending);
-                if (ret == -EIO || (ret == -EAGAIN))
-                        goto out;
+static int
+afr_update_heal_status(xlator_t *this, struct afr_reply *replies,
+                       ia_type_t ia_type, gf_boolean_t *esh, gf_boolean_t *dsh,
+                       gf_boolean_t *msh, unsigned char pending)
+{
+    int ret = -1;
+    GF_UNUSED int ret1 = 0;
+    int i = 0;
+    int io_domain_lk_count = 0;
+    int shd_domain_lk_count = 0;
+    afr_private_t *priv = NULL;
+    char *key1 = NULL;
+    char *key2 = NULL;
+
+    priv = this->private;
+    key1 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(this->name));
+    key2 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(priv->sh_domain));
+    sprintf(key1, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, this->name);
+    sprintf(key2, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, priv->sh_domain);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if ((replies[i].valid != 1) || (replies[i].op_ret != 0))
+            continue;
+        if (!io_domain_lk_count) {
+            ret1 = dict_get_int32(replies[i].xdata, key1, &io_domain_lk_count);
+        }
+        if (!shd_domain_lk_count) {
+            ret1 = dict_get_int32(replies[i].xdata, key2, &shd_domain_lk_count);
+        }
+    }
+
+    if (!pending) {
+        if ((afr_is_dirty_count_non_unary(this, replies, ia_type)) ||
+            (!io_domain_lk_count)) {
+            /* Needs heal. */
+            ret = 0;
+        } else {
+            /* No heal needed. */
+            *dsh = *esh = *msh = 0;
         }
-
-        if (esh) {
-                ret = afr_selfheal_locked_entry_inspect (frame, this, *inode,
-                                                         &esh, pending);
+    } else {
+        if (shd_domain_lk_count) {
+            ret = -EAGAIN; /*For 'possibly-healing'. */
+        } else {
+            ret = 0; /*needs heal. Just set a non -ve value so that it is
+                       assumed as the source index.*/
         }
-
-out:
-        *data_selfheal = dsh;
-        *entry_selfheal = esh;
-        *metadata_selfheal = msh;
-        if (fd)
-                fd_unref (fd);
-        return ret;
+    }
+    return ret;
 }
 
-dict_t*
-afr_set_heal_info (char *status)
-{
-        dict_t *dict = NULL;
-        int    ret   = -1;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "heal-info", status);
+/*return EIO, EAGAIN or pending*/
+int
+afr_lockless_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
+                     inode_t **inode, gf_boolean_t *entry_selfheal,
+                     gf_boolean_t *data_selfheal,
+                     gf_boolean_t *metadata_selfheal, unsigned char *pending)
+{
+    int ret = -1;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    gf_boolean_t dsh = _gf_false;
+    gf_boolean_t msh = _gf_false;
+    gf_boolean_t esh = _gf_false;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *valid_on = NULL;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+    replies = alloca0(sizeof(*replies) * priv->child_count);
+    sources = alloca0(sizeof(*sources) * priv->child_count);
+    sinks = alloca0(sizeof(*sinks) * priv->child_count);
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    valid_on = alloca0(sizeof(*valid_on) * priv->child_count);
+
+    ret = afr_selfheal_unlocked_inspect(frame, this, gfid, inode, &dsh, &msh,
+                                        &esh, replies);
+    if (ret)
+        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            valid_on[i] = 1;
+        }
+    }
+    if (msh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_METADATA_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
+        if (ret)
+            goto out;
+    }
+    if (dsh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_DATA_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
+        if (ret)
+            goto out;
+    }
+    if (esh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_ENTRY_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
         if (ret)
-                gf_msg ("", GF_LOG_WARNING, -ret,
-                        AFR_MSG_DICT_SET_FAILED,
-                        "Failed to set heal-info key to "
-                        "%s", status);
+            goto out;
+    }
+
+    ret = afr_update_heal_status(this, replies, (*inode)->ia_type, &esh, &dsh,
+                                 &msh, *pending);
 out:
-        return dict;
+    *data_selfheal = dsh;
+    *entry_selfheal = esh;
+    *metadata_selfheal = msh;
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    return ret;
 }
 
 int
-afr_get_heal_info (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        gf_boolean_t    data_selfheal     = _gf_false;
-        gf_boolean_t    metadata_selfheal = _gf_false;
-        gf_boolean_t    entry_selfheal    = _gf_false;
-        gf_boolean_t    pending           = _gf_false;
-        dict_t         *dict              = NULL;
-        int             ret               = -1;
-        int             op_errno          = 0;
-        int             size              = 0;
-        inode_t        *inode             = NULL;
-        char           *substr            = NULL;
-        char           *status            = NULL;
-
-        ret = afr_selfheal_locked_inspect (frame, this, loc->gfid, &inode,
-                                           &entry_selfheal,
-                                           &data_selfheal, &metadata_selfheal,
-                                           &pending);
-
-        if (ret == -ENOMEM) {
-                op_errno = -ret;
+afr_get_heal_info(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    gf_boolean_t data_selfheal = _gf_false;
+    gf_boolean_t metadata_selfheal = _gf_false;
+    gf_boolean_t entry_selfheal = _gf_false;
+    unsigned char pending = 0;
+    dict_t *dict = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+    inode_t *inode = NULL;
+    char *substr = NULL;
+    char *status = NULL;
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *heal_local = NULL;
+
+    /*Use frame with lk-owner set*/
+    heal_frame = afr_frame_create(frame->this, &op_errno);
+    if (!heal_frame) {
+        ret = -1;
+        goto out;
+    }
+    heal_local = heal_frame->local;
+    heal_frame->local = frame->local;
+
+    ret = afr_lockless_inspect(heal_frame, this, loc->gfid, &inode,
+                               &entry_selfheal, &data_selfheal,
+                               &metadata_selfheal, &pending);
+
+    if (ret == -ENOMEM) {
+        ret = -1;
+        goto out;
+    }
+
+    if (pending & PFLAG_PENDING) {
+        gf_asprintf(&substr, "-pending");
+        if (!substr)
+            goto out;
+    }
+
+    if (ret == -EIO) {
+        ret = gf_asprintf(&status, "split-brain%s", substr ? substr : "");
+        if (ret < 0) {
+            goto out;
+        }
+        dict = afr_set_heal_info(status);
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
+    } else if (ret == -EAGAIN) {
+        ret = gf_asprintf(&status, "possibly-healing%s", substr ? substr : "");
+        if (ret < 0) {
+            goto out;
+        }
+        dict = afr_set_heal_info(status);
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
+    } else if (ret >= 0) {
+        /* value of ret = source index
+         * so ret >= 0 and at least one of the 3 booleans set to
+         * true means a source is identified; heal is required.
+         */
+        if (!data_selfheal && !entry_selfheal && !metadata_selfheal) {
+            status = gf_strdup("no-heal");
+            if (!status) {
+                ret = -1;
+                goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
                 ret = -1;
                 goto out;
+            }
+        } else {
+            ret = gf_asprintf(&status, "heal%s", substr ? substr : "");
+            if (ret < 0) {
+                goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
+                ret = -1;
+                goto out;
+            }
         }
-
-        if (pending) {
-                size = strlen ("-pending") + 1;
-                gf_asprintf (&substr, "-pending");
-                if (!substr)
-                        goto out;
+    } else if (ret < 0) {
+        /* Apart from above checked -ve ret values, there are
+         * other possible ret values like ENOTCONN
+         * (returned when number of valid replies received are
+         * less than 2)
+         * in which case heal is required when one of the
+         * selfheal booleans is set.
+         */
+        if (data_selfheal || entry_selfheal || metadata_selfheal) {
+            ret = gf_asprintf(&status, "heal%s", substr ? substr : "");
+            if (ret < 0) {
+                goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
+                ret = -1;
+                goto out;
+            }
         }
+    }
 
-        if (ret == -EIO) {
-                size += strlen ("split-brain") + 1;
-                ret = gf_asprintf (&status, "split-brain%s",
-                                   substr? substr : "");
-                if (ret < 0)
-                        goto out;
-                dict = afr_set_heal_info (status);
-        } else if (ret == -EAGAIN) {
-                size += strlen ("possibly-healing") + 1;
-                ret = gf_asprintf (&status, "possibly-healing%s",
-                                   substr? substr : "");
-                if (ret < 0)
-                        goto out;
-                dict = afr_set_heal_info (status);
-        } else if (ret >= 0) {
-                /* value of ret = source index
-                 * so ret >= 0 and at least one of the 3 booleans set to
-                 * true means a source is identified; heal is required.
-                 */
-                if (!data_selfheal && !entry_selfheal &&
-                    !metadata_selfheal) {
-                        dict = afr_set_heal_info ("no-heal");
-                } else {
-                        size += strlen ("heal") + 1;
-                        ret = gf_asprintf (&status, "heal%s",
-                                           substr? substr : "");
-                        if (ret < 0)
-                                goto out;
-                        dict = afr_set_heal_info (status);
-                }
-        } else if (ret < 0) {
-                /* Apart from above checked -ve ret values, there are
-                 * other possible ret values like ENOTCONN
-                 * (returned when number of valid replies received are
-                 * less than 2)
-                 * in which case heal is required when one of the
-                 * selfheal booleans is set.
-                 */
-                if (data_selfheal || entry_selfheal ||
-                    metadata_selfheal) {
-                        size += strlen ("heal") + 1;
-                        ret = gf_asprintf (&status, "heal%s",
-                                           substr? substr : "");
-                        if (ret < 0)
-                                goto out;
-                        dict = afr_set_heal_info (status);
-                }
-        }
-        ret = 0;
+    ret = 0;
+    op_errno = 0;
 
 out:
-        AFR_STACK_UNWIND (getxattr, frame, ret, op_errno, dict, NULL);
-        if (dict)
-               dict_unref (dict);
-        if (inode)
-                inode_unref (inode);
-        GF_FREE (substr);
-        return ret;
+    if (heal_frame) {
+        heal_frame->local = heal_local;
+        AFR_STACK_DESTROY(heal_frame);
+    }
+    AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    if (dict)
+        dict_unref(dict);
+    if (inode)
+        inode_unref(inode);
+    GF_FREE(substr);
+    return ret;
 }
 
 int
-_afr_is_split_brain (call_frame_t *frame, xlator_t *this,
-                         struct afr_reply *replies,
-                         afr_transaction_type type,
-                         gf_boolean_t *spb)
-{
-        afr_private_t    *priv              = NULL;
-        uint64_t         *witness           = NULL;
-        unsigned char    *sources           = NULL;
-        unsigned char    *sinks             = NULL;
-        int               sources_count     = 0;
-        int               ret               = 0;
-
-        priv = this->private;
-
-        sources = alloca0 (priv->child_count);
-        sinks = alloca0 (priv->child_count);
-        witness = alloca0(priv->child_count * sizeof (*witness));
-
-        ret = afr_selfheal_find_direction (frame, this, replies,
-					   type, priv->child_up, sources,
-                                           sinks, witness, NULL);
-        if (ret)
-                return ret;
+_afr_is_split_brain(call_frame_t *frame, xlator_t *this,
+                    struct afr_reply *replies, afr_transaction_type type,
+                    gf_boolean_t *spb)
+{
+    afr_private_t *priv = NULL;
+    uint64_t *witness = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    int sources_count = 0;
+    int ret = 0;
+
+    priv = this->private;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    witness = alloca0(priv->child_count * sizeof(*witness));
+
+    ret = afr_selfheal_find_direction(frame, this, replies, type,
+                                      priv->child_up, sources, sinks, witness,
+                                      NULL);
+    if (ret)
+        return ret;
 
-        sources_count = AFR_COUNT (sources, priv->child_count);
-        if (!sources_count)
-                *spb = _gf_true;
+    sources_count = AFR_COUNT(sources, priv->child_count);
+    if (!sources_count)
+        *spb = _gf_true;
 
-        return ret;
+    return ret;
 }
 
 int
-afr_is_split_brain (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                    uuid_t gfid, gf_boolean_t *d_spb, gf_boolean_t *m_spb)
+afr_is_split_brain(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                   uuid_t gfid, gf_boolean_t *d_spb, gf_boolean_t *m_spb)
 {
-        int    ret                          = -1;
-        afr_private_t    *priv              = NULL;
-        struct afr_reply *replies           = NULL;
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        replies = alloca0 (sizeof (*replies) * priv->child_count);
+    replies = alloca0(sizeof(*replies) * priv->child_count);
 
-        ret = afr_selfheal_unlocked_discover (frame, inode, gfid, replies);
-        if (ret)
-                goto out;
+    ret = afr_selfheal_unlocked_discover(frame, inode, gfid, replies);
+    if (ret)
+        goto out;
 
-        if (!afr_can_decide_split_brain_source_sinks (replies,
-                                                      priv->child_count)) {
-                ret = -EAGAIN;
-                goto out;
-        }
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        ret = -EAGAIN;
+        goto out;
+    }
 
-        ret = _afr_is_split_brain (frame, this, replies,
-                                    AFR_DATA_TRANSACTION, d_spb);
-        if (ret)
-                goto out;
+    ret = _afr_is_split_brain(frame, this, replies, AFR_DATA_TRANSACTION,
+                              d_spb);
+    if (ret)
+        goto out;
 
-        ret = _afr_is_split_brain (frame, this, replies,
-                                    AFR_METADATA_TRANSACTION, m_spb);
+    ret = _afr_is_split_brain(frame, this, replies, AFR_METADATA_TRANSACTION,
+                              m_spb);
 out:
-        if (replies) {
-                afr_replies_wipe (replies, priv->child_count);
-                replies = NULL;
-        }
-        return ret;
+    if (replies) {
+        afr_replies_wipe(replies, priv->child_count);
+        replies = NULL;
+    }
+    return ret;
 }
 
 int
-afr_get_split_brain_status_cbk (int ret, call_frame_t *frame, void *opaque)
+afr_get_split_brain_status_cbk(int ret, call_frame_t *frame, void *opaque)
 {
-        GF_FREE (opaque);
-        return 0;
+    GF_FREE(opaque);
+    return 0;
 }
 
 int
-afr_get_split_brain_status (void *opaque)
-{
-        gf_boolean_t      d_spb             = _gf_false;
-        gf_boolean_t      m_spb             = _gf_false;
-        int               ret               = -1;
-        int               op_errno          = 0;
-        int               i                 = 0;
-        char             *choices           = NULL;
-        char             *status            = NULL;
-        dict_t           *dict              = NULL;
-        inode_t          *inode             = NULL;
-        afr_private_t    *priv              = NULL;
-        xlator_t         **children         = NULL;
-        call_frame_t     *frame             = NULL;
-        xlator_t         *this              = NULL;
-        loc_t            *loc               = NULL;
-        afr_spb_status_t *data              = NULL;
-
-        data     = opaque;
-        frame    = data->frame;
-        this     = frame->this;
-        loc      = data->loc;
-        priv     = this->private;
-        children = priv->children;
-
-        inode = afr_inode_find (this, loc->gfid);
-        if (!inode)
-                goto out;
+afr_get_split_brain_status(void *opaque)
+{
+    gf_boolean_t d_spb = _gf_false;
+    gf_boolean_t m_spb = _gf_false;
+    int ret = -1;
+    int op_errno = 0;
+    int i = 0;
+    char *choices = NULL;
+    char *status = NULL;
+    dict_t *dict = NULL;
+    inode_t *inode = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    afr_spb_status_t *data = NULL;
+
+    data = opaque;
+    frame = data->frame;
+    this = frame->this;
+    loc = data->loc;
+    priv = this->private;
+    children = priv->children;
+
+    inode = afr_inode_find(this, loc->gfid);
+    if (!inode)
+        goto out;
+
+    dict = dict_new();
+    if (!dict) {
+        op_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    /* Calculation for string length :
+     * (child_count X length of child-name) + SLEN("    Choices :")
+     * child-name consists of :
+     * a) 251 = max characters for volname according to GD_VOLUME_NAME_MAX
+     * b) strlen("-client-00,") assuming 16 replicas
+     */
+    choices = alloca0(priv->child_count * (256 + SLEN("-client-00,")) +
+                      SLEN("    Choices:"));
+
+    ret = afr_is_split_brain(frame, this, inode, loc->gfid, &d_spb, &m_spb);
+    if (ret) {
+        op_errno = -ret;
+        if (ret == -EAGAIN) {
+            ret = dict_set_sizen_str_sizen(dict, GF_AFR_SBRAIN_STATUS,
+                                           SBRAIN_HEAL_NO_GO_MSG);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, -ret,
+                       AFR_MSG_DICT_SET_FAILED,
+                       "Failed to set GF_AFR_SBRAIN_STATUS in dict");
+            }
+        }
+        ret = -1;
+        goto out;
+    }
 
-        dict = dict_new ();
-        if (!dict) {
-                op_errno = ENOMEM;
-                ret = -1;
-                goto out;
+    if (d_spb || m_spb) {
+        sprintf(choices, "    Choices:");
+        for (i = 0; i < priv->child_count; i++) {
+            strcat(choices, children[i]->name);
+            strcat(choices, ",");
         }
+        choices[strlen(choices) - 1] = '\0';
 
-        /* Calculation for string length :
-        * (child_count X length of child-name) + strlen ("    Choices :")
-        * child-name consists of :
-        * a) 256 = max characters for volname according to GD_VOLUME_NAME_MAX
-        * b) strlen ("-client-00,") assuming 16 replicas
-        */
-        choices = alloca0 (priv->child_count * (256 + strlen ("-client-00,")) +
-                           strlen ("    Choices:"));
+        ret = gf_asprintf(&status,
+                          "data-split-brain:%s    "
+                          "metadata-split-brain:%s%s",
+                          (d_spb) ? "yes" : "no", (m_spb) ? "yes" : "no",
+                          choices);
 
-        ret = afr_is_split_brain (frame, this, inode, loc->gfid, &d_spb,
-                                  &m_spb);
+        if (-1 == ret) {
+            op_errno = ENOMEM;
+            goto out;
+        }
+        ret = dict_set_dynstr_sizen(dict, GF_AFR_SBRAIN_STATUS, status);
         if (ret) {
-                op_errno = -ret;
-                if (ret == -EAGAIN) {
-                        ret = dict_set_str (dict, GF_AFR_SBRAIN_STATUS,
-                                            SBRAIN_HEAL_NO_GO_MSG);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING,
-                                        -ret, AFR_MSG_DICT_SET_FAILED,
-                                        "Failed to set GF_AFR_SBRAIN_STATUS in dict");
-                        }
-                }
-                ret = -1;
-                goto out;
+            op_errno = -ret;
+            ret = -1;
+            goto out;
         }
-
-        if (d_spb || m_spb) {
-                sprintf (choices, "    Choices:");
-                for (i = 0; i < priv->child_count; i++) {
-                        strcat (choices, children[i]->name);
-                        strcat (choices, ",");
-                }
-                choices[strlen (choices) - 1] = '\0';
-
-                ret = gf_asprintf (&status, "data-split-brain:%s    "
-                                    "metadata-split-brain:%s%s",
-                                    (d_spb) ? "yes" : "no",
-                                    (m_spb) ? "yes" : "no", choices);
-
-                if (-1 == ret) {
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-                ret = dict_set_dynstr (dict, GF_AFR_SBRAIN_STATUS, status);
-                if (ret) {
-                        op_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
-        } else {
-                ret = dict_set_str (dict, GF_AFR_SBRAIN_STATUS,
-                                    "The file is not under data or"
-                                    " metadata split-brain");
-                if (ret) {
-                        op_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
+    } else {
+        ret = dict_set_sizen_str_sizen(dict, GF_AFR_SBRAIN_STATUS,
+                                       SFILE_NOT_UNDER_DATA);
+        if (ret) {
+            op_errno = -ret;
+            ret = -1;
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        AFR_STACK_UNWIND (getxattr, frame, ret, op_errno, dict, NULL);
-        if (dict)
-               dict_unref (dict);
-        if (inode)
-                inode_unref (inode);
-        return ret;
+    AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    if (dict)
+        dict_unref(dict);
+    if (inode)
+        inode_unref(inode);
+    return ret;
 }
 
 int32_t
 afr_heal_splitbrain_file(call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-        int             ret               = 0;
-        int             op_errno          = 0;
-        dict_t         *dict              = NULL;
-        afr_local_t    *local             = NULL;
-
-        local = frame->local;
-        dict = dict_new ();
-        if (!dict) {
-                op_errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
-
-        ret = afr_selfheal_do (frame, this, loc->gfid);
-
-        if (ret == 1 || ret == 2) {
-                ret = dict_set_str (dict, "sh-fail-msg",
-                                    "File not in split-brain");
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING,
-                                -ret, AFR_MSG_DICT_SET_FAILED,
-                                "Failed to set sh-fail-msg in dict");
-                ret = 0;
-                goto out;
-        } else {
-                if (local->xdata_rsp) {
-                        /* 'sh-fail-msg' has been set in the dict during self-heal.*/
-                        dict_copy (local->xdata_rsp, dict);
-                        ret = 0;
-                } else if (ret < 0) {
-                        op_errno = -ret;
-                        ret = -1;
-                }
+    int ret = 0;
+    int op_errno = 0;
+    dict_t *dict = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *heal_local = NULL;
+    call_frame_t *heal_frame = NULL;
+
+    local = frame->local;
+    dict = dict_new();
+    if (!dict) {
+        op_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    heal_frame = afr_frame_create(this, &op_errno);
+    if (!heal_frame) {
+        ret = -1;
+        goto out;
+    }
+    heal_local = heal_frame->local;
+    heal_frame->local = frame->local;
+    /*Initiate heal with heal_frame with lk-owner set so that inodelk/entrylk
+     * work correctly*/
+    ret = afr_selfheal_do(heal_frame, this, loc->gfid);
+
+    if (ret == 1 || ret == 2) {
+        ret = dict_set_sizen_str_sizen(dict, "sh-fail-msg",
+                                       SFILE_NOT_IN_SPLIT_BRAIN);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Failed to set sh-fail-msg in dict");
+        ret = 0;
+        goto out;
+    } else {
+        if (local->xdata_rsp) {
+            /* 'sh-fail-msg' has been set in the dict during self-heal.*/
+            dict_copy(local->xdata_rsp, dict);
+            ret = 0;
+        } else if (ret < 0) {
+            op_errno = -ret;
+            ret = -1;
         }
+    }
 
 out:
-        if (local->op == GF_FOP_GETXATTR)
-                AFR_STACK_UNWIND (getxattr, frame, ret, op_errno, dict, NULL);
-        else if (local->op == GF_FOP_SETXATTR)
-                AFR_STACK_UNWIND (setxattr, frame, ret, op_errno, NULL);
-        if (dict)
-                dict_unref(dict);
-        return ret;
+    if (heal_frame) {
+        heal_frame->local = heal_local;
+        AFR_STACK_DESTROY(heal_frame);
+    }
+    if (local->op == GF_FOP_GETXATTR)
+        AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    else if (local->op == GF_FOP_SETXATTR)
+        AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
 int
-afr_get_child_index_from_name (xlator_t *this, char *name)
+afr_get_child_index_from_name(xlator_t *this, char *name)
 {
-        afr_private_t *priv  = this->private;
-        int            index = -1;
+    afr_private_t *priv = this->private;
+    int index = -1;
 
-        for (index = 0; index < priv->child_count; index++) {
-                if (!strcmp (priv->children[index]->name, name))
-                        goto out;
-        }
-        index = -1;
+    for (index = 0; index < priv->child_count; index++) {
+        if (!strcmp(priv->children[index]->name, name))
+            goto out;
+    }
+    index = -1;
 out:
-        return index;
+    return index;
 }
 
 void
-afr_priv_need_heal_set (afr_private_t *priv, gf_boolean_t need_heal)
+afr_priv_need_heal_set(afr_private_t *priv, gf_boolean_t need_heal)
 {
-        LOCK (&priv->lock);
-        {
-                priv->need_heal = need_heal;
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        priv->need_heal = need_heal;
+    }
+    UNLOCK(&priv->lock);
 }
 
 void
-afr_set_need_heal (xlator_t *this, afr_local_t *local)
+afr_set_need_heal(xlator_t *this, afr_local_t *local)
 {
-        int             i         = 0;
-        afr_private_t  *priv      = this->private;
-        gf_boolean_t    need_heal = _gf_false;
+    int i = 0;
+    afr_private_t *priv = this->private;
+    gf_boolean_t need_heal = _gf_false;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->replies[i].valid && local->replies[i].need_heal) {
-                        need_heal = _gf_true;
-                        break;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].need_heal) {
+            need_heal = _gf_true;
+            break;
         }
-        afr_priv_need_heal_set (priv, need_heal);
-        return;
+    }
+    afr_priv_need_heal_set(priv, need_heal);
+    return;
 }
 
 gf_boolean_t
-afr_get_need_heal (xlator_t *this)
+afr_get_need_heal(xlator_t *this)
 {
-        afr_private_t  *priv      = this->private;
-        gf_boolean_t    need_heal = _gf_true;
+    afr_private_t *priv = this->private;
+    gf_boolean_t need_heal = _gf_true;
 
-        LOCK (&priv->lock);
-        {
-                need_heal = priv->need_heal;
-        }
-        UNLOCK (&priv->lock);
-        return need_heal;
+    LOCK(&priv->lock);
+    {
+        need_heal = priv->need_heal;
+    }
+    UNLOCK(&priv->lock);
+    return need_heal;
 }
 
 int
-afr_get_msg_id (char *op_type)
+afr_get_msg_id(char *op_type)
 {
-
-        if (!strcmp (op_type, GF_AFR_REPLACE_BRICK))
-                return AFR_MSG_REPLACE_BRICK_STATUS;
-        else if (!strcmp (op_type, GF_AFR_ADD_BRICK))
-                return AFR_MSG_ADD_BRICK_STATUS;
-        return -1;
+    if (!strcmp(op_type, GF_AFR_REPLACE_BRICK))
+        return AFR_MSG_REPLACE_BRICK_STATUS;
+    else if (!strcmp(op_type, GF_AFR_ADD_BRICK))
+        return AFR_MSG_ADD_BRICK_STATUS;
+    return -1;
 }
 
 int
-afr_fav_child_reset_sink_xattrs_cbk (int ret, call_frame_t *heal_frame,
-                                     void *opaque)
+afr_fav_child_reset_sink_xattrs_cbk(int ret, call_frame_t *heal_frame,
+                                    void *opaque)
 {
+    call_frame_t *txn_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *heal_local = NULL;
+    xlator_t *this = NULL;
 
-        call_frame_t *txn_frame = NULL;
-        afr_local_t *local = NULL;
-        afr_local_t *heal_local = NULL;
-        xlator_t *this = NULL;
+    heal_local = heal_frame->local;
+    txn_frame = heal_local->heal_frame;
+    local = txn_frame->local;
+    this = txn_frame->this;
 
-        heal_local = heal_frame->local;
-        txn_frame = heal_local->heal_frame;
-        local = txn_frame->local;
-        this = txn_frame->this;
-
-        /* Refresh the inode agan and proceed with the transaction.*/
-        afr_inode_refresh (txn_frame, this, local->inode, NULL,
-                           local->refreshfn);
+    /* Refresh the inode agan and proceed with the transaction.*/
+    afr_inode_refresh(txn_frame, this, local->inode, NULL, local->refreshfn);
 
-        AFR_STACK_DESTROY (heal_frame);
+    AFR_STACK_DESTROY(heal_frame);
 
-        return 0;
+    return 0;
 }
 
 int
-afr_fav_child_reset_sink_xattrs (void *opaque)
-{
-        call_frame_t *heal_frame = NULL;
-        call_frame_t *txn_frame = NULL;
-        xlator_t *this = NULL;
-        gf_boolean_t d_spb = _gf_false;
-        gf_boolean_t m_spb = _gf_false;
-        afr_local_t *heal_local = NULL;
-        afr_local_t *txn_local = NULL;
-        afr_private_t *priv = NULL;
-        inode_t *inode  = NULL;
-        unsigned char *locked_on = NULL;
-        unsigned char *sources = NULL;
-        unsigned char *sinks = NULL;
-        unsigned char *healed_sinks = NULL;
-        unsigned char *undid_pending = NULL;
-        struct afr_reply *locked_replies = NULL;
-        int ret = 0;
-
-        heal_frame = (call_frame_t *) opaque;
-        heal_local = heal_frame->local;
-        txn_frame = heal_local->heal_frame;
-        txn_local = txn_frame->local;
-        this = txn_frame->this;
-        inode = txn_local->inode;
-        priv = this->private;
-        locked_on = alloca0 (priv->child_count);
-        sources = alloca0 (priv->child_count);
-        sinks = alloca0 (priv->child_count);
-        healed_sinks = alloca0 (priv->child_count);
-        undid_pending = alloca0 (priv->child_count);
-        locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-        ret = _afr_is_split_brain (txn_frame, this, txn_local->replies,
-                                   AFR_DATA_TRANSACTION, &d_spb);
-
-        ret = _afr_is_split_brain (txn_frame, this, txn_local->replies,
-                                   AFR_METADATA_TRANSACTION, &m_spb);
-
-        /* Take appropriate locks and reset sink xattrs. */
-        if (d_spb) {
-                ret = afr_selfheal_inodelk (heal_frame, this, inode, this->name,
-                                            0, 0, locked_on);
-                {
-                        if (ret < AFR_SH_MIN_PARTICIPANTS)
-                                goto data_unlock;
-                        ret = __afr_selfheal_data_prepare (heal_frame, this,
-                                                           inode, locked_on,
-                                                           sources, sinks,
-                                                           healed_sinks,
-                                                           undid_pending,
-                                                           locked_replies,
-                                                           NULL);
-                }
-data_unlock:
-                afr_selfheal_uninodelk (heal_frame, this, inode, this->name,
-                                        0, 0, locked_on);
-        }
-
-        if (m_spb) {
-                memset (locked_on, 0, sizeof (*locked_on) * priv->child_count);
-                memset (undid_pending, 0,
-                        sizeof (*undid_pending) * priv->child_count);
-                ret = afr_selfheal_inodelk (heal_frame, this, inode, this->name,
-                                            LLONG_MAX-1, 0, locked_on);
-                {
-                        if (ret < AFR_SH_MIN_PARTICIPANTS)
-                                goto mdata_unlock;
-                        ret = __afr_selfheal_metadata_prepare (heal_frame, this,
-                                                               inode, locked_on,
-                                                               sources, sinks,
-                                                               healed_sinks,
-                                                               undid_pending,
-                                                               locked_replies,
-                                                               NULL);
-
-                }
-mdata_unlock:
-                afr_selfheal_uninodelk (heal_frame, this, inode, this->name,
-                                        LLONG_MAX-1, 0, locked_on);
+afr_fav_child_reset_sink_xattrs(void *opaque)
+{
+    call_frame_t *heal_frame = NULL;
+    call_frame_t *txn_frame = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t d_spb = _gf_false;
+    gf_boolean_t m_spb = _gf_false;
+    afr_local_t *heal_local = NULL;
+    afr_local_t *txn_local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    unsigned char *locked_on = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    int ret = 0;
+
+    heal_frame = (call_frame_t *)opaque;
+    heal_local = heal_frame->local;
+    txn_frame = heal_local->heal_frame;
+    txn_local = txn_frame->local;
+    this = txn_frame->this;
+    inode = txn_local->inode;
+    priv = this->private;
+    locked_on = alloca0(priv->child_count);
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = _afr_is_split_brain(txn_frame, this, txn_local->replies,
+                              AFR_DATA_TRANSACTION, &d_spb);
+
+    ret = _afr_is_split_brain(txn_frame, this, txn_local->replies,
+                              AFR_METADATA_TRANSACTION, &m_spb);
+
+    /* Take appropriate locks and reset sink xattrs. */
+    if (d_spb) {
+        ret = afr_selfheal_inodelk(heal_frame, this, inode, this->name, 0, 0,
+                                   locked_on);
+        {
+            if (ret < priv->child_count)
+                goto data_unlock;
+            ret = __afr_selfheal_data_prepare(
+                heal_frame, this, inode, locked_on, sources, sinks,
+                healed_sinks, undid_pending, locked_replies, NULL);
+        }
+    data_unlock:
+        afr_selfheal_uninodelk(heal_frame, this, inode, this->name, 0, 0,
+                               locked_on);
+    }
+
+    if (m_spb) {
+        memset(locked_on, 0, sizeof(*locked_on) * priv->child_count);
+        memset(undid_pending, 0, sizeof(*undid_pending) * priv->child_count);
+        ret = afr_selfheal_inodelk(heal_frame, this, inode, this->name,
+                                   LLONG_MAX - 1, 0, locked_on);
+        {
+            if (ret < priv->child_count)
+                goto mdata_unlock;
+            ret = __afr_selfheal_metadata_prepare(
+                heal_frame, this, inode, locked_on, sources, sinks,
+                healed_sinks, undid_pending, locked_replies, NULL);
         }
+    mdata_unlock:
+        afr_selfheal_uninodelk(heal_frame, this, inode, this->name,
+                               LLONG_MAX - 1, 0, locked_on);
+    }
 
-        return ret;
-
+    return ret;
 }
 
 /*
  * Concatenates the xattrs in local->replies separated by a delimiter.
  */
 int
-afr_serialize_xattrs_with_delimiter (call_frame_t *frame, xlator_t *this,
-                                     char *buf, const char *default_str,
-                                     int32_t *serz_len, char delimiter)
-{
-        afr_private_t *priv      = NULL;
-        afr_local_t   *local     = NULL;
-        char          *xattr     = NULL;
-        int            i         = 0;
-        int            len       = 0;
-        int            ret       = -1;
-
-        priv = this->private;
-        local = frame->local;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->replies[i].valid || local->replies[i].op_ret) {
-                        buf = strncat (buf, default_str, strlen (default_str));
-                        len += strlen (default_str);
-                        buf[len++] = delimiter;
-                        buf[len] = '\0';
-                } else {
-                        ret = dict_get_str (local->replies[i].xattr,
-                                            local->cont.getxattr.name, &xattr);
-                        if (ret) {
-                                gf_msg ("TEST", GF_LOG_ERROR, -ret,
-                                        AFR_MSG_DICT_GET_FAILED,
-                                        "Failed to get the node_uuid of brick "
-                                        "%d", i);
-                                goto out;
-                        }
-                        buf = strncat (buf, xattr, strlen (xattr));
-                        len += strlen (xattr);
-                        buf[len++] = delimiter;
-                        buf[len] = '\0';
-                }
-        }
-        buf[--len] = '\0'; /*remove the last delimiter*/
-        if (serz_len)
-                *serz_len = ++len;
-        ret = 0;
+afr_serialize_xattrs_with_delimiter(call_frame_t *frame, xlator_t *this,
+                                    char *buf, const char *default_str,
+                                    int32_t *serz_len, char delimiter)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    char *xattr = NULL;
+    int i = 0;
+    int len = 0;
+    int keylen = 0;
+    size_t str_len = 0;
+    int ret = -1;
+
+    priv = this->private;
+    local = frame->local;
+
+    keylen = strlen(local->cont.getxattr.name);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid || local->replies[i].op_ret) {
+            str_len = strlen(default_str);
+            buf = strncat(buf, default_str, str_len);
+            len += str_len;
+            buf[len++] = delimiter;
+            buf[len] = '\0';
+        } else {
+            ret = dict_get_strn(local->replies[i].xattr,
+                                local->cont.getxattr.name, keylen, &xattr);
+            if (ret) {
+                gf_msg("TEST", GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                       "Failed to get the node_uuid of brick "
+                       "%d",
+                       i);
+                goto out;
+            }
+            str_len = strlen(xattr);
+            buf = strncat(buf, xattr, str_len);
+            len += str_len;
+            buf[len++] = delimiter;
+            buf[len] = '\0';
+        }
+    }
+    buf[--len] = '\0'; /*remove the last delimiter*/
+    if (serz_len)
+        *serz_len = ++len;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-int
-afr_write_subvol_set (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t      *local = NULL;
-        afr_private_t    *priv  = NULL;
-        unsigned char    *data_accused = NULL;
-        unsigned char    *metadata_accused = NULL;
-        unsigned char    *data_readable = NULL;
-        unsigned char    *metadata_readable = NULL;
-        uint16_t          datamap = 0;
-        uint16_t          metadatamap = 0;
-        uint64_t          val   = 0;
-        int               event = 0;
-        int               i     = 0;
+uint64_t
+afr_write_subvol_get(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    uint64_t write_subvol = 0;
 
-        local = frame->local;
-        priv = this->private;
-        data_accused = alloca0 (priv->child_count);
-        metadata_accused = alloca0 (priv->child_count);
-        data_readable = alloca0 (priv->child_count);
-        metadata_readable = alloca0 (priv->child_count);
-        event = local->event_generation;
+    local = frame->local;
+    LOCK(&local->inode->lock);
+    write_subvol = local->inode_ctx->write_subvol;
+    UNLOCK(&local->inode->lock);
 
-        afr_readables_fill (frame, this, local->inode, data_accused,
-                            metadata_accused, data_readable, metadata_readable,
-                            NULL);
+    return write_subvol;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (data_readable[i])
-                        datamap |= (1 << i);
-                if (metadata_readable[i])
-                        metadatamap |= (1 << i);
-        }
+int
+afr_write_subvol_set(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *data_accused = NULL;
+    unsigned char *metadata_accused = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint64_t val = 0;
+    int event = 0;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+    data_accused = alloca0(priv->child_count);
+    metadata_accused = alloca0(priv->child_count);
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+    event = local->event_generation;
+
+    afr_readables_fill(frame, this, local->inode, data_accused,
+                       metadata_accused, data_readable, metadata_readable,
+                       NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_readable[i])
+            datamap |= (1 << i);
+        if (metadata_readable[i])
+            metadatamap |= (1 << i);
+    }
+
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
+
+    LOCK(&local->inode->lock);
+    {
+        if (local->inode_ctx->write_subvol == 0 &&
+            local->transaction.type == AFR_DATA_TRANSACTION) {
+            local->inode_ctx->write_subvol = val;
+        }
+    }
+    UNLOCK(&local->inode->lock);
+
+    return 0;
+}
 
-        val = ((uint64_t) metadatamap) |
-              (((uint64_t) datamap) << 16) |
-              (((uint64_t) event) << 32);
+int
+afr_write_subvol_reset(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
 
-        LOCK(&local->inode->lock);
-        {
-                if (local->inode_ctx->write_subvol == 0 &&
-                    local->transaction.type == AFR_DATA_TRANSACTION) {
-                        local->inode_ctx->write_subvol = val;
-                }
-        }
-        UNLOCK (&local->inode->lock);
+    local = frame->local;
+    LOCK(&local->inode->lock);
+    {
+        GF_ASSERT(local->inode_ctx->lock_count > 0);
+        local->inode_ctx->lock_count--;
 
-        return 0;
+        if (!local->inode_ctx->lock_count)
+            local->inode_ctx->write_subvol = 0;
+    }
+    UNLOCK(&local->inode->lock);
+
+    return 0;
 }
 
 int
-afr_write_subvol_reset (call_frame_t *frame, xlator_t *this)
+afr_set_inode_local(xlator_t *this, afr_local_t *local, inode_t *inode)
 {
-        afr_local_t      *local = NULL;
+    int ret = 0;
 
-        local = frame->local;
-        LOCK(&local->inode->lock);
-        {
-                GF_ASSERT (local->inode_ctx->lock_count > 0);
-                local->inode_ctx->lock_count--;
+    local->inode = inode_ref(inode);
+    LOCK(&local->inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, local->inode, &local->inode_ctx);
+    }
+    UNLOCK(&local->inode->lock);
+    if (ret < 0) {
+        gf_msg_callingfn(
+            this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_INODE_CTX_GET_FAILED,
+            "Error getting inode ctx %s", uuid_utoa(local->inode->gfid));
+    }
+    return ret;
+}
 
-                if (!local->inode_ctx->lock_count)
-                        local->inode_ctx->write_subvol = 0;
-        }
-        UNLOCK(&local->inode->lock);
+gf_boolean_t
+afr_ta_is_fop_called_from_synctask(xlator_t *this)
+{
+    struct synctask *task = NULL;
+    gf_lkowner_t tmp_owner = {
+        0,
+    };
 
-        return 0;
+    task = synctask_get();
+    if (!task)
+        return _gf_false;
+
+    set_lk_owner_from_ptr(&tmp_owner, (void *)this);
+
+    if (!is_same_lkowner(&tmp_owner, &task->frame->root->lk_owner))
+        return _gf_false;
+
+    return _gf_true;
 }
 
 int
-afr_set_inode_local (xlator_t *this, afr_local_t *local, inode_t *inode)
+afr_ta_post_op_lock(xlator_t *this, loc_t *loc)
+{
+    int ret = 0;
+    uuid_t gfid = {
+        0,
+    };
+    afr_private_t *priv = this->private;
+    gf_boolean_t locked = _gf_false;
+    struct gf_flock flock1 = {
+        0,
+    };
+    struct gf_flock flock2 = {
+        0,
+    };
+    int32_t cmd = 0;
+
+    /* Clients must take AFR_TA_DOM_NOTIFY lock only when the previous lock
+     * has been released in afr_notify due to upcall notification from shd.
+     */
+    GF_ASSERT(priv->ta_notify_dom_lock_offset == 0);
+
+    if (!priv->shd.iamshd)
+        GF_ASSERT(afr_ta_is_fop_called_from_synctask(this));
+    flock1.l_type = F_WRLCK;
+
+    while (!locked) {
+        if (priv->shd.iamshd) {
+            cmd = F_SETLKW;
+            flock1.l_start = 0;
+            flock1.l_len = 0;
+        } else {
+            cmd = F_SETLK;
+            gf_uuid_generate(gfid);
+            flock1.l_start = gfid_to_ino(gfid);
+            if (flock1.l_start < 0)
+                flock1.l_start = -flock1.l_start;
+            flock1.l_len = 1;
+        }
+        ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                             AFR_TA_DOM_NOTIFY, loc, cmd, &flock1, NULL, NULL);
+        if (!ret) {
+            locked = _gf_true;
+            priv->ta_notify_dom_lock_offset = flock1.l_start;
+        } else if (ret == -EAGAIN) {
+            continue;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+                   "Failed to get "
+                   "AFR_TA_DOM_NOTIFY lock on %s.",
+                   loc->name);
+            goto out;
+        }
+    }
+
+    flock2.l_type = F_WRLCK;
+    flock2.l_start = 0;
+    flock2.l_len = 0;
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, loc, F_SETLKW, &flock2, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to get AFR_TA_DOM_MODIFY lock on %s.", loc->name);
+        flock1.l_type = F_UNLCK;
+        ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                             AFR_TA_DOM_NOTIFY, loc, F_SETLK, &flock1, NULL,
+                             NULL);
+    }
+out:
+    return ret;
+}
+
+int
+afr_ta_post_op_unlock(xlator_t *this, loc_t *loc)
+{
+    afr_private_t *priv = this->private;
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = 0;
+
+    if (!priv->shd.iamshd)
+        GF_ASSERT(afr_ta_is_fop_called_from_synctask(this));
+    flock.l_type = F_UNLCK;
+    flock.l_start = 0;
+    flock.l_len = 0;
+
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to unlock AFR_TA_DOM_MODIFY lock.");
+        goto out;
+    }
+
+    if (!priv->shd.iamshd)
+        /* Mounts (clients) will not release the AFR_TA_DOM_NOTIFY lock
+         * in post-op as they use it as a notification mechanism. When
+         * shd sends a lock request on TA during heal, the clients will
+         * receive a lock-contention upcall notification upon which they
+         * will release the AFR_TA_DOM_NOTIFY lock after completing the
+         * in flight I/O.*/
+        goto out;
+
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_NOTIFY, loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to unlock AFR_TA_DOM_NOTIFY lock.");
+    }
+out:
+    return ret;
+}
+
+call_frame_t *
+afr_ta_frame_create(xlator_t *this)
 {
-        int ret = 0;
+    call_frame_t *frame = NULL;
+    void *lk_owner = NULL;
 
-        local->inode = inode_ref (inode);
-        LOCK(&local->inode->lock);
-        {
-                ret = __afr_inode_ctx_get (this, local->inode,
-                                           &local->inode_ctx);
-        }
-        UNLOCK (&local->inode->lock);
-        if (ret < 0) {
-                gf_msg_callingfn (this->name, GF_LOG_ERROR, ENOMEM,
-                                  AFR_MSG_INODE_CTX_GET_FAILED,
-                                  "Error getting inode ctx %s",
-                                  uuid_utoa (local->inode->gfid));
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        return NULL;
+    lk_owner = (void *)this;
+    afr_set_lk_owner(frame, this, lk_owner);
+    return frame;
+}
+
+gf_boolean_t
+afr_ta_has_quorum(afr_private_t *priv, afr_local_t *local)
+{
+    int data_count = 0;
+
+    data_count = AFR_COUNT(local->child_up, priv->child_count);
+    if (data_count == 2) {
+        return _gf_true;
+    } else if (data_count == 1 && local->ta_child_up) {
+        return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+static gf_boolean_t
+afr_is_add_replica_mount_lookup_on_root(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+
+    if (frame->root->pid != GF_CLIENT_PID_ADD_REPLICA_MOUNT)
+        return _gf_false;
+
+    local = frame->local;
+
+    if (local->op != GF_FOP_LOOKUP)
+        /* TODO:If the replica count is being increased on a plain distribute
+         * volume that was never mounted, we need to allow setxattr on '/' with
+         * GF_CLIENT_PID_NO_ROOT_SQUASH to accomodate for DHT layout setting */
+        return _gf_false;
+
+    if (local->inode == NULL)
+        return _gf_false;
+
+    if (!__is_root_gfid(local->inode->gfid))
+        return _gf_false;
+
+    return _gf_true;
+}
+
+gf_boolean_t
+afr_lookup_has_quorum(call_frame_t *frame, const unsigned int up_children_count)
+{
+    if (frame && (up_children_count > 0) &&
+        afr_is_add_replica_mount_lookup_on_root(frame))
+        return _gf_true;
+
+    return _gf_false;
+}
+
+void
+afr_handle_replies_quorum(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    unsigned char *success_replies = NULL;
+
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
+
+    if (priv->quorum_count && !afr_has_quorum(success_replies, this, NULL)) {
+        local->op_errno = afr_final_errno(local, priv);
+        if (!local->op_errno)
+            local->op_errno = afr_quorum_errno(priv);
+        local->op_ret = -1;
+    }
+}
+
+gf_boolean_t
+afr_ta_dict_contains_pending_xattr(dict_t *dict, afr_private_t *priv, int child)
+{
+    int *pending = NULL;
+    int ret = 0;
+    int i = 0;
+
+    ret = dict_get_ptr(dict, priv->pending_key[child], (void *)&pending);
+    if (ret == 0) {
+        for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+            /* Not doing a ntoh32(pending) as we just want to check
+             * if it is non-zero or not. */
+            if (pending[i]) {
+                return _gf_true;
+            }
         }
-        return ret;
+    }
+
+    return _gf_false;
 }
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index 5218d386c79..f8bf8340dab 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -8,348 +8,339 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 #include <string.h>
 
-#include "glusterfs.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "checksum.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
 
-
 int32_t
-afr_opendir_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 fd_t *fd, dict_t *xdata)
+afr_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t   *local             = NULL;
-        int            call_count        = -1;
-        int32_t        child_index       = 0;
-	afr_fd_ctx_t  *fd_ctx = NULL;
-
-        local = frame->local;
-	fd_ctx = local->fd_ctx;
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
-                } else {
-                        local->op_ret = op_ret;
-			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-                }
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int32_t child_index = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+    child_index = (long)cookie;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+        } else {
+            local->op_ret = op_ret;
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
-        UNLOCK (&frame->lock);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        call_count = afr_frame_return (frame);
+    if (call_count == 0) {
+        afr_handle_replies_quorum(frame, this);
+        AFR_STACK_UNWIND(opendir, frame, local->op_ret, local->op_errno,
+                         local->fd, NULL);
+    }
 
-        if (call_count == 0)
-		AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-				  local->op_errno, local->fd, NULL);
-        return 0;
+    return 0;
 }
 
-
 int
-afr_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+afr_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
 {
-        afr_private_t * priv        = NULL;
-        afr_local_t   * local       = NULL;
-        int             i           = 0;
-        int             call_count  = -1;
-        int32_t         op_errno    = ENOMEM;
-	afr_fd_ctx_t *fd_ctx = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = -1;
+    int32_t op_errno = ENOMEM;
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->op = GF_FOP_OPENDIR;
-        if (!afr_is_consistent_io_possible (local, priv, &op_errno))
-		goto out;
+    local->op = GF_FOP_OPENDIR;
 
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-		goto out;
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        op_errno = afr_quorum_errno(priv);
+        goto out;
+    }
 
-        loc_copy (&local->loc, loc);
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
 
-        local->fd    = fd_ref (fd);
-	local->fd_ctx = fd_ctx;
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
+        goto out;
 
-        call_count = local->call_count;
+    loc_copy(&local->loc, loc);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_opendir_cbk,
-                                           (void*) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->opendir,
-                                           loc, fd, NULL);
+    local->fd = fd_ref(fd);
+    local->fd_ctx = fd_ctx;
 
-                        if (!--call_count)
-                                break;
-                }
+    call_count = local->call_count;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, afr_opendir_cbk, (void *)(long)i,
+                              priv->children[i],
+                              priv->children[i]->fops->opendir, loc, fd, NULL);
+
+            if (!--call_count)
+                break;
         }
+    }
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd, NULL);
-        return 0;
+    AFR_STACK_UNWIND(opendir, frame, -1, op_errno, fd, NULL);
+    return 0;
 }
 
 static int
-afr_validate_read_subvol (inode_t *inode, xlator_t *this, int par_read_subvol)
+afr_validate_read_subvol(inode_t *inode, xlator_t *this, int par_read_subvol)
 {
-	int             gen               = 0;
-        int             entry_read_subvol = 0;
-	unsigned char  *data_readable     = NULL;
-	unsigned char  *metadata_readable = NULL;
-        afr_private_t  *priv              = NULL;
-
-        priv = this->private;
-	data_readable = alloca0 (priv->child_count);
-	metadata_readable = alloca0 (priv->child_count);
-
-	afr_inode_read_subvol_get (inode, this, data_readable,
-				   metadata_readable, &gen);
-
-        if (gen != priv->event_generation ||
-                !data_readable[par_read_subvol] ||
-                !metadata_readable[par_read_subvol])
-                return -1;
-
-        /* Once the control reaches the following statement, it means that the
-         * parent's read subvol is perfectly readable. So calling
-         * either afr_data_subvol_get() or afr_metadata_subvol_get() would
-         * yield the same result. Hence, choosing afr_data_subvol_get() below.
-         */
-
-        if (!priv->consistent_metadata)
-                return 0;
-
-        /* For an inode fetched through readdirp which is yet to be linked,
-         * inode ctx would not be initialised (yet). So this function returns
-         * -1 above due to gen being 0, which is why it is OK to pass NULL for
-         *  read_subvol_args here.
-         */
-        entry_read_subvol = afr_data_subvol_get (inode, this, NULL, NULL,
-                                                 NULL, NULL);
-        if (entry_read_subvol != par_read_subvol)
-                return -1;
-
+    int gen = 0;
+    int entry_read_subvol = 0;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(inode, this, data_readable, metadata_readable,
+                              &gen);
+
+    if (gen != priv->event_generation || !data_readable[par_read_subvol] ||
+        !metadata_readable[par_read_subvol])
+        return -1;
+
+    /* Once the control reaches the following statement, it means that the
+     * parent's read subvol is perfectly readable. So calling
+     * either afr_data_subvol_get() or afr_metadata_subvol_get() would
+     * yield the same result. Hence, choosing afr_data_subvol_get() below.
+     */
+
+    if (!priv->consistent_metadata)
         return 0;
 
+    /* For an inode fetched through readdirp which is yet to be linked,
+     * inode ctx would not be initialised (yet). So this function returns
+     * -1 above due to gen being 0, which is why it is OK to pass NULL for
+     *  read_subvol_args here.
+     */
+    entry_read_subvol = afr_data_subvol_get(inode, this, NULL, NULL, NULL,
+                                            NULL);
+    if (entry_read_subvol != par_read_subvol)
+        return -1;
+
+    return 0;
 }
 
 static void
-afr_readdir_transform_entries (gf_dirent_t *subvol_entries, int subvol,
-			       gf_dirent_t *entries, fd_t *fd)
+afr_readdir_transform_entries(call_frame_t *frame, gf_dirent_t *subvol_entries,
+                              int subvol, gf_dirent_t *entries, fd_t *fd)
 {
-        int            ret   = -1;
-        gf_dirent_t   *entry = NULL;
-        gf_dirent_t   *tmp   = NULL;
-        xlator_t      *this  = NULL;
-        afr_private_t *priv  = NULL;
-        gf_boolean_t  need_heal = _gf_false;
-        gf_boolean_t  validate_subvol = _gf_false;
-
-        this = THIS;
-        priv = this->private;
-
-        need_heal = afr_get_need_heal (this);
-        validate_subvol = need_heal | priv->consistent_metadata;
-
-        list_for_each_entry_safe (entry, tmp, &subvol_entries->list, list) {
-                if (__is_root_gfid (fd->inode->gfid) &&
-                    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
-			continue;
-                }
-
-		list_del_init (&entry->list);
-		list_add_tail (&entry->list, &entries->list);
-
-                if (!validate_subvol)
-                        continue;
-
-		if (entry->inode) {
-                        ret = afr_validate_read_subvol (entry->inode, this,
-                                                        subvol);
-                        if (ret == -1) {
-				inode_unref (entry->inode);
-				entry->inode = NULL;
-                                continue;
-			}
-		}
+    int ret = -1;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+    xlator_t *this = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t need_heal = _gf_false;
+    gf_boolean_t validate_subvol = _gf_false;
+
+    this = THIS;
+    priv = this->private;
+
+    need_heal = afr_get_need_heal(this);
+    validate_subvol = need_heal | priv->consistent_metadata;
+
+    list_for_each_entry_safe(entry, tmp, &subvol_entries->list, list)
+    {
+        if (afr_is_private_directory(priv, fd->inode->gfid, entry->d_name,
+                                     frame->root->pid)) {
+            continue;
         }
-}
 
+        list_del_init(&entry->list);
+        list_add_tail(&entry->list, &entries->list);
+
+        if (!validate_subvol)
+            continue;
+
+        if (entry->inode) {
+            ret = afr_validate_read_subvol(entry->inode, this, subvol);
+            if (ret == -1) {
+                inode_unref(entry->inode);
+                entry->inode = NULL;
+                continue;
+            }
+        }
+    }
+}
 
 int32_t
-afr_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int32_t op_ret, int32_t op_errno, gf_dirent_t *subvol_entries,
-		 dict_t *xdata)
+afr_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, gf_dirent_t *subvol_entries,
+                dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	gf_dirent_t  entries;
+    afr_local_t *local = NULL;
+    gf_dirent_t entries;
 
-	INIT_LIST_HEAD (&entries.list);
+    INIT_LIST_HEAD(&entries.list);
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret < 0 && !local->cont.readdir.offset) {
-		/* failover only if this was first readdir, detected
-		   by offset == 0 */
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0 && !local->cont.readdir.offset) {
+        /* failover only if this was first readdir, detected
+           by offset == 0 */
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	if (op_ret >= 0)
-		afr_readdir_transform_entries (subvol_entries, (long) cookie,
-					       &entries, local->fd);
+    if (op_ret >= 0)
+        afr_readdir_transform_entries(frame, subvol_entries, (long)cookie,
+                                      &entries, local->fd);
 
-        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, xdata);
+    AFR_STACK_UNWIND(readdir, frame, op_ret, op_errno, &entries, xdata);
 
-        gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_readdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_readdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	afr_fd_ctx_t *fd_ctx = NULL;
-
-	priv = this->private;
-	local = frame->local;
-	fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (readdir, frame, local->op_ret,
-				  local->op_errno, 0, 0);
-		return 0;
-	}
-
-	fd_ctx->readdir_subvol = subvol;
-
-        if (local->op == GF_FOP_READDIR)
-                STACK_WIND_COOKIE (frame, afr_readdir_cbk,
-                                   (void *) (long) subvol,
-                                   priv->children[subvol],
-                                   priv->children[subvol]->fops->readdir,
-				   local->fd, local->cont.readdir.size,
-                                   local->cont.readdir.offset,
-				   local->xdata_req);
-        else
-                STACK_WIND_COOKIE (frame, afr_readdir_cbk,
-                                   (void *) (long) subvol,
-                                   priv->children[subvol],
-                                   priv->children[subvol]->fops->readdirp,
-				   local->fd, local->cont.readdir.size,
-				   local->cont.readdir.offset,
-				   local->xdata_req);
-	return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    fd_ctx = afr_fd_ctx_get(local->fd, this);
+    if (!fd_ctx) {
+        local->op_errno = EINVAL;
+        local->op_ret = -1;
+    }
+
+    if (subvol == -1 || !fd_ctx) {
+        AFR_STACK_UNWIND(readdir, frame, local->op_ret, local->op_errno, 0, 0);
+        return 0;
+    }
+
+    fd_ctx->readdir_subvol = subvol;
+
+    if (local->op == GF_FOP_READDIR)
+        STACK_WIND_COOKIE(frame, afr_readdir_cbk, (void *)(long)subvol,
+                          priv->children[subvol],
+                          priv->children[subvol]->fops->readdir, local->fd,
+                          local->cont.readdir.size, local->cont.readdir.offset,
+                          local->xdata_req);
+    else
+        STACK_WIND_COOKIE(frame, afr_readdir_cbk, (void *)(long)subvol,
+                          priv->children[subvol],
+                          priv->children[subvol]->fops->readdirp, local->fd,
+                          local->cont.readdir.size, local->cont.readdir.offset,
+                          local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-		off_t offset, int whichop, dict_t *dict)
+afr_do_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, int whichop, dict_t *dict)
 {
-        afr_local_t   *local     = NULL;
-        int32_t       op_errno   = 0;
-	int           subvol = -1;
-	afr_fd_ctx_t *fd_ctx = NULL;
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx) {
-	        op_errno = EINVAL;
-		goto out;
-        }
-
-	local->op = whichop;
-        local->fd = fd_ref (fd);
-        local->cont.readdir.size = size;
-	local->cont.readdir.offset = offset;
-        local->xdata_req = (dict)? dict_ref (dict) : NULL;
-
-	subvol = fd_ctx->readdir_subvol;
-
-	if (offset == 0 || subvol == -1) {
-		/* First readdir has option of failing over and selecting
-		   an appropriate read subvolume */
-		afr_read_txn (frame, this, fd->inode, afr_readdir_wind,
-			      AFR_DATA_TRANSACTION);
-	} else {
-		/* But continued readdirs MUST stick to the same subvolume
-		   without an option to failover */
-		afr_readdir_wind (frame, this, subvol);
-	}
-
-        return 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    int subvol = -1;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    local->op = whichop;
+    local->fd = fd_ref(fd);
+    local->cont.readdir.size = size;
+    local->cont.readdir.offset = offset;
+    local->xdata_req = (dict) ? dict_ref(dict) : NULL;
+
+    subvol = fd_ctx->readdir_subvol;
+
+    if (offset == 0 || subvol == -1) {
+        /* First readdir has option of failing over and selecting
+           an appropriate read subvolume */
+        afr_read_txn(frame, this, fd->inode, afr_readdir_wind,
+                     AFR_DATA_TRANSACTION);
+    } else {
+        /* But continued readdirs MUST stick to the same subvolume
+           without an option to failover */
+        afr_readdir_wind(frame, this, subvol);
+    }
+
+    return 0;
 out:
-        AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(readdir, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-afr_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset, dict_t *xdata)
+afr_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata)
 {
-	afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
+    afr_do_readdir(frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset, dict_t *dict)
+afr_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *dict)
 {
-        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIRP, dict);
+    afr_do_readdir(frame, this, fd, size, offset, GF_FOP_READDIRP, dict);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_releasedir (xlator_t *this, fd_t *fd)
+afr_releasedir(xlator_t *this, fd_t *fd)
 {
-        afr_cleanup_fd_ctx (this, fd);
+    afr_cleanup_fd_ctx(this, fd);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-dir-read.h b/xlators/cluster/afr/src/afr-dir-read.h
index 09456d15949..773e925ec6c 100644
--- a/xlators/cluster/afr/src/afr-dir-read.h
+++ b/xlators/cluster/afr/src/afr-dir-read.h
@@ -11,26 +11,23 @@
 #ifndef __DIR_READ_H__
 #define __DIR_READ_H__
 
-
 int32_t
-afr_opendir (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, fd_t *fd, dict_t *xdata);
+afr_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata);
 
 int32_t
-afr_releasedir (xlator_t *this, fd_t *fd);
+afr_releasedir(xlator_t *this, fd_t *fd);
 
 int32_t
-afr_readdir (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, size_t size, off_t offset, dict_t *xdata);
-
+afr_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata);
 
 int32_t
-afr_readdirp (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, size_t size, off_t offset, dict_t *dict);
+afr_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *dict);
 
 int32_t
-afr_checksum (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, int32_t flags, dict_t *xdata);
-
+afr_checksum(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             dict_t *xdata);
 
 #endif /* __DIR_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c
index 0e3ae18b46a..b7cceb79158 100644
--- a/xlators/cluster/afr/src/afr-dir-write.c
+++ b/xlators/cluster/afr/src/afr-dir-write.c
@@ -8,527 +8,493 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/list.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
 
 void
-afr_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this);
+afr_mark_entry_pending_changelog(call_frame_t *frame, xlator_t *this);
 
 int
-afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno)
+afr_build_parent_loc(loc_t *parent, loc_t *child, int32_t *op_errno)
 {
-        int     ret = -1;
-        char    *child_path = NULL;
-
-        if (!child->parent) {
-                if (op_errno)
-                        *op_errno = EINVAL;
-                goto out;
-        }
-
-        child_path = gf_strdup (child->path);
-        if (!child_path) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
-        }
-
-        parent->path = gf_strdup (dirname (child_path));
-        if (!parent->path) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
-        }
-
-        parent->inode = inode_ref (child->parent);
-	gf_uuid_copy (parent->gfid, child->pargfid);
-
-        ret = 0;
+    int ret = -1;
+    char *child_path = NULL;
+
+    if (!child->parent) {
+        if (op_errno)
+            *op_errno = EINVAL;
+        goto out;
+    }
+
+    child_path = gf_strdup(child->path);
+    if (!child_path) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    parent->path = gf_strdup(dirname(child_path));
+    if (!parent->path) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    parent->inode = inode_ref(child->parent);
+    gf_uuid_copy(parent->gfid, child->pargfid);
+
+    ret = 0;
 out:
-        GF_FREE (child_path);
+    GF_FREE(child_path);
 
-        return ret;
+    return ret;
 }
 
-
 static void
-__afr_dir_write_finalize (call_frame_t *frame, xlator_t *this)
+__afr_dir_write_finalize(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int inode_read_subvol = -1;
-	int parent_read_subvol = -1;
-	int parent2_read_subvol = -1;
-	int i = 0;
-        afr_read_subvol_args_t args = {0,};
-
-	local = frame->local;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-	        if (!local->replies[i].valid)
-	                continue;
-	        if (local->replies[i].op_ret == -1)
-	                continue;
-                gf_uuid_copy (args.gfid, local->replies[i].poststat.ia_gfid);
-                args.ia_type = local->replies[i].poststat.ia_type;
-                break;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int inode_read_subvol = -1;
+    int parent_read_subvol = -1;
+    int parent2_read_subvol = -1;
+    int i = 0;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret == -1)
+            continue;
+        gf_uuid_copy(args.gfid, local->replies[i].poststat.ia_gfid);
+        args.ia_type = local->replies[i].poststat.ia_type;
+        break;
+    }
+
+    if (local->inode) {
+        if (local->op != GF_FOP_RENAME && local->op != GF_FOP_LINK)
+            afr_replies_interpret(frame, this, local->inode, NULL);
+
+        inode_read_subvol = afr_data_subvol_get(local->inode, this, NULL, NULL,
+                                                NULL, &args);
+    }
+
+    if (local->parent)
+        parent_read_subvol = afr_data_subvol_get(local->parent, this, NULL,
+                                                 local->readable, NULL, NULL);
+
+    if (local->parent2)
+        parent2_read_subvol = afr_data_subvol_get(local->parent2, this, NULL,
+                                                  local->readable2, NULL, NULL);
+
+    local->op_ret = -1;
+    local->op_errno = afr_final_errno(local, priv);
+    afr_pick_error_xdata(local, priv, local->parent, local->readable,
+                         local->parent2, local->readable2);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0) {
+            if (local->inode)
+                afr_inode_need_refresh_set(local->inode, this);
+            if (local->parent)
+                afr_inode_need_refresh_set(local->parent, this);
+            if (local->parent2)
+                afr_inode_need_refresh_set(local->parent2, this);
+            continue;
         }
 
-	if (local->inode) {
-		afr_replies_interpret (frame, this, local->inode, NULL);
-		inode_read_subvol = afr_data_subvol_get (local->inode, this,
-                                                       NULL, NULL, NULL, &args);
-	}
-
-	if (local->parent)
-		parent_read_subvol = afr_data_subvol_get (local->parent, this,
-                                             NULL, local->readable, NULL, NULL);
-
-	if (local->parent2)
-		parent2_read_subvol = afr_data_subvol_get (local->parent2, this,
-                                            NULL, local->readable2, NULL, NULL);
-
-	local->op_ret = -1;
-	local->op_errno = afr_final_errno (local, priv);
-        afr_pick_error_xdata (local, priv, local->parent, local->readable,
-                              local->parent2, local->readable2);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].valid)
-			continue;
-		if (local->replies[i].op_ret < 0) {
-			if (local->inode)
-				afr_inode_event_gen_reset (local->inode, this);
-			if (local->parent)
-				afr_inode_event_gen_reset (local->parent,
-							     this);
-			if (local->parent2)
-				afr_inode_event_gen_reset (local->parent2,
-							     this);
-			continue;
-		}
-
-		if (local->op_ret == -1) {
-			local->op_ret = local->replies[i].op_ret;
-			local->op_errno = local->replies[i].op_errno;
-
-			local->cont.dir_fop.buf =
-				local->replies[i].poststat;
-			local->cont.dir_fop.preparent =
-				local->replies[i].preparent;
-			local->cont.dir_fop.postparent =
-				local->replies[i].postparent;
-			local->cont.dir_fop.prenewparent =
-				local->replies[i].preparent2;
-			local->cont.dir_fop.postnewparent =
-				local->replies[i].postparent2;
-                        if (local->xdata_rsp) {
-                                dict_unref (local->xdata_rsp);
-                                local->xdata_rsp = NULL;
-                        }
-
-			if (local->replies[i].xdata)
-				local->xdata_rsp =
-					dict_ref (local->replies[i].xdata);
-			continue;
-		}
-
-		if (i == inode_read_subvol) {
-			local->cont.dir_fop.buf =
-				local->replies[i].poststat;
-			if (local->replies[i].xdata) {
-				if (local->xdata_rsp)
-					dict_unref (local->xdata_rsp);
-				local->xdata_rsp =
-					dict_ref (local->replies[i].xdata);
-			}
-		}
-
-		if (i == parent_read_subvol) {
-			local->cont.dir_fop.preparent =
-				local->replies[i].preparent;
-			local->cont.dir_fop.postparent =
-				local->replies[i].postparent;
-		}
-
-		if (i == parent2_read_subvol) {
-			local->cont.dir_fop.prenewparent =
-				local->replies[i].preparent2;
-			local->cont.dir_fop.postnewparent =
-				local->replies[i].postparent2;
-		}
-	}
+        if (local->op_ret == -1) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+
+            local->cont.dir_fop.buf = local->replies[i].poststat;
+            local->cont.dir_fop.preparent = local->replies[i].preparent;
+            local->cont.dir_fop.postparent = local->replies[i].postparent;
+            local->cont.dir_fop.prenewparent = local->replies[i].preparent2;
+            local->cont.dir_fop.postnewparent = local->replies[i].postparent2;
+            if (local->xdata_rsp) {
+                dict_unref(local->xdata_rsp);
+                local->xdata_rsp = NULL;
+            }
+
+            if (local->replies[i].xdata)
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            continue;
+        }
 
-}
+        if (i == inode_read_subvol) {
+            local->cont.dir_fop.buf = local->replies[i].poststat;
+            if (local->replies[i].xdata) {
+                if (local->xdata_rsp)
+                    dict_unref(local->xdata_rsp);
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+        }
 
+        if (i == parent_read_subvol) {
+            local->cont.dir_fop.preparent = local->replies[i].preparent;
+            local->cont.dir_fop.postparent = local->replies[i].postparent;
+        }
+
+        if (i == parent2_read_subvol) {
+            local->cont.dir_fop.prenewparent = local->replies[i].preparent2;
+            local->cont.dir_fop.postnewparent = local->replies[i].postparent2;
+        }
+    }
+}
 
 static void
-__afr_dir_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
-		      int op_ret, int op_errno, struct iatt *poststat,
-		      struct iatt *preparent, struct iatt *postparent,
-		      struct iatt *preparent2, struct iatt *postparent2,
-		      dict_t *xdata)
+__afr_dir_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                     int op_ret, int op_errno, struct iatt *poststat,
+                     struct iatt *preparent, struct iatt *postparent,
+                     struct iatt *preparent2, struct iatt *postparent2,
+                     dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	afr_fd_ctx_t *fd_ctx = NULL;
-
-        local = frame->local;
-	fd_ctx = local->fd_ctx;
-
-	local->replies[child_index].valid = 1;
-	local->replies[child_index].op_ret = op_ret;
-	local->replies[child_index].op_errno = op_errno;
-        if (xdata)
-                local->replies[child_index].xdata = dict_ref (xdata);
-
-
-	if (op_ret >= 0) {
-		if (poststat)
-			local->replies[child_index].poststat = *poststat;
-		if (preparent)
-			local->replies[child_index].preparent = *preparent;
-		if (postparent)
-			local->replies[child_index].postparent = *postparent;
-		if (preparent2)
-			local->replies[child_index].preparent2 = *preparent2;
-		if (postparent2)
-			local->replies[child_index].postparent2 = *postparent2;
-		if (fd_ctx)
-			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-	} else {
-		if (op_errno != ENOTEMPTY)
-			afr_transaction_fop_failed (frame, this, child_index);
-		if (fd_ctx)
-			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
-	}
-
-        return;
+    afr_local_t *local = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
+
+    if (op_ret >= 0) {
+        if (poststat)
+            local->replies[child_index].poststat = *poststat;
+        if (preparent)
+            local->replies[child_index].preparent = *preparent;
+        if (postparent)
+            local->replies[child_index].postparent = *postparent;
+        if (preparent2)
+            local->replies[child_index].preparent2 = *preparent2;
+        if (postparent2)
+            local->replies[child_index].postparent2 = *postparent2;
+        if (fd_ctx)
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+    } else {
+        if (op_errno != ENOTEMPTY)
+            afr_transaction_fop_failed(frame, this, child_index);
+        if (fd_ctx)
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+    }
+
+    return;
 }
 
-
 static int
-__afr_dir_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int op_ret, int op_errno, struct iatt *buf,
-		     struct iatt *preparent, struct iatt *postparent,
-		     struct iatt *preparent2, struct iatt *postparent2,
-                     dict_t *xdata)
+__afr_dir_write_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, struct iatt *buf,
+                    struct iatt *preparent, struct iatt *postparent,
+                    struct iatt *preparent2, struct iatt *postparent2,
+                    dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int child_index = (long) cookie;
-        int call_count = -1;
-        afr_private_t *priv = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		__afr_dir_write_fill (frame, this, child_index, op_ret,
-				      op_errno, buf, preparent, postparent,
-				      preparent2, postparent2, xdata);
-	}
-	UNLOCK (&frame->lock);
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-		__afr_dir_write_finalize (frame, this);
-
-		if (afr_txn_nothing_failed (frame, this)) {
-                        /*if it did pre-op, it will do post-op changing ctime*/
-                        if (priv->consistent_metadata &&
-                            afr_needs_changelog_update (local))
-                                afr_zero_fill_stat (local);
-                        local->transaction.unwind (frame, this);
-                }
-
-		afr_mark_entry_pending_changelog (frame, this);
-
-                afr_transaction_resume (frame, this);
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    {
+        __afr_dir_write_fill(frame, this, child_index, op_ret, op_errno, buf,
+                             preparent, postparent, preparent2, postparent2,
+                             xdata);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        __afr_dir_write_finalize(frame, this);
+
+        if (afr_txn_nothing_failed(frame, this)) {
+            /*if it did pre-op, it will do post-op changing ctime*/
+            if (priv->consistent_metadata && afr_needs_changelog_update(local))
+                afr_zero_fill_stat(local);
+            local->transaction.unwind(frame, this);
         }
 
-        return 0;
-}
+        afr_mark_entry_pending_changelog(frame, this);
+
+        afr_transaction_resume(frame, this);
+    }
 
+    return 0;
+}
 
 int
-afr_mark_new_entry_changelog_cbk (call_frame_t *frame, void *cookie,
-				  xlator_t *this, int op_ret, int op_errno,
-                                  dict_t *xattr, dict_t *xdata)
+afr_mark_new_entry_changelog_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int op_ret, int op_errno,
+                                 dict_t *xattr, dict_t *xdata)
 {
-        int call_count = 0;
+    int call_count = 0;
 
-        call_count = afr_frame_return (frame);
+    call_count = afr_frame_return(frame);
 
-        if (call_count == 0)
-                AFR_STACK_DESTROY (frame);
+    if (call_count == 0)
+        AFR_STACK_DESTROY(frame);
 
-        return 0;
+    return 0;
 }
 
-
 void
-afr_mark_new_entry_changelog (call_frame_t *frame, xlator_t *this)
+afr_mark_new_entry_changelog(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t  *new_frame  = NULL;
-        afr_local_t   *local      = NULL;
-        afr_local_t   *new_local  = NULL;
-        afr_private_t *priv       = NULL;
-        dict_t        *xattr      = NULL;
-        int32_t       **changelog = NULL;
-        int           i           = 0;
-        int           op_errno    = ENOMEM;
-	unsigned char *pending    = NULL;
-	int           call_count   = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        new_frame = copy_frame (frame);
-        if (!new_frame)
-                goto out;
-
-	new_local = AFR_FRAME_INIT (new_frame, op_errno);
-	if (!new_local)
-		goto out;
-
-        xattr = dict_new ();
-	if (!xattr)
-		goto out;
-
-	pending = alloca0 (priv->child_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->transaction.pre_op[i] &&
-		    !local->transaction.failed_subvols[i]) {
-			call_count ++;
-			continue;
-		}
-		pending[i] = 1;
-	}
-
-        changelog = afr_mark_pending_changelog (priv, pending, xattr,
-                                            local->cont.dir_fop.buf.ia_type);
-        if (!changelog)
-                goto out;
-
-        new_local->pending = changelog;
-        gf_uuid_copy (new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid);
-        new_local->loc.inode = inode_ref (local->inode);
-
-        new_local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-		if (pending[i])
-                        continue;
-
-                STACK_WIND_COOKIE (new_frame, afr_mark_new_entry_changelog_cbk,
-                                   (void *) (long) i, priv->children[i],
-                                   priv->children[i]->fops->xattrop,
-                                   &new_local->loc, GF_XATTROP_ADD_ARRAY,
-                                   xattr, NULL);
-		if (!--call_count)
-			break;
+    call_frame_t *new_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *new_local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int32_t **changelog = NULL;
+    int i = 0;
+    int op_errno = ENOMEM;
+    unsigned char *pending = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame)
+        goto out;
+
+    new_local = AFR_FRAME_INIT(new_frame, op_errno);
+    if (!new_local)
+        goto out;
+
+    xattr = dict_new();
+    if (!xattr)
+        goto out;
+
+    pending = alloca0(priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i] &&
+            !local->transaction.failed_subvols[i]) {
+            call_count++;
+            continue;
         }
+        pending[i] = 1;
+    }
+
+    changelog = afr_mark_pending_changelog(priv, pending, xattr,
+                                           local->cont.dir_fop.buf.ia_type);
+    if (!changelog)
+        goto out;
+
+    new_local->pending = changelog;
+    gf_uuid_copy(new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid);
+    new_local->loc.inode = inode_ref(local->inode);
 
-        new_frame = NULL;
+    new_local->call_count = call_count;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (pending[i])
+            continue;
+
+        STACK_WIND_COOKIE(new_frame, afr_mark_new_entry_changelog_cbk,
+                          (void *)(long)i, priv->children[i],
+                          priv->children[i]->fops->xattrop, &new_local->loc,
+                          GF_XATTROP_ADD_ARRAY, xattr, NULL);
+        if (!--call_count)
+            break;
+    }
+
+    new_frame = NULL;
 out:
-        if (new_frame)
-                AFR_STACK_DESTROY (new_frame);
-	if (xattr)
-		dict_unref (xattr);
-        return;
+    if (new_frame)
+        AFR_STACK_DESTROY(new_frame);
+    if (xattr)
+        dict_unref(xattr);
+    return;
 }
 
-
 void
-afr_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this)
+afr_mark_entry_pending_changelog(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-	int pre_op_count = 0;
-	int failed_count = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int pre_op_count = 0;
+    int failed_count = 0;
+    unsigned char *success_replies = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        if (local->op_ret < 0)
-		return;
+    if (local->op_ret < 0)
+        return;
 
-	if (local->op != GF_FOP_CREATE && local->op != GF_FOP_MKNOD &&
-            local->op != GF_FOP_MKDIR)
-		return;
+    if (local->op != GF_FOP_CREATE && local->op != GF_FOP_MKNOD &&
+        local->op != GF_FOP_MKDIR)
+        return;
 
-	pre_op_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
-	failed_count = AFR_COUNT (local->transaction.failed_subvols,
-				  priv->child_count);
+    pre_op_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
+    failed_count = AFR_COUNT(local->transaction.failed_subvols,
+                             priv->child_count);
 
-	if (pre_op_count == priv->child_count && !failed_count)
-		return;
+    /* FOP succeeded on all bricks. */
+    if (pre_op_count == priv->child_count && !failed_count)
+        return;
 
-        afr_mark_new_entry_changelog (frame, this);
+    /* FOP did not suceed on quorum no. of bricks. */
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
+    if (!afr_has_quorum(success_replies, this, NULL))
+        return;
 
+    if (priv->thin_arbiter_count) {
+        /*Mark new entry using ta file*/
+        local->is_new_entry = _gf_true;
         return;
-}
+    }
 
+    afr_mark_new_entry_changelog(frame, this);
+
+    return;
+}
 
 /* {{{ create */
 
 int
-afr_create_unwind (call_frame_t *frame, xlator_t *this)
+afr_create_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
+    local = frame->local;
 
-        if (!main_frame)
-		return 0;
+    main_frame = afr_transaction_detach_fop_frame(frame);
 
-	AFR_STACK_UNWIND (create, main_frame, local->op_ret, local->op_errno,
-			  local->cont.create.fd, local->inode,
-			  &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(create, main_frame, local->op_ret, local->op_errno,
+                     local->cont.create.fd, local->inode,
+                     &local->cont.dir_fop.buf, &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     fd_t *fd, inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent,
-                     dict_t *xdata)
+afr_create_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_create_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_create_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-	STACK_WIND_COOKIE (frame, afr_create_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->create,
-			   &local->loc, local->cont.create.flags,
-			   local->cont.create.mode, local->umask,
-			   local->cont.create.fd, local->xdata_req);
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_create_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->create, &local->loc,
+                      local->cont.create.flags, local->cont.create.mode,
+                      local->umask, local->cont.create.fd, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-	    mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+afr_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-
-	local->fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!local->fd_ctx)
-		goto out;
-
-        local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->op                = GF_FOP_CREATE;
-        local->cont.create.flags = flags;
-        local->fd_ctx->flags     = flags;
-        local->cont.create.mode  = mode;
-        local->cont.create.fd    = fd_ref (fd);
-        local->umask  = umask;
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->transaction.wind   = afr_create_wind;
-        local->transaction.unwind = afr_create_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-            op_errno = -ret;
-            goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+
+    local->fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!local->fd_ctx)
+        goto out;
+
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->op = GF_FOP_CREATE;
+    local->cont.create.flags = flags;
+    local->fd_ctx->flags = flags;
+    local->cont.create.mode = mode;
+    local->cont.create.fd = fd_ref(fd);
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->transaction.wind = afr_create_wind;
+    local->transaction.unwind = afr_create_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 /* }}} */
@@ -536,516 +502,436 @@ out:
 /* {{{ mknod */
 
 int
-afr_mknod_unwind (call_frame_t *frame, xlator_t *this)
+afr_mknod_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (mknod, main_frame, local->op_ret, local->op_errno,
-			  local->inode, &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(mknod, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_mknod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+afr_mknod_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-	return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_mknod_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_mknod_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv  = this->private;
-
-	STACK_WIND_COOKIE (frame, afr_mknod_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->mknod,
-			   &local->loc, local->cont.mknod.mode,
-			   local->cont.mknod.dev, local->umask,
-			   local->xdata_req);
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_mknod_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->mknod, &local->loc,
+                      local->cont.mknod.mode, local->cont.mknod.dev,
+                      local->umask, local->xdata_req);
+    return 0;
 }
 
 int
-afr_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           dev_t dev, mode_t umask, dict_t *xdata)
+afr_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t dev, mode_t umask, dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-        local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->op               = GF_FOP_MKNOD;
-        local->cont.mknod.mode  = mode;
-        local->cont.mknod.dev   = dev;
-        local->umask = umask;
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->transaction.wind   = afr_mknod_wind;
-        local->transaction.unwind = afr_mknod_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->op = GF_FOP_MKNOD;
+    local->cont.mknod.mode = mode;
+    local->cont.mknod.dev = dev;
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->transaction.wind = afr_mknod_wind;
+    local->transaction.unwind = afr_mknod_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL);
-        return 0;
+    AFR_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ mkdir */
 
-
 int
-afr_mkdir_unwind (call_frame_t *frame, xlator_t *this)
+afr_mkdir_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (mkdir, main_frame, local->op_ret, local->op_errno,
-			  local->inode, &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(mkdir, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_mkdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+afr_mkdir_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-	return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_mkdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_mkdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_mkdir_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->mkdir, &local->loc,
-			   local->cont.mkdir.mode, local->umask,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_mkdir_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->mkdir, &local->loc,
+                      local->cont.mkdir.mode, local->umask, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-	   mode_t umask, dict_t *xdata)
+afr_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-        local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->cont.mkdir.mode  = mode;
-        local->umask = umask;
-
-        if (!xdata || !dict_get (xdata, "gfid-req")) {
-                op_errno = EPERM;
-                gf_msg_callingfn (this->name, GF_LOG_WARNING, op_errno,
-                                  AFR_MSG_GFID_NULL, "mkdir: %s is received "
-                                  "without gfid-req %p", loc->path, xdata);
-	        goto out;
-        }
-
-        local->xdata_req = dict_copy_with_ref (xdata, NULL);
-        if (!local->xdata_req) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        local->op = GF_FOP_MKDIR;
-        local->transaction.wind   = afr_mkdir_wind;
-        local->transaction.unwind = afr_mkdir_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.mkdir.mode = mode;
+    local->umask = umask;
+
+    if (!xdata || !dict_get_sizen(xdata, "gfid-req")) {
+        op_errno = EPERM;
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, op_errno,
+                         AFR_MSG_GFID_NULL,
+                         "mkdir: %s is received "
+                         "without gfid-req %p",
+                         loc->path, xdata);
+        goto out;
+    }
+
+    local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    if (!local->xdata_req) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->op = GF_FOP_MKDIR;
+    local->transaction.wind = afr_mkdir_wind;
+    local->transaction.unwind = afr_mkdir_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL);
-        return 0;
+    AFR_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ link */
 
-
 int
-afr_link_unwind (call_frame_t *frame, xlator_t *this)
+afr_link_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (link, main_frame, local->op_ret, local->op_errno,
-			  local->inode, &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(link, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_link_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+afr_link_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_link_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_link_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_link_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->link,
-			   &local->loc, &local->newloc, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_link_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->link, &local->loc,
+                      &local->newloc, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-	  dict_t *xdata)
+afr_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc,    oldloc);
-        loc_copy (&local->newloc, newloc);
-
-        local->inode = inode_ref (oldloc->inode);
-	local->parent = inode_ref (newloc->parent);
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_LINK;
-
-        local->transaction.wind   = afr_link_wind;
-        local->transaction.unwind = afr_link_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, newloc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame   = frame;
-        local->transaction.basename     = AFR_BASENAME (newloc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, oldloc);
+    loc_copy(&local->newloc, newloc);
 
-	return 0;
+    local->inode = inode_ref(oldloc->inode);
+    local->parent = inode_ref(newloc->parent);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_LINK;
+
+    local->transaction.wind = afr_link_wind;
+    local->transaction.unwind = afr_link_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, newloc,
+                               &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(newloc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL);
-        return 0;
+    AFR_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ symlink */
 
-
 int
-afr_symlink_unwind (call_frame_t *frame, xlator_t *this)
+afr_symlink_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (symlink, main_frame, local->op_ret, local->op_errno,
-			  local->inode, &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(symlink, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_symlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, inode_t *inode,
-                      struct iatt *buf, struct iatt *preparent,
-                      struct iatt *postparent, dict_t *xdata)
+afr_symlink_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_symlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_symlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-	STACK_WIND_COOKIE (frame, afr_symlink_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->symlink,
-			   local->cont.symlink.linkpath, &local->loc,
-			   local->umask, local->xdata_req);
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_symlink_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->symlink,
+                      local->cont.symlink.linkpath, &local->loc, local->umask,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-	     loc_t *loc, mode_t umask, dict_t *xdata)
+afr_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-        local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->cont.symlink.linkpath = gf_strdup (linkpath);
-        local->umask = umask;
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_SYMLINK;
-        local->transaction.wind   = afr_symlink_wind;
-        local->transaction.unwind = afr_symlink_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame   = frame;
-        local->transaction.basename     = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.symlink.linkpath = gf_strdup(linkpath);
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_SYMLINK;
+    local->transaction.wind = afr_symlink_wind;
+    local->transaction.unwind = afr_symlink_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
-			  NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 /* }}} */
@@ -1053,159 +939,118 @@ out:
 /* {{{ rename */
 
 int
-afr_rename_unwind (call_frame_t *frame, xlator_t *this)
+afr_rename_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (rename, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent,
-			  &local->cont.dir_fop.prenewparent,
-			  &local->cont.dir_fop.postnewparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(rename, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.buf, &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent,
+                     &local->cont.dir_fop.prenewparent,
+                     &local->cont.dir_fop.postnewparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                     struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent,
-                     dict_t *xdata)
+afr_rename_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preoldparent, postoldparent, prenewparent,
-				    postnewparent, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preoldparent, postoldparent, prenewparent,
+                               postnewparent, xdata);
 }
 
-
 int
-afr_rename_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_rename_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_rename_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->rename,
-			   &local->loc, &local->newloc, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_rename_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->rename, &local->loc,
+                      &local->newloc, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-	    dict_t *xdata)
+afr_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-        int                     nlockee                 = 0;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc,    oldloc);
-        loc_copy (&local->newloc, newloc);
-
-        local->inode = inode_ref (oldloc->inode);
-	local->parent = inode_ref (oldloc->parent);
-	local->parent2 = inode_ref (newloc->parent);
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_RENAME;
-        local->transaction.wind   = afr_rename_wind;
-        local->transaction.unwind = afr_rename_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, oldloc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-        ret = afr_build_parent_loc (&local->transaction.new_parent_loc, newloc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame   = frame;
-        local->transaction.basename     = AFR_BASENAME (oldloc->path);
-        local->transaction.new_basename = AFR_BASENAME (newloc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = nlockee = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                     &local->transaction.new_parent_loc,
-                                     local->transaction.new_basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        nlockee++;
-        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        nlockee++;
-        if (local->newloc.inode && IA_ISDIR (local->newloc.inode->ia_type)) {
-                ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                             &local->newloc,
-                                             NULL,
-                                             priv->child_count);
-                if (ret)
-                        goto out;
-
-                nlockee++;
-        }
-        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
-               afr_entry_lockee_cmp);
-        int_lock->lockee_count = nlockee;
-
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, oldloc);
+    loc_copy(&local->newloc, newloc);
+
+    local->inode = inode_ref(oldloc->inode);
+    local->parent = inode_ref(oldloc->parent);
+    local->parent2 = inode_ref(newloc->parent);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_RENAME;
+    local->transaction.wind = afr_rename_wind;
+    local->transaction.unwind = afr_rename_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, oldloc,
+                               &op_errno);
+    if (ret)
+        goto out;
+    ret = afr_build_parent_loc(&local->transaction.new_parent_loc, newloc,
+                               &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(oldloc->path);
+    local->transaction.new_basename = AFR_BASENAME(newloc->path);
+    ret = afr_transaction(transaction_frame, this,
+                          AFR_ENTRY_RENAME_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 /* }}} */
@@ -1213,259 +1058,205 @@ out:
 /* {{{ unlink */
 
 int
-afr_unlink_unwind (call_frame_t *frame, xlator_t *this)
+afr_unlink_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (unlink, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(unlink, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+afr_unlink_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, NULL,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_unlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_unlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_unlink_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->unlink,
-			   &local->loc, local->xflag, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_unlink_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->unlink, &local->loc,
+                      local->xflag, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-	    dict_t *xdata)
+afr_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-        local->xflag = xflag;
-
-        local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_UNLINK;
-        local->transaction.wind   = afr_unlink_wind;
-        local->transaction.unwind = afr_unlink_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->xflag = xflag;
+
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_UNLINK;
+    local->transaction.wind = afr_unlink_wind;
+    local->transaction.unwind = afr_unlink_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ rmdir */
 
-
-
 int
-afr_rmdir_unwind (call_frame_t *frame, xlator_t *this)
+afr_rmdir_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (rmdir, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(rmdir, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+afr_rmdir_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, NULL,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_rmdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_rmdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_rmdir_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->rmdir,
-			   &local->loc, local->cont.rmdir.flags, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_rmdir_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->rmdir, &local->loc,
+                      local->cont.rmdir.flags, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-	   dict_t *xdata)
+afr_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-        int                     nlockee                 = 0;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-
-        loc_copy (&local->loc, loc);
-        local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->cont.rmdir.flags = flags;
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_RMDIR;
-        local->transaction.wind   = afr_rmdir_wind;
-        local->transaction.unwind = afr_rmdir_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = nlockee = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        nlockee++;
-        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                     &local->loc,
-                                     NULL,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        nlockee++;
-        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
-               afr_entry_lockee_cmp);
-        int_lock->lockee_count = nlockee;
-
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.rmdir.flags = flags;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_RMDIR;
+    local->transaction.wind = afr_rmdir_wind;
+    local->transaction.unwind = afr_rmdir_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
diff --git a/xlators/cluster/afr/src/afr-dir-write.h b/xlators/cluster/afr/src/afr-dir-write.h
index 02f0a3682d9..1d88c3b9b26 100644
--- a/xlators/cluster/afr/src/afr-dir-write.h
+++ b/xlators/cluster/afr/src/afr-dir-write.h
@@ -12,36 +12,35 @@
 #define __DIR_WRITE_H__
 
 int32_t
-afr_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *xdata);
+afr_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t
-afr_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t dev, mode_t umask, dict_t *xdata);
+afr_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t dev, mode_t umask, dict_t *xdata);
 
 int32_t
-afr_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
+afr_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata);
 
 int32_t
-afr_unlink (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int xflag, dict_t *xdata);
+afr_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata);
 
 int32_t
-afr_rmdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, int flags, dict_t *xdata);
+afr_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata);
 
 int32_t
-afr_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+afr_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata);
 
 int32_t
-afr_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+afr_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata);
 
 int
-afr_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkpath, loc_t *oldloc, mode_t umask, dict_t *params);
+afr_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *oldloc, mode_t umask, dict_t *params);
 
 #endif /* __DIR_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index 8d3c823043f..c5521704de2 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <libgen.h>
 #include <unistd.h>
 #include <fnmatch.h>
@@ -16,20 +15,17 @@
 #include <stdlib.h>
 #include <signal.h>
 
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "list.h"
-#include "call-stub.h"
-#include "byte-order.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "quota-common-utils.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/list.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/quota-common-utils.h>
 
 #include "afr-transaction.h"
 #include "afr-messages.h"
@@ -44,146 +40,146 @@
  * */
 
 int
-afr_handle_quota_size (call_frame_t *frame, xlator_t *this)
+afr_handle_quota_size(call_frame_t *frame, xlator_t *this)
 {
-        unsigned char *readable   = NULL;
-        afr_local_t *local        = NULL;
-        afr_private_t *priv       = NULL;
-        struct afr_reply *replies = NULL;
-        int i                     = 0;
-        int ret                   = 0;
-        quota_meta_t size         = {0, };
-        quota_meta_t max_size     = {0, };
-        int readable_cnt          = 0;
-        int read_subvol           = -1;
-
-        local = frame->local;
-        priv = this->private;
-        replies = local->replies;
-
-        readable = alloca0 (priv->child_count);
-
-        afr_inode_read_subvol_get (local->inode, this, readable, 0, 0);
-
-        readable_cnt = AFR_COUNT (readable, priv->child_count);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-                if (readable_cnt && !readable[i])
-                        continue;
-                if (!replies[i].xdata)
-                        continue;
-                ret = quota_dict_get_meta (replies[i].xdata, QUOTA_SIZE_KEY,
-                                           &size);
-                if (ret == -1)
-                        continue;
-                if (read_subvol == -1)
-                        read_subvol = i;
-                if (size.size > max_size.size ||
-                    (size.file_count + size.dir_count) >
-                    (max_size.file_count + max_size.dir_count))
-                        read_subvol = i;
-
-                if (size.size > max_size.size)
-                        max_size.size = size.size;
-                if (size.file_count > max_size.file_count)
-                        max_size.file_count = size.file_count;
-                if (size.dir_count > max_size.dir_count)
-                        max_size.dir_count = size.dir_count;
-        }
-
-        if (max_size.size == 0 && max_size.file_count == 0 &&
-            max_size.dir_count == 0)
-                return read_subvol;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-                if (readable_cnt && !readable[i])
-                        continue;
-                if (!replies[i].xdata)
-                        continue;
-                quota_dict_set_meta (replies[i].xdata, QUOTA_SIZE_KEY,
-                                     &max_size, IA_IFDIR);
-        }
-
+    unsigned char *readable = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0;
+    int ret = 0;
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t max_size = {
+        0,
+    };
+    int readable_cnt = 0;
+    int read_subvol = -1;
+
+    local = frame->local;
+    priv = this->private;
+    replies = local->replies;
+
+    readable = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(local->inode, this, readable, 0, 0);
+
+    readable_cnt = AFR_COUNT(readable, priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (readable_cnt && !readable[i])
+            continue;
+        if (!replies[i].xdata)
+            continue;
+        ret = quota_dict_get_meta(replies[i].xdata, QUOTA_SIZE_KEY,
+                                  SLEN(QUOTA_SIZE_KEY), &size);
+        if (ret == -1)
+            continue;
+        if (read_subvol == -1)
+            read_subvol = i;
+        if (size.size > max_size.size ||
+            (size.file_count + size.dir_count) >
+                (max_size.file_count + max_size.dir_count))
+            read_subvol = i;
+
+        if (size.size > max_size.size)
+            max_size.size = size.size;
+        if (size.file_count > max_size.file_count)
+            max_size.file_count = size.file_count;
+        if (size.dir_count > max_size.dir_count)
+            max_size.dir_count = size.dir_count;
+    }
+
+    if (max_size.size == 0 && max_size.file_count == 0 &&
+        max_size.dir_count == 0)
         return read_subvol;
-}
 
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (readable_cnt && !readable[i])
+            continue;
+        if (!replies[i].xdata)
+            continue;
+        quota_dict_set_meta(replies[i].xdata, QUOTA_SIZE_KEY, &max_size,
+                            IA_IFDIR);
+    }
+
+    return read_subvol;
+}
 
 /* {{{ access */
 
 int
-afr_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int op_ret, int op_errno, dict_t *xdata)
+afr_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, dict_t *xdata)
 {
-        afr_local_t     *local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
+    AFR_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_access_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_access_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-
-	priv = this->private;
-	local = frame->local;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (access, frame, local->op_ret,
-				  local->op_errno, 0);
-		return 0;
-	}
-
-        STACK_WIND_COOKIE (frame, afr_access_cbk, (void *) (long) subvol,
-                           priv->children[subvol],
-                           priv->children[subvol]->fops->access,
-                           &local->loc, local->cont.access.mask,
-			   local->xdata_req);
-	return 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(access, frame, local->op_ret, local->op_errno, 0);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, afr_access_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->access, &local->loc,
+                      local->cont.access.mask, local->xdata_req);
+    return 0;
 }
 
 int
-afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
-	    int mask, dict_t *xdata)
+afr_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int mask,
+           dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	int op_errno = 0;
+    afr_local_t *local = NULL;
+    int op_errno = 0;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_ACCESS;
-	loc_copy (&local->loc, loc);
-	local->cont.access.mask = mask;
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_ACCESS;
+    loc_copy(&local->loc, loc);
+    local->cont.access.mask = mask;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-	afr_read_txn (frame, this, loc->inode, afr_access_wind,
-		      AFR_METADATA_TRANSACTION);
+    afr_read_txn(frame, this, loc->inode, afr_access_wind,
+                 AFR_METADATA_TRANSACTION);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+    AFR_STACK_UNWIND(access, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* }}} */
@@ -191,152 +187,140 @@ out:
 /* {{{ stat */
 
 int
-afr_stat_cbk (call_frame_t *frame, void *cookie,
-              xlator_t *this, int32_t op_ret, int32_t op_errno,
-              struct iatt *buf, dict_t *xdata)
+afr_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        afr_local_t     *local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
+    AFR_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_stat_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_stat_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-
-	priv = this->private;
-	local = frame->local;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
-				  0, 0);
-		return 0;
-	}
-
-        STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) subvol,
-                           priv->children[subvol],
-                           priv->children[subvol]->fops->stat,
-                           &local->loc, local->xdata_req);
-	return 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(stat, frame, local->op_ret, local->op_errno, 0, 0);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(
+        frame, afr_stat_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->stat, &local->loc, local->xdata_req);
+    return 0;
 }
 
 int
-afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+afr_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	int op_errno = 0;
+    afr_local_t *local = NULL;
+    int op_errno = 0;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_STAT;
-	loc_copy (&local->loc, loc);
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_STAT;
+    loc_copy(&local->loc, loc);
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-	afr_read_txn (frame, this, loc->inode, afr_stat_wind,
-		      AFR_DATA_TRANSACTION);
+    afr_read_txn(frame, this, loc->inode, afr_stat_wind, AFR_DATA_TRANSACTION);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ fstat */
 
 int
-afr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf,
-               dict_t *xdata)
+afr_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        afr_local_t     *local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
+    AFR_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_fstat_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fstat_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-
-	priv = this->private;
-	local = frame->local;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (fstat, frame, local->op_ret, local->op_errno,
-				  0, 0);
-		return 0;
-	}
-
-        STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) subvol,
-                           priv->children[subvol],
-                           priv->children[subvol]->fops->fstat,
-                           local->fd, local->xdata_req);
-	return 0;
-}
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(fstat, frame, local->op_ret, local->op_errno, 0, 0);
+        return 0;
+    }
 
+    STACK_WIND_COOKIE(
+        frame, afr_fstat_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->fstat, local->fd, local->xdata_req);
+    return 0;
+}
 
 int32_t
-afr_fstat (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, dict_t *xdata)
+afr_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	int op_errno = 0;
+    afr_local_t *local = NULL;
+    int op_errno = 0;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_FSTAT;
-	local->fd = fd_ref (fd);
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_FSTAT;
+    local->fd = fd_ref(fd);
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-	afr_read_txn (frame, this, fd->inode, afr_fstat_wind,
-		      AFR_DATA_TRANSACTION);
+    afr_read_txn(frame, this, fd->inode, afr_fstat_wind, AFR_DATA_TRANSACTION);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* }}} */
@@ -344,1563 +328,1493 @@ out:
 /* {{{ readlink */
 
 int
-afr_readlink_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  const char *buf, struct iatt *sbuf, dict_t *xdata)
+afr_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, const char *buf,
+                 struct iatt *sbuf, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = -1;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno,
-			  buf, sbuf, xdata);
-	return 0;
+    AFR_STACK_UNWIND(readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
+    return 0;
 }
 
 int
-afr_readlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_readlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (readlink, frame, local->op_ret,
-				  local->op_errno, 0, 0, 0);
-		return 0;
-	}
-
-	STACK_WIND_COOKIE (frame, afr_readlink_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->readlink,
-			   &local->loc, local->cont.readlink.size,
-			   local->xdata_req);
-	return 0;
-}
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
+    local = frame->local;
+    priv = this->private;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(readlink, frame, local->op_ret, local->op_errno, 0, 0,
+                         0);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, afr_readlink_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->readlink, &local->loc,
+                      local->cont.readlink.size, local->xdata_req);
+    return 0;
+}
 
 int
-afr_readlink (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, size_t size, dict_t *xdata)
+afr_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata)
 {
-        afr_local_t   * local      = NULL;
-        int32_t         op_errno   = 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_READLINK;
-        loc_copy (&local->loc, loc);
-        local->cont.readlink.size = size;
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_READLINK;
+    loc_copy(&local->loc, loc);
+    local->cont.readlink.size = size;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-	afr_read_txn (frame, this, loc->inode, afr_readlink_wind,
-		      AFR_DATA_TRANSACTION);
+    afr_read_txn(frame, this, loc->inode, afr_readlink_wind,
+                 AFR_DATA_TRANSACTION);
 
-        return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND(readlink, frame, -1, op_errno, 0, 0, 0);
+    AFR_STACK_UNWIND(readlink, frame, -1, op_errno, 0, 0, 0);
 
-	return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ getxattr */
 
 struct _xattr_key {
-        char *key;
-        struct list_head list;
+    char *key;
+    struct list_head list;
 };
 
-
 int
-__gather_xattr_keys (dict_t *dict, char *key, data_t *value,
-                     void *data)
+__gather_xattr_keys(dict_t *dict, char *key, data_t *value, void *data)
 {
-        struct list_head *  list  = data;
-        struct _xattr_key * xkey  = NULL;
-
-        if (!strncmp (key, AFR_XATTR_PREFIX,
-                      strlen (AFR_XATTR_PREFIX))) {
+    struct list_head *list = data;
+    struct _xattr_key *xkey = NULL;
 
-                xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
-                if (!xkey)
-                        return -1;
+    if (!strncmp(key, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX))) {
+        xkey = GF_MALLOC(sizeof(*xkey), gf_afr_mt_xattr_key);
+        if (!xkey)
+            return -1;
 
-                xkey->key = key;
-                INIT_LIST_HEAD (&xkey->list);
+        xkey->key = key;
+        INIT_LIST_HEAD(&xkey->list);
 
-                list_add_tail (&xkey->list, list);
-        }
-        return 0;
+        list_add_tail(&xkey->list, list);
+    }
+    return 0;
 }
 
-
 void
-afr_filter_xattrs (dict_t *dict)
+afr_filter_xattrs(dict_t *dict)
 {
-        struct list_head   keys = {0,};
-        struct _xattr_key *key  = NULL;
-        struct _xattr_key *tmp  = NULL;
+    struct list_head keys = {
+        0,
+    };
+    struct _xattr_key *key = NULL;
+    struct _xattr_key *tmp = NULL;
 
-        INIT_LIST_HEAD (&keys);
+    INIT_LIST_HEAD(&keys);
 
-        dict_foreach (dict, __gather_xattr_keys,
-                      (void *) &keys);
+    dict_foreach(dict, __gather_xattr_keys, (void *)&keys);
 
-        list_for_each_entry_safe (key, tmp, &keys, list) {
-                dict_del (dict, key->key);
+    list_for_each_entry_safe(key, tmp, &keys, list)
+    {
+        dict_del(dict, key->key);
 
-                list_del_init (&key->list);
+        list_del_init(&key->list);
 
-                GF_FREE (key);
-        }
+        GF_FREE(key);
+    }
 }
 
-static
-gf_boolean_t
-afr_getxattr_ignorable_errnos (int32_t op_errno)
+static gf_boolean_t
+afr_getxattr_ignorable_errnos(int32_t op_errno)
 {
-        if (op_errno == ENODATA || op_errno == ENOTSUP || op_errno == ERANGE ||
-            op_errno == ENAMETOOLONG)
-                return _gf_true;
+    if (op_errno == ENODATA || op_errno == ENOTSUP || op_errno == ERANGE ||
+        op_errno == ENAMETOOLONG)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 int
-afr_getxattr_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *dict, dict_t *xdata)
+afr_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0 && !afr_getxattr_ignorable_errnos(op_errno)) {
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0 && !afr_getxattr_ignorable_errnos(op_errno)) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	if (dict)
-		afr_filter_xattrs (dict);
+    if (dict)
+        afr_filter_xattrs(dict);
 
-	AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
+    AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_getxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_getxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (subvol == -1) {
-                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
-				  local->op_errno, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND_COOKIE (frame, afr_getxattr_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->getxattr,
-			   &local->loc, local->cont.getxattr.name,
-			   local->xdata_req);
-	return 0;
-}
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
+    local = frame->local;
+    priv = this->private;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno, NULL,
+                         NULL);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, afr_getxattr_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->getxattr, &local->loc,
+                      local->cont.getxattr.name, local->xdata_req);
+    return 0;
+}
 
 int32_t
-afr_getxattr_unwind (call_frame_t *frame, int op_ret, int op_errno,
-                     dict_t *dict, dict_t *xdata)
+afr_getxattr_unwind(call_frame_t *frame, int op_ret, int op_errno, dict_t *dict,
+                    dict_t *xdata)
 
 {
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int32_t
-afr_fgetxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno,
-                         dict_t *dict, dict_t *xdata)
+afr_fgetxattr_clrlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *dict,
+                        dict_t *xdata)
 {
-        afr_local_t     *local                  = NULL;
-        afr_private_t   *priv                   = NULL;
-        xlator_t        **children              = NULL;
-        dict_t          *xattr                  = NULL;
-        char            *tmp_report             = NULL;
-        char            lk_summary[1024]        = {0,};
-        int             serz_len                = 0;
-        int32_t         callcnt                 = 0;
-        long int        cky                     = 0;
-        int             ret                     = 0;
-
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                if (op_ret == -1)
-                        local->replies[cky].op_errno = op_errno;
-
-                if (!local->dict)
-                        local->dict = dict_new ();
-                if (local->dict) {
-                        ret = dict_get_str (dict, local->cont.getxattr.name,
-                                            &tmp_report);
-                        if (ret)
-                                goto unlock;
-                        ret = dict_set_dynstr (local->dict,
-                                               children[cky]->name,
-                                               gf_strdup (tmp_report));
-                        if (ret)
-                                goto unlock;
-                }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    dict_t *xattr = NULL;
+    char *tmp_report = NULL;
+    char lk_summary[1024] = {
+        0,
+    };
+    int serz_len = 0;
+    int32_t callcnt = 0;
+    long int cky = 0;
+    int ret = 0;
+    int keylen = 0;
+    int children_keylen = 0;
+
+    priv = this->private;
+    children = priv->children;
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    children_keylen = strlen(children[cky]->name);
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        if (op_ret == -1)
+            local->replies[cky].op_errno = op_errno;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (local->dict) {
+            ret = dict_get_strn(dict, local->cont.getxattr.name, keylen,
+                                &tmp_report);
+            if (ret)
+                goto unlock;
+            ret = dict_set_dynstrn(local->dict, children[cky]->name,
+                                   children_keylen, gf_strdup(tmp_report));
+            if (ret)
+                goto unlock;
         }
+    }
 unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                xattr = dict_new ();
-                if (!xattr) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-                ret = dict_serialize_value_with_delim (local->dict,
-                                                       lk_summary,
-                                                       &serz_len, '\n');
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-                if (serz_len == -1)
-                        snprintf (lk_summary, sizeof (lk_summary),
-                                  "No locks cleared.");
-                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
-                                       gf_strdup (lk_summary));
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        gf_msg (this->name, GF_LOG_ERROR,
-                                ENOMEM, AFR_MSG_DICT_SET_FAILED,
-                                "Error setting dictionary");
-                        goto unwind;
-                }
+    UNLOCK(&frame->lock);
+
+    if (!callcnt) {
+        xattr = dict_new();
+        if (!xattr) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        ret = dict_serialize_value_with_delim(local->dict, lk_summary,
+                                              &serz_len, '\n');
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        if (serz_len == -1)
+            snprintf(lk_summary, sizeof(lk_summary), "No locks cleared.");
+        ret = dict_set_dynstrn(xattr, local->cont.getxattr.name, keylen,
+                               gf_strdup(lk_summary));
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting dictionary");
+            goto unwind;
+        }
 
-                op_errno = afr_final_errno (local, priv);
+        op_errno = afr_final_errno(local, priv);
 
-unwind:
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, xattr,
-                                  xdata);
-                if (xattr)
-                        dict_unref (xattr);
-        }
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, xattr, xdata);
+        if (xattr)
+            dict_unref(xattr);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
-                        xlator_t *this, int32_t op_ret, int32_t op_errno,
-                        dict_t *dict, dict_t *xdata)
+afr_getxattr_clrlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
-        afr_local_t     *local                  = NULL;
-        afr_private_t   *priv                   = NULL;
-        xlator_t        **children              = NULL;
-        dict_t          *xattr                  = NULL;
-        char            *tmp_report             = NULL;
-        char            lk_summary[1024]        = {0,};
-        int             serz_len                = 0;
-        int32_t         callcnt                 = 0;
-        long int        cky                     = 0;
-        int             ret                     = 0;
-
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                if (op_ret == -1)
-                        local->replies[cky].op_errno = op_errno;
-
-                if (!local->dict)
-                        local->dict = dict_new ();
-                if (local->dict) {
-                        ret = dict_get_str (dict, local->cont.getxattr.name,
-                                            &tmp_report);
-                        if (ret)
-                                goto unlock;
-                        ret = dict_set_dynstr (local->dict,
-                                               children[cky]->name,
-                                               gf_strdup (tmp_report));
-                        if (ret)
-                                goto unlock;
-                }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    dict_t *xattr = NULL;
+    char *tmp_report = NULL;
+    char lk_summary[1024] = {
+        0,
+    };
+    int serz_len = 0;
+    int32_t callcnt = 0;
+    long int cky = 0;
+    int ret = 0;
+    int keylen = 0;
+    int children_keylen = 0;
+
+    priv = this->private;
+    children = priv->children;
+
+    local = frame->local;
+    cky = (long)cookie;
+
+    keylen = strlen(local->cont.getxattr.name);
+    children_keylen = strlen(children[cky]->name);
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        if (op_ret == -1)
+            local->replies[cky].op_errno = op_errno;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (local->dict) {
+            ret = dict_get_strn(dict, local->cont.getxattr.name, keylen,
+                                &tmp_report);
+            if (ret)
+                goto unlock;
+            ret = dict_set_dynstrn(local->dict, children[cky]->name,
+                                   children_keylen, gf_strdup(tmp_report));
+            if (ret)
+                goto unlock;
         }
+    }
 unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                xattr = dict_new ();
-                if (!xattr) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-                ret = dict_serialize_value_with_delim (local->dict,
-                                                       lk_summary,
-                                                       &serz_len, '\n');
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-                if (serz_len == -1)
-                        snprintf (lk_summary, sizeof (lk_summary),
-                                  "No locks cleared.");
-                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
-                                       gf_strdup (lk_summary));
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        gf_msg (this->name, GF_LOG_ERROR,
-                                ENOMEM, AFR_MSG_DICT_SET_FAILED,
-                                "Error setting dictionary");
-                        goto unwind;
-                }
+    UNLOCK(&frame->lock);
+
+    if (!callcnt) {
+        xattr = dict_new();
+        if (!xattr) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        ret = dict_serialize_value_with_delim(local->dict, lk_summary,
+                                              &serz_len, '\n');
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        if (serz_len == -1)
+            snprintf(lk_summary, sizeof(lk_summary), "No locks cleared.");
+        ret = dict_set_dynstrn(xattr, local->cont.getxattr.name, keylen,
+                               gf_strdup(lk_summary));
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting dictionary");
+            goto unwind;
+        }
 
-                op_errno = afr_final_errno (local, priv);
+        op_errno = afr_final_errno(local, priv);
 
-unwind:
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
 
-                if (xattr)
-                        dict_unref (xattr);
-        }
+        if (xattr)
+            dict_unref(xattr);
+    }
 
-        return ret;
+    return ret;
 }
 
 /**
  * node-uuid cbk uses next child querying mechanism
  */
 int32_t
-afr_getxattr_node_uuid_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *dict, dict_t *xdata)
+afr_getxattr_node_uuid_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        afr_private_t  *priv            = NULL;
-        afr_local_t    *local           = NULL;
-        xlator_t      **children        = NULL;
-        int             unwind          = 1;
-        int             curr_call_child = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    xlator_t **children = NULL;
+    int unwind = 1;
+    int curr_call_child = 0;
 
-        priv = this->private;
-        children = priv->children;
+    priv = this->private;
+    children = priv->children;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret == -1) { /** query the _next_ child */
-
-                /**
-                 * _current_ becomes _next_
-                 * If done with all childs and yet no success; give up !
-                 */
-                curr_call_child = (int) ((long)cookie);
-                if (++curr_call_child == priv->child_count)
-                        goto unwind;
-
-                gf_msg_debug (this->name, op_errno,
-                              "op_ret (-1): Re-querying afr-child (%d/%d)",
-                              curr_call_child, priv->child_count);
-
-                unwind = 0;
-                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
-                                   (void *) (long) curr_call_child,
-                                   children[curr_call_child],
-                                   children[curr_call_child]->fops->getxattr,
-                                   &local->loc,
-                                   local->cont.getxattr.name,
-                                   local->xdata_req);
-        }
+    if (op_ret == -1) { /** query the _next_ child */
 
- unwind:
-        if (unwind)
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,
-                                  xdata);
+        /**
+         * _current_ becomes _next_
+         * If done with all children and yet no success; give up !
+         */
+        curr_call_child = (int)((long)cookie);
+        if (++curr_call_child == priv->child_count)
+            goto unwind;
+
+        gf_msg_debug(this->name, op_errno,
+                     "op_ret (-1): Re-querying afr-child (%d/%d)",
+                     curr_call_child, priv->child_count);
+
+        unwind = 0;
+        STACK_WIND_COOKIE(
+            frame, afr_getxattr_node_uuid_cbk, (void *)(long)curr_call_child,
+            children[curr_call_child],
+            children[curr_call_child]->fops->getxattr, &local->loc,
+            local->cont.getxattr.name, local->xdata_req);
+    }
 
-        return 0;
+unwind:
+    if (unwind)
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+
+    return 0;
 }
 
 /**
  * list-node-uuids cbk returns the list of node_uuids for the subvolume.
  */
 int32_t
-afr_getxattr_list_node_uuids_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this, int32_t op_ret,
-                                  int32_t op_errno, dict_t *dict, dict_t *xdata)
+afr_getxattr_list_node_uuids_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        afr_local_t    *local          = NULL;
-        afr_private_t  *priv           = NULL;
-        int32_t         callcnt        = 0;
-        int             ret            = 0;
-        char           *xattr_serz     = NULL;
-        long            cky            = 0;
-        int32_t         tlen           = 0;
-
-        local = frame->local;
-        priv = this->private;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                local->replies[cky].valid = 1;
-                local->replies[cky].op_ret = op_ret;
-                local->replies[cky].op_errno = op_errno;
-
-                if (op_ret < 0)
-                        goto unlock;
-
-                local->op_ret = 0;
-
-                if (!local->xdata_rsp && xdata)
-                                local->xdata_rsp = dict_ref (xdata);
-                local->replies[cky].xattr = dict_ref (dict);
-        }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr_serz = NULL;
+    long cky = 0;
+    int32_t tlen = 0;
+
+    local = frame->local;
+    priv = this->private;
+    cky = (long)cookie;
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        local->replies[cky].valid = 1;
+        local->replies[cky].op_ret = op_ret;
+        local->replies[cky].op_errno = op_errno;
+
+        if (op_ret < 0)
+            goto unlock;
+
+        local->op_ret = 0;
+
+        if (!local->xdata_rsp && xdata)
+            local->xdata_rsp = dict_ref(xdata);
+        local->replies[cky].xattr = dict_ref(dict);
+    }
 
 unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
+    UNLOCK(&frame->lock);
 
-                if (local->op_ret != 0) {
-                        /* All bricks gave an error. */
-                        local->op_errno = afr_final_errno (local, priv);
-                        goto unwind;
-                }
+    if (!callcnt) {
+        if (local->op_ret != 0) {
+            /* All bricks gave an error. */
+            local->op_errno = afr_final_errno(local, priv);
+            goto unwind;
+        }
 
-                /*Since we store the UUID0_STR as node uuid for down bricks and
-                 *for non zero op_ret, assigning length to  priv->child_count
-                 *number of uuids*/
-                local->cont.getxattr.xattr_len = (strlen (UUID0_STR) + 2) *
-                                                  priv->child_count;
-
-                if (!local->dict)
-                        local->dict = dict_new ();
-                if (!local->dict) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
+        /*Since we store the UUID0_STR as node uuid for down bricks and
+         *for non zero op_ret, assigning length to  priv->child_count
+         *number of uuids*/
+        local->cont.getxattr.xattr_len = (SLEN(UUID0_STR) + 2) *
+                                         priv->child_count;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (!local->dict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
-                                        sizeof (char), gf_common_mt_char);
+        xattr_serz = GF_CALLOC(local->cont.getxattr.xattr_len, sizeof(char),
+                               gf_common_mt_char);
 
-                if (!xattr_serz) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-                ret = afr_serialize_xattrs_with_delimiter (frame, this,
-                                                           xattr_serz,
-                                                           UUID0_STR, &tlen,
-                                                           ' ');
-                if (ret) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
-                ret = dict_set_dynstr (local->dict,
-                                       GF_XATTR_LIST_NODE_UUIDS_KEY,
-                                       xattr_serz);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR,
-                                -ret, AFR_MSG_DICT_SET_FAILED,
-                                "Cannot set node_uuid key in dict");
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                } else {
-                        local->op_ret = local->cont.getxattr.xattr_len - 1;
-                        local->op_errno = 0;
-                }
+        if (!xattr_serz) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-unwind:
-                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
-                                  local->op_errno, local->dict,
-                                  local->xdata_rsp);
+        ret = afr_serialize_xattrs_with_delimiter(frame, this, xattr_serz,
+                                                  UUID0_STR, &tlen, ' ');
+        if (ret) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            GF_FREE(xattr_serz);
+            goto unwind;
+        }
+        ret = dict_set_dynstr_sizen(local->dict, GF_XATTR_LIST_NODE_UUIDS_KEY,
+                                    xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set node_uuid key in dict");
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
+        } else {
+            local->op_ret = local->cont.getxattr.xattr_len - 1;
+            local->op_errno = 0;
         }
 
-        return ret;
-}
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         local->dict, local->xdata_rsp);
+    }
 
+    return ret;
+}
 
 int32_t
-afr_getxattr_quota_size_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             dict_t *dict, dict_t *xdata)
+afr_getxattr_quota_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *dict,
+                            dict_t *xdata)
 {
-        int         idx         = (long) cookie;
-        int         call_count  = 0;
-        afr_local_t *local      = frame->local;
-        int         read_subvol = -1;
-
-        local->replies[idx].valid = 1;
-        local->replies[idx].op_ret = op_ret;
-        local->replies[idx].op_errno = op_errno;
-        if (dict)
-                local->replies[idx].xdata = dict_ref (dict);
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                local->inode = inode_ref (local->loc.inode);
-                read_subvol = afr_handle_quota_size (frame, this);
-                if (read_subvol != -1) {
-                        op_ret = local->replies[read_subvol].op_ret;
-                        op_errno = local->replies[read_subvol].op_errno;
-                        dict = local->replies[read_subvol].xdata;
-                }
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,
-                                  xdata);
+    int idx = (long)cookie;
+    int call_count = 0;
+    afr_local_t *local = frame->local;
+    int read_subvol = -1;
+
+    local->replies[idx].valid = 1;
+    local->replies[idx].op_ret = op_ret;
+    local->replies[idx].op_errno = op_errno;
+    if (dict)
+        local->replies[idx].xdata = dict_ref(dict);
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        local->inode = inode_ref(local->loc.inode);
+        read_subvol = afr_handle_quota_size(frame, this);
+        if (read_subvol != -1) {
+            op_ret = local->replies[read_subvol].op_ret;
+            op_errno = local->replies[read_subvol].op_errno;
+            dict = local->replies[read_subvol].xdata;
         }
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    }
 
-        return 0;
+    return 0;
 }
 
 int32_t
-afr_getxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
-                           xlator_t *this, int32_t op_ret, int32_t op_errno,
-                           dict_t *dict, dict_t *xdata)
+afr_getxattr_lockinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *dict,
+                          dict_t *xdata)
 {
-        int          call_cnt     = 0, len = 0;
-        char        *lockinfo_buf = NULL;
-        dict_t      *lockinfo     = NULL, *newdict = NULL;
-        afr_local_t *local        = NULL;
+    int call_cnt = 0, len = 0;
+    char *lockinfo_buf = NULL;
+    dict_t *lockinfo = NULL, *newdict = NULL;
+    afr_local_t *local = NULL;
 
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
-
-                call_cnt = --local->call_count;
+    LOCK(&frame->lock);
+    {
+        local = frame->local;
 
-                if ((op_ret < 0) || (!dict && !xdata)) {
-                        goto unlock;
-                }
+        call_cnt = --local->call_count;
 
-                if (xdata) {
-                        if (!local->xdata_rsp) {
-                                local->xdata_rsp = dict_new ();
-                                if (!local->xdata_rsp) {
-                                        local->op_ret = -1;
-                                        local->op_errno = ENOMEM;
-                                        goto unlock;
-                                }
-                        }
-                }
+        if ((op_ret < 0) || (!dict && !xdata)) {
+            goto unlock;
+        }
 
-                if (!dict) {
-                        goto unlock;
+        if (xdata) {
+            if (!local->xdata_rsp) {
+                local->xdata_rsp = dict_new();
+                if (!local->xdata_rsp) {
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    goto unlock;
                 }
+            }
+        }
 
-                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
-                                               (void **)&lockinfo_buf, &len);
+        if (!dict) {
+            goto unlock;
+        }
 
-                if (!lockinfo_buf) {
-                        goto unlock;
-                }
+        op_ret = dict_get_ptr_and_len(dict, GF_XATTR_LOCKINFO_KEY,
+                                      (void **)&lockinfo_buf, &len);
 
-                if (!local->dict) {
-                        local->dict = dict_new ();
-                        if (!local->dict) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto unlock;
-                        }
-                }
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (lockinfo_buf != NULL) {
-                lockinfo = dict_new ();
-                if (lockinfo == NULL) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                } else {
-                        op_ret = dict_unserialize (lockinfo_buf, len,
-                                                   &lockinfo);
-
-                        if (lockinfo && local->dict) {
-                                dict_copy (lockinfo, local->dict);
-                        }
-                }
+        if (!lockinfo_buf) {
+            goto unlock;
         }
 
-        if (xdata && local->xdata_rsp) {
-                dict_copy (xdata, local->xdata_rsp);
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto unlock;
+            }
         }
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (!call_cnt) {
-                newdict = dict_new ();
-                if (!newdict) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-                len = dict_serialized_length (local->dict);
-                if (len <= 0) {
-                        goto unwind;
-                }
-
-                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
-                if (!lockinfo_buf) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
+    if (lockinfo_buf != NULL) {
+        lockinfo = dict_new();
+        if (lockinfo == NULL) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+        } else {
+            op_ret = dict_unserialize(lockinfo_buf, len, &lockinfo);
 
-                op_ret = dict_serialize (local->dict, lockinfo_buf);
-                if (op_ret < 0) {
-                        local->op_ret = -1;
-                        local->op_errno = -op_ret;
-                }
+            if (lockinfo && local->dict) {
+                dict_copy(lockinfo, local->dict);
+            }
+        }
+    }
+
+    if (xdata && local->xdata_rsp) {
+        dict_copy(xdata, local->xdata_rsp);
+    }
+
+    if (!call_cnt) {
+        newdict = dict_new();
+        if (!newdict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
-                                          (void *)lockinfo_buf, len);
-                if (op_ret < 0) {
-                        local->op_ret = -1;
-                        local->op_errno = -op_ret;
-                        goto unwind;
-                }
+        op_ret = dict_allocate_and_serialize(
+            local->dict, (char **)&lockinfo_buf, (unsigned int *)&len);
+        if (op_ret != 0) {
+            local->op_ret = -1;
+            goto unwind;
+        }
 
-        unwind:
-                AFR_STACK_UNWIND (getxattr, frame, op_ret,
-                                  op_errno, newdict,
-                                  local->xdata_rsp);
+        op_ret = dict_set_dynptr(newdict, GF_XATTR_LOCKINFO_KEY,
+                                 (void *)lockinfo_buf, len);
+        if (op_ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = -op_ret;
+            goto unwind;
         }
 
-        dict_unref (lockinfo);
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, newdict,
+                         local->xdata_rsp);
+    }
 
-        return 0;
+    dict_unref(lockinfo);
+
+    return 0;
 }
 
 int32_t
-afr_fgetxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *dict, dict_t *xdata)
+afr_fgetxattr_lockinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        int          call_cnt     = 0, len = 0;
-        char        *lockinfo_buf = NULL;
-        dict_t      *lockinfo     = NULL, *newdict = NULL;
-        afr_local_t *local        = NULL;
+    int call_cnt = 0, len = 0;
+    char *lockinfo_buf = NULL;
+    dict_t *lockinfo = NULL, *newdict = NULL;
+    afr_local_t *local = NULL;
 
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
-
-                call_cnt = --local->call_count;
+    LOCK(&frame->lock);
+    {
+        local = frame->local;
 
-                if ((op_ret < 0) || (!dict && !xdata)) {
-                        goto unlock;
-                }
+        call_cnt = --local->call_count;
 
-                if (xdata) {
-                        if (!local->xdata_rsp) {
-                                local->xdata_rsp = dict_new ();
-                                if (!local->xdata_rsp) {
-                                        local->op_ret = -1;
-                                        local->op_errno = ENOMEM;
-                                        goto unlock;
-                                }
-                        }
-                }
+        if ((op_ret < 0) || (!dict && !xdata)) {
+            goto unlock;
+        }
 
-                if (!dict) {
-                        goto unlock;
+        if (xdata) {
+            if (!local->xdata_rsp) {
+                local->xdata_rsp = dict_new();
+                if (!local->xdata_rsp) {
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    goto unlock;
                 }
+            }
+        }
 
-                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
-                                               (void **)&lockinfo_buf, &len);
+        if (!dict) {
+            goto unlock;
+        }
 
-                if (!lockinfo_buf) {
-                        goto unlock;
-                }
+        op_ret = dict_get_ptr_and_len(dict, GF_XATTR_LOCKINFO_KEY,
+                                      (void **)&lockinfo_buf, &len);
 
-                if (!local->dict) {
-                        local->dict = dict_new ();
-                        if (!local->dict) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto unlock;
-                        }
-                }
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (lockinfo_buf != NULL) {
-                lockinfo = dict_new ();
-                if (lockinfo == NULL) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                } else {
-                        op_ret = dict_unserialize (lockinfo_buf, len,
-                                                   &lockinfo);
-
-                        if (lockinfo && local->dict) {
-                                dict_copy (lockinfo, local->dict);
-                        }
-                }
+        if (!lockinfo_buf) {
+            goto unlock;
         }
 
-        if (xdata && local->xdata_rsp) {
-                dict_copy (xdata, local->xdata_rsp);
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto unlock;
+            }
         }
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (!call_cnt) {
-                newdict = dict_new ();
-                if (!newdict) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-                len = dict_serialized_length (local->dict);
-                if (len <= 0) {
-                        goto unwind;
-                }
-
-                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
-                if (!lockinfo_buf) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
+    if (lockinfo_buf != NULL) {
+        lockinfo = dict_new();
+        if (lockinfo == NULL) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+        } else {
+            op_ret = dict_unserialize(lockinfo_buf, len, &lockinfo);
 
-                op_ret = dict_serialize (local->dict, lockinfo_buf);
-                if (op_ret < 0) {
-                        local->op_ret = -1;
-                        local->op_errno = -op_ret;
-                }
+            if (lockinfo && local->dict) {
+                dict_copy(lockinfo, local->dict);
+            }
+        }
+    }
+
+    if (xdata && local->xdata_rsp) {
+        dict_copy(xdata, local->xdata_rsp);
+    }
+
+    if (!call_cnt) {
+        newdict = dict_new();
+        if (!newdict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
-                                          (void *)lockinfo_buf, len);
-                if (op_ret < 0) {
-                        local->op_ret = -1;
-                        local->op_errno = -op_ret;
-                        goto unwind;
-                }
+        op_ret = dict_allocate_and_serialize(
+            local->dict, (char **)&lockinfo_buf, (unsigned int *)&len);
+        if (op_ret != 0) {
+            local->op_ret = -1;
+            goto unwind;
+        }
 
-        unwind:
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret,
-                                  op_errno, newdict,
-                                  local->xdata_rsp);
+        op_ret = dict_set_dynptr(newdict, GF_XATTR_LOCKINFO_KEY,
+                                 (void *)lockinfo_buf, len);
+        if (op_ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = -op_ret;
+            goto unwind;
         }
 
-        dict_unref (lockinfo);
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, newdict,
+                         local->xdata_rsp);
+    }
 
-        return 0;
+    dict_unref(lockinfo);
+
+    return 0;
 }
 
 int32_t
-afr_fgetxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *dict, dict_t *xdata)
+afr_fgetxattr_pathinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        afr_local_t *local          = NULL;
-        int32_t      callcnt        = 0;
-        int          ret            = 0;
-        char        *xattr          = NULL;
-        char        *xattr_serz     = NULL;
-        char        xattr_cky[1024] = {0,};
-        dict_t      *nxattr         = NULL;
-        long         cky            = 0;
-        int32_t      padding        = 0;
-        int32_t      tlen           = 0;
-
-        if (!frame || !frame->local || !this) {
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        AFR_MSG_INVALID_ARG, "possible NULL deref");
-                goto out;
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr = NULL;
+    char *xattr_serz = NULL;
+    int keylen = 0;
+    char xattr_cky[1024] = {
+        0,
+    };
+    int xattr_cky_len = 0;
+    dict_t *nxattr = NULL;
+    long cky = 0;
+    int32_t padding = 0;
+    int32_t tlen = 0;
+
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    xattr_cky_len = snprintf(xattr_cky, sizeof(xattr_cky), "%s-%ld",
+                             local->cont.getxattr.name, cky);
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+
+        if (op_ret < 0) {
+            local->op_errno = op_errno;
+        } else {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
 
-        local = frame->local;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret < 0) {
-                        local->op_errno = op_errno;
-                } else {
-                        local->op_ret = op_ret;
-                        if (!local->xdata_rsp && xdata)
-                                local->xdata_rsp = dict_ref (xdata);
-                }
+        if (!dict || (op_ret < 0))
+            goto unlock;
 
-                if (!dict || (op_ret < 0))
-                        goto unlock;
-
-                if (!local->dict)
-                        local->dict = dict_new ();
-
-                if (local->dict) {
-                        ret = dict_get_str (dict,
-                                            local->cont.getxattr.name,
-                                            &xattr);
-                        if (ret)
-                                goto unlock;
-
-                        xattr = gf_strdup (xattr);
-
-                        (void)snprintf (xattr_cky, 1024, "%s-%ld",
-                                        local->cont.getxattr.name, cky);
-                        ret = dict_set_dynstr (local->dict,
-                                               xattr_cky, xattr);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR,
-                                        -ret, AFR_MSG_DICT_SET_FAILED,
-                                        "Cannot set xattr cookie key");
-                                goto unlock;
-                        }
-
-                        local->cont.getxattr.xattr_len
-                                += strlen (xattr) + 1;
-                }
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict)
+                goto unlock;
+        }
+        ret = dict_get_strn(dict, local->cont.getxattr.name, keylen, &xattr);
+        if (ret)
+            goto unlock;
+
+        xattr = gf_strdup(xattr);
+
+        ret = dict_set_dynstrn(local->dict, xattr_cky, xattr_cky_len, xattr);
+        if (ret) {
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set xattr cookie key");
+            goto post_unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (!local->cont.getxattr.xattr_len)
-                        goto unwind;
-
-                nxattr = dict_new ();
-                if (!nxattr)
-                        goto unwind;
-
-                /* extra bytes for decorations (brackets and <>'s) */
-                padding += strlen (this->name)
-                        + strlen (AFR_PATHINFO_HEADER) + 4;
-                local->cont.getxattr.xattr_len += (padding + 2);
-
-                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
-                                        sizeof (char), gf_common_mt_char);
-
-                if (!xattr_serz)
-                        goto unwind;
-
-                /* the xlator info */
-                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
-                                this->name);
-
-                /* actual series of pathinfo */
-                ret = dict_serialize_value_with_delim (local->dict,
-                                                       xattr_serz
-                                                       + strlen (xattr_serz),
-                                                       &tlen, ' ');
-                if (ret) {
-                        goto unwind;
-                }
 
-                /* closing part */
-                *(xattr_serz + padding + tlen) = ')';
-                *(xattr_serz + padding + tlen + 1) = '\0';
+        local->cont.getxattr.xattr_len += strlen(xattr) + 1;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!callcnt) {
+        if (!local->cont.getxattr.xattr_len)
+            goto unwind;
+
+        nxattr = dict_new();
+        if (!nxattr)
+            goto unwind;
+
+        /* extra bytes for decorations (brackets and <>'s) */
+        padding += strlen(this->name) + SLEN(AFR_PATHINFO_HEADER) + 4;
+        local->cont.getxattr.xattr_len += (padding + 2);
+
+        xattr_serz = GF_MALLOC(local->cont.getxattr.xattr_len,
+                               gf_common_mt_char);
+
+        if (!xattr_serz)
+            goto unwind;
+
+        /* the xlator info */
+        int xattr_serz_len = sprintf(
+            xattr_serz, "(<" AFR_PATHINFO_HEADER "%s> ", this->name);
+
+        /* actual series of pathinfo */
+        ret = dict_serialize_value_with_delim(
+            local->dict, xattr_serz + xattr_serz_len, &tlen, ' ');
+        if (ret) {
+            GF_FREE(xattr_serz);
+            goto unwind;
+        }
 
-                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
-                                       xattr_serz);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR,
-                                -ret, AFR_MSG_DICT_SET_FAILED,
-                                "Cannot set pathinfo key in dict");
+        /* closing part */
+        *(xattr_serz + padding + tlen) = ')';
+        *(xattr_serz + padding + tlen + 1) = '\0';
+
+        ret = dict_set_dynstrn(nxattr, local->cont.getxattr.name, keylen,
+                               xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set pathinfo key in dict");
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
+        }
 
-        unwind:
-                AFR_STACK_UNWIND (fgetxattr, frame, local->op_ret,
-                                  local->op_errno, nxattr, local->xdata_rsp);
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, local->op_ret, local->op_errno,
+                         nxattr, local->xdata_rsp);
 
-                if (nxattr)
-                        dict_unref (nxattr);
-        }
+        if (nxattr)
+            dict_unref(nxattr);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
-                           xlator_t *this, int32_t op_ret, int32_t op_errno,
-                           dict_t *dict, dict_t *xdata)
+afr_getxattr_pathinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *dict,
+                          dict_t *xdata)
 {
-        afr_local_t *local          = NULL;
-        int32_t      callcnt        = 0;
-        int          ret            = 0;
-        char        *xattr          = NULL;
-        char        *xattr_serz     = NULL;
-        char        xattr_cky[1024] = {0,};
-        dict_t      *nxattr         = NULL;
-        long         cky            = 0;
-        int32_t      padding        = 0;
-        int32_t      tlen           = 0;
-
-        if (!frame || !frame->local || !this) {
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        AFR_MSG_INVALID_ARG, "possible NULL deref");
-                goto out;
-        }
-
-        local = frame->local;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-                {
-                        callcnt = --local->call_count;
-
-                        if (op_ret < 0) {
-                                local->op_errno = op_errno;
-                        } else {
-                                local->op_ret = op_ret;
-                                if (!local->xdata_rsp && xdata)
-                                        local->xdata_rsp = dict_ref (xdata);
-                        }
-
-                        if (!dict || (op_ret < 0))
-                                goto unlock;
-
-                        if (!local->dict)
-                                local->dict = dict_new ();
-
-                        if (local->dict) {
-                                ret = dict_get_str (dict,
-                                                    local->cont.getxattr.name,
-                                                    &xattr);
-                                if (ret)
-                                        goto unlock;
-
-                                xattr = gf_strdup (xattr);
-
-                                (void)snprintf (xattr_cky, 1024, "%s-%ld",
-                                                local->cont.getxattr.name, cky);
-                                ret = dict_set_dynstr (local->dict,
-                                                       xattr_cky, xattr);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR,
-                                                -ret,
-                                                AFR_MSG_DICT_SET_FAILED,
-                                                "Cannot set xattr "
-                                                "cookie key");
-                                        goto unlock;
-                                }
-
-                                local->cont.getxattr.xattr_len += strlen (xattr) + 1;
-                        }
-                }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (!local->cont.getxattr.xattr_len)
-                        goto unwind;
-
-                nxattr = dict_new ();
-                if (!nxattr)
-                        goto unwind;
-
-                /* extra bytes for decorations (brackets and <>'s) */
-                padding += strlen (this->name) + strlen (AFR_PATHINFO_HEADER) + 4;
-                local->cont.getxattr.xattr_len += (padding + 2);
-
-                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
-                                        sizeof (char), gf_common_mt_char);
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr = NULL;
+    char *xattr_serz = NULL;
+    char xattr_cky[1024] = {
+        0,
+    };
+    int keylen = 0;
+    int xattr_cky_len = 0;
+    dict_t *nxattr = NULL;
+    long cky = 0;
+    int32_t padding = 0;
+    int32_t tlen = 0;
+
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    xattr_cky_len = snprintf(xattr_cky, sizeof(xattr_cky), "%s-%ld",
+                             local->cont.getxattr.name, cky);
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
 
-                if (!xattr_serz)
-                        goto unwind;
+        if (op_ret < 0) {
+            local->op_errno = op_errno;
+        } else {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        }
 
-                /* the xlator info */
-                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
-                                this->name);
+        if (!dict || (op_ret < 0))
+            goto unlock;
 
-                /* actual series of pathinfo */
-                ret = dict_serialize_value_with_delim (local->dict,
-                                                       xattr_serz + strlen (xattr_serz),
-                                                       &tlen, ' ');
-                if (ret) {
-                        goto unwind;
-                }
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict)
+                goto unlock;
+        }
+        ret = dict_get_strn(dict, local->cont.getxattr.name, keylen, &xattr);
+        if (ret)
+            goto unlock;
+
+        xattr = gf_strdup(xattr);
+
+        ret = dict_set_dynstrn(local->dict, xattr_cky, xattr_cky_len, xattr);
+        if (ret) {
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set xattr cookie key");
+            goto post_unlock;
+        }
 
-                /* closing part */
-                *(xattr_serz + padding + tlen) = ')';
-                *(xattr_serz + padding + tlen + 1) = '\0';
+        local->cont.getxattr.xattr_len += strlen(xattr) + 1;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!callcnt) {
+        if (!local->cont.getxattr.xattr_len)
+            goto unwind;
+
+        nxattr = dict_new();
+        if (!nxattr)
+            goto unwind;
+
+        /* extra bytes for decorations (brackets and <>'s) */
+        padding += strlen(this->name) + SLEN(AFR_PATHINFO_HEADER) + 4;
+        local->cont.getxattr.xattr_len += (padding + 2);
+
+        xattr_serz = GF_MALLOC(local->cont.getxattr.xattr_len,
+                               gf_common_mt_char);
+
+        if (!xattr_serz)
+            goto unwind;
+
+        /* the xlator info */
+        int xattr_serz_len = sprintf(
+            xattr_serz, "(<" AFR_PATHINFO_HEADER "%s> ", this->name);
+
+        /* actual series of pathinfo */
+        ret = dict_serialize_value_with_delim(
+            local->dict, xattr_serz + xattr_serz_len, &tlen, ' ');
+        if (ret) {
+            GF_FREE(xattr_serz);
+            goto unwind;
+        }
 
-                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
-                                       xattr_serz);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR,
-                                -ret, AFR_MSG_DICT_SET_FAILED,
-                                "Cannot set pathinfo key in dict");
+        /* closing part */
+        *(xattr_serz + padding + tlen) = ')';
+        *(xattr_serz + padding + tlen + 1) = '\0';
+
+        ret = dict_set_dynstrn(nxattr, local->cont.getxattr.name, keylen,
+                               xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set pathinfo key in dict");
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
+        }
 
-        unwind:
-                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
-                                  local->op_errno, nxattr, local->xdata_rsp);
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         nxattr, local->xdata_rsp);
 
-                if (nxattr)
-                        dict_unref (nxattr);
-        }
+        if (nxattr)
+            dict_unref(nxattr);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-afr_aggregate_stime_xattr (dict_t *this, char *key, data_t *value, void *data)
+afr_aggregate_stime_xattr(dict_t *this, char *key, data_t *value, void *data)
 {
-        int ret = 0;
+    int ret = 0;
 
-        if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
-                ret = gf_get_max_stime (THIS, data, key, value);
+    if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
+        ret = gf_get_max_stime(THIS, data, key, value);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-afr_common_getxattr_stime_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               dict_t *dict, dict_t *xdata)
+afr_common_getxattr_stime_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, dict_t *dict,
+                              dict_t *xdata)
 {
-        afr_local_t *local          = NULL;
-        int32_t      callcnt        = 0;
-
-        if (!frame || !frame->local || !this) {
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        AFR_MSG_INVALID_ARG, "possible NULL deref");
-                goto out;
-        }
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
 
-        local = frame->local;
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
 
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
+    local = frame->local;
 
-                if (!dict || (op_ret < 0)) {
-                        local->op_errno = op_errno;
-                        goto cleanup;
-                }
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
 
-                if (!local->dict)
-                        local->dict = dict_copy_with_ref (dict, NULL);
-                else
-                        dict_foreach (dict, afr_aggregate_stime_xattr,
-                                      local->dict);
-                local->op_ret = 0;
+        if (!dict || (op_ret < 0)) {
+            local->op_errno = op_errno;
+            goto cleanup;
         }
 
+        if (!local->dict)
+            local->dict = dict_copy_with_ref(dict, NULL);
+        else
+            dict_foreach(dict, afr_aggregate_stime_xattr, local->dict);
+        local->op_ret = 0;
+    }
+
 cleanup:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 
-        if (!callcnt) {
-                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
-                                  local->op_errno, local->dict, xdata);
-        }
+    if (!callcnt) {
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         local->dict, xdata);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-
 static gf_boolean_t
-afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk,
-                      gf_boolean_t is_fgetxattr)
+afr_is_special_xattr(const char *name, fop_getxattr_cbk_t *cbk,
+                     gf_boolean_t is_fgetxattr)
 {
-        gf_boolean_t    is_spl = _gf_true;
-
-        GF_ASSERT (cbk);
-        if (!cbk || !name) {
-                is_spl = _gf_false;
-                goto out;
+    gf_boolean_t is_spl = _gf_true;
+
+    GF_ASSERT(cbk);
+    if (!cbk || !name) {
+        is_spl = _gf_false;
+        goto out;
+    }
+
+    if (!strcmp(name, GF_XATTR_PATHINFO_KEY) ||
+        !strcmp(name, GF_XATTR_USER_PATHINFO_KEY)) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_pathinfo_cbk;
+        } else {
+            *cbk = afr_getxattr_pathinfo_cbk;
         }
-
-        if (!strcmp (name, GF_XATTR_PATHINFO_KEY) ||
-	    !strcmp (name, GF_XATTR_USER_PATHINFO_KEY)) {
-                if (is_fgetxattr) {
-                        *cbk = afr_fgetxattr_pathinfo_cbk;
-                } else {
-                        *cbk = afr_getxattr_pathinfo_cbk;
-                }
-        } else if (!strncmp (name, GF_XATTR_CLRLK_CMD,
-                             strlen (GF_XATTR_CLRLK_CMD))) {
-                if (is_fgetxattr) {
-                        *cbk = afr_fgetxattr_clrlk_cbk;
-                } else {
-                        *cbk = afr_getxattr_clrlk_cbk;
-                }
-        } else if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
-                             strlen (GF_XATTR_LOCKINFO_KEY))) {
-                if (is_fgetxattr) {
-                        *cbk = afr_fgetxattr_lockinfo_cbk;
-                } else {
-                        *cbk = afr_getxattr_lockinfo_cbk;
-                }
-        } else if (fnmatch (GF_XATTR_STIME_PATTERN, name, FNM_NOESCAPE) == 0) {
-                *cbk = afr_common_getxattr_stime_cbk;
-        } else if (strcmp (name, QUOTA_SIZE_KEY) == 0) {
-                *cbk = afr_getxattr_quota_size_cbk;
-        } else if (!strcmp (name, GF_XATTR_LIST_NODE_UUIDS_KEY)) {
-                *cbk = afr_getxattr_list_node_uuids_cbk;
+    } else if (!strncmp(name, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD))) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_clrlk_cbk;
         } else {
-                is_spl = _gf_false;
+            *cbk = afr_getxattr_clrlk_cbk;
         }
+    } else if (!strncmp(name, GF_XATTR_LOCKINFO_KEY,
+                        SLEN(GF_XATTR_LOCKINFO_KEY))) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_lockinfo_cbk;
+        } else {
+            *cbk = afr_getxattr_lockinfo_cbk;
+        }
+    } else if (fnmatch(GF_XATTR_STIME_PATTERN, name, FNM_NOESCAPE) == 0) {
+        *cbk = afr_common_getxattr_stime_cbk;
+    } else if (strcmp(name, QUOTA_SIZE_KEY) == 0) {
+        *cbk = afr_getxattr_quota_size_cbk;
+    } else if (!strcmp(name, GF_XATTR_LIST_NODE_UUIDS_KEY)) {
+        *cbk = afr_getxattr_list_node_uuids_cbk;
+    } else {
+        is_spl = _gf_false;
+    }
 
 out:
-        return is_spl;
+    return is_spl;
 }
 
 static void
-afr_getxattr_all_subvols (xlator_t *this, call_frame_t *frame,
-			  const char *name, loc_t *loc,
-			  fop_getxattr_cbk_t cbk)
+afr_getxattr_all_subvols(xlator_t *this, call_frame_t *frame, const char *name,
+                         loc_t *loc, fop_getxattr_cbk_t cbk)
 {
-        afr_private_t   *priv           = NULL;
-        afr_local_t     *local          = NULL;
-        int             i               = 0;
-        int             call_count      = 0;
-
-        priv     = this->private;
-
-        local = frame->local;
-        //local->call_count set in afr_local_init
-        call_count = local->call_count;
-
-        if (!strcmp (name, GF_XATTR_LIST_NODE_UUIDS_KEY)) {
-                GF_FREE (local->cont.getxattr.name);
-                local->cont.getxattr.name = gf_strdup (GF_XATTR_NODE_UUID_KEY);
-        }
-
-        //If up-children count is 0, afr_local_init would have failed already
-        //and the call would have unwound so not handling it here.
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, cbk,
-                                           (void *) (long) i, priv->children[i],
-                                           priv->children[i]->fops->getxattr,
-                                           loc, local->cont.getxattr.name,
-                                           NULL);
-                        if (!--call_count)
-                                break;
-                }
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = 0;
+
+    priv = this->private;
+
+    local = frame->local;
+    // local->call_count set in afr_local_init
+    call_count = local->call_count;
+
+    if (!strcmp(name, GF_XATTR_LIST_NODE_UUIDS_KEY)) {
+        GF_FREE(local->cont.getxattr.name);
+        local->cont.getxattr.name = gf_strdup(GF_XATTR_NODE_UUID_KEY);
+    }
+
+    // If up-children count is 0, afr_local_init would have failed already
+    // and the call would have unwound so not handling it here.
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, cbk, (void *)(long)i, priv->children[i],
+                              priv->children[i]->fops->getxattr, loc,
+                              local->cont.getxattr.name, NULL);
+            if (!--call_count)
+                break;
         }
-        return;
+    }
+    return;
 }
 
 int
-afr_marker_populate_args (call_frame_t *frame, int type, int *gauge,
-                          xlator_t **subvols)
+afr_marker_populate_args(call_frame_t *frame, int type, int *gauge,
+                         xlator_t **subvols)
 {
-        xlator_t *this = frame->this;
-        afr_private_t *priv = this->private;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
 
-        memcpy (subvols, priv->children, sizeof (*subvols) * priv->child_count);
+    memcpy(subvols, priv->children, sizeof(*subvols) * priv->child_count);
 
-        if (type == MARKER_XTIME_TYPE) {
-                /*Don't error out on ENOENT/ENOTCONN */
-                gauge[MCNT_NOTFOUND] = 0;
-                gauge[MCNT_ENOTCONN] = 0;
-        }
-        return priv->child_count;
+    if (type == MARKER_XTIME_TYPE) {
+        /*Don't error out on ENOENT/ENOTCONN */
+        gauge[MCNT_NOTFOUND] = 0;
+        gauge[MCNT_ENOTCONN] = 0;
+    }
+    return priv->child_count;
 }
 
 static int
-afr_handle_heal_xattrs (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                        const char *heal_op)
+afr_handle_heal_xattrs(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       const char *heal_op)
 {
-        int                     ret     = -1;
-        afr_spb_status_t       *data    = NULL;
+    int ret = -1;
+    afr_spb_status_t *data = NULL;
 
-        if (!strcmp (heal_op, GF_HEAL_INFO)) {
-                afr_get_heal_info (frame, this, loc);
-                ret = 0;
-                goto out;
-        }
+    if (!strcmp(heal_op, GF_HEAL_INFO)) {
+        afr_get_heal_info(frame, this, loc);
+        ret = 0;
+        goto out;
+    }
 
-        if (!strcmp (heal_op, GF_AFR_HEAL_SBRAIN)) {
-                afr_heal_splitbrain_file (frame, this, loc);
-                ret = 0;
-                goto out;
+    if (!strcmp(heal_op, GF_AFR_HEAL_SBRAIN)) {
+        afr_heal_splitbrain_file(frame, this, loc);
+        ret = 0;
+        goto out;
+    }
+
+    if (!strcmp(heal_op, GF_AFR_SBRAIN_STATUS)) {
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_spb_status_t);
+        if (!data) {
+            ret = 1;
+            goto out;
         }
-
-        if (!strcmp (heal_op, GF_AFR_SBRAIN_STATUS)) {
-                data = GF_CALLOC (1, sizeof (*data), gf_afr_mt_spb_status_t);
-                if (!data) {
-                        ret = 1;
-                        goto out;
-                }
-                data->frame = frame;
-                data->loc = loc;
-                ret = synctask_new (this->ctx->env,
-                                    afr_get_split_brain_status,
-                                    afr_get_split_brain_status_cbk,
-                                    NULL, data);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN_STATUS,
-                                "Failed to create"
-                                " synctask. Unable to fetch split-brain status"
-                                " for %s.", loc->name);
-                        ret = 1;
-                        goto out;
-                }
-                goto out;
+        data->frame = frame;
+        data->loc = loc;
+        ret = synctask_new(this->ctx->env, afr_get_split_brain_status,
+                           afr_get_split_brain_status_cbk, NULL, data);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                   "Failed to create"
+                   " synctask. Unable to fetch split-brain status"
+                   " for %s.",
+                   loc->name);
+            ret = 1;
+            goto out;
         }
+        goto out;
+    }
 
 out:
-        if (ret == 1) {
-                AFR_STACK_UNWIND (getxattr, frame, -1, ENOMEM, NULL, NULL);
-                if (data)
-                        GF_FREE (data);
-                ret = 0;
-        }
-        return ret;
+    if (ret == 1) {
+        AFR_STACK_UNWIND(getxattr, frame, -1, ENOMEM, NULL, NULL);
+        if (data)
+            GF_FREE(data);
+        ret = 0;
+    }
+    return ret;
 }
 
 int32_t
-afr_getxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, const char *name, dict_t *xdata)
+afr_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata)
 {
-        afr_private_t           *priv         = NULL;
-        afr_local_t             *local        = NULL;
-        xlator_t                **children    = NULL;
-        int                     i             = 0;
-        int32_t                 op_errno      = 0;
-        int                     ret           = -1;
-        fop_getxattr_cbk_t      cbk           = NULL;
-
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    xlator_t **children = NULL;
+    int i = 0;
+    int32_t op_errno = 0;
+    int ret = -1;
+    fop_getxattr_cbk_t cbk = NULL;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        priv     = this->private;
+    priv = this->private;
 
-        children = priv->children;
+    children = priv->children;
 
-        loc_copy (&local->loc, loc);
+    loc_copy(&local->loc, loc);
 
-	local->op = GF_FOP_GETXATTR;
+    local->op = GF_FOP_GETXATTR;
 
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        if (!name)
-                goto no_name;
+    if (!name)
+        goto no_name;
 
-        local->cont.getxattr.name = gf_strdup (name);
+    local->cont.getxattr.name = gf_strdup(name);
 
-	if (!local->cont.getxattr.name) {
-		op_errno = ENOMEM;
-		goto out;
-	}
-
-        if (!strncmp (name, AFR_XATTR_PREFIX,
-                      strlen (AFR_XATTR_PREFIX))) {
-                op_errno = ENODATA;
-                goto out;
-        }
+    if (!local->cont.getxattr.name) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-        if (cluster_handle_marker_getxattr (frame, loc, name, priv->vol_uuid,
-                                            afr_getxattr_unwind,
-                                            afr_marker_populate_args) == 0)
-                return 0;
+    if (!strncmp(name, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX))) {
+        op_errno = ENODATA;
+        goto out;
+    }
 
-        ret = afr_handle_heal_xattrs (frame, this, &local->loc, name);
-        if (ret == 0)
-                return 0;
+    if (cluster_handle_marker_getxattr(frame, loc, name, priv->vol_uuid,
+                                       afr_getxattr_unwind,
+                                       afr_marker_populate_args) == 0)
+        return 0;
 
-        /*
-         * Heal daemons don't have IO threads ... and as a result they
-         * send this getxattr down and eventually crash :(
-         */
-        op_errno = -1;
-        GF_CHECK_XATTR_KEY_AND_GOTO(name, IO_THREADS_QUEUE_SIZE_KEY, op_errno,
-                                    out);
+    ret = afr_handle_heal_xattrs(frame, this, &local->loc, name);
+    if (ret == 0)
+        return 0;
 
-        /*
-         * Special xattrs which need responses from all subvols
-         */
-        if (afr_is_special_xattr (name, &cbk, 0)) {
-                afr_getxattr_all_subvols (this, frame, name, loc, cbk);
-                return 0;
-        }
+    /*
+     * Heal daemons don't have IO threads ... and as a result they
+     * send this getxattr down and eventually crash :(
+     */
+    op_errno = -1;
+    GF_CHECK_XATTR_KEY_AND_GOTO(name, IO_THREADS_QUEUE_SIZE_KEY, op_errno, out);
+
+    /*
+     * Special xattrs which need responses from all subvols
+     */
+    if (afr_is_special_xattr(name, &cbk, 0)) {
+        afr_getxattr_all_subvols(this, frame, name, loc, cbk);
+        return 0;
+    }
 
-        if (XATTR_IS_NODE_UUID (name)) {
-                i = 0;
-                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
-                                   (void *) (long) i,
-                                   children[i],
-                                   children[i]->fops->getxattr,
-                                   loc, name, xdata);
-                return 0;
-        }
+    if (XATTR_IS_NODE_UUID(name)) {
+        i = 0;
+        STACK_WIND_COOKIE(frame, afr_getxattr_node_uuid_cbk, (void *)(long)i,
+                          children[i], children[i]->fops->getxattr, loc, name,
+                          xdata);
+        return 0;
+    }
 
 no_name:
 
-	afr_read_txn (frame, this, local->loc.inode, afr_getxattr_wind,
-		      AFR_METADATA_TRANSACTION);
+    afr_read_txn(frame, this, local->loc.inode, afr_getxattr_wind,
+                 AFR_METADATA_TRANSACTION);
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    if (ret < 0)
+        AFR_STACK_UNWIND(getxattr, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
 /* {{{ fgetxattr */
 
-
 int32_t
-afr_fgetxattr_cbk (call_frame_t *frame, void *cookie,
-                   xlator_t *this, int32_t op_ret, int32_t op_errno,
-                   dict_t *dict, dict_t *xdata)
+afr_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = -1;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	if (dict)
-		afr_filter_xattrs (dict);
+    if (dict)
+        afr_filter_xattrs(dict);
 
-	AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, dict, xdata);
 
-	return 0;
+    return 0;
 }
 
 int
-afr_fgetxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fgetxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (fgetxattr, frame, local->op_ret,
-				  local->op_errno, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND_COOKIE (frame, afr_fgetxattr_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fgetxattr,
-			   local->fd, local->cont.getxattr.name,
-			   local->xdata_req);
-	return 0;
-}
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
+    local = frame->local;
+    priv = this->private;
 
-static void
-afr_fgetxattr_all_subvols (xlator_t *this, call_frame_t *frame,
-			   fop_fgetxattr_cbk_t cbk)
-{
-        afr_private_t   *priv           = NULL;
-        afr_local_t     *local          = NULL;
-        int             i               = 0;
-        int             call_count      = 0;
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(fgetxattr, frame, local->op_ret, local->op_errno, NULL,
+                         NULL);
+        return 0;
+    }
 
-        priv     = this->private;
+    STACK_WIND_COOKIE(frame, afr_fgetxattr_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fgetxattr, local->fd,
+                      local->cont.getxattr.name, local->xdata_req);
+    return 0;
+}
 
-        local = frame->local;
-        //local->call_count set in afr_local_init
-        call_count = local->call_count;
-
-        //If up-children count is 0, afr_local_init would have failed already
-        //and the call would have unwound so not handling it here.
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fgetxattr,
-                                           local->fd, local->cont.getxattr.name,
-					   NULL);
-                        if (!--call_count)
-                                break;
-                }
+static void
+afr_fgetxattr_all_subvols(xlator_t *this, call_frame_t *frame,
+                          fop_fgetxattr_cbk_t cbk)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = 0;
+
+    priv = this->private;
+
+    local = frame->local;
+    // local->call_count set in afr_local_init
+    call_count = local->call_count;
+
+    // If up-children count is 0, afr_local_init would have failed already
+    // and the call would have unwound so not handling it here.
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, cbk, (void *)(long)i, priv->children[i],
+                              priv->children[i]->fops->fgetxattr, local->fd,
+                              local->cont.getxattr.name, NULL);
+            if (!--call_count)
+                break;
         }
+    }
 
-        return;
+    return;
 }
 
-
 int
-afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name, dict_t *xdata)
+afr_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata)
 {
-        afr_local_t          *local      = NULL;
-        int32_t               op_errno   = 0;
-        fop_fgetxattr_cbk_t   cbk        = NULL;
-
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-	local->op = GF_FOP_FGETXATTR;
-        local->fd = fd_ref (fd);
-        if (name) {
-                local->cont.getxattr.name = gf_strdup (name);
-		if (!local->cont.getxattr.name) {
-			op_errno = ENOMEM;
-			goto out;
-		}
-	}
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
-
-        /* pathinfo gets handled only in getxattr(), but we need to handle
-         * lockinfo.
-         * If we are doing fgetxattr with lockinfo as the key then we
-         * collect information from all children.
-         */
-        if (afr_is_special_xattr (name, &cbk, 1)) {
-                afr_fgetxattr_all_subvols (this, frame, cbk);
-                return 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    fop_fgetxattr_cbk_t cbk = NULL;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_FGETXATTR;
+    local->fd = fd_ref(fd);
+    if (name) {
+        local->cont.getxattr.name = gf_strdup(name);
+        if (!local->cont.getxattr.name) {
+            op_errno = ENOMEM;
+            goto out;
         }
+    }
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    /* pathinfo gets handled only in getxattr(), but we need to handle
+     * lockinfo.
+     * If we are doing fgetxattr with lockinfo as the key then we
+     * collect information from all children.
+     */
+    if (afr_is_special_xattr(name, &cbk, 1)) {
+        afr_fgetxattr_all_subvols(this, frame, cbk);
+        return 0;
+    }
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-	afr_read_txn (frame, this, fd->inode, afr_fgetxattr_wind,
-		      AFR_METADATA_TRANSACTION);
+    afr_read_txn(frame, this, fd->inode, afr_fgetxattr_wind,
+                 AFR_METADATA_TRANSACTION);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ readv */
 
 int
-afr_readv_cbk (call_frame_t *frame, void *cookie,
-               xlator_t *this, int32_t op_ret, int32_t op_errno,
-               struct iovec *vector, int32_t count, struct iatt *buf,
-               struct iobref *iobref, dict_t *xdata)
+afr_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iovec *vector, int32_t count,
+              struct iatt *buf, struct iobref *iobref, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = -1;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (readv, frame, op_ret, op_errno,
-			  vector, count, buf, iobref, xdata);
-	return 0;
+    AFR_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, buf, iobref,
+                     xdata);
+    return 0;
 }
 
-
 int
-afr_readv_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_readv_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (readv, frame, local->op_ret, local->op_errno,
-				  0, 0, 0, 0, 0);
-		return 0;
-	}
-
-	STACK_WIND_COOKIE (frame, afr_readv_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->readv,
-			   local->fd, local->cont.readv.size,
-			   local->cont.readv.offset, local->cont.readv.flags,
-			   local->xdata_req);
-	return 0;
-}
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
+    local = frame->local;
+    priv = this->private;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(readv, frame, local->op_ret, local->op_errno, 0, 0, 0,
+                         0, 0);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(
+        frame, afr_readv_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->readv, local->fd, local->cont.readv.size,
+        local->cont.readv.offset, local->cont.readv.flags, local->xdata_req);
+    return 0;
+}
 
 int
-afr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-	   off_t offset, uint32_t flags, dict_t *xdata)
+afr_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
-        afr_local_t   * local      = NULL;
-        int32_t         op_errno   = 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_READ;
-        local->fd = fd_ref (fd);
-        local->cont.readv.size = size;
-        local->cont.readv.offset = offset;
-        local->cont.readv.flags = flags;
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_READ;
+    local->fd = fd_ref(fd);
+    local->cont.readv.size = size;
+    local->cont.readv.offset = offset;
+    local->cont.readv.flags = flags;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-	afr_read_txn (frame, this, fd->inode, afr_readv_wind,
-		      AFR_DATA_TRANSACTION);
+    afr_read_txn(frame, this, fd->inode, afr_readv_wind, AFR_DATA_TRANSACTION);
 
-        return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND(readv, frame, -1, op_errno, 0, 0, 0, 0, 0);
+    AFR_STACK_UNWIND(readv, frame, -1, op_errno, 0, 0, 0, 0, 0);
 
-	return 0;
+    return 0;
 }
 
 /* }}} */
@@ -1908,77 +1822,73 @@ out:
 /* {{{ seek */
 
 int
-afr_seek_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, off_t offset, dict_t *xdata)
+afr_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, off_t offset, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret < 0) {
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-
-                afr_read_txn_continue (frame, this, (long) cookie);
-                return 0;
-        }
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-        AFR_STACK_UNWIND (seek, frame, op_ret, op_errno, offset, xdata);
+        afr_read_txn_continue(frame, this, (long)cookie);
         return 0;
-}
+    }
 
+    AFR_STACK_UNWIND(seek, frame, op_ret, op_errno, offset, xdata);
+    return 0;
+}
 
 int
-afr_seek_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_seek_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv = this->private;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        if (subvol == -1) {
-                AFR_STACK_UNWIND (seek, frame, local->op_ret, local->op_errno,
-                                  0, NULL);
-                return 0;
-        }
+    local = frame->local;
+    priv = this->private;
 
-        STACK_WIND_COOKIE (frame, afr_seek_cbk, (void *) (long) subvol,
-                           priv->children[subvol],
-                           priv->children[subvol]->fops->seek,
-                           local->fd, local->cont.seek.offset,
-                           local->cont.seek.what, local->xdata_req);
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(seek, frame, local->op_ret, local->op_errno, 0, NULL);
         return 0;
-}
+    }
 
+    STACK_WIND_COOKIE(
+        frame, afr_seek_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->seek, local->fd, local->cont.seek.offset,
+        local->cont.seek.what, local->xdata_req);
+    return 0;
+}
 
 int
-afr_seek (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-          gf_seek_what_t what, dict_t *xdata)
+afr_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+         gf_seek_what_t what, dict_t *xdata)
 {
-        afr_local_t    *local      = NULL;
-        int32_t         op_errno   = 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->op = GF_FOP_SEEK;
-        local->fd = fd_ref (fd);
-        local->cont.seek.offset = offset;
-        local->cont.seek.what = what;
-        if (xdata)
-                local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_SEEK;
+    local->fd = fd_ref(fd);
+    local->cont.seek.offset = offset;
+    local->cont.seek.what = what;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-        afr_read_txn (frame, this, fd->inode, afr_seek_wind,
-                      AFR_DATA_TRANSACTION);
+    afr_read_txn(frame, this, fd->inode, afr_seek_wind, AFR_DATA_TRANSACTION);
 
-        return 0;
+    return 0;
 out:
-        AFR_STACK_UNWIND (seek, frame, -1, op_errno, 0, NULL);
+    AFR_STACK_UNWIND(seek, frame, -1, op_errno, 0, NULL);
 
-        return 0;
+    return 0;
 }
 /* }}} */
diff --git a/xlators/cluster/afr/src/afr-inode-read.h b/xlators/cluster/afr/src/afr-inode-read.h
index d128134ef2a..8c982bc7e6f 100644
--- a/xlators/cluster/afr/src/afr-inode-read.h
+++ b/xlators/cluster/afr/src/afr-inode-read.h
@@ -12,34 +12,34 @@
 #define __INODE_READ_H__
 
 int32_t
-afr_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask, dict_t *xdata);
+afr_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata);
 
 int32_t
-afr_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc, dict_t *xdata);
+afr_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
 
 int32_t
-afr_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, dict_t *xdata);
+afr_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
 
 int32_t
-afr_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size, dict_t *xdata);
+afr_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata);
 
 int32_t
-afr_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata);
+afr_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata);
 
 int32_t
-afr_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name, dict_t *xdata);
+afr_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata);
 
 int32_t
-afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name, dict_t *xdata);
-
+afr_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata);
 
 int
-afr_handle_quota_size (call_frame_t *frame, xlator_t *this);
+afr_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+         gf_seek_what_t what, dict_t *xdata);
+int
+afr_handle_quota_size(call_frame_t *frame, xlator_t *this);
 #endif /* __INODE_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index 9a9ef2e90a5..1d6e4f3570a 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -8,798 +8,779 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
 #include "protocol-common.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "afr-transaction.h"
 #include "afr-self-heal.h"
 #include "afr-messages.h"
 
 static void
-__afr_inode_write_finalize (call_frame_t *frame, xlator_t *this)
-{
-	int                       i               = 0;
-        int                       ret             = 0;
-	int                       read_subvol     = 0;
-        struct iatt              *stbuf           = NULL;
-	afr_local_t              *local           = NULL;
-	afr_private_t            *priv            = NULL;
-        afr_read_subvol_args_t    args            = {0,};
-
-	local = frame->local;
-	priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, local->inode, out);
-
-        /*This code needs to stay till DHT sends fops on linked
-         * inodes*/
-        if (!inode_is_linked (local->inode)) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!local->replies[i].valid)
-                                continue;
-                        if (local->replies[i].op_ret == -1)
-                                continue;
-                        if (!gf_uuid_is_null
-                                        (local->replies[i].poststat.ia_gfid)) {
-                                gf_uuid_copy (args.gfid,
-                                            local->replies[i].poststat.ia_gfid);
-                                args.ia_type =
-                                        local->replies[i].poststat.ia_type;
-                                break;
-                        } else {
-                                ret = dict_get_bin (local->replies[i].xdata,
-                                                    DHT_IATT_IN_XDATA_KEY,
-                                                    (void **) &stbuf);
-                                if (ret)
-                                        continue;
-                                gf_uuid_copy (args.gfid, stbuf->ia_gfid);
-                                args.ia_type = stbuf->ia_type;
-                                break;
-                        }
-                }
+__afr_inode_write_finalize(call_frame_t *frame, xlator_t *this)
+{
+    int i = 0;
+    int ret = 0;
+    int read_subvol = 0;
+    struct iatt *stbuf = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+
+    local = frame->local;
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, local->inode, out);
+
+    /*This code needs to stay till DHT sends fops on linked
+     * inodes*/
+    if (!inode_is_linked(local->inode)) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!local->replies[i].valid)
+                continue;
+            if (local->replies[i].op_ret == -1)
+                continue;
+            if (!gf_uuid_is_null(local->replies[i].poststat.ia_gfid)) {
+                gf_uuid_copy(args.gfid, local->replies[i].poststat.ia_gfid);
+                args.ia_type = local->replies[i].poststat.ia_type;
+                break;
+            } else {
+                ret = dict_get_bin(local->replies[i].xdata,
+                                   DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
+                if (ret)
+                    continue;
+                gf_uuid_copy(args.gfid, stbuf->ia_gfid);
+                args.ia_type = stbuf->ia_type;
+                break;
+            }
         }
-
-        if (local->transaction.type == AFR_METADATA_TRANSACTION) {
-                read_subvol = afr_metadata_subvol_get (local->inode, this,
-                                                       NULL, local->readable,
-                                                       NULL, &args);
-	} else {
-                read_subvol = afr_data_subvol_get (local->inode, this,
-                                                   NULL, local->readable,
-                                                   NULL, &args);
+    }
+
+    if (local->transaction.type == AFR_METADATA_TRANSACTION) {
+        read_subvol = afr_metadata_subvol_get(local->inode, this, NULL,
+                                              local->readable, NULL, &args);
+    } else {
+        read_subvol = afr_data_subvol_get(local->inode, this, NULL,
+                                          local->readable, NULL, &args);
+    }
+
+    local->op_ret = -1;
+    local->op_errno = afr_final_errno(local, priv);
+    afr_pick_error_xdata(local, priv, local->inode, local->readable, NULL,
+                         NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0)
+            continue;
+
+        /* Order of checks in the compound conditional
+           below is important.
+
+           - Highest precedence: largest op_ret
+           - Next precedence: if all op_rets are equal, read subvol
+           - Least precedence: any succeeded subvol
+        */
+        if ((local->op_ret < local->replies[i].op_ret) ||
+            ((local->op_ret == local->replies[i].op_ret) &&
+             (i == read_subvol))) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+
+            local->cont.inode_wfop.prebuf = local->replies[i].prestat;
+            local->cont.inode_wfop.postbuf = local->replies[i].poststat;
+
+            if (local->replies[i].xdata) {
+                if (local->xdata_rsp)
+                    dict_unref(local->xdata_rsp);
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+            if (local->replies[i].xattr) {
+                if (local->xattr_rsp)
+                    dict_unref(local->xattr_rsp);
+                local->xattr_rsp = dict_ref(local->replies[i].xattr);
+            }
         }
+    }
 
-	local->op_ret = -1;
-	local->op_errno = afr_final_errno (local, priv);
-        afr_pick_error_xdata (local, priv, local->inode, local->readable, NULL,
-                              NULL);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].valid)
-			continue;
-		if (local->replies[i].op_ret < 0)
-			continue;
-
-		/* Order of checks in the compound conditional
-		   below is important.
-
-		   - Highest precedence: largest op_ret
-		   - Next precendence: if all op_rets are equal, read subvol
-		   - Least precedence: any succeeded subvol
-		*/
-		if ((local->op_ret < local->replies[i].op_ret) ||
-		    ((local->op_ret == local->replies[i].op_ret) &&
-		     (i == read_subvol))) {
-
-			local->op_ret = local->replies[i].op_ret;
-			local->op_errno = local->replies[i].op_errno;
-
-			local->cont.inode_wfop.prebuf =
-				local->replies[i].prestat;
-			local->cont.inode_wfop.postbuf =
-				local->replies[i].poststat;
-
-			if (local->replies[i].xdata) {
-				if (local->xdata_rsp)
-					dict_unref (local->xdata_rsp);
-				local->xdata_rsp =
-					dict_ref (local->replies[i].xdata);
-			}
-			if (local->replies[i].xattr) {
-				if (local->xattr_rsp)
-					dict_unref (local->xattr_rsp);
-				local->xattr_rsp =
-					dict_ref (local->replies[i].xattr);
-			}
-		}
-	}
-
-        afr_set_in_flight_sb_status (this, frame, local->inode);
+    afr_set_in_flight_sb_status(this, frame, local->inode);
 out:
-        return;
+    return;
 }
 
-
 static void
-__afr_inode_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
-			int op_ret, int op_errno,
-			struct iatt *prebuf, struct iatt *postbuf,
-			dict_t *xattr, dict_t *xdata)
+__afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                       int op_ret, int op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	local->replies[child_index].valid = 1;
+    local->replies[child_index].valid = 1;
 
-        if (AFR_IS_ARBITER_BRICK(priv, child_index) && op_ret == 1)
-                op_ret = iov_length (local->cont.writev.vector,
-                                     local->cont.writev.count);
+    if (AFR_IS_ARBITER_BRICK(priv, child_index) && op_ret == 1)
+        op_ret = iov_length(local->cont.writev.vector,
+                            local->cont.writev.count);
 
-	local->replies[child_index].op_ret = op_ret;
-	local->replies[child_index].op_errno = op_errno;
-        if (xdata)
-                local->replies[child_index].xdata = dict_ref (xdata);
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
 
-	if (op_ret >= 0) {
-		if (prebuf)
-			local->replies[child_index].prestat = *prebuf;
-		if (postbuf)
-			local->replies[child_index].poststat = *postbuf;
-		if (xattr)
-			local->replies[child_index].xattr = dict_ref (xattr);
-	} else {
-		afr_transaction_fop_failed (frame, this, child_index);
-	}
+    if (op_ret >= 0) {
+        if (prebuf)
+            local->replies[child_index].prestat = *prebuf;
+        if (postbuf)
+            local->replies[child_index].poststat = *postbuf;
+        if (xattr)
+            local->replies[child_index].xattr = dict_ref(xattr);
+    } else {
+        afr_transaction_fop_failed(frame, this, child_index);
+    }
 
-        return;
+    return;
 }
 
-
 static int
-__afr_inode_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xattr, dict_t *xdata)
-{
-        afr_local_t *local = NULL;
-        int child_index = (long) cookie;
-        int call_count = -1;
-        afr_private_t *priv = NULL;
-
-        priv = this->private;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                __afr_inode_write_fill (frame, this, child_index, op_ret,
-					op_errno, prebuf, postbuf, xattr,
-					xdata);
+__afr_inode_write_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xattr, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    {
+        __afr_inode_write_fill(frame, this, child_index, op_ret, op_errno,
+                               prebuf, postbuf, xattr, xdata);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        __afr_inode_write_finalize(frame, this);
+
+        if (afr_txn_nothing_failed(frame, this)) {
+            /*if it did pre-op, it will do post-op changing ctime*/
+            if (priv->consistent_metadata && afr_needs_changelog_update(local))
+                afr_zero_fill_stat(local);
+            local->transaction.unwind(frame, this);
         }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-		__afr_inode_write_finalize (frame, this);
 
-		if (afr_txn_nothing_failed (frame, this)) {
-                        /*if it did pre-op, it will do post-op changing ctime*/
-                        if (priv->consistent_metadata &&
-                            afr_needs_changelog_update (local))
-                                afr_zero_fill_stat (local);
-                        local->transaction.unwind (frame, this);
-                }
+        afr_transaction_resume(frame, this);
+    }
 
-                afr_transaction_resume (frame, this);
-        }
-
-        return 0;
+    return 0;
 }
 
 /* {{{ writev */
 
 void
-afr_writev_copy_outvars (call_frame_t *src_frame, call_frame_t *dst_frame)
+afr_writev_copy_outvars(call_frame_t *src_frame, call_frame_t *dst_frame)
 {
-        afr_local_t *src_local = NULL;
-        afr_local_t *dst_local = NULL;
+    afr_local_t *src_local = NULL;
+    afr_local_t *dst_local = NULL;
 
-        src_local = src_frame->local;
-        dst_local = dst_frame->local;
+    src_local = src_frame->local;
+    dst_local = dst_frame->local;
 
-        dst_local->op_ret = src_local->op_ret;
-        dst_local->op_errno = src_local->op_errno;
-        dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
-        dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
-	if (src_local->xdata_rsp)
-		dst_local->xdata_rsp = dict_ref (src_local->xdata_rsp);
+    dst_local->op_ret = src_local->op_ret;
+    dst_local->op_errno = src_local->op_errno;
+    dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
+    dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
+    if (src_local->xdata_rsp)
+        dst_local->xdata_rsp = dict_ref(src_local->xdata_rsp);
 }
 
 void
-afr_writev_unwind (call_frame_t *frame, xlator_t *this)
+afr_writev_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t *priv = this->private;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
 
-        local = frame->local;
+    local = frame->local;
 
-       if (priv->consistent_metadata)
-               afr_zero_fill_stat (local);
+    if (priv->consistent_metadata)
+        afr_zero_fill_stat(local);
 
-        AFR_STACK_UNWIND (writev, frame,
-                          local->op_ret, local->op_errno,
-                          &local->cont.inode_wfop.prebuf,
-                          &local->cont.inode_wfop.postbuf,
-                          local->xdata_rsp);
+    AFR_STACK_UNWIND(writev, frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
 }
 
-
 int
-afr_transaction_writev_unwind (call_frame_t *frame, xlator_t *this)
+afr_transaction_writev_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *fop_frame = NULL;
+    call_frame_t *fop_frame = NULL;
 
-        fop_frame = afr_transaction_detach_fop_frame (frame);
+    fop_frame = afr_transaction_detach_fop_frame(frame);
 
-        if (fop_frame) {
-                afr_writev_copy_outvars (frame, fop_frame);
-                afr_writev_unwind (fop_frame, this);
-        }
-        return 0;
+    if (fop_frame) {
+        afr_writev_copy_outvars(frame, fop_frame);
+        afr_writev_unwind(fop_frame, this);
+    }
+    return 0;
 }
 
 static void
-afr_writev_handle_short_writes (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int           i      = 0;
-
-        local = frame->local;
-        priv = this->private;
-        /*
-         * We already have the best case result of the writev calls staged
-         * as the return value. Any writev that returns some value less
-         * than the best case is now out of sync, so mark the fop as
-         * failed. Note that fops that have returned with errors have
-         * already been marked as failed.
-         */
-        for (i = 0; i < priv->child_count; i++) {
-                if ((!local->replies[i].valid) ||
-                    (local->replies[i].op_ret == -1))
-                        continue;
-
-                if (local->replies[i].op_ret < local->op_ret)
-                        afr_transaction_fop_failed (frame, this, i);
-        }
+afr_writev_handle_short_writes(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+    /*
+     * We already have the best case result of the writev calls staged
+     * as the return value. Any writev that returns some value less
+     * than the best case is now out of sync, so mark the fop as
+     * failed. Note that fops that have returned with errors have
+     * already been marked as failed.
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if ((!local->replies[i].valid) || (local->replies[i].op_ret == -1))
+            continue;
+
+        if (local->replies[i].op_ret < local->op_ret)
+            afr_transaction_fop_failed(frame, this, i);
+    }
 }
 
 void
-afr_inode_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
+afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                      struct iatt *postbuf, dict_t *xdata)
 {
-        int ret = 0;
-        afr_local_t *local = frame->local;
-        uint32_t open_fd_count = 0;
-        uint32_t write_is_append = 0;
-
-        LOCK (&frame->lock);
-        {
-                __afr_inode_write_fill (frame, this, child_index, op_ret,
-					op_errno, prebuf, postbuf, NULL, xdata);
-		if (op_ret == -1 || !xdata)
-			goto unlock;
-
-		write_is_append = 0;
-		ret = dict_get_uint32 (xdata, GLUSTERFS_WRITE_IS_APPEND,
-				       &write_is_append);
-		if (ret || !write_is_append)
-			local->append_write = _gf_false;
-
-                ret = dict_get_uint32 (xdata, GLUSTERFS_ACTIVE_FD_COUNT,
-                                       &open_fd_count);
-                if (ret < 0)
-                        goto unlock;
-		if (open_fd_count > local->open_fd_count) {
-                        local->open_fd_count = open_fd_count;
-                        local->update_open_fd_count = _gf_true;
-		}
+    int ret = 0;
+    afr_local_t *local = frame->local;
+    uint32_t open_fd_count = 0;
+    uint32_t write_is_append = 0;
+    int32_t num_inodelks = 0;
+
+    LOCK(&frame->lock);
+    {
+        __afr_inode_write_fill(frame, this, child_index, op_ret, op_errno,
+                               prebuf, postbuf, NULL, xdata);
+        if (op_ret == -1 || !xdata)
+            goto unlock;
+
+        write_is_append = 0;
+        ret = dict_get_uint32(xdata, GLUSTERFS_WRITE_IS_APPEND,
+                              &write_is_append);
+        if (ret || !write_is_append)
+            local->append_write = _gf_false;
+
+        ret = dict_get_uint32(xdata, GLUSTERFS_ACTIVE_FD_COUNT, &open_fd_count);
+        if (ret < 0)
+            goto unlock;
+        if (open_fd_count > local->open_fd_count) {
+            local->open_fd_count = open_fd_count;
+            local->update_open_fd_count = _gf_true;
         }
+
+        ret = dict_get_int32_sizen(xdata, GLUSTERFS_INODELK_COUNT,
+                                   &num_inodelks);
+        if (ret < 0)
+            goto unlock;
+        if (num_inodelks > local->num_inodelks) {
+            local->num_inodelks = num_inodelks;
+            local->update_num_inodelks = _gf_true;
+        }
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 }
 
 void
-afr_process_post_writev (call_frame_t *frame, xlator_t *this)
+afr_process_post_writev(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t     *local = NULL;
+    afr_local_t *local = NULL;
+    afr_lock_t *lock = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (!local->stable_write && !local->append_write)
-                /* An appended write removes the necessity to
-                   fsync() the file. This is because self-heal
-                   has the logic to check for larger file when
-                   the xattrs are not reliably pointing at
-                   a stale file.
-                */
-                afr_fd_report_unstable_write (this, local);
+    if (!local->stable_write && !local->append_write)
+        /* An appended write removes the necessity to
+           fsync() the file. This is because self-heal
+           has the logic to check for larger file when
+           the xattrs are not reliably pointing at
+           a stale file.
+        */
+        afr_fd_report_unstable_write(this, local);
 
-        __afr_inode_write_finalize (frame, this);
+    __afr_inode_write_finalize(frame, this);
 
-        afr_writev_handle_short_writes (frame, this);
-
-        if (local->update_open_fd_count)
-                local->inode_ctx->open_fd_count = local->open_fd_count;
+    afr_writev_handle_short_writes(frame, this);
 
+    if (local->update_open_fd_count)
+        local->inode_ctx->open_fd_count = local->open_fd_count;
+    if (local->update_num_inodelks &&
+        local->transaction.type == AFR_DATA_TRANSACTION) {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        lock->num_inodelks = local->num_inodelks;
+    }
 }
 
 int
-afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
+afr_writev_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
 {
-        call_frame_t    *fop_frame = NULL;
-        int child_index = (long) cookie;
-        int call_count  = -1;
-
-        afr_inode_write_fill (frame, this, child_index, op_ret, op_errno,
-                              prebuf, postbuf, xdata);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_process_post_writev (frame, this);
-
-                if (!afr_txn_nothing_failed (frame, this)) {
-                        //Don't unwind until post-op is complete
-                        afr_transaction_resume (frame, this);
-                } else {
-                /*
-                 * Generally inode-write fops do transaction.unwind then
-                 * transaction.resume, but writev needs to make sure that
-                 * delayed post-op frame is placed in fdctx before unwind
-                 * happens. This prevents the race of flush doing the
-                 * changelog wakeup first in fuse thread and then this
-                 * writev placing its delayed post-op frame in fdctx.
-                 * This helps flush make sure all the delayed post-ops are
-                 * completed.
-                 */
-
-                        fop_frame = afr_transaction_detach_fop_frame (frame);
-                        afr_writev_copy_outvars (frame, fop_frame);
-                        afr_transaction_resume (frame, this);
-                        afr_writev_unwind (fop_frame, this);
-                }
+    call_frame_t *fop_frame = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+
+    afr_inode_write_fill(frame, this, child_index, op_ret, op_errno, prebuf,
+                         postbuf, xdata);
+
+    call_count = afr_frame_return(frame);
+
+    if (call_count == 0) {
+        afr_process_post_writev(frame, this);
+
+        if (!afr_txn_nothing_failed(frame, this)) {
+            // Don't unwind until post-op is complete
+            afr_transaction_resume(frame, this);
+        } else {
+            /*
+             * Generally inode-write fops do transaction.unwind then
+             * transaction.resume, but writev needs to make sure that
+             * delayed post-op frame is placed in fdctx before unwind
+             * happens. This prevents the race of flush doing the
+             * changelog wakeup first in fuse thread and then this
+             * writev placing its delayed post-op frame in fdctx.
+             * This helps flush make sure all the delayed post-ops are
+             * completed.
+             */
+
+            fop_frame = afr_transaction_detach_fop_frame(frame);
+            afr_writev_copy_outvars(frame, fop_frame);
+            afr_transaction_resume(frame, this);
+            afr_writev_unwind(fop_frame, this);
         }
-        return 0;
+    }
+    return 0;
 }
 
 static int
-afr_arbiter_writev_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_arbiter_writev_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = frame->local;
-        afr_private_t *priv = this->private;
-        static char byte = 0xFF;
-        static struct iovec vector = {&byte, 1};
-        int32_t count = 1;
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    static char byte = 0xFF;
+    static struct iovec vector = {&byte, 1};
+    int32_t count = 1;
 
-        STACK_WIND_COOKIE (frame, afr_writev_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->writev,
-			   local->fd, &vector, count, local->cont.writev.offset,
-			   local->cont.writev.flags, local->cont.writev.iobref,
-			   local->xdata_req);
+    STACK_WIND_COOKIE(
+        frame, afr_writev_wind_cbk, (void *)(long)subvol,
+        priv->children[subvol], priv->children[subvol]->fops->writev, local->fd,
+        &vector, count, local->cont.writev.offset, local->cont.writev.flags,
+        local->cont.writev.iobref, local->xdata_req);
 
-        return 0;
+    return 0;
 }
 
 int
-afr_writev_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_writev_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        if (AFR_IS_ARBITER_BRICK(priv, subvol)) {
-                afr_arbiter_writev_wind (frame, this, subvol);
-                return 0;
-        }
-
-	STACK_WIND_COOKIE (frame, afr_writev_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->writev,
-			   local->fd, local->cont.writev.vector,
-			   local->cont.writev.count, local->cont.writev.offset,
-			   local->cont.writev.flags, local->cont.writev.iobref,
-			   local->xdata_req);
+    if (AFR_IS_ARBITER_BRICK(priv, subvol)) {
+        afr_arbiter_writev_wind(frame, this, subvol);
         return 0;
-}
+    }
 
+    STACK_WIND_COOKIE(frame, afr_writev_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->writev, local->fd,
+                      local->cont.writev.vector, local->cont.writev.count,
+                      local->cont.writev.offset, local->cont.writev.flags,
+                      local->cont.writev.iobref, local->xdata_req);
+    return 0;
+}
 
 int
-afr_do_writev (call_frame_t *frame, xlator_t *this)
+afr_do_writev(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t    *transaction_frame = NULL;
-        afr_local_t     *local             = NULL;
-        int             ret   = -1;
-        int             op_errno = ENOMEM;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    call_frame_t *transaction_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        local = frame->local;
-        transaction_frame->local = local;
-	frame->local = NULL;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	if (!AFR_FRAME_INIT (frame, op_errno))
-		goto out;
+    local = frame->local;
+    transaction_frame->local = local;
+    frame->local = NULL;
 
-        local->op = GF_FOP_WRITE;
+    if (!AFR_FRAME_INIT(frame, op_errno))
+        goto out;
 
-        local->transaction.wind   = afr_writev_wind;
-        local->transaction.unwind = afr_transaction_writev_unwind;
+    local->op = GF_FOP_WRITE;
 
-        local->transaction.main_frame = frame;
+    local->transaction.wind = afr_writev_wind;
+    local->transaction.unwind = afr_transaction_writev_unwind;
 
-        if (local->fd->flags & O_APPEND) {
-               /*
-                * Backend vfs ignores the 'offset' for append mode fd so
-                * locking just the region provided for the writev does not
-                * give consistency guarantee. The actual write may happen at a
-                * completely different range than the one provided by the
-                * offset, len in the fop. So lock the entire file.
-                */
-                local->transaction.start   = 0;
-                local->transaction.len     = 0;
-        } else {
-                local->transaction.start   = local->cont.writev.offset;
-                local->transaction.len     = iov_length (local->cont.writev.vector,
-                                                         local->cont.writev.count);
-        }
-
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-            op_errno = -ret;
-            goto out;
-        }
+    local->transaction.main_frame = frame;
 
-	return 0;
+    if (local->fd->flags & O_APPEND) {
+        /*
+         * Backend vfs ignores the 'offset' for append mode fd so
+         * locking just the region provided for the writev does not
+         * give consistency guarantee. The actual write may happen at a
+         * completely different range than the one provided by the
+         * offset, len in the fop. So lock the entire file.
+         */
+        local->transaction.start = 0;
+        local->transaction.len = 0;
+    } else {
+        local->transaction.start = local->cont.writev.offset;
+        local->transaction.len = iov_length(local->cont.writev.vector,
+                                            local->cont.writev.count);
+    }
+
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int
-afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref, dict_t *xdata)
+afr_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int op_errno = ENOMEM;
-        int ret = -1;
+    afr_local_t *local = NULL;
+    int op_errno = ENOMEM;
+    int ret = -1;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.writev.vector = iov_dup (vector, count);
-	if (!local->cont.writev.vector)
-		goto out;
-        local->cont.writev.count      = count;
-        local->cont.writev.offset     = offset;
-        local->cont.writev.flags      = flags;
-        local->cont.writev.iobref     = iobref_ref (iobref);
+    local->cont.writev.vector = iov_dup(vector, count);
+    if (!local->cont.writev.vector)
+        goto out;
+    local->cont.writev.count = count;
+    local->cont.writev.offset = offset;
+    local->cont.writev.flags = flags;
+    local->cont.writev.iobref = iobref_ref(iobref);
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->fd = fd_ref (fd);
-        ret = afr_set_inode_local (this, local, fd->inode);
-        if (ret)
-                goto out;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        if (dict_set_uint32 (local->xdata_req, GLUSTERFS_ACTIVE_FD_COUNT, 4)) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    if (dict_set_uint32(local->xdata_req, GLUSTERFS_ACTIVE_FD_COUNT, 4)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (dict_set_str_sizen(local->xdata_req, GLUSTERFS_INODELK_DOM_COUNT,
+                           this->name)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-	if (dict_set_uint32 (local->xdata_req, GLUSTERFS_WRITE_IS_APPEND, 4)) {
-		op_errno = ENOMEM;
-		goto out;
-	}
+    if (dict_set_uint32(local->xdata_req, GLUSTERFS_WRITE_IS_APPEND, 4)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-	/* Set append_write to be true speculatively. If on any
-	   server it turns not be true, we unset it in the
-	   callback.
-	*/
-	local->append_write = _gf_true;
+    /* Set append_write to be true speculatively. If on any
+       server it turns not be true, we unset it in the
+       callback.
+    */
+    local->append_write = _gf_true;
 
-	/* detect here, but set it in writev_wind_cbk *after* the unstable
-	   write is performed
-	*/
-	local->stable_write = !!((fd->flags|flags)&(O_SYNC|O_DSYNC));
+    /* detect here, but set it in writev_wind_cbk *after* the unstable
+       write is performed
+    */
+    local->stable_write = !!((fd->flags | flags) & (O_SYNC | O_DSYNC));
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-        afr_do_writev (frame, this);
+    afr_do_writev(frame, this);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+    AFR_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ truncate */
 
 int
-afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
+afr_truncate_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (truncate, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(truncate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
+afr_truncate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
-		local->stable_write = _gf_false;
+    if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+        local->stable_write = _gf_false;
 
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_truncate_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_truncate_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_truncate_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->truncate,
-			   &local->loc, local->cont.truncate.offset,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_truncate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->truncate, &local->loc,
+                      local->cont.truncate.offset, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_truncate (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, off_t offset, dict_t *xdata)
+afr_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
 {
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-	int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.truncate.offset  = offset;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.truncate.offset = offset;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-	local->transaction.wind   = afr_truncate_wind;
-        local->transaction.unwind = afr_truncate_unwind;
+    local->transaction.wind = afr_truncate_wind;
+    local->transaction.unwind = afr_truncate_unwind;
 
-        loc_copy (&local->loc, loc);
-        ret = afr_set_inode_local (this, local, loc->inode);
-        if (ret)
-                goto out;
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-        local->op = GF_FOP_TRUNCATE;
+    local->op = GF_FOP_TRUNCATE;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = offset;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = offset;
+    local->transaction.len = 0;
 
-	/* Set it true speculatively, will get reset in afr_truncate_wind_cbk
-	   if truncate was not a NOP */
-	local->stable_write = _gf_true;
+    /* Set it true speculatively, will get reset in afr_truncate_wind_cbk
+       if truncate was not a NOP */
+    local->stable_write = _gf_true;
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ ftruncate */
 
-
 int
-afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
+afr_ftruncate_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(ftruncate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+afr_ftruncate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
-		local->stable_write = _gf_false;
+    if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+        local->stable_write = _gf_false;
 
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_ftruncate_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_ftruncate_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_ftruncate_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->ftruncate,
-			   local->fd, local->cont.ftruncate.offset,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_ftruncate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->ftruncate, local->fd,
+                      local->cont.ftruncate.offset, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	       dict_t *xdata)
+afr_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	call_frame_t *transaction_frame = NULL;
-	int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        local = AFR_FRAME_INIT (transaction_frame, op_errno);
-        if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.ftruncate.offset  = offset;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.ftruncate.offset = offset;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->fd = fd_ref (fd);
-        ret = afr_set_inode_local (this, local, fd->inode);
-        if (ret)
-                goto out;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        local->op = GF_FOP_FTRUNCATE;
+    local->op = GF_FOP_FTRUNCATE;
 
-	local->transaction.wind   = afr_ftruncate_wind;
-        local->transaction.unwind = afr_ftruncate_unwind;
+    local->transaction.wind = afr_ftruncate_wind;
+    local->transaction.unwind = afr_ftruncate_unwind;
 
-        local->transaction.main_frame = frame;
+    local->transaction.main_frame = frame;
 
-        local->transaction.start   = local->cont.ftruncate.offset;
-        local->transaction.len     = 0;
+    local->transaction.start = local->cont.ftruncate.offset;
+    local->transaction.len = 0;
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-	/* Set it true speculatively, will get reset in afr_ftruncate_wind_cbk
-	   if truncate was not a NOP */
-	local->stable_write = _gf_true;
+    /* Set it true speculatively, will get reset in afr_ftruncate_wind_cbk
+       if truncate was not a NOP */
+    local->stable_write = _gf_true;
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+    AFR_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* }}} */
@@ -807,1813 +788,1778 @@ out:
 /* {{{ setattr */
 
 int
-afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_setattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (setattr, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf,
-			  local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(setattr, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno,
-                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
+afr_setattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, struct iatt *preop,
+                     struct iatt *postop, dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      preop, postop, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, preop,
+                                 postop, NULL, xdata);
 }
 
-
 int
-afr_setattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_setattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_setattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->setattr,
-			   &local->loc, &local->cont.setattr.in_buf,
-			   local->cont.setattr.valid, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_setattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->setattr, &local->loc,
+                      &local->cont.setattr.in_buf, local->cont.setattr.valid,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
-	     int32_t valid, dict_t *xdata)
+afr_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+            int32_t valid, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.setattr.in_buf = *buf;
-        local->cont.setattr.valid  = valid;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.setattr.in_buf = *buf;
+    local->cont.setattr.valid = valid;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_setattr_wind;
-        local->transaction.unwind = afr_setattr_unwind;
+    local->transaction.wind = afr_setattr_wind;
+    local->transaction.unwind = afr_setattr_unwind;
 
-        loc_copy (&local->loc, loc);
-        ret = afr_set_inode_local (this, local, loc->inode);
-        if (ret)
-                goto out;
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_SETATTR;
+    local->op = GF_FOP_SETATTR;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* {{{ fsetattr */
 
 int
-afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fsetattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(fsetattr, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
+afr_fsetattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *preop,
+                      struct iatt *postop, dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      preop, postop, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, preop,
+                                 postop, NULL, xdata);
 }
 
-
 int
-afr_fsetattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fsetattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_fsetattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fsetattr,
-			   local->fd, &local->cont.fsetattr.in_buf,
-			   local->cont.fsetattr.valid, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_fsetattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsetattr, local->fd,
+                      &local->cont.fsetattr.in_buf, local->cont.fsetattr.valid,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata)
+afr_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
+             int32_t valid, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.fsetattr.in_buf = *buf;
-        local->cont.fsetattr.valid  = valid;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.fsetattr.in_buf = *buf;
+    local->cont.fsetattr.valid = valid;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_fsetattr_wind;
-        local->transaction.unwind = afr_fsetattr_unwind;
+    local->transaction.wind = afr_fsetattr_wind;
+    local->transaction.unwind = afr_fsetattr_unwind;
 
-        local->fd                 = fd_ref (fd);
-        ret = afr_set_inode_local (this, local, fd->inode);
-        if (ret)
-                goto out;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_FSETATTR;
+    local->op = GF_FOP_FSETATTR;
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /* {{{ setxattr */
 
-
 int
-afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_setxattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (setxattr, main_frame, local->op_ret, local->op_errno,
-			  local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(setxattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_setxattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
 }
 
-
 int
-afr_setxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_setxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t      *local         = NULL;
-        afr_private_t    *priv          = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->setxattr,
-			   &local->loc, local->cont.setxattr.dict,
-			   local->cont.setxattr.flags, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_setxattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->setxattr, &local->loc,
+                      local->cont.setxattr.dict, local->cont.setxattr.flags,
+                      local->xdata_req);
+    return 0;
 }
 
 int
-afr_emptyb_set_pending_changelog_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this, int op_ret, int op_errno,
-                                  dict_t *xattr, dict_t *xdata)
+afr_emptyb_set_pending_changelog_cbk(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int op_ret, int op_errno,
+                                     dict_t *xattr, dict_t *xdata)
 
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int i, ret = 0;
-        char *op_type = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i, ret = 0;
+    char *op_type = NULL;
 
-        local = frame->local;
-        priv = this->private;
-        i = (long) cookie;
+    local = frame->local;
+    priv = this->private;
+    i = (long)cookie;
 
-        local->replies[i].valid = 1;
-        local->replies[i].op_ret = op_ret;
-        local->replies[i].op_errno = op_errno;
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
 
-        ret = dict_get_str (local->xdata_req, "replicate-brick-op", &op_type);
-        if (ret)
-                goto out;
+    ret = dict_get_str_sizen(local->xdata_req, "replicate-brick-op", &op_type);
+    if (ret)
+        goto out;
 
-        gf_msg (this->name, op_ret ? GF_LOG_ERROR : GF_LOG_INFO,
-                op_ret ? op_errno : 0,
-                afr_get_msg_id (op_type),
-                "Set of pending xattr %s on"
-                " %s.", op_ret ? "failed" : "succeeded",
-                priv->children[i]->name);
+    gf_smsg(this->name, op_ret ? GF_LOG_ERROR : GF_LOG_INFO,
+            op_ret ? op_errno : 0, AFR_MSG_SET_PEND_XATTR, "name=%s",
+            priv->children[i]->name, "op_ret=%s",
+            op_ret ? "failed" : "succeeded", NULL);
 
 out:
-        syncbarrier_wake (&local->barrier);
-        return 0;
+    syncbarrier_wake(&local->barrier);
+    return 0;
 }
 
 int
-afr_emptyb_set_pending_changelog (call_frame_t *frame, xlator_t *this,
-                                unsigned char *locked_nodes)
+afr_emptyb_set_pending_changelog(call_frame_t *frame, xlator_t *this,
+                                 unsigned char *locked_nodes)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int ret = 0, i = 0;
-
-        local = frame->local;
-        priv = this->private;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int ret = 0, i = 0;
 
-        AFR_ONLIST (locked_nodes, frame, afr_emptyb_set_pending_changelog_cbk,
-                    xattrop, &local->loc, GF_XATTROP_ADD_ARRAY,
-                    local->xattr_req, NULL);
-
-        /* It is sufficient if xattrop was successful on one child */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->replies[i].valid)
-                        continue;
-
-                if (local->replies[i].op_ret == 0) {
-                        ret = 0;
-                        goto out;
-                } else {
-                        ret = afr_higher_errno (ret,
-                                                local->replies[i].op_errno);
-                }
-        }
-out:
-        return -ret;
-}
+    local = frame->local;
+    priv = this->private;
 
-int
-_afr_handle_empty_brick_type (xlator_t *this, call_frame_t *frame,
-                            loc_t *loc, int empty_index,
-                            afr_transaction_type type,
-                            char *op_type)
-{
-        int              count            = 0;
-        int              ret              = -ENOMEM;
-        int              idx              = -1;
-        int              d_idx            = -1;
-        unsigned char   *locked_nodes     = NULL;
-        afr_local_t     *local            = NULL;
-        afr_private_t   *priv             = NULL;
+    AFR_ONLIST(locked_nodes, frame, afr_emptyb_set_pending_changelog_cbk,
+               xattrop, &local->loc, GF_XATTROP_ADD_ARRAY, local->xattr_req,
+               NULL);
 
-        priv = this->private;
-        local = frame->local;
+    /* It is sufficient if xattrop was successful on one child */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
 
-        locked_nodes = alloca0 (priv->child_count);
-
-        idx = afr_index_for_transaction_type (type);
-        d_idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-
-        local->pending = afr_matrix_create (priv->child_count,
-                                            AFR_NUM_CHANGE_LOGS);
-        if (!local->pending)
-                goto out;
-
-        local->pending[empty_index][idx] = hton32 (1);
-
-        if ((priv->esh_granular) && (type == AFR_ENTRY_TRANSACTION))
-                        local->pending[empty_index][d_idx] = hton32 (1);
-
-        local->xdata_req = dict_new ();
-        if (!local->xdata_req)
-                goto out;
-
-        ret = dict_set_str (local->xdata_req, "replicate-brick-op", op_type);
-        if (ret)
-                goto out;
-
-        local->xattr_req = dict_new ();
-        if (!local->xattr_req)
-                goto out;
-
-        ret = afr_set_pending_dict (priv, local->xattr_req, local->pending);
-        if (ret < 0)
-                goto out;
-
-        if (AFR_ENTRY_TRANSACTION == type) {
-                count = afr_selfheal_entrylk (frame, this, loc->inode,
-                                              this->name, NULL, locked_nodes);
+        if (local->replies[i].op_ret == 0) {
+            ret = 0;
+            goto out;
         } else {
-                count = afr_selfheal_inodelk (frame, this, loc->inode,
-                                              this->name, LLONG_MAX - 1, 0,
-                                              locked_nodes);
-        }
-
-        if (!count) {
-                gf_msg (this->name, GF_LOG_ERROR, EAGAIN,
-                        AFR_MSG_REPLACE_BRICK_STATUS, "Couldn't acquire lock on"
-                        " any child.");
-                ret = -EAGAIN;
-                goto unlock;
+            ret = afr_higher_errno(ret, local->replies[i].op_errno);
         }
+    }
+out:
+    return -ret;
+}
 
-        ret = afr_emptyb_set_pending_changelog (frame, this, locked_nodes);
-        if (ret)
-                goto unlock;
-        ret = 0;
+static int
+_afr_handle_empty_brick_type(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                             int empty_index, afr_transaction_type type,
+                             char *op_type, const int op_type_len)
+{
+    int count = 0;
+    int ret = -ENOMEM;
+    int idx = -1;
+    int d_idx = -1;
+    unsigned char *locked_nodes = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    locked_nodes = alloca0(priv->child_count);
+
+    idx = afr_index_for_transaction_type(type);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+
+    local->pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!local->pending)
+        goto out;
+
+    local->pending[empty_index][idx] = hton32(1);
+
+    if ((priv->esh_granular) && (type == AFR_ENTRY_TRANSACTION))
+        local->pending[empty_index][d_idx] = hton32(1);
+
+    local->xdata_req = dict_new();
+    if (!local->xdata_req)
+        goto out;
+
+    ret = dict_set_nstrn(local->xdata_req, "replicate-brick-op",
+                         SLEN("replicate-brick-op"), op_type, op_type_len);
+    if (ret)
+        goto out;
+
+    local->xattr_req = dict_new();
+    if (!local->xattr_req)
+        goto out;
+
+    ret = afr_set_pending_dict(priv, local->xattr_req, local->pending);
+    if (ret < 0)
+        goto out;
+
+    if (AFR_ENTRY_TRANSACTION == type) {
+        count = afr_selfheal_entrylk(frame, this, loc->inode, this->name, NULL,
+                                     locked_nodes);
+    } else {
+        count = afr_selfheal_inodelk(frame, this, loc->inode, this->name,
+                                     LLONG_MAX - 1, 0, locked_nodes);
+    }
+
+    if (!count) {
+        gf_smsg(this->name, GF_LOG_ERROR, EAGAIN, AFR_MSG_REPLACE_BRICK_STATUS,
+                NULL);
+        ret = -EAGAIN;
+        goto unlock;
+    }
+
+    ret = afr_emptyb_set_pending_changelog(frame, this, locked_nodes);
+    if (ret)
+        goto unlock;
+    ret = 0;
 unlock:
-        if (AFR_ENTRY_TRANSACTION == type) {
-                afr_selfheal_unentrylk (frame, this, loc->inode, this->name,
-                                        NULL, locked_nodes, NULL);
-        } else {
-                afr_selfheal_uninodelk (frame, this, loc->inode, this->name,
-                                        LLONG_MAX - 1, 0, locked_nodes);
-        }
+    if (AFR_ENTRY_TRANSACTION == type) {
+        afr_selfheal_unentrylk(frame, this, loc->inode, this->name, NULL,
+                               locked_nodes, NULL);
+    } else {
+        afr_selfheal_uninodelk(frame, this, loc->inode, this->name,
+                               LLONG_MAX - 1, 0, locked_nodes);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 void
-afr_brick_args_cleanup (void *opaque)
-{
-        afr_empty_brick_args_t *data = NULL;
-
-        data = opaque;
-        loc_wipe (&data->loc);
-        GF_FREE (data);
-}
-
-int
-_afr_handle_empty_brick_cbk (int ret, call_frame_t *frame, void *opaque)
-{
-        afr_brick_args_cleanup (opaque);
-        return 0;
-}
-
-int
-_afr_handle_empty_brick (void *opaque)
-{
-
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
-        int              empty_index       = -1;
-        int              ret            = -1;
-        int              op_errno       = ENOMEM;
-        call_frame_t    *frame          = NULL;
-        xlator_t        *this           = NULL;
-        char            *op_type        = NULL;
-        afr_empty_brick_args_t *data  = NULL;
-
-        data = opaque;
-        frame = data->frame;
-        empty_index = data->empty_index;
-        op_type = data->op_type;
-        this = frame->this;
-        priv = this->private;
-
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local)
-                goto out;
-
-        loc_copy (&local->loc, &data->loc);
-
-        gf_msg (this->name, GF_LOG_INFO, 0, 0, "New brick is : %s",
-                priv->children[empty_index]->name);
-
-        ret = _afr_handle_empty_brick_type (this, frame, &local->loc, empty_index,
-                                          AFR_METADATA_TRANSACTION, op_type);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-
-        dict_unref (local->xdata_req);
-        dict_unref (local->xattr_req);
-        afr_matrix_cleanup (local->pending, priv->child_count);
-        local->pending = NULL;
-        local->xattr_req = NULL;
-        local->xdata_req = NULL;
-
-        ret = _afr_handle_empty_brick_type (this, frame, &local->loc, empty_index,
-                                          AFR_ENTRY_TRANSACTION, op_type);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
+afr_brick_args_cleanup(void *opaque)
+{
+    afr_empty_brick_args_t *data = NULL;
+
+    data = opaque;
+    loc_wipe(&data->loc);
+    GF_FREE(data);
+}
+
+int
+_afr_handle_empty_brick_cbk(int ret, call_frame_t *frame, void *opaque)
+{
+    afr_brick_args_cleanup(opaque);
+    return 0;
+}
+
+int
+_afr_handle_empty_brick(void *opaque)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int empty_index = -1;
+    int ret = -1;
+    int op_errno = ENOMEM;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    char *op_type = NULL;
+    int op_type_len = 0;
+    afr_empty_brick_args_t *data = NULL;
+    call_frame_t *op_frame = NULL;
+
+    data = opaque;
+    frame = data->frame;
+    empty_index = data->empty_index;
+    if (!data->op_type)
+        goto out;
+
+    op_frame = copy_frame(frame);
+    if (!op_frame) {
+        ret = -1;
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_type = data->op_type;
+    op_type_len = strlen(op_type);
+    this = op_frame->this;
+    priv = this->private;
+
+    afr_set_lk_owner(op_frame, this, op_frame->root);
+    local = AFR_FRAME_INIT(op_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, &data->loc);
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, AFR_MSG_NEW_BRICK, "name=%s",
+            priv->children[empty_index]->name, NULL);
+
+    ret = _afr_handle_empty_brick_type(this, op_frame, &local->loc, empty_index,
+                                       AFR_METADATA_TRANSACTION, op_type,
+                                       op_type_len);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    dict_unref(local->xdata_req);
+    dict_unref(local->xattr_req);
+    afr_matrix_cleanup(local->pending, priv->child_count);
+    local->pending = NULL;
+    local->xattr_req = NULL;
+    local->xdata_req = NULL;
+
+    ret = _afr_handle_empty_brick_type(this, op_frame, &local->loc, empty_index,
+                                       AFR_ENTRY_TRANSACTION, op_type,
+                                       op_type_len);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 out:
-        AFR_STACK_UNWIND (setxattr, frame, ret, op_errno, NULL);
-        return 0;
+    if (op_frame) {
+        AFR_STACK_DESTROY(op_frame);
+    }
+    AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    return 0;
+}
+
+int
+afr_split_brain_resolve_do(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                           char *data)
+{
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = EINVAL;
+
+    local = frame->local;
+    local->xdata_req = dict_new();
+
+    if (!local->xdata_req) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(local->xdata_req, "heal-op",
+                               GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_str_sizen(local->xdata_req, "child-name", data);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    /* set spb choice to -1 whether heal succeeds or not:
+     * If heal succeeds : spb-choice should be set to -1 as
+     *                    it is no longer valid; file is not
+     *                    in split-brain anymore.
+     * If heal doesn't succeed:
+     *                    spb-choice should be set to -1
+     *                    otherwise reads will be served
+     *                    from spb-choice which is misleading.
+     */
+    ret = afr_inode_split_brain_choice_set(loc->inode, this, -1);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN_SET_FAILED,
+                NULL);
+    afr_heal_splitbrain_file(frame, this, loc);
+    ret = 0;
+out:
+    if (ret < 0)
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-
 int
-afr_split_brain_resolve_do (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                            char *data)
+afr_get_split_brain_child_index(xlator_t *this, void *value, size_t len)
 {
-        afr_local_t    *local             = NULL;
-        int     ret                       = -1;
-        int     op_errno                  = EINVAL;
+    int spb_child_index = -1;
+    char *spb_child_str = NULL;
 
-        local = frame->local;
-        local->xdata_req = dict_new ();
+    spb_child_str = alloca0(len + 1);
+    memcpy(spb_child_str, value, len);
 
-        if (!local->xdata_req) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    if (!strcmp(spb_child_str, "none"))
+        return -2;
 
-        ret = dict_set_int32 (local->xdata_req, "heal-op",
-                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-        ret = dict_set_str (local->xdata_req, "child-name", data);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-        /* set spb choice to -1 whether heal succeeds or not:
-         * If heal succeeds : spb-choice should be set to -1 as
-         *                    it is no longer valid; file is not
-         *                    in split-brain anymore.
-         * If heal doesn't succeed:
-         *                    spb-choice should be set to -1
-         *                    otherwise reads will be served
-         *                    from spb-choice which is misleading.
-         */
-        ret = afr_inode_split_brain_choice_set (loc->inode, this, -1);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR, "Failed to set"
-                        "split-brain choice to -1");
-        afr_heal_splitbrain_file (frame, this, loc);
-        ret = 0;
-out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-        return 0;
+    spb_child_index = afr_get_child_index_from_name(this, spb_child_str);
+    if (spb_child_index < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                "subvol=%s", spb_child_str, NULL);
+    }
+    return spb_child_index;
 }
 
 int
-afr_get_split_brain_child_index (xlator_t *this, void *value, size_t len)
+afr_can_set_split_brain_choice(void *opaque)
 {
-        int             spb_child_index   = -1;
-        char           *spb_child_str     = NULL;
+    afr_spbc_timeout_t *data = opaque;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    int ret = -1;
 
-        spb_child_str =  alloca0 (len + 1);
-        memcpy (spb_child_str, value, len);
+    frame = data->frame;
+    loc = data->loc;
+    this = frame->this;
 
-        if (!strcmp (spb_child_str, "none"))
-                return -2;
+    ret = afr_is_split_brain(frame, this, loc->inode, loc->gfid, &data->d_spb,
+                             &data->m_spb);
 
-        spb_child_index = afr_get_child_index_from_name (this,
-                                                         spb_child_str);
-        if (spb_child_index < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        AFR_MSG_INVALID_SUBVOL, "Invalid subvol: %s",
-                        spb_child_str);
-        }
-        return spb_child_index;
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED, "gfid=%s",
+                uuid_utoa(loc->gfid), NULL);
+    return ret;
 }
 
 int
-afr_can_set_split_brain_choice (void *opaque)
+afr_handle_split_brain_commands(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                                dict_t *dict)
 {
-        afr_spbc_timeout_t        *data         = opaque;
-        call_frame_t              *frame        = NULL;
-        xlator_t                  *this         = NULL;
-        loc_t                     *loc          = NULL;
-        int                        ret          = -1;
+    void *choice_value = NULL;
+    void *resolve_value = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_spbc_timeout_t *data = NULL;
+    int len = 0;
+    int spb_child_index = -1;
+    int ret = -1;
+    int op_errno = EINVAL;
 
-        frame = data->frame;
-        loc = data->loc;
-        this = frame->this;
+    priv = this->private;
 
-        ret = afr_is_split_brain (frame, this, loc->inode, loc->gfid,
-                                  &data->d_spb, &data->m_spb);
+    ret = dict_get_ptr_and_len(dict, GF_AFR_SBRAIN_CHOICE, &choice_value, &len);
+    ret = dict_get_ptr_and_len(dict, GF_AFR_SBRAIN_RESOLVE, &resolve_value,
+                               &len);
+    if (!choice_value && !resolve_value) {
+        ret = -1;
+        goto out;
+    }
 
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
-                        "Failed to determine if %s"
-                        " is in split-brain. "
-                        "Aborting split-brain-choice set.",
-                        uuid_utoa (loc->gfid));
-        return ret;
-}
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local) {
+        ret = 1;
+        goto out;
+    }
 
-int
-afr_handle_split_brain_commands (xlator_t *this, call_frame_t *frame,
-                                loc_t *loc, dict_t *dict)
-{
-        void           *value             = NULL;
-        afr_private_t  *priv              = NULL;
-        afr_local_t    *local             = NULL;
-        afr_spbc_timeout_t *data          = NULL;
-        int             len               = 0;
-        int             spb_child_index   = -1;
-        int             ret               = -1;
-        int             op_errno          = EINVAL;
-
-        priv = this->private;
+    local->op = GF_FOP_SETXATTR;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local) {
+    if (choice_value) {
+        spb_child_index = afr_get_split_brain_child_index(this, choice_value,
+                                                          len);
+        if (spb_child_index < 0) {
+            /* Case where value was "none" */
+            if (spb_child_index == -2)
+                spb_child_index = -1;
+            else {
                 ret = 1;
+                op_errno = EINVAL;
                 goto out;
+            }
         }
 
-        local->op = GF_FOP_SETXATTR;
-
-        ret =  dict_get_ptr_and_len (dict, GF_AFR_SBRAIN_CHOICE, &value,
-                                     &len);
-        if (value) {
-                spb_child_index = afr_get_split_brain_child_index (this, value,
-                                                                   len);
-                if (spb_child_index < 0) {
-                        /* Case where value was "none" */
-                        if (spb_child_index == -2)
-                                spb_child_index = -1;
-                        else {
-                                ret = 1;
-                                op_errno = EINVAL;
-                                goto out;
-                        }
-                }
-
-                data = GF_CALLOC (1, sizeof (*data), gf_afr_mt_spbc_timeout_t);
-                if (!data) {
-                        ret = 1;
-                        goto out;
-                }
-                data->spb_child_index = spb_child_index;
-                data->frame = frame;
-                loc_copy (&local->loc, loc);
-                data->loc = &local->loc;
-                ret = synctask_new (this->ctx->env,
-                                    afr_can_set_split_brain_choice,
-                                    afr_set_split_brain_choice, NULL, data);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
-                                "Failed to create"
-                                " synctask. Aborting split-brain choice set"
-                                " for %s", loc->name);
-                        ret = 1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-                ret = 0;
-                goto out;
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_spbc_timeout_t);
+        if (!data) {
+            ret = 1;
+            goto out;
         }
+        data->spb_child_index = spb_child_index;
+        data->frame = frame;
+        loc_copy(&local->loc, loc);
+        data->loc = &local->loc;
+        ret = synctask_new(this->ctx->env, afr_can_set_split_brain_choice,
+                           afr_set_split_brain_choice, NULL, data);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                    "name=%s", loc->name, NULL);
+            ret = 1;
+            op_errno = ENOMEM;
+            goto out;
+        }
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_get_ptr_and_len (dict, GF_AFR_SBRAIN_RESOLVE, &value, &len);
-        if (value) {
-                spb_child_index = afr_get_split_brain_child_index (this, value,
-                                                                   len);
-                if (spb_child_index < 0) {
-                        ret = 1;
-                        goto out;
-                }
-
-                afr_split_brain_resolve_do (frame, this, loc,
-                                            priv->children[spb_child_index]->name);
-                ret = 0;
+    if (resolve_value) {
+        spb_child_index = afr_get_split_brain_child_index(this, resolve_value,
+                                                          len);
+        if (spb_child_index < 0) {
+            ret = 1;
+            goto out;
         }
+
+        afr_split_brain_resolve_do(frame, this, loc,
+                                   priv->children[spb_child_index]->name);
+        ret = 0;
+    }
 out:
-        /* key was correct but value was invalid when ret == 1 */
-        if (ret == 1) {
-                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-                if (data)
-                        GF_FREE (data);
-                ret = 0;
-        }
-        return ret;
+    /* key was correct but value was invalid when ret == 1 */
+    if (ret == 1) {
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+        if (data)
+            GF_FREE(data);
+        ret = 0;
+    }
+    return ret;
 }
 
 int
-afr_handle_spb_choice_timeout (xlator_t *this, call_frame_t *frame,
-                               dict_t *dict)
+afr_handle_spb_choice_timeout(xlator_t *this, call_frame_t *frame, dict_t *dict)
 {
-        int             ret               = -1;
-        int             op_errno          = 0;
-        uint64_t        timeout           = 0;
-        afr_private_t  *priv              = NULL;
+    int ret = -1;
+    int op_errno = 0;
+    uint64_t timeout = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        ret = dict_get_uint64 (dict, GF_AFR_SPB_CHOICE_TIMEOUT, &timeout);
-        if (!ret) {
-                priv->spb_choice_timeout = timeout * 60;
-                AFR_STACK_UNWIND (setxattr, frame, ret, op_errno, NULL);
-        }
+    ret = dict_get_uint64(dict, GF_AFR_SPB_CHOICE_TIMEOUT, &timeout);
+    if (!ret) {
+        priv->spb_choice_timeout = timeout * 60;
+        AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-afr_handle_empty_brick (xlator_t *this, call_frame_t *frame, loc_t *loc,
-                        dict_t *dict)
+afr_handle_empty_brick(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                       dict_t *dict)
 {
-        int             ret               = -1;
-        int             ab_ret            = -1;
-        int             empty_index        = -1;
-        int             op_errno          = EPERM;
-        char           *empty_brick         = NULL;
-        char           *op_type           = NULL;
-        afr_empty_brick_args_t *data        = NULL;
+    int ret = -1;
+    int ab_ret = -1;
+    int empty_index = -1;
+    int op_errno = EPERM;
+    char *empty_brick = NULL;
+    char *op_type = NULL;
+    afr_empty_brick_args_t *data = NULL;
 
-        ret =  dict_get_str (dict, GF_AFR_REPLACE_BRICK, &empty_brick);
-        if (!ret)
-                op_type = GF_AFR_REPLACE_BRICK;
+    ret = dict_get_str_sizen(dict, GF_AFR_REPLACE_BRICK, &empty_brick);
+    if (!ret)
+        op_type = GF_AFR_REPLACE_BRICK;
 
-        ab_ret = dict_get_str (dict, GF_AFR_ADD_BRICK, &empty_brick);
-        if (!ab_ret)
-                op_type = GF_AFR_ADD_BRICK;
+    ab_ret = dict_get_str_sizen(dict, GF_AFR_ADD_BRICK, &empty_brick);
+    if (!ab_ret)
+        op_type = GF_AFR_ADD_BRICK;
 
-        if (ret && ab_ret)
-                goto out;
+    if (ret && ab_ret)
+        goto out;
 
-        if (frame->root->pid != GF_CLIENT_PID_SELF_HEALD) {
-                gf_msg (this->name, GF_LOG_ERROR, EPERM,
-                        afr_get_msg_id (op_type),
-                        "'%s' is an internal extended attribute.",
-                        op_type);
-                ret = 1;
-                goto out;
+    if (frame->root->pid != GF_CLIENT_PID_ADD_REPLICA_MOUNT) {
+        gf_smsg(this->name, GF_LOG_ERROR, EPERM, AFR_MSG_INTERNAL_ATTR,
+                "op_type=%s", op_type, NULL);
+        ret = 1;
+        goto out;
+    }
+    empty_index = afr_get_child_index_from_name(this, empty_brick);
+
+    if (empty_index < 0) {
+        /* Didn't belong to this replica pair
+         * Just do a no-op
+         */
+        AFR_STACK_UNWIND(setxattr, frame, 0, 0, NULL);
+        return 0;
+    } else {
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_empty_brick_t);
+        if (!data) {
+            ret = 1;
+            op_errno = ENOMEM;
+            goto out;
         }
-        empty_index = afr_get_child_index_from_name (this, empty_brick);
-
-        if (empty_index < 0) {
-                 /* Didn't belong to this replica pair
-                  * Just do a no-op
-                  */
-                AFR_STACK_UNWIND (setxattr, frame, 0, 0, NULL);
-                return 0;
-        } else {
-                data = GF_CALLOC (1, sizeof (*data),
-                                  gf_afr_mt_empty_brick_t);
-                if (!data) {
-                        ret = 1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-                data->frame = frame;
-                loc_copy (&data->loc, loc);
-                data->empty_index = empty_index;
-                data->op_type = op_type;
-                ret = synctask_new (this->ctx->env,
-                                    _afr_handle_empty_brick,
-                                    _afr_handle_empty_brick_cbk,
-                                    NULL, data);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                afr_get_msg_id (op_type),
-                                "Failed to create synctask.");
-                        ret = 1;
-                        op_errno = ENOMEM;
-                        afr_brick_args_cleanup (data);
-                        goto out;
-                }
+        data->frame = frame;
+        loc_copy(&data->loc, loc);
+        data->empty_index = empty_index;
+        data->op_type = op_type;
+        ret = synctask_new(this->ctx->env, _afr_handle_empty_brick,
+                           _afr_handle_empty_brick_cbk, NULL, data);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                    NULL);
+            ret = 1;
+            op_errno = ENOMEM;
+            afr_brick_args_cleanup(data);
+            goto out;
         }
-        ret = 0;
+    }
+    ret = 0;
 out:
-        if (ret == 1) {
-                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-                ret = 0;
-        }
-        return ret;
+    if (ret == 1) {
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+        ret = 0;
+    }
+    return ret;
 }
 
 static int
-afr_handle_special_xattr (xlator_t *this, call_frame_t *frame, loc_t *loc,
-                          dict_t *dict)
+afr_handle_special_xattr(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                         dict_t *dict)
 {
-        int     ret     = -1;
+    int ret = -1;
 
-        ret = afr_handle_split_brain_commands (this, frame, loc, dict);
-        if (ret == 0)
-                goto out;
+    ret = afr_handle_split_brain_commands(this, frame, loc, dict);
+    if (ret == 0)
+        goto out;
 
-        ret = afr_handle_spb_choice_timeout (this, frame, dict);
-        if (ret == 0)
-                goto out;
+    ret = afr_handle_spb_choice_timeout(this, frame, dict);
+    if (ret == 0)
+        goto out;
 
-        /* Applicable for replace-brick and add-brick commands */
-        ret = afr_handle_empty_brick (this, frame, loc, dict);
+    /* Applicable for replace-brick and add-brick commands */
+    ret = afr_handle_empty_brick(this, frame, loc, dict);
 out:
-        return ret;
+    return ret;
 }
 
 int
-afr_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-	      int32_t flags, dict_t *xdata)
+afr_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata)
 {
-        afr_local_t    *local             = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int             ret               = -1;
-        int             op_errno          = EINVAL;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = EINVAL;
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   op_errno, out);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.afr.*", dict, op_errno, out);
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   op_errno, out);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.glusterfs.afr.*", dict, op_errno, out);
 
-        ret = afr_handle_special_xattr (this, frame, loc, dict);
-        if (ret == 0)
-                return 0;
+    ret = afr_handle_special_xattr(this, frame, loc, dict);
+    if (ret == 0)
+        return 0;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-                goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.setxattr.dict  = dict_ref (dict);
-        local->cont.setxattr.flags = flags;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.setxattr.dict = dict_ref(dict);
+    local->cont.setxattr.flags = flags;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_setxattr_wind;
-        local->transaction.unwind = afr_setxattr_unwind;
+    local->transaction.wind = afr_setxattr_wind;
+    local->transaction.unwind = afr_setxattr_unwind;
 
-        loc_copy (&local->loc, loc);
-        ret = afr_set_inode_local (this, local, loc->inode);
-        if (ret)
-                goto out;
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-	local->op = GF_FOP_SETXATTR;
+    local->op = GF_FOP_SETXATTR;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+    AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* {{{ fsetxattr */
 
-
 int
-afr_fsetxattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fsetxattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t    *local         = NULL;
-        call_frame_t   *main_frame    = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (fsetxattr, main_frame, local->op_ret, local->op_errno,
-			  local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(fsetxattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_fsetxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_fsetxattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
 }
 
-
 int
-afr_fsetxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fsetxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t        *local       = NULL;
-        afr_private_t      *priv        = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_fsetxattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fsetxattr,
-			   local->fd, local->cont.fsetxattr.dict,
-			   local->cont.fsetxattr.flags, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_fsetxattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsetxattr, local->fd,
+                      local->cont.fsetxattr.dict, local->cont.fsetxattr.flags,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
+afr_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata)
 {
-        afr_local_t      *local             = NULL;
-        call_frame_t     *transaction_frame = NULL;
-        int               ret               = -1;
-        int               op_errno          = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   op_errno, out);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.afr.*", dict, op_errno, out);
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   op_errno, out);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.glusterfs.afr.*", dict, op_errno, out);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.fsetxattr.dict  = dict_ref (dict);
-        local->cont.fsetxattr.flags = flags;
+    local->cont.fsetxattr.dict = dict_ref(dict);
+    local->cont.fsetxattr.flags = flags;
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_fsetxattr_wind;
-        local->transaction.unwind = afr_fsetxattr_unwind;
+    local->transaction.wind = afr_fsetxattr_wind;
+    local->transaction.unwind = afr_fsetxattr_unwind;
 
-        local->fd                 = fd_ref (fd);
-        ret = afr_set_inode_local (this, local, fd->inode);
-        if (ret)
-                goto out;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_FSETXATTR;
+    local->op = GF_FOP_FSETXATTR;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start  = LLONG_MAX - 1;
-        local->transaction.len    = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
-        return 0;
+    AFR_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
 /* }}} */
 
-
 /* {{{ removexattr */
 
-
 int
-afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_removexattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (removexattr, main_frame, local->op_ret, local->op_errno,
-			  local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(removexattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_removexattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
 }
 
-
 int
-afr_removexattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_removexattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_removexattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->removexattr,
-			   &local->loc, local->cont.removexattr.name,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_removexattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->removexattr, &local->loc,
+                      local->cont.removexattr.name, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name, dict_t *xdata)
+afr_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
 {
-        afr_local_t     *local             = NULL;
-        call_frame_t    *transaction_frame = NULL;
-        int              ret               = -1;
-        int              op_errno          = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
-                                 name, op_errno, out);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.afr.*", name, op_errno, out);
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
-                                 name, op_errno, out);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.glusterfs.afr.*", name, op_errno, out);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.removexattr.name = gf_strdup (name);
+    local->cont.removexattr.name = gf_strdup(name);
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_removexattr_wind;
-        local->transaction.unwind = afr_removexattr_unwind;
+    local->transaction.wind = afr_removexattr_wind;
+    local->transaction.unwind = afr_removexattr_unwind;
 
-        loc_copy (&local->loc, loc);
-        ret = afr_set_inode_local (this, local, loc->inode);
-        if (ret)
-                goto out;
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_REMOVEXATTR;
+    local->op = GF_FOP_REMOVEXATTR;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
-        return 0;
+    AFR_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
 /* ffremovexattr */
 int
-afr_fremovexattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fremovexattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (fremovexattr, main_frame, local->op_ret, local->op_errno,
-			  local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(fremovexattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+afr_fremovexattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
 }
 
-
 int
-afr_fremovexattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fremovexattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_fremovexattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fremovexattr,
-			   local->fd, local->cont.removexattr.name,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_fremovexattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fremovexattr, local->fd,
+                      local->cont.removexattr.name, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		  const char *name, dict_t *xdata)
+afr_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
-                                 name, op_errno, out);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.afr.*", name, op_errno, out);
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
-                                 name, op_errno, out);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.glusterfs.afr.*", name, op_errno, out);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-                goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.removexattr.name = gf_strdup (name);
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.removexattr.name = gf_strdup(name);
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_fremovexattr_wind;
-        local->transaction.unwind = afr_fremovexattr_unwind;
+    local->transaction.wind = afr_fremovexattr_wind;
+    local->transaction.unwind = afr_fremovexattr_unwind;
 
-        local->fd = fd_ref (fd);
-        ret = afr_set_inode_local (this, local, fd->inode);
-        if (ret)
-                goto out;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_FREMOVEXATTR;
+    local->op = GF_FOP_FREMOVEXATTR;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
+    AFR_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_fallocate_unwind (call_frame_t *frame, xlator_t *this)
+afr_fallocate_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (fallocate, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(fallocate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_fallocate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+afr_fallocate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_fallocate_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fallocate_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_fallocate_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fallocate,
-			   local->fd, local->cont.fallocate.mode,
-			   local->cont.fallocate.offset,
-			   local->cont.fallocate.len, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_fallocate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fallocate, local->fd,
+                      local->cont.fallocate.mode, local->cont.fallocate.offset,
+                      local->cont.fallocate.len, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-               off_t offset, size_t len, dict_t *xdata)
+afr_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata)
 {
-        call_frame_t *transaction_frame = NULL;
-        afr_local_t *local = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    call_frame_t *transaction_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.fallocate.mode = mode;
-        local->cont.fallocate.offset  = offset;
-        local->cont.fallocate.len = len;
+    local->cont.fallocate.mode = mode;
+    local->cont.fallocate.offset = offset;
+    local->cont.fallocate.len = len;
 
-        local->fd = fd_ref (fd);
-        ret = afr_set_inode_local (this, local, fd->inode);
-        if (ret)
-                goto out;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->op = GF_FOP_FALLOCATE;
+    local->op = GF_FOP_FALLOCATE;
 
-        local->transaction.wind   = afr_fallocate_wind;
-        local->transaction.unwind = afr_fallocate_unwind;
+    local->transaction.wind = afr_fallocate_wind;
+    local->transaction.unwind = afr_fallocate_unwind;
 
-        local->transaction.main_frame = frame;
+    local->transaction.main_frame = frame;
 
-        local->transaction.start   = local->cont.fallocate.offset;
-        local->transaction.len     = 0;
+    local->transaction.start = local->cont.fallocate.offset;
+    local->transaction.len = 0;
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ discard */
 
 int
-afr_discard_unwind (call_frame_t *frame, xlator_t *this)
+afr_discard_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (discard, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(discard, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_discard_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+afr_discard_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_discard_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_discard_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_discard_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->discard,
-			   local->fd, local->cont.discard.offset,
-			   local->cont.discard.len, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_discard_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->discard, local->fd,
+                      local->cont.discard.offset, local->cont.discard.len,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-             size_t len, dict_t *xdata)
+afr_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.discard.offset  = offset;
-        local->cont.discard.len = len;
+    local->cont.discard.offset = offset;
+    local->cont.discard.len = len;
 
-        local->fd = fd_ref (fd);
-        ret = afr_set_inode_local (this, local, fd->inode);
-        if (ret)
-                goto out;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->op = GF_FOP_DISCARD;
+    local->op = GF_FOP_DISCARD;
 
-        local->transaction.wind   = afr_discard_wind;
-        local->transaction.unwind = afr_discard_unwind;
+    local->transaction.wind = afr_discard_wind;
+    local->transaction.unwind = afr_discard_unwind;
 
-        local->transaction.main_frame = frame;
+    local->transaction.main_frame = frame;
 
-        local->transaction.start   = local->cont.discard.offset;
-        local->transaction.len     = 0;
+    local->transaction.start = local->cont.discard.offset;
+    local->transaction.len = 0;
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /* {{{ zerofill */
 
 int
-afr_zerofill_unwind (call_frame_t *frame, xlator_t *this)
+afr_zerofill_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (discard, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(discard, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_zerofill_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+afr_zerofill_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                       struct iatt *postbuf, dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_zerofill_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_zerofill_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_zerofill_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->zerofill,
-			   local->fd, local->cont.zerofill.offset,
-			   local->cont.zerofill.len, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_zerofill_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->zerofill, local->fd,
+                      local->cont.zerofill.offset, local->cont.zerofill.len,
+                      local->xdata_req);
+    return 0;
 }
 
 int
-afr_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
              size_t len, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.zerofill.offset  = offset;
-        local->cont.zerofill.len = len;
+    local->cont.zerofill.offset = offset;
+    local->cont.zerofill.len = len;
 
-        local->fd = fd_ref (fd);
-        ret = afr_set_inode_local (this, local, fd->inode);
-        if (ret)
-                goto out;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->op = GF_FOP_ZEROFILL;
+    local->op = GF_FOP_ZEROFILL;
 
-        local->transaction.wind   = afr_zerofill_wind;
-        local->transaction.unwind = afr_zerofill_unwind;
+    local->transaction.wind = afr_zerofill_wind;
+    local->transaction.unwind = afr_zerofill_unwind;
 
-        local->transaction.main_frame = frame;
+    local->transaction.main_frame = frame;
 
-        local->transaction.start   = local->cont.discard.offset;
-        local->transaction.len     = len;
+    local->transaction.start = local->cont.zerofill.offset;
+    local->transaction.len = len;
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 int32_t
-afr_xattrop_wind_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int32_t op_ret, int32_t op_errno,
-                      dict_t *xattr, dict_t *xdata)
+afr_xattrop_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                     dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xattr, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, xattr, xdata);
 }
 
 int
-afr_xattrop_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_xattrop_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_xattrop_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->xattrop,
-			   &local->loc, local->cont.xattrop.optype,
-			   local->cont.xattrop.xattr, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_xattrop_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->xattrop, &local->loc,
+                      local->cont.xattrop.optype, local->cont.xattrop.xattr,
+                      local->xdata_req);
+    return 0;
 }
 
 int
-afr_xattrop_unwind (call_frame_t *frame, xlator_t *this)
+afr_xattrop_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (xattrop, main_frame, local->op_ret, local->op_errno,
-			  local->xattr_rsp, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(xattrop, main_frame, local->op_ret, local->op_errno,
+                     local->xattr_rsp, local->xdata_rsp);
+    return 0;
 }
 
 int32_t
-afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+afr_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.xattrop.xattr = dict_ref (xattr);
-        local->cont.xattrop.optype = optype;
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->cont.xattrop.xattr = dict_ref(xattr);
+    local->cont.xattrop.optype = optype;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        local->transaction.wind   = afr_xattrop_wind;
-        local->transaction.unwind = afr_xattrop_unwind;
+    local->transaction.wind = afr_xattrop_wind;
+    local->transaction.unwind = afr_xattrop_unwind;
 
-        loc_copy (&local->loc, loc);
-        ret = afr_set_inode_local (this, local, loc->inode);
-        if (ret)
-                goto out;
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_XATTROP;
+    local->op = GF_FOP_XATTROP;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
 int32_t
-afr_fxattrop_wind_cbk (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       dict_t *xattr, dict_t *xdata)
+afr_fxattrop_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                      dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xattr, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, xattr, xdata);
 }
 
 int
-afr_fxattrop_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fxattrop_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_fxattrop_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fxattrop,
-			   local->fd, local->cont.xattrop.optype,
-			   local->cont.xattrop.xattr, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_fxattrop_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fxattrop, local->fd,
+                      local->cont.xattrop.optype, local->cont.xattrop.xattr,
+                      local->xdata_req);
+    return 0;
 }
 
 int
-afr_fxattrop_unwind (call_frame_t *frame, xlator_t *this)
+afr_fxattrop_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (fxattrop, main_frame, local->op_ret, local->op_errno,
-			  local->xattr_rsp, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(fxattrop, main_frame, local->op_ret, local->op_errno,
+                     local->xattr_rsp, local->xdata_rsp);
+    return 0;
 }
 
 int32_t
-afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+afr_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.xattrop.xattr = dict_ref (xattr);
-        local->cont.xattrop.optype = optype;
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->cont.xattrop.xattr = dict_ref(xattr);
+    local->cont.xattrop.optype = optype;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        local->transaction.wind   = afr_fxattrop_wind;
-        local->transaction.unwind = afr_fxattrop_unwind;
+    local->transaction.wind = afr_fxattrop_wind;
+    local->transaction.unwind = afr_fxattrop_unwind;
 
-	local->fd = fd_ref (fd);
-        ret = afr_set_inode_local (this, local, fd->inode);
-        if (ret)
-                goto out;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_FXATTROP;
+    local->op = GF_FOP_FXATTROP;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this,
-                               AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
-
 int
-afr_fsync_unwind (call_frame_t *frame, xlator_t *this)
+afr_fsync_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-                return 0;
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-        AFR_STACK_UNWIND (fsync, main_frame, local->op_ret, local->op_errno,
-                          &local->cont.inode_wfop.prebuf,
-                          &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    AFR_STACK_UNWIND(fsync, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_fsync_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf, dict_t *xdata)
+afr_fsync_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-                                      prebuf, postbuf, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_fsync_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fsync_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        STACK_WIND_COOKIE (frame, afr_fsync_wind_cbk, (void *)(long) subvol,
-                           priv->children[subvol],
-                           priv->children[subvol]->fops->fsync,
-                           local->fd, local->cont.fsync.datasync,
-                           local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_fsync_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsync, local->fd,
+                      local->cont.fsync.datasync, local->xdata_req);
+    return 0;
 }
 
 int
-afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
-	   dict_t *xdata)
+afr_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int32_t op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int32_t op_errno = ENOMEM;
+    int8_t last_fsync = 0;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-        else
-                local->xdata_req = dict_new ();
+    if (xdata) {
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+        if (dict_get_int8(xdata, "last-fsync", &last_fsync) == 0) {
+            if (last_fsync) {
+                local->transaction.disable_delayed_post_op = _gf_true;
+            }
+        }
+    } else {
+        local->xdata_req = dict_new();
+    }
 
-        if (!local->xdata_req)
-                goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->fd = fd_ref (fd);
-        ret = afr_set_inode_local (this, local, fd->inode);
-        if (ret)
-                goto out;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        local->op = GF_FOP_FSYNC;
-        local->cont.fsync.datasync = datasync;
+    local->op = GF_FOP_FSYNC;
+    local->cont.fsync.datasync = datasync;
 
-	if (afr_fd_has_witnessed_unstable_write (this, fd->inode)) {
-		/* don't care. we only wanted to CLEAR the bit */
-	}
+    if (afr_fd_has_witnessed_unstable_write(this, fd->inode)) {
+        /* don't care. we only wanted to CLEAR the bit */
+    }
 
-        local->transaction.wind   = afr_fsync_wind;
-        local->transaction.unwind = afr_fsync_unwind;
+    local->transaction.wind = afr_fsync_wind;
+    local->transaction.unwind = afr_fsync_unwind;
 
-        local->transaction.main_frame = frame;
+    local->transaction.main_frame = frame;
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+    AFR_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-inode-write.h b/xlators/cluster/afr/src/afr-inode-write.h
index 1e8bb5c12b3..a787069b7a1 100644
--- a/xlators/cluster/afr/src/afr-inode-write.h
+++ b/xlators/cluster/afr/src/afr-inode-write.h
@@ -12,83 +12,83 @@
 #define __INODE_WRITE_H__
 
 int32_t
-afr_chmod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dict_t *xdata);
+afr_chmod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dict_t *xdata);
 
 int32_t
-afr_chown (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, uid_t uid, gid_t gid, dict_t *xdata);
+afr_chown(call_frame_t *frame, xlator_t *this, loc_t *loc, uid_t uid, gid_t gid,
+          dict_t *xdata);
 
 int
-afr_fchown (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, uid_t uid, gid_t gid, dict_t *xdata);
+afr_fchown(call_frame_t *frame, xlator_t *this, fd_t *fd, uid_t uid, gid_t gid,
+           dict_t *xdata);
 
 int32_t
-afr_fchmod (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, mode_t mode, dict_t *xdata);
+afr_fchmod(call_frame_t *frame, xlator_t *this, fd_t *fd, mode_t mode,
+           dict_t *xdata);
 
 int32_t
-afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	    struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref, dict_t *xdata);
+afr_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata);
 
 int32_t
-afr_truncate (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, off_t offset, dict_t *xdata);
+afr_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata);
 
 int32_t
-afr_ftruncate (call_frame_t *frame, xlator_t *this,
-	       fd_t *fd, off_t offset, dict_t *xdata);
+afr_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata);
 
 int32_t
-afr_utimens (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, struct timespec tv[2], dict_t *xdata);
+afr_utimens(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            struct timespec tv[2], dict_t *xdata);
 
 int
-afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid, dict_t *xdata);
+afr_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+            int32_t valid, dict_t *xdata);
 
 int
-afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata);
+afr_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
+             int32_t valid, dict_t *xdata);
 
 int32_t
-afr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata);
+afr_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata);
 
 int32_t
-afr_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata);
+afr_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata);
 
 int32_t
-afr_removexattr (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, const char *name, dict_t *xdata);
+afr_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata);
 
 int32_t
-afr_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *name, dict_t *xdata);
+afr_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata);
 
 int
-afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-             size_t len, dict_t *xdata);
+afr_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata);
 
 int
-afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-               off_t offset, size_t len, dict_t *xdata);
+afr_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata);
 
 int
 afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
              off_t len, dict_t *xdata);
 
 int32_t
-afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+afr_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
 
 int32_t
-afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+afr_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
 
 int
-afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
-	   dict_t *xdata);
+afr_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata);
 #endif /* __INODE_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c
index be3de01924d..bc8eabe0f43 100644
--- a/xlators/cluster/afr/src/afr-lk-common.c
+++ b/xlators/cluster/afr/src/afr-lk-common.c
@@ -8,9 +8,9 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "dict.h"
-#include "byte-order.h"
-#include "common-utils.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/common-utils.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
@@ -18,1121 +18,774 @@
 
 #include <signal.h>
 
+#define LOCKED_NO 0x0    /* no lock held */
+#define LOCKED_YES 0x1   /* for DATA, METADATA, ENTRY and higher_path */
+#define LOCKED_LOWER 0x2 /* for lower path */
 
-#define LOCKED_NO       0x0        /* no lock held */
-#define LOCKED_YES      0x1        /* for DATA, METADATA, ENTRY and higher_path */
-#define LOCKED_LOWER    0x2        /* for lower path */
+void
+afr_lockee_cleanup(afr_lockee_t *lockee)
+{
+    if (lockee->fd) {
+        fd_unref(lockee->fd);
+        lockee->fd = NULL;
+    } else {
+        loc_wipe(&lockee->loc);
+    }
+
+    GF_FREE(lockee->basename);
+    lockee->basename = NULL;
+    GF_FREE(lockee->locked_nodes);
+    lockee->locked_nodes = NULL;
+
+    return;
+}
 
-int
-afr_entry_lockee_cmp (const void *l1, const void *l2)
+void
+afr_lockees_cleanup(afr_internal_lock_t *int_lock)
 {
-        const afr_entry_lockee_t       *r1 = l1;
-        const afr_entry_lockee_t       *r2 = l2;
-        int                            ret = 0;
-        uuid_t                         gfid1 = {0};
-        uuid_t                         gfid2 = {0};
-
-        loc_gfid ((loc_t*)&r1->loc, gfid1);
-        loc_gfid ((loc_t*)&r2->loc, gfid2);
-        ret = gf_uuid_compare (gfid1, gfid2);
-        /*Entrylks with NULL basename are the 'smallest'*/
-        if (ret == 0) {
-                if (!r1->basename)
-                        return -1;
-                if (!r2->basename)
-                        return 1;
-                ret = strcmp (r1->basename, r2->basename);
-        }
+    int i = 0;
+
+    for (i = 0; i < int_lock->lockee_count; i++) {
+        afr_lockee_cleanup(&int_lock->lockee[i]);
+    }
 
-        if (ret <= 0)
-                return -1;
-        else
-                return 1;
+    return;
+}
+int
+afr_entry_lockee_cmp(const void *l1, const void *l2)
+{
+    const afr_lockee_t *r1 = l1;
+    const afr_lockee_t *r2 = l2;
+    int ret = 0;
+    uuid_t gfid1 = {0};
+    uuid_t gfid2 = {0};
+
+    loc_gfid((loc_t *)&r1->loc, gfid1);
+    loc_gfid((loc_t *)&r2->loc, gfid2);
+    ret = gf_uuid_compare(gfid1, gfid2);
+    /*Entrylks with NULL basename are the 'smallest'*/
+    if (ret == 0) {
+        if (!r1->basename)
+            return -1;
+        if (!r2->basename)
+            return 1;
+        ret = strcmp(r1->basename, r2->basename);
+    }
+
+    if (ret <= 0)
+        return -1;
+    else
+        return 1;
 }
 
-int afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
+int
+afr_lock_blocking(call_frame_t *frame, xlator_t *this, int child_index);
 
 void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner)
+afr_set_lk_owner(call_frame_t *frame, xlator_t *this, void *lk_owner)
 {
-        gf_msg_trace (this->name, 0,
-                      "Setting lk-owner=%llu",
-                      (unsigned long long) (unsigned long)lk_owner);
+    gf_msg_trace(this->name, 0, "Setting lk-owner=%llu",
+                 (unsigned long long)(unsigned long)lk_owner);
 
-        set_lk_owner_from_ptr (&frame->root->lk_owner, lk_owner);
+    set_lk_owner_from_ptr(&frame->root->lk_owner, lk_owner);
 }
 
 int32_t
-internal_lock_count (call_frame_t *frame, xlator_t *this)
+internal_lock_count(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int32_t call_count = 0;
-        int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int32_t call_count = 0;
+    int i = 0;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i])
-                        ++call_count;
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i])
+            ++call_count;
+    }
 
-        return call_count;
+    return call_count;
 }
 
 int
-afr_is_inodelk_transaction(afr_transaction_type type)
+afr_add_entry_lockee(afr_local_t *local, loc_t *loc, char *basename,
+                     int child_count)
 {
-        int ret = 0;
+    int ret = -ENOMEM;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    afr_lockee_t *lockee = &int_lock->lockee[int_lock->lockee_count];
 
-        switch (type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                ret = 1;
-                break;
+    GF_ASSERT(int_lock->lockee_count < AFR_LOCKEE_COUNT_MAX);
+    loc_copy(&lockee->loc, loc);
+    lockee->basename = (basename) ? gf_strdup(basename) : NULL;
+    if (basename && !lockee->basename)
+        goto out;
 
-        case AFR_ENTRY_RENAME_TRANSACTION:
-        case AFR_ENTRY_TRANSACTION:
-                ret = 0;
-                break;
+    lockee->locked_count = 0;
+    lockee->locked_nodes = GF_CALLOC(child_count, sizeof(*lockee->locked_nodes),
+                                     gf_afr_mt_afr_node_character);
 
-        }
+    if (!lockee->locked_nodes)
+        goto out;
 
-        return ret;
+    ret = 0;
+    int_lock->lockee_count++;
+out:
+    if (ret) {
+        afr_lockee_cleanup(lockee);
+    }
+    return ret;
 }
 
 int
-afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
-                       loc_t *loc, char *basename, int child_count)
+afr_add_inode_lockee(afr_local_t *local, int child_count)
 {
-        int     ret     = -1;
+    int ret = -ENOMEM;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    afr_lockee_t *lockee = &int_lock->lockee[int_lock->lockee_count];
 
-        loc_copy (&lockee->loc, loc);
-        lockee->basename        = (basename)? gf_strdup (basename): NULL;
-        if (basename && !lockee->basename)
-                goto out;
+    if (local->fd) {
+        lockee->fd = fd_ref(local->fd);
+    } else {
+        loc_copy(&lockee->loc, &local->loc);
+    }
 
-        lockee->locked_count    = 0;
-        lockee->locked_nodes    = GF_CALLOC (child_count,
-                                             sizeof (*lockee->locked_nodes),
-                                             gf_afr_mt_afr_node_character);
+    lockee->locked_count = 0;
+    lockee->locked_nodes = GF_CALLOC(child_count, sizeof(*lockee->locked_nodes),
+                                     gf_afr_mt_afr_node_character);
 
-        if (!lockee->locked_nodes)
-                goto out;
+    if (!lockee->locked_nodes)
+        goto out;
 
-        ret = 0;
+    ret = 0;
+    int_lock->lockee_count++;
 out:
-        return ret;
-
-}
-
-void
-afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock)
-{
-        int     i   = 0;
-
-        for (i = 0; i < int_lock->lockee_count; i++) {
-                loc_wipe (&int_lock->lockee[i].loc);
-                if (int_lock->lockee[i].basename)
-                        GF_FREE (int_lock->lockee[i].basename);
-                if (int_lock->lockee[i].locked_nodes)
-                       GF_FREE (int_lock->lockee[i].locked_nodes);
-        }
-
-        return;
-}
-
-static int
-initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-
-        int i = 0;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->entrylk_lock_count = 0;
-        int_lock->lock_op_ret        = -1;
-        int_lock->lock_op_errno      = 0;
-
-        for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {
-                if (!int_lock->lockee[i].locked_nodes)
-                        break;
-                int_lock->lockee[i].locked_count = 0;
-                memset (int_lock->lockee[i].locked_nodes, 0,
-                        sizeof (*int_lock->lockee[i].locked_nodes) *
-                        priv->child_count);
-        }
-
-        return 0;
+    if (ret) {
+        afr_lockee_cleanup(lockee);
+    }
+    return ret;
 }
 
 static int
-initialize_inodelk_variables (call_frame_t *frame, xlator_t *this)
+initialize_internal_lock_variables(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->lock_count    = 0;
-        int_lock->lk_attempted_count = 0;
-        int_lock->lock_op_ret   = -1;
-        int_lock->lock_op_errno = 0;
-
-        memset (int_lock->locked_nodes, 0,
-                sizeof (*int_lock->locked_nodes) * priv->child_count);
-
-        return 0;
+    afr_local_t *local = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_private_t *priv = NULL;
+
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
+
+    int_lock->lock_count = 0;
+    int_lock->lock_op_ret = -1;
+    int_lock->lock_op_errno = 0;
+    int_lock->lk_attempted_count = 0;
+
+    for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {
+        if (!int_lock->lockee[i].locked_nodes)
+            break;
+        int_lock->lockee[i].locked_count = 0;
+        memset(int_lock->lockee[i].locked_nodes, 0,
+               sizeof(*int_lock->lockee[i].locked_nodes) * priv->child_count);
+    }
+
+    return 0;
 }
 
 int
-afr_lockee_locked_nodes_count (afr_internal_lock_t *int_lock)
+afr_lockee_locked_nodes_count(afr_internal_lock_t *int_lock)
 {
-        int call_count  = 0;
-        int i           = 0;
+    int call_count = 0;
+    int i = 0;
 
-        for (i = 0; i < int_lock->lockee_count; i++)
-                call_count += int_lock->lockee[i].locked_count;
+    for (i = 0; i < int_lock->lockee_count; i++)
+        call_count += int_lock->lockee[i].locked_count;
 
-        return call_count;
+    return call_count;
 }
 
 int
-afr_locked_nodes_count (unsigned char *locked_nodes, int child_count)
+afr_locked_nodes_count(unsigned char *locked_nodes, int child_count)
 
 {
-        int i = 0;
-        int call_count = 0;
+    int i = 0;
+    int call_count = 0;
 
-        for (i = 0; i < child_count; i++) {
-                if (locked_nodes[i] & LOCKED_YES)
-                        call_count++;
-        }
+    for (i = 0; i < child_count; i++) {
+        if (locked_nodes[i] & LOCKED_YES)
+            call_count++;
+    }
 
-        return call_count;
+    return call_count;
 }
 
-/* FIXME: What if UNLOCK fails */
-static int32_t
-afr_unlock_common_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static void
+afr_log_locks_failure(call_frame_t *frame, char *where, char *what,
+                      int op_errno)
 {
-        afr_local_t             *local          = NULL;
-        afr_internal_lock_t     *int_lock       = NULL;
-        int                      call_count     = 0;
-        int                      ret            = 0;
+    xlator_t *this = frame->this;
+    gf_lkowner_t *lk_owner = &frame->root->lk_owner;
+    afr_local_t *local = frame->local;
+    const char *fop = NULL;
+    char *gfid = NULL;
+    const char *name = NULL;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        if (local->transaction.type == AFR_DATA_TRANSACTION && op_ret != 1)
-                ret = afr_write_subvol_reset (frame, this);
-
-        LOCK (&frame->lock);
-        {
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (call_count == 0) {
-                gf_msg_trace (this->name, 0,
-                              "All internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-        }
-
-        return ret;
-}
-
-void
-afr_update_uninodelk (afr_local_t *local, afr_internal_lock_t *int_lock,
-                    int32_t child_index)
-{
-        int_lock->locked_nodes[child_index] &= LOCKED_NO;
+    fop = gf_fop_list[local->op];
 
+    switch (local->transaction.type) {
+        case AFR_ENTRY_RENAME_TRANSACTION:
+        case AFR_ENTRY_TRANSACTION:
+            switch (local->op) {
+                case GF_FOP_LINK:
+                    gfid = uuid_utoa(local->newloc.pargfid);
+                    name = local->newloc.name;
+                    break;
+                default:
+                    gfid = uuid_utoa(local->loc.pargfid);
+                    name = local->loc.name;
+                    break;
+            }
+            gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                   AFR_MSG_INTERNAL_LKS_FAILED,
+                   "Unable to do entry %s with lk-owner:%s on %s "
+                   "while attempting %s on {pgfid:%s, name:%s}.",
+                   what, lkowner_utoa(lk_owner), where, fop, gfid, name);
+            break;
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            gfid = uuid_utoa(local->inode->gfid);
+            gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                   AFR_MSG_INTERNAL_LKS_FAILED,
+                   "Unable to do inode %s with lk-owner:%s on %s "
+                   "while attempting %s on gfid:%s.",
+                   what, lkowner_utoa(lk_owner), where, fop, gfid);
+            break;
+    }
 }
 
 static int32_t
-afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_local_t         *local = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        int32_t             child_index = (long)cookie;
-        afr_private_t       *priv = NULL;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-
-        priv = this->private;
-
-        if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        AFR_MSG_UNLOCK_FAIL,
-                        "path=%s gfid=%s: unlock failed on subvolume %s "
-                        "with lock owner %s", local->loc.path,
-                        loc_gfid_utoa (&(local->loc)),
-                        priv->children[child_index]->name,
-                        lkowner_utoa (&frame->root->lk_owner));
-        }
-
-        afr_update_uninodelk (local, int_lock, child_index);
-
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-
-        return 0;
-
-}
-
-static int
-afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
+afr_unlock_common_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        struct gf_flock flock = {0,};
-        int call_count = 0;
-        int i = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        flock.l_start = int_lock->flock.l_start;
-        flock.l_len   = int_lock->flock.l_len;
-        flock.l_type  = F_UNLCK;
-
-        call_count = afr_locked_nodes_count (int_lock->locked_nodes,
-                                             priv->child_count);
-
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count) {
-                GF_ASSERT (!local->transaction.do_eager_unlock);
-                gf_msg_trace (this->name, 0,
-                              "No internal locks unlocked");
-
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if ((int_lock->locked_nodes[i] & LOCKED_YES) != LOCKED_YES)
-                        continue;
-
-                if (local->fd) {
-                        STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
-                                           (void *) (long)i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->finodelk,
-                                           int_lock->domain, local->fd,
-                                           F_SETLK, &flock, NULL);
-                } else {
-                        STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
-                                           (void *) (long)i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->inodelk,
-                                           int_lock->domain, &local->loc,
-                                           F_SETLK, &flock, NULL);
-                }
-
-                if (!--call_count)
-                        break;
-        }
-out:
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    int lockee_num = 0;
+    int call_count = 0;
+    int child_index = 0;
+    int ret = 0;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+    lockee_num = (int)((long)cookie) / priv->child_count;
+    child_index = (int)((long)cookie) % priv->child_count;
+
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        afr_log_locks_failure(frame, priv->children[child_index]->name,
+                              "unlock", op_errno);
+    }
+
+    int_lock->lockee[lockee_num].locked_nodes[child_index] &= LOCKED_NO;
+    if (local->transaction.type == AFR_DATA_TRANSACTION && op_ret != 1)
+        ret = afr_write_subvol_reset(frame, this);
+
+    LOCK(&frame->lock);
+    {
+        call_count = --int_lock->lk_call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        int_lock->lock_cbk(frame, this);
+    }
+
+    return ret;
 }
 
-static int32_t
-afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+void
+afr_internal_lock_wind(call_frame_t *frame,
+                       int32_t (*cbk)(call_frame_t *, void *, xlator_t *,
+                                      int32_t, int32_t, dict_t *),
+                       void *cookie, int child, int lockee_num,
+                       gf_boolean_t blocking, gf_boolean_t unlock)
 {
-        afr_local_t         *local = NULL;
-        afr_private_t       *priv  = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        int32_t             child_index = 0;
-        int                 lockee_no   = 0;
-
-        priv = this->private;
-        lockee_no = (int)((long) cookie) / priv->child_count;
-        child_index = (int) ((long) cookie) % priv->child_count;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-
-        if (op_ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        AFR_MSG_ENTRY_UNLOCK_FAIL,
-                        "%s: unlock failed on %s", local->loc.path,
-                        priv->children[child_index]->name);
-        }
-
-        int_lock->lockee[lockee_no].locked_nodes[child_index] &= LOCKED_NO;
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, NULL);
+    afr_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    entrylk_cmd cmd = ENTRYLK_LOCK_NB;
+    int32_t cmd1 = F_SETLK;
+    struct gf_flock flock = {
+        0,
+    };
+
+    switch (local->transaction.type) {
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+            if (unlock) {
+                cmd = ENTRYLK_UNLOCK;
+            } else if (blocking) { /*Doesn't make sense to have blocking
+                                      unlock*/
+                cmd = ENTRYLK_LOCK;
+            }
+
+            if (local->fd) {
+                STACK_WIND_COOKIE(frame, cbk, cookie, priv->children[child],
+                                  priv->children[child]->fops->fentrylk,
+                                  int_lock->domain,
+                                  int_lock->lockee[lockee_num].fd,
+                                  int_lock->lockee[lockee_num].basename, cmd,
+                                  ENTRYLK_WRLCK, NULL);
+            } else {
+                STACK_WIND_COOKIE(frame, cbk, cookie, priv->children[child],
+                                  priv->children[child]->fops->entrylk,
+                                  int_lock->domain,
+                                  &int_lock->lockee[lockee_num].loc,
+                                  int_lock->lockee[lockee_num].basename, cmd,
+                                  ENTRYLK_WRLCK, NULL);
+            }
+            break;
 
-        return 0;
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            flock = int_lock->lockee[lockee_num].flock;
+            if (unlock) {
+                flock.l_type = F_UNLCK;
+            } else if (blocking) { /*Doesn't make sense to have blocking
+                                      unlock*/
+                cmd1 = F_SETLKW;
+            }
+
+            if (local->fd) {
+                STACK_WIND_COOKIE(
+                    frame, cbk, cookie, priv->children[child],
+                    priv->children[child]->fops->finodelk, int_lock->domain,
+                    int_lock->lockee[lockee_num].fd, cmd1, &flock, NULL);
+            } else {
+                STACK_WIND_COOKIE(
+                    frame, cbk, cookie, priv->children[child],
+                    priv->children[child]->fops->inodelk, int_lock->domain,
+                    &int_lock->lockee[lockee_num].loc, cmd1, &flock, NULL);
+            }
+            break;
+    }
 }
 
 static int
-afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
+afr_unlock_now(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t     *int_lock       = NULL;
-        afr_local_t             *local          = NULL;
-        afr_private_t           *priv           = NULL;
-        int                     call_count      = 0;
-        int                     index           = 0;
-        int                     lockee_no       = 0;
-        int                     copies          = 0;
-        int                     i               = -1;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-        copies   = priv->child_count;
-
-        call_count = afr_lockee_locked_nodes_count (int_lock);
-
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count){
-                gf_msg_trace (this->name, 0,
-                              "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
-
-        for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
-                lockee_no = i / copies;
-                index     = i % copies;
-                if (int_lock->lockee[lockee_no].locked_nodes[index] & LOCKED_YES) {
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[index],
-                                           priv->children[index]->fops->entrylk,
-                                           int_lock->domain,
-                                           &int_lock->lockee[lockee_no].loc,
-                                           int_lock->lockee[lockee_no].basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int child_index = 0;
+    int lockee_num = 0;
+    int i = -1;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+
+    call_count = afr_lockee_locked_nodes_count(int_lock);
+
+    int_lock->lk_call_count = call_count;
+
+    if (!call_count) {
+        gf_msg_trace(this->name, 0, "No internal locks unlocked");
+        int_lock->lock_cbk(frame, this);
+        goto out;
+    }
+
+    for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+        lockee_num = i / priv->child_count;
+        child_index = i % priv->child_count;
+        if (int_lock->lockee[lockee_num].locked_nodes[child_index] &
+            LOCKED_YES) {
+            afr_internal_lock_wind(frame, afr_unlock_common_cbk,
+                                   (void *)(long)i, child_index, lockee_num,
+                                   _gf_false, _gf_true);
+            if (!--call_count)
+                break;
         }
+    }
 
 out:
-        return 0;
-
-}
-
-int32_t
-afr_unlock_now (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = frame->local;
-
-        if (afr_is_inodelk_transaction(local->transaction.type))
-                afr_unlock_inodelk (frame, this);
-        else
-                afr_unlock_entrylk (frame, this);
-        return 0;
+    return 0;
 }
 
 static int32_t
-afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
 {
-        afr_internal_lock_t     *int_lock       = NULL;
-        afr_local_t             *local          = NULL;
-        afr_private_t           *priv           = NULL;
-        int                     cky             = (long) cookie;
-        int                     child_index     = 0;
-        int                     lockee_no       = 0;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        child_index = ((int)cky) % priv->child_count;
-        lockee_no   = ((int)cky) / priv->child_count;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno == ENOSYS) {
-                                /* return ENOTSUP */
-                                gf_msg (this->name, GF_LOG_ERROR, ENOSYS,
-                                        AFR_MSG_LOCK_XLATOR_NOT_LOADED,
-                                        "subvolume does not support locking. "
-                                        "please load features/locks xlator on server");
-                                local->op_ret = op_ret;
-                                int_lock->lock_op_ret = op_ret;
-                        }
-
-                        local->op_errno              = op_errno;
-                        int_lock->lock_op_errno      = op_errno;
-                }
-
-		int_lock->lk_attempted_count++;
-        }
-        UNLOCK (&frame->lock);
-
-        if ((op_ret == -1) &&
-            (op_errno == ENOSYS)) {
-                afr_unlock_now (frame, this);
-        } else {
-                if (op_ret == 0) {
-                        if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
-                            local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
-                                int_lock->lockee[lockee_no].locked_nodes[child_index] |= LOCKED_YES;
-                                int_lock->lockee[lockee_no].locked_count++;
-                                int_lock->entrylk_lock_count++;
-                        } else {
-                                int_lock->locked_nodes[child_index] |= LOCKED_YES;
-                                int_lock->lock_count++;
-
-                                if (local->transaction.type ==
-                                    AFR_DATA_TRANSACTION) {
-                                        LOCK(&local->inode->lock);
-                                        {
-                                                local->inode_ctx->lock_count++;
-                                        }
-                                        UNLOCK (&local->inode->lock);
-                                }
-                        }
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int cky = (long)cookie;
+    int child_index = 0;
+    int lockee_num = 0;
+
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
+
+    child_index = ((int)cky) % priv->child_count;
+    lockee_num = ((int)cky) / priv->child_count;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if (op_errno == ENOSYS) {
+                /* return ENOTSUP */
+                gf_msg(this->name, GF_LOG_ERROR, ENOSYS,
+                       AFR_MSG_LOCK_XLATOR_NOT_LOADED,
+                       "subvolume does not support locking. "
+                       "please load features/locks xlator on server");
+                local->op_ret = op_ret;
+                int_lock->lock_op_ret = op_ret;
+            }
+
+            local->op_errno = op_errno;
+            int_lock->lock_op_errno = op_errno;
+        }
+
+        int_lock->lk_attempted_count++;
+    }
+    UNLOCK(&frame->lock);
+
+    if ((op_ret == -1) && (op_errno == ENOSYS)) {
+        afr_unlock_now(frame, this);
+    } else {
+        if (op_ret == 0) {
+            int_lock->lockee[lockee_num]
+                .locked_nodes[child_index] |= LOCKED_YES;
+            int_lock->lockee[lockee_num].locked_count++;
+            int_lock->lock_count++;
+            if (local->transaction.type == AFR_DATA_TRANSACTION) {
+                LOCK(&local->inode->lock);
+                {
+                    local->inode_ctx->lock_count++;
                 }
-                afr_lock_blocking (frame, this, cky + 1);
+                UNLOCK(&local->inode->lock);
+            }
         }
+        afr_lock_blocking(frame, this, cky + 1);
+    }
 
-        return 0;
-}
-
-static int32_t
-afr_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-        return 0;
-
-}
-
-static int32_t
-afr_blocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-        return 0;
-}
-
-static gf_boolean_t
-afr_is_entrylk (afr_transaction_type trans_type)
-{
-        if (afr_is_inodelk_transaction (trans_type))
-                return _gf_false;
-        return _gf_true;
+    return 0;
 }
 
 static gf_boolean_t
-_is_lock_wind_needed (afr_local_t *local, int child_index)
-{
-        if (!local->child_up[child_index])
-                return _gf_false;
-
-        return _gf_true;
-}
-
-static void
-afr_log_entry_locks_failure(xlator_t *this, afr_local_t *local,
-                            afr_internal_lock_t *int_lock)
+_is_lock_wind_needed(afr_local_t *local, int child_index)
 {
-        const char *fop = NULL;
-        char *pargfid = NULL;
-        const char *name = NULL;
-
-        fop = gf_fop_list[local->op];
+    if (!local->child_up[child_index])
+        return _gf_false;
 
-        switch (local->op) {
-        case GF_FOP_LINK:
-                pargfid = uuid_utoa(local->newloc.pargfid);
-                name = local->newloc.name;
-                break;
-        default:
-                pargfid = uuid_utoa(local->loc.pargfid);
-                name = local->loc.name;
-                break;
-        }
-
-        gf_msg (this->name, GF_LOG_WARNING, 0, AFR_MSG_BLOCKING_LKS_FAILED,
-                "Unable to obtain sufficient blocking entry locks on at least "
-                "one child while attempting %s on {pgfid:%s, name:%s}.", fop,
-                pargfid, name);
+    return _gf_true;
 }
 
 static gf_boolean_t
-is_blocking_locks_count_sufficient (call_frame_t *frame, xlator_t *this)
+is_blocking_locks_count_sufficient(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t  *local = NULL;
-        afr_private_t *priv = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        gf_boolean_t is_entrylk = _gf_false;
-        int child = 0;
-        int nlockee = 0;
-        int lockee_count = 0;
-        gf_boolean_t ret = _gf_true;
-
-        local = frame->local;
-        priv = this->private;
-        int_lock = &local->internal_lock;
-        lockee_count = int_lock->lockee_count;
-        is_entrylk = afr_is_entrylk (local->transaction.type);
-
-        if (!is_entrylk) {
-                if (int_lock->lock_count == 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_BLOCKING_LKS_FAILED, "Unable to obtain "
-                                "blocking inode lock on even one child for "
-                                "gfid:%s.", uuid_utoa (local->inode->gfid));
-                        return _gf_false;
-                } else {
-                        /*inodelk succeded on atleast one child. */
-                        return _gf_true;
-                }
-
-        } else {
-                if (int_lock->entrylk_lock_count == 0) {
-                        afr_log_entry_locks_failure (this, local, int_lock);
-                        return _gf_false;
-                }
-                /* For FOPS that take multiple sets of locks (mkdir, rename),
-                 * there must be atleast one brick on which the locks from
-                 * all lock sets were successful. */
-                for (child = 0; child < priv->child_count; child++) {
-                        ret = _gf_true;
-                        for (nlockee = 0; nlockee < lockee_count; nlockee++) {
-                                if (!(int_lock->lockee[nlockee].locked_nodes[child] & LOCKED_YES))
-                                        ret = _gf_false;
-                        }
-                        if (ret)
-                                return ret;
-                }
-                if (!ret)
-                        afr_log_entry_locks_failure (this, local, int_lock);
-        }
-
-        return ret;
-
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    int child = 0;
+    int nlockee = 0;
+    int lockee_count = 0;
+    gf_boolean_t ret = _gf_true;
+
+    local = frame->local;
+    priv = this->private;
+    int_lock = &local->internal_lock;
+    lockee_count = int_lock->lockee_count;
+
+    if (int_lock->lock_count == 0) {
+        afr_log_locks_failure(frame, "any subvolume", "lock",
+                              int_lock->lock_op_errno);
+        return _gf_false;
+    }
+    /* For FOPS that take multiple sets of locks (mkdir, rename),
+     * there must be at least one brick on which the locks from
+     * all lock sets were successful. */
+    for (child = 0; child < priv->child_count; child++) {
+        ret = _gf_true;
+        for (nlockee = 0; nlockee < lockee_count; nlockee++) {
+            if (!(int_lock->lockee[nlockee].locked_nodes[child] & LOCKED_YES))
+                ret = _gf_false;
+        }
+        if (ret)
+            return ret;
+    }
+    if (!ret)
+        afr_log_locks_failure(frame, "all", "lock", int_lock->lock_op_errno);
+
+    return ret;
 }
 
 int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int cookie)
+afr_lock_blocking(call_frame_t *frame, xlator_t *this, int cookie)
 {
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-        afr_private_t       *priv        = NULL;
-        struct gf_flock flock = {0,};
-        uint64_t ctx = 0;
-        int ret = 0;
-        int child_index = 0;
-        int lockee_no   = 0;
-        gf_boolean_t is_entrylk = _gf_false;
-
-        local         = frame->local;
-        int_lock      = &local->internal_lock;
-        priv          = this->private;
-        child_index   = cookie % priv->child_count;
-        lockee_no     = cookie / priv->child_count;
-        is_entrylk    = afr_is_entrylk (local->transaction.type);
-
-
-        if (!is_entrylk) {
-                flock.l_start = int_lock->flock.l_start;
-                flock.l_len   = int_lock->flock.l_len;
-                flock.l_type  = int_lock->flock.l_type;
-        }
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
+    int child_index = 0;
+    int lockee_num = 0;
 
-        if (local->fd) {
-                ret = fd_ctx_get (local->fd, this, &ctx);
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+    child_index = cookie % priv->child_count;
+    lockee_num = cookie / priv->child_count;
 
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                AFR_MSG_FD_CTX_GET_FAILED,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
+    if (local->fd) {
+        ret = fd_ctx_get(local->fd, this, &ctx);
 
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_FD_CTX_GET_FAILED,
+                   "unable to get fd ctx for fd=%p", local->fd);
 
-                        afr_unlock_now (frame, this);
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
 
-                        return 0;
-                }
-        }
-
-        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
-                if (!is_blocking_locks_count_sufficient (frame, this)) {
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
+            afr_unlock_now(frame, this);
 
-                        afr_unlock_now(frame, this);
-
-                        return 0;
-                }
+            return 0;
         }
+    }
 
-        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
-                /* we're done locking */
-
-                gf_msg_debug (this->name, 0,
-                              "we're done locking");
+    if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+        if (!is_blocking_locks_count_sufficient(frame, this)) {
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
 
-                int_lock->lock_op_ret = 0;
-                int_lock->lock_cbk (frame, this);
-                return 0;
-        }
+            afr_unlock_now(frame, this);
 
-        if (!_is_lock_wind_needed (local, child_index)) {
-                afr_lock_blocking (frame, this, cookie + 1);
-                return 0;
+            return 0;
         }
+    }
 
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
+    if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+        /* we're done locking */
 
-                if (local->fd) {
-                        STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
-                                           (void *) (long) child_index,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->finodelk,
-                                           int_lock->domain, local->fd,
-                                           F_SETLKW, &flock, NULL);
-
-                } else {
-                        STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
-                                           (void *) (long) child_index,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->inodelk,
-                                           int_lock->domain, &local->loc,
-                                           F_SETLKW, &flock, NULL);
-                }
+        gf_msg_debug(this->name, 0, "we're done locking");
 
-                break;
+        int_lock->lock_op_ret = 0;
+        int_lock->lock_cbk(frame, this);
+        return 0;
+    }
 
-        case AFR_ENTRY_RENAME_TRANSACTION:
-        case AFR_ENTRY_TRANSACTION:
-                /*Accounting for child_index increments on 'down'
-                 *and 'fd-less' children */
-
-                if (local->fd) {
-                        STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) cookie,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->fentrylk,
-                                           int_lock->domain, local->fd,
-                                           int_lock->lockee[lockee_no].basename,
-                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-                } else {
-                        STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) cookie,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->entrylk,
-                                           int_lock->domain,
-                                           &int_lock->lockee[lockee_no].loc,
-                                           int_lock->lockee[lockee_no].basename,
-                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-                }
+    if (!_is_lock_wind_needed(local, child_index)) {
+        afr_lock_blocking(frame, this, cookie + 1);
+        return 0;
+    }
 
-                break;
-        }
+    afr_internal_lock_wind(frame, afr_lock_cbk, (void *)(long)cookie,
+                           child_index, lockee_num, _gf_true, _gf_false);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-afr_blocking_lock (call_frame_t *frame, xlator_t *this)
+afr_blocking_lock(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        int                  up_count = 0;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int up_count = 0;
 
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                initialize_inodelk_variables (frame, this);
-                break;
-
-        case AFR_ENTRY_RENAME_TRANSACTION:
-        case AFR_ENTRY_TRANSACTION:
-                up_count = AFR_COUNT (local->child_up, priv->child_count);
-                int_lock->lk_call_count = int_lock->lk_expected_count
-                                        = (int_lock->lockee_count *
-                                           up_count);
-                initialize_entrylk_variables (frame, this);
-                break;
-        }
+    up_count = AFR_COUNT(local->child_up, priv->child_count);
+    int_lock->lk_call_count = int_lock->lk_expected_count =
+        (int_lock->lockee_count * up_count);
+    initialize_internal_lock_variables(frame, this);
 
-        afr_lock_blocking (frame, this, 0);
+    afr_lock_blocking(frame, this, 0);
 
-        return 0;
+    return 0;
 }
 
 static int32_t
-afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        int call_count          = 0;
-        int child_index         = (long) cookie;
-        int copies              = 0;
-        int index               = 0;
-        int lockee_no           = 0;
-        afr_private_t       *priv = NULL;
-
-        priv = this->private;
-
-        copies = priv->child_count;
-        index = child_index % copies;
-        lockee_no = child_index / copies;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret < 0 ) {
-			if (op_errno == ENOSYS) {
-                        /* return ENOTSUP */
-			        gf_msg (this->name, GF_LOG_ERROR,
-                                        ENOSYS, AFR_MSG_LOCK_XLATOR_NOT_LOADED,
-                                        "subvolume does not support "
-                                        "locking. please load features/locks"
-                                        " xlator on server");
-				local->op_ret         = op_ret;
-				int_lock->lock_op_ret = op_ret;
-
-				int_lock->lock_op_errno      = op_errno;
-				local->op_errno              = op_errno;
-			}
-		} else if (op_ret == 0) {
-			int_lock->lockee[lockee_no].locked_nodes[index] |= \
-				LOCKED_YES;
-			int_lock->lockee[lockee_no].locked_count++;
-			int_lock->entrylk_lock_count++;
-		}
-
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (call_count == 0) {
-                gf_msg_trace (this->name, 0,
-                              "Last locking reply received");
-                /* all locks successful. Proceed to call FOP */
-                if (int_lock->entrylk_lock_count ==
-                                int_lock->lk_expected_count) {
-                        gf_msg_trace (this->name, 0,
-                                      "All servers locked. Calling the cbk");
-                        int_lock->lock_op_ret = 0;
-                        int_lock->lock_cbk (frame, this);
-                }
-                /* Not all locks were successful. Unlock and try locking
-                   again, this time with serially blocking locks */
-                else {
-                        gf_msg_trace (this->name, 0,
-                                      "%d servers locked. Trying again "
-                                      "with blocking calls",
-                                      int_lock->lock_count);
-
-                        afr_unlock_now(frame, this);
-                }
-        }
-
-        return 0;
-}
-
-int
-afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
+afr_nb_internal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock   = NULL;
-        afr_local_t         *local      = NULL;
-        afr_private_t       *priv       = NULL;
-        afr_fd_ctx_t        *fd_ctx     = NULL;
-        int                 copies      = 0;
-        int                 index       = 0;
-        int                 lockee_no   = 0;
-        int32_t             call_count  = 0;
-        int i = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        copies = priv->child_count;
-        initialize_entrylk_variables (frame, this);
-
-        if (local->fd) {
-                fd_ctx = afr_fd_ctx_get (local->fd, this);
-                if (!fd_ctx) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                AFR_MSG_FD_CTX_GET_FAILED,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
-                        local->op_errno         = EINVAL;
-                        int_lock->lock_op_errno = EINVAL;
-
-			afr_unlock_now (frame, this);
-                        return -1;
-                }
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    int child_index = 0;
+    int lockee_num = 0;
+    afr_private_t *priv = NULL;
 
-                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
-                int_lock->lk_call_count = call_count;
-                int_lock->lk_expected_count = call_count;
+    priv = this->private;
 
-                if (!call_count) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                AFR_MSG_INFO_COMMON,
-                                "fd not open on any subvolumes. aborting.");
-                        afr_unlock_now (frame, this);
-                        goto out;
-                }
+    child_index = ((long)cookie) % priv->child_count;
+    lockee_num = ((long)cookie) / priv->child_count;
 
-                /* Send non-blocking entrylk calls only on up children
-                   and where the fd has been opened */
-                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
-                        index = i%copies;
-                        lockee_no = i/copies;
-                        if (local->child_up[index]) {
-                                STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[index],
-                                                   priv->children[index]->fops->fentrylk,
-                                                   this->name, local->fd,
-                                                   int_lock->lockee[lockee_no].basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
-                                                   NULL);
-                                if (!--call_count)
-                                        break;
-                        }
-                }
-        } else {
-                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
-                int_lock->lk_call_count = call_count;
-                int_lock->lk_expected_count = call_count;
-
-                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
-                        index = i%copies;
-                        lockee_no = i/copies;
-                        if (local->child_up[index]) {
-                                STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[index],
-                                                   priv->children[index]->fops->entrylk,
-                                                   this->name, &int_lock->lockee[lockee_no].loc,
-                                                   int_lock->lockee[lockee_no].basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
-                                                   NULL);
-
-                                if (!--call_count)
-                                        break;
-                        }
-                }
-        }
-out:
-        return 0;
-}
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-int32_t
-afr_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-        int                  call_count  = 0;
-        int                  child_index = (long) cookie;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        LOCK (&frame->lock);
+    if (op_ret == 0 && local->transaction.type == AFR_DATA_TRANSACTION) {
+        LOCK(&local->inode->lock);
         {
-		if (op_ret < 0) {
-			if (op_errno == ENOSYS) {
-				/* return ENOTSUP */
-		                gf_msg (this->name, GF_LOG_ERROR, ENOSYS,
-                                        AFR_MSG_LOCK_XLATOR_NOT_LOADED,
-					"subvolume does not support "
-                                        "locking. please load features/locks"
-                                        " xlator on server");
-				local->op_ret                = op_ret;
-				int_lock->lock_op_ret        = op_ret;
-				int_lock->lock_op_errno      = op_errno;
-				local->op_errno              = op_errno;
-			}
-		} else {
-			int_lock->locked_nodes[child_index] |= LOCKED_YES;
-			int_lock->lock_count++;
-		}
-
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret == 0 && local->transaction.type == AFR_DATA_TRANSACTION) {
-                LOCK (&local->inode->lock);
-                {
-                        local->inode_ctx->lock_count++;
-                }
-                UNLOCK (&local->inode->lock);
-        }
-        if (call_count == 0) {
-                gf_msg_trace (this->name, 0,
-                              "Last inode locking reply received");
-                /* all locks successful. Proceed to call FOP */
-                if (int_lock->lock_count == int_lock->lk_expected_count) {
-                        gf_msg_trace (this->name, 0,
-                                      "All servers locked. Calling the cbk");
-                        int_lock->lock_op_ret = 0;
-                        int_lock->lock_cbk (frame, this);
-                }
-                /* Not all locks were successful. Unlock and try locking
-                   again, this time with serially blocking locks */
-                else {
-                        gf_msg_trace (this->name, 0,
-                                      "%d servers locked. "
-                                      "Trying again with blocking calls",
-                                      int_lock->lock_count);
-
-                        afr_unlock_now(frame, this);
-                }
+            local->inode_ctx->lock_count++;
         }
+        UNLOCK(&local->inode->lock);
+    }
 
-        return 0;
+    LOCK(&frame->lock);
+    {
+        if (op_ret < 0) {
+            if (op_errno == ENOSYS) {
+                /* return ENOTSUP */
+                gf_msg(this->name, GF_LOG_ERROR, ENOSYS,
+                       AFR_MSG_LOCK_XLATOR_NOT_LOADED,
+                       "subvolume does not support "
+                       "locking. please load features/locks"
+                       " xlator on server");
+                local->op_ret = op_ret;
+                int_lock->lock_op_ret = op_ret;
+
+                int_lock->lock_op_errno = op_errno;
+                local->op_errno = op_errno;
+            }
+        } else if (op_ret == 0) {
+            int_lock->lockee[lockee_num]
+                .locked_nodes[child_index] |= LOCKED_YES;
+            int_lock->lockee[lockee_num].locked_count++;
+            int_lock->lock_count++;
+        }
+
+        call_count = --int_lock->lk_call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        gf_msg_trace(this->name, 0, "Last locking reply received");
+        /* all locks successful. Proceed to call FOP */
+        if (int_lock->lock_count == int_lock->lk_expected_count) {
+            gf_msg_trace(this->name, 0, "All servers locked. Calling the cbk");
+            int_lock->lock_op_ret = 0;
+            int_lock->lock_cbk(frame, this);
+        }
+        /* Not all locks were successful. Unlock and try locking
+           again, this time with serially blocking locks */
+        else {
+            gf_msg_trace(this->name, 0,
+                         "%d servers locked. Trying again "
+                         "with blocking calls",
+                         int_lock->lock_count);
+
+            afr_unlock_now(frame, this);
+        }
+    }
+
+    return 0;
 }
 
 int
-afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
+afr_lock_nonblocking(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_fd_ctx_t        *fd_ctx   = NULL;
-        int32_t             call_count = 0;
-        int                 i          = 0;
-        int                 ret        = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        initialize_inodelk_variables (frame, this);
-
-        if (local->fd) {
-                fd_ctx = afr_fd_ctx_get (local->fd, this);
-                if (!fd_ctx) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                AFR_MSG_FD_CTX_GET_FAILED,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
-                        local->op_errno         = EINVAL;
-                        int_lock->lock_op_errno = EINVAL;
-
-			afr_unlock_now (frame, this);
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        call_count = internal_lock_count (frame, this);
-        int_lock->lk_call_count = call_count;
-        int_lock->lk_expected_count = call_count;
-
-        if (!call_count) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        AFR_MSG_SUBVOLS_DOWN,
-                        "All bricks are down, aborting.");
-                afr_unlock_now (frame, this);
-                goto out;
-        }
-
-        /* Send non-blocking inodelk calls only on up children
-           and where the fd has been opened */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->child_up[i])
-                        continue;
-
-                if (local->fd) {
-                        STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->finodelk,
-                                           int_lock->domain, local->fd,
-                                           F_SETLK, &int_lock->flock, NULL);
-                } else {
-
-                        STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->inodelk,
-                                           int_lock->domain, &local->loc,
-                                           F_SETLK, &int_lock->flock, NULL);
-                }
-                if (!--call_count)
-                        break;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int child = 0;
+    int lockee_num = 0;
+    int32_t call_count = 0;
+    int i = 0;
+    int ret = 0;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+
+    initialize_internal_lock_variables(frame, this);
+
+    if (local->fd) {
+        fd_ctx = afr_fd_ctx_get(local->fd, this);
+        if (!fd_ctx) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_FD_CTX_GET_FAILED,
+                   "unable to get fd ctx for fd=%p", local->fd);
+
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
+            local->op_errno = EINVAL;
+            int_lock->lock_op_errno = EINVAL;
+
+            afr_unlock_now(frame, this);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    call_count = int_lock->lockee_count * internal_lock_count(frame, this);
+    int_lock->lk_call_count = call_count;
+    int_lock->lk_expected_count = call_count;
+
+    if (!call_count) {
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_INFO_COMMON,
+               "fd not open on any subvolumes. aborting.");
+        afr_unlock_now(frame, this);
+        goto out;
+    }
+
+    /* Send non-blocking lock calls only on up children
+       and where the fd has been opened */
+    for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+        child = i % priv->child_count;
+        lockee_num = i / priv->child_count;
+        if (local->child_up[child]) {
+            afr_internal_lock_wind(frame, afr_nb_internal_lock_cbk,
+                                   (void *)(long)i, child, lockee_num,
+                                   _gf_false, _gf_false);
+            if (!--call_count)
+                break;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-afr_unlock (call_frame_t *frame, xlator_t *this)
+afr_unlock(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_lock_t  *lock  = NULL;
-
-        local = frame->local;
-
-        if (!local->transaction.eager_lock_on)
-                goto out;
-        lock = &local->inode_ctx->lock[local->transaction.type];
-        LOCK (&local->inode->lock);
-        {
-                list_del_init (&local->transaction.owner_list);
-                if (list_empty (&lock->owners) && list_empty (&lock->post_op)) {
-                        local->transaction.do_eager_unlock = _gf_true;
-        /*TODO: Need to get metadata use on_disk and inherit/uninherit
-         *GF_ASSERT (!local->inode_ctx->on_disk[local->transaction.type]);
-         *GF_ASSERT (!local->inode_ctx->inherited[local->transaction.type]);
-        */
-                        GF_ASSERT (lock->release);
-                }
-        }
-        UNLOCK (&local->inode->lock);
-        if (!local->transaction.do_eager_unlock) {
-                local->internal_lock.lock_cbk (frame, this);
-                return 0;
-        }
+    afr_local_t *local = NULL;
+    afr_lock_t *lock = NULL;
+
+    local = frame->local;
+
+    if (!local->transaction.eager_lock_on)
+        goto out;
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    LOCK(&local->inode->lock);
+    {
+        list_del_init(&local->transaction.owner_list);
+        if (list_empty(&lock->owners) && list_empty(&lock->post_op)) {
+            local->transaction.do_eager_unlock = _gf_true;
+            /*TODO: Need to get metadata use on_disk and inherit/uninherit
+             *GF_ASSERT (!local->inode_ctx->on_disk[local->transaction.type]);
+             *GF_ASSERT (!local->inode_ctx->inherited[local->transaction.type]);
+             */
+            GF_ASSERT(lock->release);
+        }
+    }
+    UNLOCK(&local->inode->lock);
+    if (!local->transaction.do_eager_unlock) {
+        local->internal_lock.lock_cbk(frame, this);
+        return 0;
+    }
 
 out:
-        afr_unlock_now (frame, this);
-        return 0;
+    afr_unlock_now(frame, this);
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index 2e1117fc18c..816065fb57a 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -8,47 +8,31 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __AFR_MEM_TYPES_H__
 #define __AFR_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_afr_mem_types_ {
-        gf_afr_mt_iovec  = gf_common_mt_end + 1,
-        gf_afr_mt_afr_fd_ctx_t,
-        gf_afr_mt_afr_private_t,
-        gf_afr_mt_int32_t,
-        gf_afr_mt_char,
-        gf_afr_mt_xattr_key,
-        gf_afr_mt_dict_t,
-        gf_afr_mt_xlator_t,
-        gf_afr_mt_iatt,
-        gf_afr_mt_int,
-        gf_afr_mt_afr_node_character,
-        gf_afr_mt_sh_diff_loop_state,
-        gf_afr_mt_uint8_t,
-        gf_afr_mt_loc_t,
-        gf_afr_mt_entry_name,
-        gf_afr_mt_pump_priv,
-        gf_afr_mt_locked_fd,
-        gf_afr_mt_inode_ctx_t,
-        gf_afr_fd_paused_call_t,
-        gf_afr_mt_crawl_data_t,
-        gf_afr_mt_brick_pos_t,
-        gf_afr_mt_shd_bool_t,
-        gf_afr_mt_shd_timer_t,
-        gf_afr_mt_shd_event_t,
-        gf_afr_mt_time_t,
-        gf_afr_mt_pos_data_t,
-	gf_afr_mt_reply_t,
-	gf_afr_mt_subvol_healer_t,
-	gf_afr_mt_spbc_timeout_t,
-        gf_afr_mt_spb_status_t,
-        gf_afr_mt_empty_brick_t,
-        gf_afr_mt_child_latency_t,
-        gf_afr_mt_atomic_t,
+    gf_afr_mt_afr_fd_ctx_t = gf_common_mt_end + 1,
+    gf_afr_mt_afr_private_t,
+    gf_afr_mt_int32_t,
+    gf_afr_mt_char,
+    gf_afr_mt_xattr_key,
+    gf_afr_mt_dict_t,
+    gf_afr_mt_xlator_t,
+    gf_afr_mt_afr_node_character,
+    gf_afr_mt_inode_ctx_t,
+    gf_afr_mt_shd_event_t,
+    gf_afr_mt_reply_t,
+    gf_afr_mt_subvol_healer_t,
+    gf_afr_mt_spbc_timeout_t,
+    gf_afr_mt_spb_status_t,
+    gf_afr_mt_empty_brick_t,
+    gf_afr_mt_child_latency_t,
+    gf_afr_mt_atomic_t,
+    gf_afr_mt_lk_heal_info_t,
+    gf_afr_mt_gf_lock,
     gf_afr_mt_end
 };
 #endif
-
diff --git a/xlators/cluster/afr/src/afr-messages.h b/xlators/cluster/afr/src/afr-messages.h
index 8aa94730158..e73fd997765 100644
--- a/xlators/cluster/afr/src/afr-messages.h
+++ b/xlators/cluster/afr/src/afr-messages.h
@@ -11,7 +11,7 @@
 #ifndef _AFR_MESSAGES_H_
 #define _AFR_MESSAGES_H_
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
 
 /* To add new message IDs, append new identifiers at the end of the list.
  *
@@ -23,51 +23,145 @@
  * glfs-message-id.h.
  */
 
-GLFS_MSGID(AFR,
-        AFR_MSG_QUORUM_FAIL,
-        AFR_MSG_QUORUM_MET,
-        AFR_MSG_QUORUM_OVERRIDE,
-        AFR_MSG_INVALID_CHILD_UP,
-        AFR_MSG_SUBVOL_UP,
-        AFR_MSG_SUBVOLS_DOWN,
-        AFR_MSG_ENTRY_UNLOCK_FAIL,
-        AFR_MSG_SPLIT_BRAIN,
-        AFR_MSG_OPEN_FAIL,
-        AFR_MSG_UNLOCK_FAIL,
-        AFR_MSG_REPLACE_BRICK_STATUS,
-        AFR_MSG_GFID_NULL,
-        AFR_MSG_FD_CREATE_FAILED,
-        AFR_MSG_DICT_SET_FAILED,
-        AFR_MSG_EXPUNGING_FILE_OR_DIR,
-        AFR_MSG_MIGRATION_IN_PROGRESS,
-        AFR_MSG_CHILD_MISCONFIGURED,
-        AFR_MSG_VOL_MISCONFIGURED,
-        AFR_MSG_BLOCKING_LKS_FAILED,
-        AFR_MSG_INVALID_FD,
-        AFR_MSG_LOCK_INFO,
-        AFR_MSG_LOCK_XLATOR_NOT_LOADED,
-        AFR_MSG_FD_CTX_GET_FAILED,
-        AFR_MSG_INVALID_SUBVOL,
-        AFR_MSG_PUMP_XLATOR_ERROR,
-        AFR_MSG_SELF_HEAL_INFO,
-        AFR_MSG_READ_SUBVOL_ERROR,
-        AFR_MSG_DICT_GET_FAILED,
-        AFR_MSG_INFO_COMMON,
-        AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
-        AFR_MSG_LOCAL_CHILD,
-        AFR_MSG_INVALID_DATA,
-        AFR_MSG_INVALID_ARG,
-        AFR_MSG_INDEX_DIR_GET_FAILED,
-        AFR_MSG_FSYNC_FAILED,
-        AFR_MSG_FAVORITE_CHILD,
-        AFR_MSG_SELF_HEAL_FAILED,
-        AFR_MSG_SPLIT_BRAIN_STATUS,
-        AFR_MSG_ADD_BRICK_STATUS,
-        AFR_MSG_NO_CHANGELOG,
-        AFR_MSG_TIMER_CREATE_FAIL,
-        AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
-        AFR_MSG_INODE_CTX_GET_FAILED,
-        AFR_MSG_THIN_ARB
-);
+GLFS_MSGID(
+    AFR, AFR_MSG_QUORUM_FAIL, AFR_MSG_QUORUM_MET, AFR_MSG_QUORUM_OVERRIDE,
+    AFR_MSG_INVALID_CHILD_UP, AFR_MSG_SUBVOL_UP, AFR_MSG_SUBVOLS_DOWN,
+    AFR_MSG_ENTRY_UNLOCK_FAIL, AFR_MSG_SPLIT_BRAIN, AFR_MSG_OPEN_FAIL,
+    AFR_MSG_UNLOCK_FAIL, AFR_MSG_REPLACE_BRICK_STATUS, AFR_MSG_GFID_NULL,
+    AFR_MSG_FD_CREATE_FAILED, AFR_MSG_DICT_SET_FAILED,
+    AFR_MSG_EXPUNGING_FILE_OR_DIR, AFR_MSG_MIGRATION_IN_PROGRESS,
+    AFR_MSG_CHILD_MISCONFIGURED, AFR_MSG_VOL_MISCONFIGURED,
+    AFR_MSG_INTERNAL_LKS_FAILED, AFR_MSG_INVALID_FD, AFR_MSG_LOCK_INFO,
+    AFR_MSG_LOCK_XLATOR_NOT_LOADED, AFR_MSG_FD_CTX_GET_FAILED,
+    AFR_MSG_INVALID_SUBVOL, AFR_MSG_PUMP_XLATOR_ERROR, AFR_MSG_SELF_HEAL_INFO,
+    AFR_MSG_READ_SUBVOL_ERROR, AFR_MSG_DICT_GET_FAILED, AFR_MSG_INFO_COMMON,
+    AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR, AFR_MSG_LOCAL_CHILD, AFR_MSG_INVALID_DATA,
+    AFR_MSG_INVALID_ARG, AFR_MSG_INDEX_DIR_GET_FAILED, AFR_MSG_FSYNC_FAILED,
+    AFR_MSG_FAVORITE_CHILD, AFR_MSG_SELF_HEAL_FAILED,
+    AFR_MSG_SPLIT_BRAIN_STATUS, AFR_MSG_ADD_BRICK_STATUS, AFR_MSG_NO_CHANGELOG,
+    AFR_MSG_TIMER_CREATE_FAIL, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+    AFR_MSG_INODE_CTX_GET_FAILED, AFR_MSG_THIN_ARB,
+    AFR_MSG_THIN_ARB_XATTROP_FAILED, AFR_MSG_THIN_ARB_LOC_POP_FAILED,
+    AFR_MSG_GET_PEND_VAL, AFR_MSG_THIN_ARB_SKIP_SHD, AFR_MSG_UNKNOWN_SET,
+    AFR_MSG_NO_XL_ID, AFR_MSG_SELF_HEAL_INFO_START,
+    AFR_MSG_SELF_HEAL_INFO_FINISH, AFR_MSG_INCRE_COUNT,
+    AFR_MSG_ADD_TO_OUTPUT_FAILED, AFR_MSG_SET_TIME_FAILED,
+    AFR_MSG_GFID_MISMATCH_DETECTED, AFR_MSG_GFID_HEAL_MSG,
+    AFR_MSG_THIN_ARB_LOOKUP_FAILED, AFR_MSG_DICT_CREATE_FAILED,
+    AFR_MSG_NO_MAJORITY_TO_RESOLVE, AFR_MSG_TYPE_MISMATCH,
+    AFR_MSG_SIZE_POLICY_NOT_APPLICABLE, AFR_MSG_NO_CHILD_SELECTED,
+    AFR_MSG_INVALID_CHILD, AFR_MSG_RESOLVE_CONFLICTING_DATA,
+    SERROR_GETTING_SRC_BRICK, SNO_DIFF_IN_MTIME, SNO_BIGGER_FILE,
+    SALL_BRICKS_UP_TO_RESOLVE, AFR_MSG_UNLOCK_FAILED, AFR_MSG_POST_OP_FAILED,
+    AFR_MSG_TA_FRAME_CREATE_FAILED, AFR_MSG_SET_KEY_XATTROP_FAILED,
+    AFR_MSG_BLOCKING_ENTRYLKS_FAILED, AFR_MSG_FOP_FAILED,
+    AFR_MSG_CLEAN_UP_FAILED, AFR_MSG_UNABLE_TO_FETCH, AFR_MSG_XATTR_SET_FAILED,
+    AFR_MSG_SPLIT_BRAIN_REPLICA, AFR_MSG_INODE_CTX_FAILED,
+    AFR_MSG_LOOKUP_FAILED, AFR_MSG_ALL_SUBVOLS_DOWN,
+    AFR_MSG_RELEASE_LOCK_FAILED, AFR_MSG_CLEAR_TIME_SPLIT_BRAIN,
+    AFR_MSG_READ_FAILED, AFR_MSG_LAUNCH_FAILED, AFR_MSG_READ_SUBVOL_NOT_UP,
+    AFR_MSG_LK_HEAL_DOM, AFR_MSG_NEW_BRICK, AFR_MSG_SPLIT_BRAIN_SET_FAILED,
+    AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED, AFR_MSG_HEALER_SPAWN_FAILED,
+    AFR_MSG_ADD_CRAWL_EVENT_FAILED, AFR_MSG_NULL_DEREF, AFR_MSG_SET_PEND_XATTR,
+    AFR_MSG_INTERNAL_ATTR);
 
+#define AFR_MSG_DICT_GET_FAILED_STR "Dict get failed"
+#define AFR_MSG_DICT_SET_FAILED_STR "Dict set failed"
+#define AFR_MSG_HEALER_SPAWN_FAILED_STR "Healer spawn failed"
+#define AFR_MSG_ADD_CRAWL_EVENT_FAILED_STR "Adding crawl event failed"
+#define AFR_MSG_INVALID_ARG_STR "Invalid argument"
+#define AFR_MSG_INDEX_DIR_GET_FAILED_STR "unable to get index-dir on "
+#define AFR_MSG_THIN_ARB_LOOKUP_FAILED_STR "Failed lookup on file"
+#define AFR_MSG_DICT_CREATE_FAILED_STR "Failed to create dict."
+#define AFR_MSG_THIN_ARB_XATTROP_FAILED_STR "Xattrop failed."
+#define AFR_MSG_THIN_ARB_LOC_POP_FAILED_STR                                    \
+    "Failed to populate loc for thin-arbiter"
+#define AFR_MSG_GET_PEND_VAL_STR "Error getting value of pending"
+#define AFR_MSG_THIN_ARB_SKIP_SHD_STR "I am not the god shd. skipping."
+#define AFR_MSG_UNKNOWN_SET_STR "Unknown set"
+#define AFR_MSG_NO_XL_ID_STR "xl does not have id"
+#define AFR_MSG_SELF_HEAL_INFO_START_STR "starting full sweep on"
+#define AFR_MSG_SELF_HEAL_INFO_FINISH_STR "finished full sweep on"
+#define AFR_MSG_INCRE_COUNT_STR "Could not increment the counter."
+#define AFR_MSG_ADD_TO_OUTPUT_FAILED_STR "Could not add to output"
+#define AFR_MSG_SET_TIME_FAILED_STR "Could not set time"
+#define AFR_MSG_GFID_HEAL_MSG_STR "Error setting gfid-heal-msg dict"
+#define AFR_MSG_NO_MAJORITY_TO_RESOLVE_STR                                     \
+    "No majority to resolve gfid split brain"
+#define AFR_MSG_GFID_MISMATCH_DETECTED_STR "Gfid mismatch dectected"
+#define AFR_MSG_SELF_HEAL_INFO_STR "performing selfheal"
+#define AFR_MSG_TYPE_MISMATCH_STR "TYPE mismatch"
+#define AFR_MSG_SIZE_POLICY_NOT_APPLICABLE_STR                                 \
+    "Size policy is not applicable to directories."
+#define AFR_MSG_NO_CHILD_SELECTED_STR                                          \
+    "No child selected by favorite-child policy"
+#define AFR_MSG_INVALID_CHILD_STR "Invalid child"
+#define AFR_MSG_RESOLVE_CONFLICTING_DATA_STR                                   \
+    "selected as authentic to resolve conflicting data"
+#define SERROR_GETTING_SRC_BRICK_STR "Error getting the source brick"
+#define SNO_DIFF_IN_MTIME_STR "No difference in mtime"
+#define SNO_BIGGER_FILE_STR "No bigger file"
+#define SALL_BRICKS_UP_TO_RESOLVE_STR                                          \
+    "All the bricks should be up to resolve the gfid split brain"
+#define AFR_MSG_UNLOCK_FAILED_STR "Failed to unlock"
+#define AFR_MSG_POST_OP_FAILED_STR "Post-op on thin-arbiter failed"
+#define AFR_MSG_TA_FRAME_CREATE_FAILED_STR "Failed to create ta_frame"
+#define AFR_MSG_SET_KEY_XATTROP_FAILED_STR "Could not set key during xattrop"
+#define AFR_MSG_BLOCKING_ENTRYLKS_FAILED_STR "Blocking entrylks failed"
+#define AFR_MSG_FSYNC_FAILED_STR "fsync failed"
+#define AFR_MSG_QUORUM_FAIL_STR "quorum is not met"
+#define AFR_MSG_FOP_FAILED_STR "Failing Fop"
+#define AFR_MSG_INVALID_SUBVOL_STR "not a subvolume"
+#define AFR_MSG_VOL_MISCONFIGURED_STR "Volume is dangling"
+#define AFR_MSG_CHILD_MISCONFIGURED_STR                                        \
+    "replicate translator needs more than one subvolume defined"
+#define AFR_MSG_CLEAN_UP_FAILED_STR "Failed to clean up healer threads"
+#define AFR_MSG_QUORUM_OVERRIDE_STR "overriding quorum-count"
+#define AFR_MSG_UNABLE_TO_FETCH_STR                                            \
+    "Unable to fetch afr-pending-xattr option from volfile. Falling back to "  \
+    "using client translator names"
+#define AFR_MSG_NULL_DEREF_STR "possible NULL deref"
+#define AFR_MSG_XATTR_SET_FAILED_STR "Cannot set xattr cookie key"
+#define AFR_MSG_SPLIT_BRAIN_STATUS_STR "Failed to create synctask"
+#define AFR_MSG_SUBVOLS_DOWN_STR "All subvolumes are not up"
+#define AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR_STR                                   \
+    "Failed to cancel split-brain choice"
+#define AFR_MSG_SPLIT_BRAIN_REPLICA_STR                                        \
+    "Cannot set replica. File is not in data/metadata split-brain"
+#define AFR_MSG_INODE_CTX_FAILED_STR "Failed to get inode_ctx"
+#define AFR_MSG_READ_SUBVOL_ERROR_STR "no read subvols"
+#define AFR_MSG_LOCAL_CHILD_STR "selecting local read-child"
+#define AFR_MSG_LOOKUP_FAILED_STR "Failed to lookup/create thin-arbiter id file"
+#define AFR_MSG_TIMER_CREATE_FAIL_STR                                          \
+    "Cannot create timer for delayed initialization"
+#define AFR_MSG_SUBVOL_UP_STR "Subvolume came back up; going online"
+#define AFR_MSG_ALL_SUBVOLS_DOWN_STR                                           \
+    "All subvolumes are down. Going offline until atleast one of them is up"
+#define AFR_MSG_RELEASE_LOCK_FAILED_STR "Failed to release lock"
+#define AFR_MSG_INVALID_CHILD_UP_STR "Received child_up from invalid subvolume"
+#define AFR_MSG_QUORUM_MET_STR "Client-quorum is met"
+#define AFR_MSG_EXPUNGING_FILE_OR_DIR_STR "expunging file or dir"
+#define AFR_MSG_SELF_HEAL_FAILED_STR "Invalid"
+#define AFR_MSG_SPLIT_BRAIN_STR "Skipping conservative mergeon the file"
+#define AFR_MSG_CLEAR_TIME_SPLIT_BRAIN_STR "clear time split brain"
+#define AFR_MSG_READ_FAILED_STR "Failing read since good brick is down"
+#define AFR_MSG_LAUNCH_FAILED_STR "Failed to launch synctask"
+#define AFR_MSG_READ_SUBVOL_NOT_UP_STR                                         \
+    "read subvolume in this generation is not up"
+#define AFR_MSG_INTERNAL_LKS_FAILED_STR                                        \
+    "Unable to work with lk-owner while attempting fop"
+#define AFR_MSG_LOCK_XLATOR_NOT_LOADED_STR                                     \
+    "subvolume does not support locking. please load features/locks xlator "   \
+    "on server."
+#define AFR_MSG_FD_CTX_GET_FAILED_STR "unable to get fd ctx"
+#define AFR_MSG_INFO_COMMON_STR "fd not open on any subvolumes, aborting."
+#define AFR_MSG_REPLACE_BRICK_STATUS_STR "Couldn't acquire lock on any child."
+#define AFR_MSG_NEW_BRICK_STR "New brick"
+#define AFR_MSG_SPLIT_BRAIN_SET_FAILED_STR                                     \
+    "Failed to set split-brain choice to -1"
+#define AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED_STR                               \
+    "Failed to determine split-brain. Aborting split-brain-choice set"
+#define AFR_MSG_OPEN_FAIL_STR "Failed to open subvolume"
+#define AFR_MSG_SET_PEND_XATTR_STR "Set of pending xattr"
+#define AFR_MSG_INTERNAL_ATTR_STR "is an internal extended attribute"
 #endif /* !_AFR_MESSAGES_H_ */
diff --git a/xlators/cluster/afr/src/afr-open.c b/xlators/cluster/afr/src/afr-open.c
index de125296bb3..64856042b65 100644
--- a/xlators/cluster/afr/src/afr-open.c
+++ b/xlators/cluster/afr/src/afr-open.c
@@ -8,364 +8,346 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-#include "statedump.h"
-
-#include "afr-inode-read.h"
-#include "afr-inode-write.h"
-#include "afr-dir-read.h"
-#include "afr-dir-write.h"
-#include "afr-transaction.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/statedump.h>
 
+#include "afr-transaction.h"
 
 gf_boolean_t
-afr_is_fd_fixable (fd_t *fd)
+afr_is_fd_fixable(fd_t *fd)
 {
-        if (!fd || !fd->inode)
-                return _gf_false;
-        else if (fd_is_anonymous (fd))
-                return _gf_false;
-        else if (gf_uuid_is_null (fd->inode->gfid))
-                return _gf_false;
-
-        return _gf_true;
+    if (!fd || !fd->inode)
+        return _gf_false;
+    else if (fd_is_anonymous(fd))
+        return _gf_false;
+    else if (gf_uuid_is_null(fd->inode->gfid))
+        return _gf_false;
+
+    return _gf_true;
 }
 
-
 int
-afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+afr_open_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t * local = frame->local;
+    afr_local_t *local = frame->local;
 
-        AFR_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-                          local->fd, xdata);
-        return 0;
+    AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno,
+                     local->cont.open.fd, xdata);
+    return 0;
 }
 
 int
-afr_open_cbk (call_frame_t *frame, void *cookie,
-              xlator_t *this, int32_t op_ret, int32_t op_errno,
-              fd_t *fd, dict_t *xdata)
+afr_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t   *local           = NULL;
-        int           call_count       = -1;
-        int           child_index      = (long) cookie;
-        afr_fd_ctx_t  *fd_ctx          = NULL;
-
-        local = frame->local;
-	fd_ctx = local->fd_ctx;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
-                } else {
-                        local->op_ret = op_ret;
-			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-                }
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = (long)cookie;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+        } else {
+            local->op_ret = op_ret;
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if ((fd_ctx->flags & O_TRUNC) && (local->op_ret >= 0)) {
-                        STACK_WIND (frame, afr_open_ftruncate_cbk,
-                                    this, this->fops->ftruncate,
-                                    fd, 0, NULL);
-                } else {
-                        AFR_STACK_UNWIND (open, frame, local->op_ret,
-                                          local->op_errno, local->cont.open.fd,
-                                          local->xdata_rsp);
-                }
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        afr_handle_replies_quorum(frame, this);
+        if (local->op_ret == -1) {
+            AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno, NULL,
+                             NULL);
+        } else if (fd_ctx->flags & O_TRUNC) {
+            STACK_WIND(frame, afr_open_ftruncate_cbk, this,
+                       this->fops->ftruncate, fd, 0, NULL);
+        } else {
+            AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno,
+                             local->cont.open.fd, local->xdata_rsp);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_open_continue (call_frame_t *frame, xlator_t *this, int err)
+afr_open_continue(call_frame_t *frame, xlator_t *this, int err)
 {
-        afr_local_t   *local     = NULL;
-        afr_private_t *priv      = NULL;
-        int           call_count = 0;
-        int           i          = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int i = 0;
 
-        local  = frame->local;
-        priv   = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        if (err) {
-                AFR_STACK_UNWIND (open, frame, -1, -err, NULL, NULL);
-        } else {
-                local->call_count = AFR_COUNT (local->child_up,
-                                               priv->child_count);
-                call_count = local->call_count;
-
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i]) {
-                                STACK_WIND_COOKIE (frame, afr_open_cbk,
-                                                   (void *)(long)i,
-                                                   priv->children[i],
-                                                  priv->children[i]->fops->open,
-                                                   &local->loc,
-                                            (local->cont.open.flags & ~O_TRUNC),
-                                                   local->cont.open.fd,
-                                                   local->xdata_req);
-                                if (!--call_count)
-                                        break;
-                        }
-                }
+    if (err) {
+        AFR_STACK_UNWIND(open, frame, -1, err, NULL, NULL);
+    } else {
+        local->call_count = AFR_COUNT(local->child_up, priv->child_count);
+        call_count = local->call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->child_up[i]) {
+                STACK_WIND_COOKIE(frame, afr_open_cbk, (void *)(long)i,
+                                  priv->children[i],
+                                  priv->children[i]->fops->open, &local->loc,
+                                  (local->cont.open.flags & ~O_TRUNC),
+                                  local->cont.open.fd, local->xdata_req);
+                if (!--call_count)
+                    break;
+            }
         }
-        return 0;
+    }
+    return 0;
 }
 
 int
-afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, dict_t *xdata)
+afr_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
 {
-        afr_private_t *priv            = NULL;
-        afr_local_t   *local           = NULL;
-        int           spb_choice       = 0;
-        int           event_generation = 0;
-        int           ret              = 0;
-        int32_t       op_errno         = 0;
-        afr_fd_ctx_t  *fd_ctx          = NULL;
-
-        //We can't let truncation to happen outside transaction.
-
-        priv = this->private;
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-        local->op = GF_FOP_OPEN;
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx) {
-		op_errno = ENOMEM;
-		goto out;
-	}
-
-        if (!afr_is_consistent_io_possible (local, priv, &op_errno))
-		goto out;
-
-        local->inode = inode_ref (loc->inode);
-        loc_copy (&local->loc, loc);
-	local->fd_ctx = fd_ctx;
-	fd_ctx->flags = flags;
-        if (xdata)
-                local->xdata_req = dict_ref (xdata);
-
-        local->cont.open.flags = flags;
-        local->cont.open.fd = fd_ref (fd);
-
-        ret = afr_inode_get_readable (frame, local->inode, this,
-                                      NULL, &event_generation,
-                                      AFR_DATA_TRANSACTION);
-        if ((ret < 0) &&
-            (afr_inode_split_brain_choice_get (local->inode,
-                                             this, &spb_choice) == 0) &&
-            spb_choice < 0) {
-                afr_inode_refresh (frame, this, local->inode,
-                                   local->inode->gfid, afr_open_continue);
-        } else {
-                afr_open_continue (frame, this, 0);
-        }
-
-	return 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int spb_subvol = 0;
+    int event_generation = 0;
+    int ret = 0;
+    int32_t op_errno = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    // We can't let truncation to happen outside transaction.
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_OPEN;
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        op_errno = afr_quorum_errno(priv);
+        goto out;
+    }
+
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
+
+    local->inode = inode_ref(loc->inode);
+    loc_copy(&local->loc, loc);
+    local->fd_ctx = fd_ctx;
+    fd_ctx->flags = flags;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    local->cont.open.flags = flags;
+    local->cont.open.fd = fd_ref(fd);
+
+    ret = afr_inode_get_readable(frame, local->inode, this, NULL,
+                                 &event_generation, AFR_DATA_TRANSACTION);
+    if ((ret < 0) &&
+        (afr_split_brain_read_subvol_get(local->inode, this, NULL,
+                                         &spb_subvol) == 0) &&
+        spb_subvol < 0) {
+        afr_inode_refresh(frame, this, local->inode, local->inode->gfid,
+                          afr_open_continue);
+    } else {
+        afr_open_continue(frame, this, 0);
+    }
+
+    return 0;
 out:
-	AFR_STACK_UNWIND (open, frame, -1, op_errno, fd, NULL);
+    AFR_STACK_UNWIND(open, frame, -1, op_errno, fd, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-afr_openfd_fix_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, fd_t *fd,
-                         dict_t *xdata)
+afr_openfd_fix_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, fd_t *fd,
+                        dict_t *xdata)
 {
-        afr_local_t   *local      = NULL;
-        afr_private_t *priv       = NULL;
-        afr_fd_ctx_t  *fd_ctx     = NULL;
-        int           call_count  = 0;
-        int           child_index = (long) cookie;
-
-        priv     = this->private;
-        local    = frame->local;
-
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int call_count = 0;
+    int child_index = (long)cookie;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (op_ret >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "fd for %s opened "
+                     "successfully on subvolume %s",
+                     local->loc.path, priv->children[child_index]->name);
+    } else {
+        gf_smsg(this->name, fop_log_level(GF_FOP_OPEN, op_errno), op_errno,
+                AFR_MSG_OPEN_FAIL, "path=%s", local->loc.path, "subvolume=%s",
+                priv->children[child_index]->name, NULL);
+    }
+
+    fd_ctx = local->fd_ctx;
+
+    LOCK(&local->fd->lock);
+    {
         if (op_ret >= 0) {
-                gf_msg_debug (this->name, 0, "fd for %s opened "
-                              "successfully on subvolume %s", local->loc.path,
-                              priv->children[child_index]->name);
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
         } else {
-                gf_msg (this->name, fop_log_level (GF_FOP_OPEN, op_errno),
-                        op_errno, AFR_MSG_OPEN_FAIL, "Failed to open %s on "
-                        "subvolume %s", local->loc.path,
-                        priv->children[child_index]->name);
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
         }
+    }
+    UNLOCK(&local->fd->lock);
 
-        fd_ctx = local->fd_ctx;
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        AFR_STACK_DESTROY(frame);
 
-        LOCK (&local->fd->lock);
-        {
-                if (op_ret >= 0) {
-                        fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-                } else {
-                        fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
-                }
-        }
-        UNLOCK (&local->fd->lock);
+    return 0;
+}
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0)
-                AFR_STACK_DESTROY (frame);
+static int
+afr_fd_ctx_need_open(fd_t *fd, xlator_t *this, unsigned char *need_open)
+{
+    afr_fd_ctx_t *fd_ctx = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int count = 0;
 
+    priv = this->private;
+
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
         return 0;
-}
 
+    LOCK(&fd->lock);
+    {
+        for (i = 0; i < priv->child_count; i++) {
+            if (fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED &&
+                priv->child_up[i]) {
+                fd_ctx->opened_on[i] = AFR_FD_OPENING;
+                need_open[i] = 1;
+                count++;
+            } else {
+                need_open[i] = 0;
+            }
+        }
+    }
+    UNLOCK(&fd->lock);
 
-static int
-afr_fd_ctx_need_open (fd_t *fd, xlator_t *this, unsigned char *need_open)
-{
-	afr_fd_ctx_t *fd_ctx = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-	int count = 0;
-
-	priv = this->private;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-		return 0;
-
-	LOCK (&fd->lock);
-	{
-		for (i = 0; i < priv->child_count; i++) {
-			if (fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED &&
-			    priv->child_up[i]) {
-				fd_ctx->opened_on[i] = AFR_FD_OPENING;
-				need_open[i] = 1;
-				count++;
-			} else {
-				need_open[i] = 0;
-			}
-		}
-	}
-	UNLOCK (&fd->lock);
-
-	return count;
+    return count;
 }
 
-
 void
-afr_fix_open (fd_t *fd, xlator_t *this)
+afr_fix_open(fd_t *fd, xlator_t *this)
 {
-        afr_private_t *priv    = NULL;
-        int           i        = 0;
-        call_frame_t  *frame   = NULL;
-        afr_local_t   *local   = NULL;
-        int           ret      = -1;
-        int32_t       op_errno = 0;
-        afr_fd_ctx_t  *fd_ctx  = NULL;
-	unsigned char *need_open = NULL;
-	int call_count = 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int32_t op_errno = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    unsigned char *need_open = NULL;
+    int call_count = 0;
 
-        priv  = this->private;
+    priv = this->private;
 
-        if (!afr_is_fd_fixable (fd))
-                goto out;
+    if (!afr_is_fd_fixable(fd))
+        goto out;
 
-        fd_ctx = afr_fd_ctx_get (fd, this);
-        if (!fd_ctx)
-                goto out;
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
+        goto out;
 
-	need_open = alloca0 (priv->child_count);
+    need_open = alloca0(priv->child_count);
 
-	call_count = afr_fd_ctx_need_open (fd, this, need_open);
-	if (!call_count)
-		goto out;
+    call_count = afr_fd_ctx_need_open(fd, this, need_open);
+    if (!call_count)
+        goto out;
 
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->loc.inode = inode_ref (fd->inode);
-        ret = loc_path (&local->loc, NULL);
-        if (ret < 0)
-                goto out;
+    local->loc.inode = inode_ref(fd->inode);
+    ret = loc_path(&local->loc, NULL);
+    if (ret < 0)
+        goto out;
 
-        local->fd = fd_ref (fd);
-	local->fd_ctx = fd_ctx;
+    local->fd = fd_ref(fd);
+    local->fd_ctx = fd_ctx;
 
-        local->call_count = call_count;
+    local->call_count = call_count;
 
-        gf_msg_debug (this->name, 0, "need open count: %d",
-                      call_count);
+    gf_msg_debug(this->name, 0, "need open count: %d", call_count);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!need_open[i])
-                        continue;
-
-                if (IA_IFDIR == fd->inode->ia_type) {
-                        gf_msg_debug (this->name, 0,
-                                      "opening fd for dir %s on subvolume %s",
-                                      local->loc.path, priv->children[i]->name);
-
-                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
-                                           (void*) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->opendir,
-                                           &local->loc, local->fd,
-                                           NULL);
-                } else {
-                        gf_msg_debug (this->name, 0,
-                                      "opening fd for file %s on subvolume %s",
-                                      local->loc.path, priv->children[i]->name);
-
-                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
-                                           (void *)(long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->open,
-                                           &local->loc,
-                                           fd_ctx->flags & (~O_TRUNC),
-                                           local->fd, NULL);
-                }
-
-		if (!--call_count)
-			break;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!need_open[i])
+            continue;
+
+        if (IA_IFDIR == fd->inode->ia_type) {
+            gf_msg_debug(this->name, 0, "opening fd for dir %s on subvolume %s",
+                         local->loc.path, priv->children[i]->name);
+
+            STACK_WIND_COOKIE(frame, afr_openfd_fix_open_cbk, (void *)(long)i,
+                              priv->children[i],
+                              priv->children[i]->fops->opendir, &local->loc,
+                              local->fd, NULL);
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "opening fd for file %s on subvolume %s",
+                         local->loc.path, priv->children[i]->name);
+
+            STACK_WIND_COOKIE(frame, afr_openfd_fix_open_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->open,
+                              &local->loc, fd_ctx->flags & (~O_TRUNC),
+                              local->fd, NULL);
         }
 
-	return;
+        if (!--call_count)
+            break;
+    }
+
+    return;
 out:
-        if (frame)
-                AFR_STACK_DESTROY (frame);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
 }
diff --git a/xlators/cluster/afr/src/afr-read-txn.c b/xlators/cluster/afr/src/afr-read-txn.c
index a8a4090efd1..6fc2c75145c 100644
--- a/xlators/cluster/afr/src/afr-read-txn.c
+++ b/xlators/cluster/afr/src/afr-read-txn.c
@@ -13,140 +13,327 @@
 #include "afr-messages.h"
 
 void
-afr_pending_read_increment (afr_private_t *priv, int child_index)
+afr_pending_read_increment(afr_private_t *priv, int child_index)
 {
-        if (child_index < 0 || child_index > priv->child_count)
-                return;
+    if (child_index < 0 || child_index > priv->child_count)
+        return;
 
-        GF_ATOMIC_INC(priv->pending_reads[child_index]);
+    GF_ATOMIC_INC(priv->pending_reads[child_index]);
 }
 
 void
-afr_pending_read_decrement (afr_private_t *priv, int child_index)
+afr_pending_read_decrement(afr_private_t *priv, int child_index)
 {
-        if (child_index < 0 || child_index > priv->child_count)
-                return;
+    if (child_index < 0 || child_index > priv->child_count)
+        return;
 
-        GF_ATOMIC_DEC(priv->pending_reads[child_index]);
+    GF_ATOMIC_DEC(priv->pending_reads[child_index]);
 }
 
 void
-afr_read_txn_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_read_txn_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        afr_pending_read_decrement (priv, local->read_subvol);
-        local->read_subvol = subvol;
-        afr_pending_read_increment (priv, subvol);
-        local->readfn (frame, this, subvol);
+    afr_pending_read_decrement(priv, local->read_subvol);
+    local->read_subvol = subvol;
+    afr_pending_read_increment(priv, subvol);
+    local->readfn(frame, this, subvol);
 }
 
 int
-afr_read_txn_next_subvol (call_frame_t *frame, xlator_t *this)
+afr_read_txn_next_subvol(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-	int subvol = -1;
-
-	local = frame->local;
-	priv = this->private;
-
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->readable[i]) {
-			/* don't even bother trying here.
-			   just mark as attempted and move on. */
-			local->read_attempted[i] = 1;
-			continue;
-		}
-
-		if (!local->read_attempted[i]) {
-			subvol = i;
-			break;
-		}
-	}
-
-	/* If no more subvols were available for reading, we leave
-	   @subvol as -1, which is an indication we have run out of
-	   readable subvols. */
-	if (subvol != -1)
-		local->read_attempted[subvol] = 1;
-	afr_read_txn_wind (frame, this, subvol);
-
-	return 0;
-}
-
-int
-afr_read_txn_refresh_done (call_frame_t *frame, xlator_t *this, int err)
-{
-	afr_local_t *local = NULL;
-	int read_subvol = 0;
-	inode_t *inode = NULL;
-	int ret = -1;
-        int spb_choice = -1;
-
-	local = frame->local;
-	inode = local->inode;
-
-        if (err) {
-                read_subvol = -1;
-                goto readfn;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int subvol = -1;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->readable[i]) {
+            /* don't even bother trying here.
+               just mark as attempted and move on. */
+            local->read_attempted[i] = 1;
+            continue;
         }
 
-	read_subvol = afr_read_subvol_select_by_policy (inode, this,
-							local->readable, NULL);
-	if (read_subvol == -1) {
-                err = -EIO;
-                goto readfn;
+        if (!local->read_attempted[i]) {
+            subvol = i;
+            break;
         }
+    }
 
-	if (local->read_attempted[read_subvol]) {
-		afr_read_txn_next_subvol (frame, this);
-		return 0;
-	}
+    /* If no more subvols were available for reading, we leave
+       @subvol as -1, which is an indication we have run out of
+       readable subvols. */
+    if (subvol != -1)
+        local->read_attempted[subvol] = 1;
+    afr_read_txn_wind(frame, this, subvol);
 
-	local->read_attempted[read_subvol] = 1;
-readfn:
-        if (read_subvol == -1) {
-                ret = afr_inode_split_brain_choice_get (inode, this,
-                                                        &spb_choice);
-                if ((ret == 0) && spb_choice >= 0)
-                        read_subvol = spb_choice;
-        }
+    return 0;
+}
 
-        if (read_subvol == -1) {
-                AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN (-1, -err);
-        }
-        afr_read_txn_wind (frame, this, read_subvol);
+static int
+afr_ta_read_txn_done(int ret, call_frame_t *ta_frame, void *opaque)
+{
+    STACK_DESTROY(ta_frame->root);
+    return 0;
+}
+
+static int
+afr_ta_read_txn(void *opaque)
+{
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    int read_subvol = -1;
+    int query_child = AFR_CHILD_UNKNOWN;
+    int possible_bad_child = AFR_CHILD_UNKNOWN;
+    int ret = 0;
+    int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    dict_t *xdata_req = NULL;
+    dict_t *xdata_rsp = NULL;
+    int **pending = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    frame = (call_frame_t *)opaque;
+    this = frame->this;
+    local = frame->local;
+    priv = this->private;
+    query_child = local->read_txn_query_child;
+
+    if (query_child == AFR_CHILD_ZERO) {
+        possible_bad_child = AFR_CHILD_ONE;
+    } else if (query_child == AFR_CHILD_ONE) {
+        possible_bad_child = AFR_CHILD_ZERO;
+    } else {
+        /*read_txn_query_child is AFR_CHILD_UNKNOWN*/
+        goto out;
+    }
+
+    /* Ask the query_child to see if it blames the possibly bad one. */
+    xdata_req = dict_new();
+    if (!xdata_req)
+        goto out;
+
+    pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!pending)
+        goto out;
+
+    ret = afr_set_pending_dict(priv, xdata_req, pending);
+    if (ret < 0)
+        goto out;
+
+    if (local->fd) {
+        ret = syncop_fxattrop(priv->children[query_child], local->fd,
+                              GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                              NULL);
+    } else {
+        ret = syncop_xattrop(priv->children[query_child], &local->loc,
+                             GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                             NULL);
+    }
+    if (ret || !xdata_rsp) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed xattrop for gfid %s on %s",
+               uuid_utoa(local->inode->gfid),
+               priv->children[query_child]->name);
+        op_errno = -ret;
+        goto out;
+    }
+
+    if (afr_ta_dict_contains_pending_xattr(xdata_rsp, priv,
+                                           possible_bad_child)) {
+        read_subvol = query_child;
+        goto out;
+    }
+    dict_unref(xdata_rsp);
+    xdata_rsp = NULL;
+
+    /* It doesn't. So query thin-arbiter to see if it blames any data brick. */
+    ret = afr_fill_ta_loc(this, &loc, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        goto out;
+    }
+    flock.l_type = F_WRLCK; /*start and length are already zero. */
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, &loc, F_SETLKW, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed to get AFR_TA_DOM_MODIFY lock on %s.",
+               uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+        op_errno = -ret;
+        goto out;
+    }
+
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
+                         GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                         NULL);
+    if (ret || !xdata_rsp) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed xattrop on %s.", uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+        op_errno = -ret;
+        goto unlock;
+    }
+
+    if (!afr_ta_dict_contains_pending_xattr(xdata_rsp, priv, query_child)) {
+        read_subvol = query_child;
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, EIO, AFR_MSG_THIN_ARB,
+               "Failing read for gfid %s since good brick %s is down",
+               uuid_utoa(local->inode->gfid),
+               priv->children[possible_bad_child]->name);
+        op_errno = EIO;
+    }
+
+unlock:
+    flock.l_type = F_UNLCK;
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, &loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed to unlock AFR_TA_DOM_MODIFY lock on "
+               "%s.",
+               uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+    }
+out:
+    if (xdata_req)
+        dict_unref(xdata_req);
+    if (xdata_rsp)
+        dict_unref(xdata_rsp);
+    if (pending)
+        afr_matrix_cleanup(pending, priv->child_count);
+    loc_wipe(&loc);
+
+    if (read_subvol == -1) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    }
+    afr_read_txn_wind(frame, this, read_subvol);
+    return ret;
+}
 
-	return 0;
+void
+afr_ta_read_txn_synctask(call_frame_t *frame, xlator_t *this)
+{
+    call_frame_t *ta_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    ta_frame = afr_ta_frame_create(this);
+    if (!ta_frame) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to create ta_frame");
+        goto out;
+    }
+    ret = synctask_new(this->ctx->env, afr_ta_read_txn, afr_ta_read_txn_done,
+                       ta_frame, frame);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to launch "
+               "afr_ta_read_txn synctask for gfid %s.",
+               uuid_utoa(local->inode->gfid));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        STACK_DESTROY(ta_frame->root);
+        goto out;
+    }
+    return;
+out:
+    afr_read_txn_wind(frame, this, -1);
 }
 
+int
+afr_read_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int read_subvol = -1;
+    inode_t *inode = NULL;
+    int ret = -1;
+    int spb_subvol = -1;
+
+    local = frame->local;
+    inode = local->inode;
+    priv = this->private;
+
+    if (err) {
+        if (!priv->thin_arbiter_count)
+            goto readfn;
+        if (err != EINVAL)
+            goto readfn;
+        /* We need to query the good bricks and/or thin-arbiter.*/
+        afr_ta_read_txn_synctask(frame, this);
+        return 0;
+    }
+
+    read_subvol = afr_read_subvol_select_by_policy(inode, this, local->readable,
+                                                   NULL);
+    if (read_subvol == -1) {
+        err = EIO;
+        goto readfn;
+    }
+
+    if (local->read_attempted[read_subvol]) {
+        afr_read_txn_next_subvol(frame, this);
+        return 0;
+    }
+
+    local->read_attempted[read_subvol] = 1;
+readfn:
+    if (read_subvol == -1) {
+        ret = afr_split_brain_read_subvol_get(inode, this, frame, &spb_subvol);
+        if ((ret == 0) && spb_subvol >= 0)
+            read_subvol = spb_subvol;
+    }
+
+    if (read_subvol == -1) {
+        AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(-1, err);
+    }
+    afr_read_txn_wind(frame, this, read_subvol);
+
+    return 0;
+}
 
 int
-afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol)
+afr_read_txn_continue(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (!local->refreshed) {
-		local->refreshed = _gf_true;
-		afr_inode_refresh (frame, this, local->inode, NULL,
-				   afr_read_txn_refresh_done);
-	} else {
-		afr_read_txn_next_subvol (frame, this);
-	}
+    if (!local->refreshed) {
+        local->refreshed = _gf_true;
+        afr_inode_refresh(frame, this, local->inode, NULL,
+                          afr_read_txn_refresh_done);
+    } else {
+        afr_read_txn_next_subvol(frame, this);
+    }
 
-	return 0;
+    return 0;
 }
 
-
 /* afr_read_txn_wipe:
 
    clean internal variables in @local in order to make
@@ -155,27 +342,26 @@ afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol)
 */
 
 void
-afr_read_txn_wipe (call_frame_t *frame, xlator_t *this)
+afr_read_txn_wipe(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	local->readfn = NULL;
+    local->readfn = NULL;
 
-	if (local->inode)
-		inode_unref (local->inode);
+    if (local->inode)
+        inode_unref(local->inode);
 
-	for (i = 0; i < priv->child_count; i++) {
-		local->read_attempted[i] = 0;
-		local->readable[i] = 0;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        local->read_attempted[i] = 0;
+        local->readable[i] = 0;
+    }
 }
 
-
 /*
   afr_read_txn:
 
@@ -204,87 +390,105 @@ afr_read_txn_wipe (call_frame_t *frame, xlator_t *this)
 */
 
 int
-afr_read_txn (call_frame_t *frame, xlator_t *this, inode_t *inode,
-	      afr_read_txn_wind_t readfn, afr_transaction_type type)
+afr_read_txn(call_frame_t *frame, xlator_t *this, inode_t *inode,
+             afr_read_txn_wind_t readfn, afr_transaction_type type)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-        unsigned char *data = NULL;
-        unsigned char *metadata = NULL;
-	int read_subvol = -1;
-	int event_generation = 0;
-	int ret = -1;
-
-	priv = this->private;
-	local = frame->local;
-        data = alloca0 (priv->child_count);
-        metadata = alloca0 (priv->child_count);
-
-	afr_read_txn_wipe (frame, this);
-
-	local->readfn = readfn;
-	local->inode = inode_ref (inode);
-        local->is_read_txn = _gf_true;
-
-        if (priv->quorum_count && !afr_has_quorum (local->child_up, this)) {
-                local->op_ret = -1;
-                local->op_errno = ENOTCONN;
-                read_subvol = -1;
-                goto read;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *data = NULL;
+    unsigned char *metadata = NULL;
+    int read_subvol = -1;
+    int event_generation = 0;
+    int ret = -1;
+
+    priv = this->private;
+    local = frame->local;
+    data = alloca0(priv->child_count);
+    metadata = alloca0(priv->child_count);
+
+    afr_read_txn_wipe(frame, this);
+
+    local->readfn = readfn;
+    local->inode = inode_ref(inode);
+    local->is_read_txn = _gf_true;
+    local->transaction.type = type;
+
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_quorum_errno(priv);
+        goto read;
+    }
+
+    if (!afr_is_consistent_io_possible(local, priv, &local->op_errno)) {
+        local->op_ret = -1;
+        goto read;
+    }
+
+    if (priv->thin_arbiter_count && !afr_ta_has_quorum(priv, local)) {
+        local->op_ret = -1;
+        local->op_errno = -afr_quorum_errno(priv);
+        goto read;
+    }
+
+    if (priv->thin_arbiter_count &&
+        AFR_COUNT(local->child_up, priv->child_count) != priv->child_count) {
+        if (local->child_up[0]) {
+            local->read_txn_query_child = AFR_CHILD_ZERO;
+        } else if (local->child_up[1]) {
+            local->read_txn_query_child = AFR_CHILD_ONE;
         }
-
-        if (!afr_is_consistent_io_possible (local, priv, &local->op_errno)) {
-                local->op_ret = -1;
-                read_subvol = -1;
-                goto read;
-        }
-
-	local->transaction.type = type;
-        ret = afr_inode_read_subvol_get (inode, this, data, metadata,
-                                         &event_generation);
-	if (ret == -1)
-		/* very first transaction on this inode */
-		goto refresh;
-        AFR_INTERSECT (local->readable, data, metadata, priv->child_count);
-
-        gf_msg_debug (this->name, 0, "%s: generation now vs cached: %d, "
-                      "%d", uuid_utoa (inode->gfid), local->event_generation,
-                      event_generation);
-	if (afr_is_inode_refresh_reqd (inode, this, local->event_generation,
-                                       event_generation))
-		/* servers have disconnected / reconnected, and possibly
-		   rebooted, very likely changing the state of freshness
-		   of copies */
-		goto refresh;
-
-	read_subvol = afr_read_subvol_select_by_policy (inode, this,
-							local->readable, NULL);
-
-	if (read_subvol < 0 || read_subvol > priv->child_count) {
-	        gf_msg_debug (this->name, 0, "Unreadable subvolume %d found "
-                       "with event generation %d for gfid %s.",
-                        read_subvol, event_generation, uuid_utoa(inode->gfid));
-		goto refresh;
-	}
-
-	if (!local->child_up[read_subvol]) {
-		/* should never happen, just in case */
-	        gf_msg (this->name, GF_LOG_WARNING, 0,
-                        AFR_MSG_READ_SUBVOL_ERROR, "subvolume %d is the "
-			"read subvolume in this generation, but is not up",
-			read_subvol);
-		goto refresh;
-	}
-
-	local->read_attempted[read_subvol] = 1;
+        afr_ta_read_txn_synctask(frame, this);
+        return 0;
+    }
+
+    ret = afr_inode_read_subvol_get(inode, this, data, metadata,
+                                    &event_generation);
+    if (ret == -1)
+        /* very first transaction on this inode */
+        goto refresh;
+    AFR_INTERSECT(local->readable, data, metadata, priv->child_count);
+
+    gf_msg_debug(this->name, 0,
+                 "%s: generation now vs cached: %d, "
+                 "%d",
+                 uuid_utoa(inode->gfid), local->event_generation,
+                 event_generation);
+    if (afr_is_inode_refresh_reqd(inode, this, local->event_generation,
+                                  event_generation))
+        /* servers have disconnected / reconnected, and possibly
+           rebooted, very likely changing the state of freshness
+           of copies */
+        goto refresh;
+
+    read_subvol = afr_read_subvol_select_by_policy(inode, this, local->readable,
+                                                   NULL);
+
+    if (read_subvol < 0 || read_subvol > priv->child_count) {
+        gf_msg_debug(this->name, 0,
+                     "Unreadable subvolume %d found "
+                     "with event generation %d for gfid %s.",
+                     read_subvol, event_generation, uuid_utoa(inode->gfid));
+        goto refresh;
+    }
+
+    if (!local->child_up[read_subvol]) {
+        /* should never happen, just in case */
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_READ_SUBVOL_ERROR,
+               "subvolume %d is the "
+               "read subvolume in this generation, but is not up",
+               read_subvol);
+        goto refresh;
+    }
+
+    local->read_attempted[read_subvol] = 1;
 
 read:
-	afr_read_txn_wind (frame, this, read_subvol);
+    afr_read_txn_wind(frame, this, read_subvol);
 
-	return 0;
+    return 0;
 
 refresh:
-	afr_inode_refresh (frame, this, inode, NULL, afr_read_txn_refresh_done);
+    afr_inode_refresh(frame, this, inode, NULL, afr_read_txn_refresh_done);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index dc380c6d280..a580a1584cc 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -8,749 +8,808 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "afr.h"
 #include "afr-self-heal.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "protocol-common.h"
 #include "afr-messages.h"
-#include "events.h"
+#include <glusterfs/events.h>
 
 void
-afr_heal_synctask (xlator_t *this, afr_local_t *local);
+afr_heal_synctask(xlator_t *this, afr_local_t *local);
 
 int
-afr_lookup_and_heal_gfid (xlator_t *this, inode_t *parent, const char *name,
-                          inode_t *inode, struct afr_reply *replies,
-                          int source,  void *gfid)
-{
-        afr_private_t *priv = NULL;
-        call_frame_t   *frame    = NULL;
-        afr_local_t *local = NULL;
-        unsigned char *wind_on = NULL;
-        ia_type_t ia_type = IA_INVAL;
-        dict_t *xdata = NULL;
-        loc_t loc = {0, };
-        int ret = 0;
-        int i = 0;
-
-        priv = this->private;
-        wind_on = alloca0 (priv->child_count);
+afr_lookup_and_heal_gfid(xlator_t *this, inode_t *parent, const char *name,
+                         inode_t *inode, struct afr_reply *replies, int source,
+                         unsigned char *sources, void *gfid, int *gfid_idx)
+{
+    afr_private_t *priv = NULL;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    unsigned char *wind_on = NULL;
+    ia_type_t ia_type = IA_INVAL;
+    dict_t *xdata = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
+    int i = 0;
+
+    priv = this->private;
+    wind_on = alloca0(priv->child_count);
+    if (source >= 0 && replies[source].valid && replies[source].op_ret == 0)
         ia_type = replies[source].poststat.ia_type;
 
-        /* gfid heal on those subvolumes that do not have gfid associated
-         * with the inode and update those replies.
-         */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret != 0)
-                        continue;
-                if (!gf_uuid_is_null (replies[i].poststat.ia_gfid) ||
-                    replies[i].poststat.ia_type != ia_type)
-                        continue;
-
-                wind_on[i] = 1;
+    if (ia_type != IA_INVAL)
+        goto heal;
+
+    /* If ia_type is still invalid, it means either
+     * (a)'source' was -1, i.e. parent dir pending xattrs are in split-brain
+     * (or) (b) The parent dir pending xattrs are all zeroes (i.e. all bricks
+     * are sources) and the 'source' we selected earlier might be the one where
+     * the file is not actually present.
+     *
+     * In both cases, let us pick a brick with a successful reply and use its
+     * ia_type.
+     * */
+    for (i = 0; i < priv->child_count; i++) {
+        if (source == -1) {
+            /* case (a) above. */
+            if (replies[i].valid && replies[i].op_ret == 0 &&
+                replies[i].poststat.ia_type != IA_INVAL) {
+                ia_type = replies[i].poststat.ia_type;
+                break;
+            }
+        } else {
+            /* case (b) above. */
+            if (i == source)
+                continue;
+            if (sources[i] && replies[i].valid && replies[i].op_ret == 0 &&
+                replies[i].poststat.ia_type != IA_INVAL) {
+                ia_type = replies[i].poststat.ia_type;
+                break;
+            }
         }
+    }
 
-        if (AFR_COUNT(wind_on, priv->child_count) == 0)
-                return 0;
+heal:
+    /* gfid heal on those subvolumes that do not have gfid associated
+     * with the inode and update those replies.
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
 
-        xdata = dict_new ();
-        if (!xdata) {
-                ret = -ENOMEM;
-                goto out;
-        }
+        if (gf_uuid_is_null(gfid) &&
+            !gf_uuid_is_null(replies[i].poststat.ia_gfid) &&
+            replies[i].poststat.ia_type == ia_type)
+            gfid = replies[i].poststat.ia_gfid;
 
-        ret = dict_set_gfuuid (xdata, "gfid-req", gfid, true);
-        if (ret) {
-                ret = -ENOMEM;
-                goto out;
-        }
+        if (!gf_uuid_is_null(replies[i].poststat.ia_gfid) ||
+            replies[i].poststat.ia_type != ia_type)
+            continue;
 
-        frame = afr_frame_create (this, &ret);
-        if (!frame) {
-                ret = -ret;
-                goto out;
-        }
+        wind_on[i] = 1;
+    }
 
-        local = frame->local;
-        loc.parent = inode_ref (parent);
-        gf_uuid_copy (loc.pargfid, parent->gfid);
-        loc.name = name;
-        loc.inode = inode_ref (inode);
+    if (AFR_COUNT(wind_on, priv->child_count) == 0)
+        return 0;
 
-        AFR_ONLIST (wind_on, frame, afr_selfheal_discover_cbk, lookup,
-                    &loc, xdata);
+    xdata = dict_new();
+    if (!xdata) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
+    ret = dict_set_gfuuid(xdata, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    local = frame->local;
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = name;
+    loc.inode = inode_ref(inode);
+
+    AFR_ONLIST(wind_on, frame, afr_selfheal_discover_cbk, lookup, &loc, xdata);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!wind_on[i])
+            continue;
+        afr_reply_wipe(&replies[i]);
+        afr_reply_copy(&replies[i], &local->replies[i]);
+    }
+    if (gfid_idx && (*gfid_idx == -1)) {
+        /*Pick a brick where the gifd heal was successful.*/
         for (i = 0; i < priv->child_count; i++) {
-                if (!wind_on[i])
-                        continue;
-                afr_reply_wipe (&replies[i]);
-                afr_reply_copy (&replies[i], &local->replies[i]);
+            if (!wind_on[i])
+                continue;
+            if (replies[i].valid && replies[i].op_ret == 0 &&
+                !gf_uuid_is_null(replies[i].poststat.ia_gfid)) {
+                *gfid_idx = i;
+                break;
+            }
         }
+    }
 out:
-        loc_wipe (&loc);
-        if (frame)
-                AFR_STACK_DESTROY (frame);
-        if (xdata)
-                dict_unref (xdata);
+    if (gfid_idx && (*gfid_idx == -1) && (ret == 0) && local) {
+        ret = -afr_final_errno(local, priv);
+    }
+    loc_wipe(&loc);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    if (xdata)
+        dict_unref(xdata);
 
-        return ret;
+    return ret;
 }
 
 int
-afr_gfid_sbrain_source_from_src_brick (xlator_t *this,
-                                       struct afr_reply *replies,
-                                       char *src_brick)
+afr_gfid_sbrain_source_from_src_brick(xlator_t *this, struct afr_reply *replies,
+                                      char *src_brick)
 {
-        int             i        = 0;
-        afr_private_t  *priv     = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-                if (strcmp (priv->children[i]->name, src_brick) == 0)
-                        return i;
-        }
-        return -1;
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (strcmp(priv->children[i]->name, src_brick) == 0)
+            return i;
+    }
+    return -1;
 }
 
 int
-afr_selfheal_gfid_mismatch_by_majority (struct afr_reply *replies,
-                                        int child_count)
+afr_selfheal_gfid_mismatch_by_majority(struct afr_reply *replies,
+                                       int child_count)
 {
-        int             j                  = 0;
-        int             i                  = 0;
-        int             src                = -1;
-        int             votes[child_count];
-
-        for (i = 0; i < child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-
-                votes[i] = 1;
-                for (j = i+1; j < child_count; j++) {
-                        if ((!gf_uuid_compare (replies[i].poststat.ia_gfid,
-                            replies[j].poststat.ia_gfid)))
-                                votes[i]++;
-                        if (votes[i] > child_count / 2) {
-                                src = i;
-                                goto out;
-                        }
-                }
+    int j = 0;
+    int i = 0;
+    int votes;
+
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+
+        votes = 1;
+        for (j = i + 1; j < child_count; j++) {
+            if ((!gf_uuid_compare(replies[i].poststat.ia_gfid,
+                                  replies[j].poststat.ia_gfid)))
+                votes++;
+            if (votes > child_count / 2)
+                return i;
         }
+    }
 
-out:
-        return src;
+    return -1;
 }
 
-int afr_gfid_sbrain_source_from_bigger_file (struct afr_reply *replies,
-                                             int child_count)
+int
+afr_gfid_sbrain_source_from_bigger_file(struct afr_reply *replies,
+                                        int child_count)
 {
-        int       i       = 0;
-        int       src     = -1;
-        uint64_t  size    = 0;
+    int i = 0;
+    int src = -1;
+    uint64_t size = 0;
 
-        for (i = 0; i < child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-                if (size < replies[i].poststat.ia_size) {
-                        src = i;
-                        size = replies[i].poststat.ia_size;
-                } else if (replies[i].poststat.ia_size == size) {
-                        src = -1;
-                }
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (size < replies[i].poststat.ia_size) {
+            src = i;
+            size = replies[i].poststat.ia_size;
+        } else if (replies[i].poststat.ia_size == size) {
+            src = -1;
         }
-        return src;
-}
-
-int afr_gfid_sbrain_source_from_latest_mtime (struct afr_reply *replies,
-                                              int child_count)
-{
-        int       i             = 0;
-        int       src           = -1;
-        uint32_t  mtime         = 0;
-        uint32_t  mtime_nsec    = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret != 0)
-                        continue;
-                if ((mtime < replies[i].poststat.ia_mtime) ||
-                    ((mtime == replies[i].poststat.ia_mtime) &&
-                     (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
-                        src = i;
-                        mtime = replies[i].poststat.ia_mtime;
-                        mtime_nsec = replies[i].poststat.ia_mtime_nsec;
-                } else if ((mtime == replies[i].poststat.ia_mtime) &&
-                           (mtime_nsec == replies[i].poststat.ia_mtime_nsec)) {
-                        src = -1;
-                }
-        }
-        return src;
+    }
+    return src;
 }
 
 int
-afr_gfid_split_brain_source (xlator_t *this, struct afr_reply *replies,
-                             inode_t *inode, uuid_t pargfid, const char *bname,
-                             int src_idx, int child_idx,
-                             unsigned char *locked_on, int *src, dict_t *xdata)
-{
-        afr_private_t   *priv      = NULL;
-        char             g1[64]    = {0,};
-        char             g2[64]    = {0,};
-        int              up_count  = 0;
-        int              heal_op   = -1;
-        int              ret       = -1;
-        char            *src_brick = NULL;
-
-        *src = -1;
-        priv = this->private;
-        up_count = AFR_COUNT (locked_on, priv->child_count);
-        if (up_count != priv->child_count) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
-                        "All the bricks should be up to resolve the gfid split "
-                        "barin");
-                if (xdata) {
-                        ret = dict_set_str (xdata, "gfid-heal-msg", "All the "
-                                            "bricks should be up to resolve the"
-                                            " gfid split barin");
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        AFR_MSG_DICT_SET_FAILED, "Error setting"
-                                        " gfid-heal-msg dict");
-                }
-                goto out;
+afr_gfid_sbrain_source_from_latest_mtime(struct afr_reply *replies,
+                                         int child_count)
+{
+    int i = 0;
+    int src = -1;
+    uint32_t mtime = 0;
+    uint32_t mtime_nsec = 0;
+
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
+        if ((mtime < replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
+            src = i;
+            mtime = replies[i].poststat.ia_mtime;
+            mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+        } else if ((mtime == replies[i].poststat.ia_mtime) &&
+                   (mtime_nsec == replies[i].poststat.ia_mtime_nsec)) {
+            src = -1;
         }
+    }
+    return src;
+}
 
+int
+afr_gfid_split_brain_source(xlator_t *this, struct afr_reply *replies,
+                            inode_t *inode, uuid_t pargfid, const char *bname,
+                            int src_idx, int child_idx,
+                            unsigned char *locked_on, int *src, dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    char g1[64] = {
+        0,
+    };
+    char g2[64] = {
+        0,
+    };
+    int up_count = 0;
+    int heal_op = -1;
+    int ret = -1;
+    char *src_brick = NULL;
+
+    *src = -1;
+    priv = this->private;
+    up_count = AFR_COUNT(locked_on, priv->child_count);
+    if (up_count != priv->child_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "All the bricks should be up to resolve the gfid split "
+               "barin");
         if (xdata) {
-                ret = dict_get_int32 (xdata, "heal-op", &heal_op);
-                if (ret)
-                        goto fav_child;
-        } else {
-                goto fav_child;
+            ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                           SALL_BRICKS_UP_TO_RESOLVE);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                       "Error setting"
+                       " gfid-heal-msg dict");
         }
+        goto out;
+    }
+
+    if (xdata) {
+        ret = dict_get_int32_sizen(xdata, "heal-op", &heal_op);
+        if (ret)
+            goto fav_child;
+    } else {
+        goto fav_child;
+    }
 
-        switch (heal_op) {
+    switch (heal_op) {
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
-                *src = afr_gfid_sbrain_source_from_bigger_file (replies,
-                                                                priv->child_count);
-                if (*src == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN, "No bigger file");
-                        if (xdata) {
-                                ret = dict_set_str (xdata, "gfid-heal-msg",
-                                                    "No bigger file");
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                AFR_MSG_DICT_SET_FAILED, "Error"
-                                                " setting gfid-heal-msg dict");
-                        }
+            *src = afr_gfid_sbrain_source_from_bigger_file(replies,
+                                                           priv->child_count);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SNO_BIGGER_FILE);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SNO_BIGGER_FILE);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               " setting gfid-heal-msg dict");
                 }
-                break;
+            }
+            break;
 
         case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
-                *src = afr_gfid_sbrain_source_from_latest_mtime (replies,
-                                                                 priv->child_count);
-                if (*src == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN, "No difference in mtime");
-                        if (xdata) {
-                                ret = dict_set_str (xdata, "gfid-heal-msg",
-                                                    "No difference in mtime");
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                AFR_MSG_DICT_SET_FAILED, "Error"
-                                                "setting gfid-heal-msg dict");
-                        }
+            *src = afr_gfid_sbrain_source_from_latest_mtime(replies,
+                                                            priv->child_count);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SNO_DIFF_IN_MTIME);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SNO_DIFF_IN_MTIME);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               "setting gfid-heal-msg dict");
                 }
-                break;
+            }
+            break;
 
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
-                ret = dict_get_str (xdata, "child-name", &src_brick);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN, "Error getting the source "
-                                "brick");
-                        break;
-                }
-                *src = afr_gfid_sbrain_source_from_src_brick (this, replies,
-                                                              src_brick);
-                if (*src == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN, "Error getting the source "
-                                "brick");
-                        if (xdata) {
-                                ret = dict_set_str (xdata, "gfid-heal-msg",
-                                                    "Error getting the source "
-                                                    "brick");
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                AFR_MSG_DICT_SET_FAILED, "Error"
-                                                " setting gfid-heal-msg dict");
-                        }
-                }
+            ret = dict_get_str_sizen(xdata, "child-name", &src_brick);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       "Error getting the source "
+                       "brick");
                 break;
+            }
+            *src = afr_gfid_sbrain_source_from_src_brick(this, replies,
+                                                         src_brick);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SERROR_GETTING_SRC_BRICK);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SERROR_GETTING_SRC_BRICK);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               " setting gfid-heal-msg dict");
+                }
+            }
+            break;
 
         default:
-                break;
-        }
-        goto out;
+            break;
+    }
+    goto out;
 
 fav_child:
-        switch (priv->fav_child_policy) {
+    switch (priv->fav_child_policy) {
         case AFR_FAV_CHILD_BY_SIZE:
-                *src = afr_sh_fav_by_size (this, replies, inode);
-                break;
+            *src = afr_sh_fav_by_size(this, replies, inode);
+            break;
         case AFR_FAV_CHILD_BY_MTIME:
-                *src = afr_sh_fav_by_mtime (this, replies, inode);
-                break;
+            *src = afr_sh_fav_by_mtime(this, replies, inode);
+            break;
         case AFR_FAV_CHILD_BY_CTIME:
-                *src = afr_sh_fav_by_ctime(this, replies, inode);
-                break;
+            *src = afr_sh_fav_by_ctime(this, replies, inode);
+            break;
         case AFR_FAV_CHILD_BY_MAJORITY:
-                if (priv->child_count != 2)
-                        *src = afr_selfheal_gfid_mismatch_by_majority (replies,
-                                                                       priv->child_count);
-                else
-                        *src = -1;
-
-                if (*src == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN, "No majority to resolve "
-                                "gfid split brain");
-                }
-                break;
+            if (priv->child_count != 2)
+                *src = afr_selfheal_gfid_mismatch_by_majority(
+                    replies, priv->child_count);
+            else
+                *src = -1;
+
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       "No majority to resolve "
+                       "gfid split brain");
+            }
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
 out:
-        if (*src == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
-                        "Gfid mismatch detected for <gfid:%s>/%s>, %s on %s and"
-                        " %s on %s.", uuid_utoa (pargfid), bname,
-                        uuid_utoa_r (replies[child_idx].poststat.ia_gfid, g1),
-                        priv->children[child_idx]->name,
-                        uuid_utoa_r (replies[src_idx].poststat.ia_gfid, g2),
-                        priv->children[src_idx]->name);
-                gf_event (EVENT_AFR_SPLIT_BRAIN, "subvol=%s;type=gfid;file="
-                          "<gfid:%s>/%s>;count=2;child-%d=%s;gfid-%d=%s;"
-                          "child-%d=%s;gfid-%d=%s", this->name,
-                          uuid_utoa (pargfid), bname, child_idx,
-                          priv->children[child_idx]->name, child_idx,
-                          uuid_utoa_r (replies[child_idx].poststat.ia_gfid, g1),
-                          src_idx, priv->children[src_idx]->name, src_idx,
-                          uuid_utoa_r (replies[src_idx].poststat.ia_gfid, g2));
-                return -1;
-        }
-        return 0;
+    if (*src == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "Gfid mismatch detected for <gfid:%s>/%s>, %s on %s and"
+               " %s on %s.",
+               uuid_utoa(pargfid), bname,
+               uuid_utoa_r(replies[child_idx].poststat.ia_gfid, g1),
+               priv->children[child_idx]->name,
+               uuid_utoa_r(replies[src_idx].poststat.ia_gfid, g2),
+               priv->children[src_idx]->name);
+        gf_event(EVENT_AFR_SPLIT_BRAIN,
+                 "client-pid=%d;"
+                 "subvol=%s;type=gfid;file="
+                 "<gfid:%s>/%s>;count=2;child-%d=%s;gfid-%d=%s;"
+                 "child-%d=%s;gfid-%d=%s",
+                 this->ctx->cmd_args.client_pid, this->name, uuid_utoa(pargfid),
+                 bname, child_idx, priv->children[child_idx]->name, child_idx,
+                 uuid_utoa_r(replies[child_idx].poststat.ia_gfid, g1), src_idx,
+                 priv->children[src_idx]->name, src_idx,
+                 uuid_utoa_r(replies[src_idx].poststat.ia_gfid, g2));
+        return -1;
+    }
+    return 0;
 }
 
-
 int
-afr_selfheal_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+afr_selfheal_post_op_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-        local->op_ret = op_ret;
-        local->op_errno = op_errno;
-	syncbarrier_wake (&local->barrier);
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+    syncbarrier_wake(&local->barrier);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      int subvol, dict_t *xattr, dict_t *xdata)
+afr_selfheal_post_op(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     int subvol, dict_t *xattr, dict_t *xdata)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	loc_t loc = {0, };
-        int ret = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
 
-	priv = this->private;
-	local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-        local->op_ret = 0;
+    local->op_ret = 0;
 
-	STACK_WIND (frame, afr_selfheal_post_op_cbk, priv->children[subvol],
-		    priv->children[subvol]->fops->xattrop, &loc,
-		    GF_XATTROP_ADD_ARRAY, xattr, xdata);
+    STACK_WIND(frame, afr_selfheal_post_op_cbk, priv->children[subvol],
+               priv->children[subvol]->fops->xattrop, &loc,
+               GF_XATTROP_ADD_ARRAY, xattr, xdata);
 
-	syncbarrier_wait (&local->barrier, 1);
-        if (local->op_ret < 0)
-                ret = -local->op_errno;
+    syncbarrier_wait(&local->barrier, 1);
+    if (local->op_ret < 0)
+        ret = -local->op_errno;
 
-        loc_wipe (&loc);
-        local->op_ret = 0;
+    loc_wipe(&loc);
+    local->op_ret = 0;
 
-	return ret;
+    return ret;
 }
 
 int
-afr_check_stale_error (struct afr_reply *replies, afr_private_t *priv)
+afr_check_stale_error(struct afr_reply *replies, afr_private_t *priv)
 {
-        int i = 0;
-        int op_errno = 0;
-        int tmp_errno = 0;
-        int stale_count = 0;
+    int i = 0;
+    int op_errno = 0;
+    int tmp_errno = 0;
+    int stale_count = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                tmp_errno = replies[i].op_errno;
-                if (tmp_errno == ENOENT || tmp_errno == ESTALE) {
-                        op_errno = afr_higher_errno (op_errno, tmp_errno);
-                        stale_count++;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        tmp_errno = replies[i].op_errno;
+        if (tmp_errno == ENOENT || tmp_errno == ESTALE) {
+            op_errno = afr_higher_errno(op_errno, tmp_errno);
+            stale_count++;
         }
-        if (stale_count != priv->child_count)
-                return -ENOTCONN;
-        else
-                return -op_errno;
+    }
+    if (stale_count != priv->child_count)
+        return -ENOTCONN;
+    else
+        return -op_errno;
 }
 
 int
-afr_sh_generic_fop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-          int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
-          dict_t *xdata)
+afr_sh_generic_fop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *pre,
+                       struct iatt *post, dict_t *xdata)
 {
-        int i = (long) cookie;
-        afr_local_t *local = NULL;
+    int i = (long)cookie;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->replies[i].valid = 1;
-        local->replies[i].op_ret = op_ret;
-        local->replies[i].op_errno = op_errno;
-        if (pre)
-                local->replies[i].prestat = *pre;
-        if (post)
-                local->replies[i].poststat = *post;
-        if (xdata)
-                local->replies[i].xdata = dict_ref (xdata);
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (pre)
+        local->replies[i].prestat = *pre;
+    if (post)
+        local->replies[i].poststat = *post;
+    if (xdata)
+        local->replies[i].xdata = dict_ref(xdata);
 
-        syncbarrier_wake (&local->barrier);
+    syncbarrier_wake(&local->barrier);
 
-        return 0;
+    return 0;
 }
 
 int
-afr_selfheal_restore_time (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                           int source, unsigned char *healed_sinks,
-                           struct afr_reply *replies)
+afr_selfheal_restore_time(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          int source, unsigned char *healed_sinks,
+                          struct afr_reply *replies)
 {
-        loc_t loc = {0, };
+    loc_t loc = {
+        0,
+    };
 
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-        AFR_ONLIST (healed_sinks, frame, afr_sh_generic_fop_cbk, setattr, &loc,
-                    &replies[source].poststat,
-                    (GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME), NULL);
+    AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, setattr, &loc,
+               &replies[source].poststat,
+               (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME),
+               NULL);
 
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        return 0;
+    return 0;
 }
 
 dict_t *
-afr_selfheal_output_xattr (xlator_t *this, gf_boolean_t is_full_crawl,
-                           afr_transaction_type type, int *output_dirty,
-                           int **output_matrix, int subvol,
-                           int **full_heal_mtx_out)
-{
-	int                j     = 0;
-	int                idx   = 0;
-	int                d_idx = 0;
-	int                ret   = 0;
-	int               *raw   = 0;
-	dict_t            *xattr = NULL;
-	afr_private_t     *priv  = NULL;
-
-	priv = this->private;
-	idx = afr_index_for_transaction_type (type);
-        d_idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-
-	xattr = dict_new ();
-	if (!xattr)
-		return NULL;
-
-	/* clear dirty */
-	raw = GF_CALLOC (sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
-	if (!raw)
-		goto err;
-
-	raw[idx] = hton32 (output_dirty[subvol]);
-	ret = dict_set_bin (xattr, AFR_DIRTY, raw,
-			    sizeof(int) * AFR_NUM_CHANGE_LOGS);
-	if (ret) {
-                GF_FREE (raw);
-		goto err;
-        }
+afr_selfheal_output_xattr(xlator_t *this, gf_boolean_t is_full_crawl,
+                          afr_transaction_type type, int *output_dirty,
+                          int **output_matrix, int subvol,
+                          int **full_heal_mtx_out)
+{
+    int j = 0;
+    int idx = 0;
+    int d_idx = 0;
+    int ret = 0;
+    int *raw = 0;
+    dict_t *xattr = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    idx = afr_index_for_transaction_type(type);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+
+    xattr = dict_new();
+    if (!xattr)
+        return NULL;
 
-	/* clear/set pending */
-	for (j = 0; j < priv->child_count; j++) {
-		raw = GF_CALLOC (sizeof(int), AFR_NUM_CHANGE_LOGS,
-				 gf_afr_mt_int32_t);
-		if (!raw)
-			goto err;
-
-		raw[idx] = hton32 (output_matrix[subvol][j]);
-                if (is_full_crawl)
-                        raw[d_idx] = hton32 (full_heal_mtx_out[subvol][j]);
-
-		ret = dict_set_bin (xattr, priv->pending_key[j],
-				    raw, sizeof(int) * AFR_NUM_CHANGE_LOGS);
-		if (ret) {
-                        GF_FREE (raw);
-			goto err;
-                }
-	}
+    /* clear dirty */
+    raw = GF_CALLOC(sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
+    if (!raw)
+        goto err;
+
+    raw[idx] = hton32(output_dirty[subvol]);
+    ret = dict_set_bin(xattr, AFR_DIRTY, raw,
+                       sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    if (ret) {
+        GF_FREE(raw);
+        goto err;
+    }
+
+    /* clear/set pending */
+    for (j = 0; j < priv->child_count; j++) {
+        raw = GF_CALLOC(sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
+        if (!raw)
+            goto err;
+
+        raw[idx] = hton32(output_matrix[subvol][j]);
+        if (is_full_crawl)
+            raw[d_idx] = hton32(full_heal_mtx_out[subvol][j]);
+
+        ret = dict_set_bin(xattr, priv->pending_key[j], raw,
+                           sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret) {
+            GF_FREE(raw);
+            goto err;
+        }
+    }
 
-	return xattr;
+    return xattr;
 err:
-	if (xattr)
-		dict_unref (xattr);
-	return NULL;
+    if (xattr)
+        dict_unref(xattr);
+    return NULL;
 }
 
-
 int
-afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			   unsigned char *sources, unsigned char *sinks,
-			   unsigned char *healed_sinks,
-                           unsigned char *undid_pending,
-                           afr_transaction_type type, struct afr_reply *replies,
-                           unsigned char *locked_on)
-{
-	afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-	int i = 0;
-	int j = 0;
-	unsigned char *pending = NULL;
-	int *input_dirty = NULL;
-	int **input_matrix = NULL;
-	int **full_heal_mtx_in = NULL;
-	int **full_heal_mtx_out = NULL;
-	int *output_dirty = NULL;
-	int **output_matrix = NULL;
-	dict_t *xattr = NULL;
-	dict_t *xdata = NULL;
-
-	priv = this->private;
-        local = frame->local;
-
-	pending = alloca0 (priv->child_count);
-
-	input_dirty = alloca0 (priv->child_count * sizeof (int));
-	input_matrix = ALLOC_MATRIX (priv->child_count, int);
-	full_heal_mtx_in = ALLOC_MATRIX (priv->child_count, int);
-	full_heal_mtx_out = ALLOC_MATRIX (priv->child_count, int);
-	output_dirty = alloca0 (priv->child_count * sizeof (int));
-	output_matrix = ALLOC_MATRIX (priv->child_count, int);
-
-        xdata = dict_new ();
-        if (!xdata)
-                return -1;
+afr_selfheal_undo_pending(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          unsigned char *sources, unsigned char *sinks,
+                          unsigned char *healed_sinks,
+                          unsigned char *undid_pending,
+                          afr_transaction_type type, struct afr_reply *replies,
+                          unsigned char *locked_on)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int j = 0;
+    unsigned char *pending = NULL;
+    int *input_dirty = NULL;
+    int **input_matrix = NULL;
+    int **full_heal_mtx_in = NULL;
+    int **full_heal_mtx_out = NULL;
+    int *output_dirty = NULL;
+    int **output_matrix = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xdata = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    pending = alloca0(priv->child_count);
+
+    input_dirty = alloca0(priv->child_count * sizeof(int));
+    input_matrix = ALLOC_MATRIX(priv->child_count, int);
+    full_heal_mtx_in = ALLOC_MATRIX(priv->child_count, int);
+    full_heal_mtx_out = ALLOC_MATRIX(priv->child_count, int);
+    output_dirty = alloca0(priv->child_count * sizeof(int));
+    output_matrix = ALLOC_MATRIX(priv->child_count, int);
+
+    xdata = dict_new();
+    if (!xdata)
+        return -1;
 
-	afr_selfheal_extract_xattr (this, replies, type, input_dirty,
-				    input_matrix);
-
-        if (local->need_full_crawl)
-                afr_selfheal_extract_xattr (this, replies, AFR_DATA_TRANSACTION,
-                                            NULL, full_heal_mtx_in);
-
-	for (i = 0; i < priv->child_count; i++)
-		if (sinks[i] && !healed_sinks[i])
-			pending[i] = 1;
-
-	for (i = 0; i < priv->child_count; i++) {
-		for (j = 0; j < priv->child_count; j++) {
-			if (pending[j]) {
-				output_matrix[i][j] = 1;
-                                if (type == AFR_ENTRY_TRANSACTION)
-                                        full_heal_mtx_out[i][j] = 1;
-			} else if (locked_on[j]) {
-				output_matrix[i][j] = -input_matrix[i][j];
-                                if (type == AFR_ENTRY_TRANSACTION)
-                                        full_heal_mtx_out[i][j] = -full_heal_mtx_in[i][j];
-                        }
-		}
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!pending[i])
-			output_dirty[i] = -input_dirty[i];
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!locked_on[i])
-			/* perform post-op only on subvols we had locked
-			   and inspected on.
-			*/
-			continue;
-                if (undid_pending[i])
-                        /* We already unset the pending xattrs in
-                         * _afr_fav_child_reset_sink_xattrs(). */
-                        continue;
-
-		xattr = afr_selfheal_output_xattr (this, local->need_full_crawl,
-                                                   type, output_dirty,
-                                                   output_matrix, i,
-                                                   full_heal_mtx_out);
-		if (!xattr) {
-			continue;
-		}
-
-                if ((type == AFR_ENTRY_TRANSACTION) && (priv->esh_granular)) {
-                        if (xdata &&
-                            dict_set_int8 (xdata, GF_XATTROP_PURGE_INDEX, 1))
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        AFR_MSG_DICT_SET_FAILED, "Failed to set"
-                                        " dict value for %s",
-                                        GF_XATTROP_PURGE_INDEX);
-                }
+    afr_selfheal_extract_xattr(this, replies, type, input_dirty, input_matrix);
+
+    if (local->need_full_crawl)
+        afr_selfheal_extract_xattr(this, replies, AFR_DATA_TRANSACTION, NULL,
+                                   full_heal_mtx_in);
+
+    for (i = 0; i < priv->child_count; i++)
+        if (sinks[i] && !healed_sinks[i])
+            pending[i] = 1;
+
+    for (i = 0; i < priv->child_count; i++) {
+        for (j = 0; j < priv->child_count; j++) {
+            if (pending[j]) {
+                output_matrix[i][j] = 1;
+                if (type == AFR_ENTRY_TRANSACTION)
+                    full_heal_mtx_out[i][j] = 1;
+            } else if (locked_on[j]) {
+                output_matrix[i][j] = -input_matrix[i][j];
+                if (type == AFR_ENTRY_TRANSACTION)
+                    full_heal_mtx_out[i][j] = -full_heal_mtx_in[i][j];
+            }
+        }
+    }
 
-		afr_selfheal_post_op (frame, this, inode, i, xattr, xdata);
-		dict_unref (xattr);
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (!pending[i])
+            output_dirty[i] = -input_dirty[i];
+    }
 
-        if (xdata)
-                dict_unref (xdata);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!locked_on[i])
+            /* perform post-op only on subvols we had locked
+               and inspected on.
+            */
+            continue;
+        if (undid_pending[i])
+            /* We already unset the pending xattrs in
+             * _afr_fav_child_reset_sink_xattrs(). */
+            continue;
 
-	return 0;
+        xattr = afr_selfheal_output_xattr(this, local->need_full_crawl, type,
+                                          output_dirty, output_matrix, i,
+                                          full_heal_mtx_out);
+        if (!xattr) {
+            continue;
+        }
+
+        if ((type == AFR_ENTRY_TRANSACTION) && (priv->esh_granular)) {
+            if (xdata && dict_set_int8(xdata, GF_XATTROP_PURGE_INDEX, 1))
+                gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_DICT_SET_FAILED,
+                       "Failed to set"
+                       " dict value for %s",
+                       GF_XATTROP_PURGE_INDEX);
+        }
+
+        afr_selfheal_post_op(frame, this, inode, i, xattr, xdata);
+        dict_unref(xattr);
+    }
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
 }
 
 void
-afr_reply_copy (struct afr_reply *dst, struct afr_reply *src)
-{
-	dict_t *xdata = NULL;
-
-        dst->valid = src->valid;
-	dst->op_ret = src->op_ret;
-	dst->op_errno = src->op_errno;
-	dst->prestat = src->prestat;
-	dst->poststat = src->poststat;
-	dst->preparent = src->preparent;
-	dst->postparent = src->postparent;
-	dst->preparent2 = src->preparent2;
-	dst->postparent2 = src->postparent2;
-	if (src->xdata)
-		xdata = dict_ref (src->xdata);
-	else
-		xdata = NULL;
-	if (dst->xdata)
-		dict_unref (dst->xdata);
-	dst->xdata = xdata;
-        if (xdata && dict_get_str_boolean (xdata, "fips-mode-rchecksum",
-            _gf_false) == _gf_true) {
-                memcpy (dst->checksum, src->checksum, SHA256_DIGEST_LENGTH);
-        } else {
-                memcpy (dst->checksum, src->checksum, MD5_DIGEST_LENGTH);
-        }
-        dst->fips_mode_rchecksum = src->fips_mode_rchecksum;
+afr_reply_copy(struct afr_reply *dst, struct afr_reply *src)
+{
+    dict_t *xdata = NULL;
+
+    dst->valid = src->valid;
+    dst->op_ret = src->op_ret;
+    dst->op_errno = src->op_errno;
+    dst->prestat = src->prestat;
+    dst->poststat = src->poststat;
+    dst->preparent = src->preparent;
+    dst->postparent = src->postparent;
+    dst->preparent2 = src->preparent2;
+    dst->postparent2 = src->postparent2;
+    if (src->xdata)
+        xdata = dict_ref(src->xdata);
+    else
+        xdata = NULL;
+    if (dst->xdata)
+        dict_unref(dst->xdata);
+    dst->xdata = xdata;
+    if (xdata && dict_get_str_boolean(xdata, "fips-mode-rchecksum",
+                                      _gf_false) == _gf_true) {
+        memcpy(dst->checksum, src->checksum, SHA256_DIGEST_LENGTH);
+    } else {
+        memcpy(dst->checksum, src->checksum, MD5_DIGEST_LENGTH);
+    }
+    dst->fips_mode_rchecksum = src->fips_mode_rchecksum;
 }
 
 void
-afr_replies_copy (struct afr_reply *dst, struct afr_reply *src, int count)
+afr_replies_copy(struct afr_reply *dst, struct afr_reply *src, int count)
 {
-	int i = 0;
+    int i = 0;
 
-	if (dst == src)
-		return;
+    if (dst == src)
+        return;
 
-	for (i = 0; i < count; i++) {
-                afr_reply_copy (&dst[i], &src[i]);
-	}
+    for (i = 0; i < count; i++) {
+        afr_reply_copy(&dst[i], &src[i]);
+    }
 }
 
 int
-afr_selfheal_fill_dirty (xlator_t *this, int *dirty, int subvol,
-			 int idx, dict_t *xdata)
+afr_selfheal_fill_dirty(xlator_t *this, int *dirty, int subvol, int idx,
+                        dict_t *xdata)
 {
-	void *pending_raw = NULL;
-	int pending[3] = {0, };
+    void *pending_raw = NULL;
+    int pending[3] = {
+        0,
+    };
 
-        if (!dirty)
-                return 0;
+    if (!dirty)
+        return 0;
 
-	if (dict_get_ptr (xdata, AFR_DIRTY, &pending_raw))
-		return -1;
+    if (dict_get_ptr(xdata, AFR_DIRTY, &pending_raw))
+        return -1;
 
-	if (!pending_raw)
-		return -1;
+    if (!pending_raw)
+        return -1;
 
-	memcpy (pending, pending_raw, sizeof(pending));
+    memcpy(pending, pending_raw, sizeof(pending));
 
-	dirty[subvol] = ntoh32 (pending[idx]);
+    dirty[subvol] = ntoh32(pending[idx]);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_selfheal_fill_matrix (xlator_t *this, int **matrix, int subvol,
-			  int idx, dict_t *xdata)
+afr_selfheal_fill_matrix(xlator_t *this, int **matrix, int subvol, int idx,
+                         dict_t *xdata)
 {
-	int i = 0;
-	void *pending_raw = NULL;
-	int pending[3] = {0, };
-	afr_private_t *priv = NULL;
+    int i = 0;
+    void *pending_raw = NULL;
+    int pending[3] = {
+        0,
+    };
+    afr_private_t *priv = NULL;
 
-	priv = this->private;
+    priv = this->private;
 
-        if (!matrix)
-                return 0;
+    if (!matrix)
+        return 0;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (dict_get_ptr (xdata, priv->pending_key[i], &pending_raw))
-			continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
+            continue;
 
-		if (!pending_raw)
-			continue;
+        if (!pending_raw)
+            continue;
 
-		memcpy (pending, pending_raw, sizeof(pending));
+        memcpy(pending, pending_raw, sizeof(pending));
 
-		matrix[subvol][i] = ntoh32 (pending[idx]);
-	}
+        matrix[subvol][i] = ntoh32(pending[idx]);
+    }
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
-			    afr_transaction_type type, int *dirty, int **matrix)
+afr_selfheal_extract_xattr(xlator_t *this, struct afr_reply *replies,
+                           afr_transaction_type type, int *dirty, int **matrix)
 {
-	afr_private_t *priv = NULL;
-	int i = 0;
-	dict_t *xdata = NULL;
-	int idx = -1;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    dict_t *xdata = NULL;
+    int idx = -1;
 
-	idx = afr_index_for_transaction_type (type);
+    idx = afr_index_for_transaction_type(type);
 
-	priv = this->private;
+    priv = this->private;
 
-	for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret != 0)
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
 
-		if (!replies[i].xdata)
-			continue;
+        if (!replies[i].xdata)
+            continue;
 
-		xdata = replies[i].xdata;
+        xdata = replies[i].xdata;
 
-		afr_selfheal_fill_dirty (this, dirty, i, idx, xdata);
-		afr_selfheal_fill_matrix (this, matrix, i, idx, xdata);
-	}
+        afr_selfheal_fill_dirty(this, dirty, i, idx, xdata);
+        afr_selfheal_fill_matrix(this, matrix, i, idx, xdata);
+    }
 
-	return 0;
+    return 0;
 }
 
 /*
@@ -760,560 +819,566 @@ afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
  * This can happen if data was directly modified in the backend or for snapshots
  */
 void
-afr_mark_largest_file_as_source (xlator_t *this, unsigned char *sources,
-                                 struct afr_reply *replies)
+afr_mark_largest_file_as_source(xlator_t *this, unsigned char *sources,
+                                struct afr_reply *replies)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
-        uint64_t size = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint64_t size = 0;
 
-        /* Find source with biggest file size */
-        priv = this->private;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (!replies[i].valid || replies[i].op_ret != 0) {
-                        sources[i] = 0;
-                        continue;
-                }
-                if (size <= replies[i].poststat.ia_size) {
-                        size = replies[i].poststat.ia_size;
-                }
+    /* Find source with biggest file size */
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (!replies[i].valid || replies[i].op_ret != 0) {
+            sources[i] = 0;
+            continue;
         }
-
-        /* Mark sources with less size as not source */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (size > replies[i].poststat.ia_size)
-                        sources[i] = 0;
+        if (size <= replies[i].poststat.ia_size) {
+            size = replies[i].poststat.ia_size;
         }
+    }
+
+    /* Mark sources with less size as not source */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (size > replies[i].poststat.ia_size)
+            sources[i] = 0;
+    }
 }
 
 void
-afr_mark_latest_mtime_file_as_source (xlator_t *this, unsigned char *sources,
-                                 struct afr_reply *replies)
-{
-        int i = 0;
-        afr_private_t *priv = NULL;
-        uint32_t mtime = 0;
-        uint32_t mtime_nsec = 0;
-
-        priv = this->private;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (!replies[i].valid || replies[i].op_ret != 0) {
-                        sources[i] = 0;
-                        continue;
-                }
-                if ((mtime < replies[i].poststat.ia_mtime) ||
-                    ((mtime == replies[i].poststat.ia_mtime) &&
-                     (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
-                        mtime = replies[i].poststat.ia_mtime;
-                        mtime_nsec = replies[i].poststat.ia_mtime_nsec;
-                }
-        }
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if ((mtime > replies[i].poststat.ia_mtime) ||
-                    ((mtime == replies[i].poststat.ia_mtime) &&
-                     (mtime_nsec > replies[i].poststat.ia_mtime_nsec))) {
-                        sources[i] = 0;
-                }
-        }
+afr_mark_latest_mtime_file_as_source(xlator_t *this, unsigned char *sources,
+                                     struct afr_reply *replies)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint32_t mtime = 0;
+    uint32_t mtime_nsec = 0;
+
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (!replies[i].valid || replies[i].op_ret != 0) {
+            sources[i] = 0;
+            continue;
+        }
+        if ((mtime < replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
+            mtime = replies[i].poststat.ia_mtime;
+            mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+        }
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if ((mtime > replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec > replies[i].poststat.ia_mtime_nsec))) {
+            sources[i] = 0;
+        }
+    }
 }
 
 void
-afr_mark_active_sinks (xlator_t *this, unsigned char *sources,
-                       unsigned char *locked_on, unsigned char *sinks)
+afr_mark_active_sinks(xlator_t *this, unsigned char *sources,
+                      unsigned char *locked_on, unsigned char *sinks)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        memset (sinks, 0, sizeof (*sinks) * priv->child_count);
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i] && locked_on[i])
-                        sinks[i] = 1;
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i] && locked_on[i])
+            sinks[i] = 1;
+        else
+            sinks[i] = 0;
+    }
 }
 
 gf_boolean_t
-afr_dict_contains_heal_op (call_frame_t *frame)
+afr_dict_contains_heal_op(call_frame_t *frame)
 {
-        afr_local_t   *local     = NULL;
-        dict_t        *xdata_req = NULL;
-        int            ret       = 0;
-        int            heal_op   = -1;
+    afr_local_t *local = NULL;
+    dict_t *xdata_req = NULL;
+    int ret = 0;
+    int heal_op = -1;
 
-        local = frame->local;
-        xdata_req = local->xdata_req;
-        ret = dict_get_int32 (xdata_req, "heal-op", &heal_op);
-        if (ret)
-                return _gf_false;
-        if (local->xdata_rsp == NULL) {
-                local->xdata_rsp = dict_new();
-                if (!local->xdata_rsp)
-                        return _gf_true;
-        }
-        ret = dict_set_str (local->xdata_rsp, "sh-fail-msg",
-                            "File not in split-brain");
+    local = frame->local;
+    xdata_req = local->xdata_req;
+    ret = dict_get_int32_sizen(xdata_req, "heal-op", &heal_op);
+    if (ret)
+        return _gf_false;
+    if (local->xdata_rsp == NULL) {
+        local->xdata_rsp = dict_new();
+        if (!local->xdata_rsp)
+            return _gf_true;
+    }
+    ret = dict_set_sizen_str_sizen(local->xdata_rsp, "sh-fail-msg",
+                                   SFILE_NOT_IN_SPLIT_BRAIN);
 
-        return _gf_true;
+    return _gf_true;
 }
 
 gf_boolean_t
-afr_can_decide_split_brain_source_sinks (struct afr_reply *replies,
-                                         int child_count)
+afr_can_decide_split_brain_source_sinks(struct afr_reply *replies,
+                                        int child_count)
 {
-        int i = 0;
+    int i = 0;
 
-        for (i = 0; i < child_count; i++)
-                if (replies[i].valid != 1 || replies[i].op_ret != 0)
-                        return _gf_false;
+    for (i = 0; i < child_count; i++)
+        if (replies[i].valid != 1 || replies[i].op_ret != 0)
+            return _gf_false;
 
-        return _gf_true;
+    return _gf_true;
 }
 
 int
-afr_mark_split_brain_source_sinks_by_heal_op (call_frame_t *frame,
-                                   xlator_t *this, unsigned char *sources,
-                                   unsigned char *sinks,
-                                   unsigned char *healed_sinks,
-                                   unsigned char *locked_on,
-                                   struct afr_reply *replies,
-                                   afr_transaction_type type, int heal_op)
-{
-        afr_local_t   *local     = NULL;
-        afr_private_t *priv      = NULL;
-        dict_t        *xdata_req = NULL;
-        dict_t        *xdata_rsp = NULL;
-        int            ret       = 0;
-        int            i         = 0;
-        char          *name      = NULL;
-        int            source     = -1;
-
-        local = frame->local;
-        priv = this->private;
-        xdata_req = local->xdata_req;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (locked_on[i])
-                        if (sources[i] || !sinks[i] || !healed_sinks[i]) {
-                                ret = -1;
-                                goto out;
-                        }
-        }
-        if (local->xdata_rsp == NULL) {
-                local->xdata_rsp = dict_new();
-                if (!local->xdata_rsp) {
-                        ret = -1;
-                        goto out;
-                }
-        }
-        xdata_rsp = local->xdata_rsp;
-
-        if (!afr_can_decide_split_brain_source_sinks (replies,
-                                                      priv->child_count)) {
-                ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                    SBRAIN_HEAL_NO_GO_MSG);
+afr_mark_split_brain_source_sinks_by_heal_op(
+    call_frame_t *frame, xlator_t *this, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type, int heal_op)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata_req = NULL;
+    dict_t *xdata_rsp = NULL;
+    int ret = 0;
+    int i = 0;
+    char *name = NULL;
+    int source = -1;
+
+    local = frame->local;
+    priv = this->private;
+    xdata_req = local->xdata_req;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (locked_on[i])
+            if (sources[i] || !sinks[i] || !healed_sinks[i]) {
                 ret = -1;
                 goto out;
-        }
+            }
+    }
+    if (local->xdata_rsp == NULL) {
+        local->xdata_rsp = dict_new();
+        if (!local->xdata_rsp) {
+            ret = -1;
+            goto out;
+        }
+    }
+    xdata_rsp = local->xdata_rsp;
+
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                       SBRAIN_HEAL_NO_GO_MSG);
+        ret = -1;
+        goto out;
+    }
 
-        for (i = 0 ; i < priv->child_count; i++)
-                if (locked_on[i])
-                        sources[i] = 1;
-        switch (heal_op) {
+    for (i = 0; i < priv->child_count; i++)
+        if (locked_on[i])
+            sources[i] = 1;
+    switch (heal_op) {
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
-                if (type == AFR_METADATA_TRANSACTION) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "Use source-brick option to"
-                                            " heal metadata split-brain");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                afr_mark_largest_file_as_source (this, sources, replies);
-                if (AFR_COUNT (sources, priv->child_count) != 1) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "No bigger file");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                break;
+            if (type == AFR_METADATA_TRANSACTION) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SUSE_SOURCE_BRICK_TO_HEAL);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            afr_mark_largest_file_as_source(this, sources, replies);
+            if (AFR_COUNT(sources, priv->child_count) != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SNO_BIGGER_FILE);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            break;
         case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
-                if (type == AFR_METADATA_TRANSACTION) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "Use source-brick option to"
-                                            " heal metadata split-brain");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                afr_mark_latest_mtime_file_as_source (this, sources, replies);
-                if (AFR_COUNT (sources, priv->child_count) != 1) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "No difference in mtime");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                break;
+            if (type == AFR_METADATA_TRANSACTION) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SUSE_SOURCE_BRICK_TO_HEAL);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            afr_mark_latest_mtime_file_as_source(this, sources, replies);
+            if (AFR_COUNT(sources, priv->child_count) != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SNO_DIFF_IN_MTIME);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            break;
         case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
-                ret = dict_get_str (xdata_req, "child-name", &name);
-                if (ret)
-                        goto out;
-                source = afr_get_child_index_from_name (this, name);
-                if (source < 0) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "Invalid brick name");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                if (locked_on[source] != 1) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "Brick is not up");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                memset (sources, 0, sizeof (*sources) * priv->child_count);
-                sources[source] = 1;
-                break;
-        default:
-                ret = -1;
+            ret = dict_get_str_sizen(xdata_req, "child-name", &name);
+            if (ret)
                 goto out;
-        }
-        for (i = 0 ; i < priv->child_count; i++) {
-                if (sources[i]) {
-                        source = i;
-                        break;
-                }
-        }
-        sinks[source] = 0;
-        healed_sinks[source] = 0;
-        ret = source;
+            source = afr_get_child_index_from_name(this, name);
+            if (source < 0) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SINVALID_BRICK_NAME);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            if (locked_on[source] != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SBRICK_IS_NOT_UP);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            memset(sources, 0, sizeof(*sources) * priv->child_count);
+            sources[source] = 1;
+            break;
+        default:
+            ret = -1;
+            goto out;
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i]) {
+            source = i;
+            break;
+        }
+    }
+    sinks[source] = 0;
+    healed_sinks[source] = 0;
+    ret = source;
 out:
-        if (ret < 0)
-                memset (sources, 0, sizeof (*sources) * priv->child_count);
-        return ret;
-
+    if (ret < 0)
+        memset(sources, 0, sizeof(*sources) * priv->child_count);
+    return ret;
 }
 
 int
-afr_sh_fav_by_majority (xlator_t *this, struct afr_reply *replies,
-                        inode_t *inode)
-{
-        afr_private_t   *priv;
-        int             vote_count = -1;
-        int             fav_child = -1;
-        int             i = 0;
-        int             k = 0;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (replies[i].valid == 1) {
-                        gf_msg_debug (this->name, 0, "Child:%s "
-                                "mtime_sec = %ld, size = %lu for gfid %s",
-                                priv->children[i]->name,
-                                replies[i].poststat.ia_mtime,
-                                replies[i].poststat.ia_size,
-                                uuid_utoa (inode->gfid));
-                                vote_count = 0;
-                        for (k = 0; k < priv->child_count; k++) {
-                                if ((replies[k].poststat.ia_mtime ==
-                                     replies[i].poststat.ia_mtime) &&
-                                    (replies[k].poststat.ia_size ==
-                                     replies[i].poststat.ia_size)
-                                   ) {
-                                        vote_count++;
-                                }
-                        }
-                        if (vote_count > priv->child_count/2) {
-                                fav_child = i;
-                                break;
-                        }
+afr_sh_fav_by_majority(xlator_t *this, struct afr_reply *replies,
+                       inode_t *inode)
+{
+    afr_private_t *priv;
+    int vote_count = -1;
+    int fav_child = -1;
+    int i = 0;
+    int k = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s mtime_sec = %" PRId64 ", size = %" PRIu64
+                         " for gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_mtime,
+                         replies[i].poststat.ia_size, uuid_utoa(inode->gfid));
+            vote_count = 0;
+            for (k = 0; k < priv->child_count; k++) {
+                if ((replies[k].poststat.ia_mtime ==
+                     replies[i].poststat.ia_mtime) &&
+                    (replies[k].poststat.ia_size ==
+                     replies[i].poststat.ia_size)) {
+                    vote_count++;
                 }
+            }
+            if (vote_count > priv->child_count / 2) {
+                fav_child = i;
+                break;
+            }
         }
-        return fav_child;
+    }
+    return fav_child;
 }
 
 /*
  * afr_sh_fav_by_mtime: Choose favorite child by mtime.
  */
 int
-afr_sh_fav_by_mtime (xlator_t *this, struct afr_reply *replies, inode_t *inode)
-{
-        afr_private_t *priv;
-        int fav_child = -1;
-        int i = 0;
-        uint32_t cmp_mtime = 0;
-        uint32_t cmp_mtime_nsec = 0;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (replies[i].valid == 1) {
-                        gf_msg_debug (this->name, 0, "Child:%s "
-                                "mtime = %ld, mtime_nsec = %d for gfid %s",
-                                priv->children[i]->name,
-                                replies[i].poststat.ia_mtime,
-                                replies[i].poststat.ia_mtime_nsec,
-                                uuid_utoa (inode->gfid));
-                        if (replies[i].poststat.ia_mtime > cmp_mtime) {
-                                cmp_mtime = replies[i].poststat.ia_mtime;
-                                cmp_mtime_nsec =
-                                        replies[i].poststat.ia_mtime_nsec;
-                                fav_child = i;
-                        } else if ((replies[i].poststat.ia_mtime == cmp_mtime)
-                                    && (replies[i].poststat.ia_mtime_nsec >
-                                    cmp_mtime_nsec)) {
-                                cmp_mtime = replies[i].poststat.ia_mtime;
-                                cmp_mtime_nsec =
-                                        replies[i].poststat.ia_mtime_nsec;
-                                fav_child = i;
-                        }
-                }
-        }
-        return fav_child;
+afr_sh_fav_by_mtime(xlator_t *this, struct afr_reply *replies, inode_t *inode)
+{
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint32_t cmp_mtime = 0;
+    uint32_t cmp_mtime_nsec = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s mtime = %" PRId64
+                         ", mtime_nsec = %d for "
+                         "gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_mtime,
+                         replies[i].poststat.ia_mtime_nsec,
+                         uuid_utoa(inode->gfid));
+            if (replies[i].poststat.ia_mtime > cmp_mtime) {
+                cmp_mtime = replies[i].poststat.ia_mtime;
+                cmp_mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+                fav_child = i;
+            } else if ((replies[i].poststat.ia_mtime == cmp_mtime) &&
+                       (replies[i].poststat.ia_mtime_nsec > cmp_mtime_nsec)) {
+                cmp_mtime = replies[i].poststat.ia_mtime;
+                cmp_mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+                fav_child = i;
+            }
+        }
+    }
+    return fav_child;
 }
 
 /*
  * afr_sh_fav_by_ctime: Choose favorite child by ctime.
  */
 int
-afr_sh_fav_by_ctime (xlator_t *this, struct afr_reply *replies, inode_t *inode)
-{
-        afr_private_t *priv;
-        int fav_child = -1;
-        int i = 0;
-        uint32_t cmp_ctime = 0;
-        uint32_t cmp_ctime_nsec = 0;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (replies[i].valid == 1) {
-                        gf_msg_debug (this->name, 0, "Child:%s "
-                                "ctime = %ld, ctime_nsec = %d for gfid %s",
-                                priv->children[i]->name,
-                                replies[i].poststat.ia_ctime,
-                                replies[i].poststat.ia_ctime_nsec,
-                                uuid_utoa (inode->gfid));
-                        if (replies[i].poststat.ia_ctime > cmp_ctime) {
-                                cmp_ctime = replies[i].poststat.ia_ctime;
-                                cmp_ctime_nsec =
-                                        replies[i].poststat.ia_ctime_nsec;
-                                fav_child = i;
-                        } else if ((replies[i].poststat.ia_ctime == cmp_ctime)
-                                    && (replies[i].poststat.ia_ctime_nsec >
-                                    cmp_ctime_nsec)) {
-                                cmp_ctime = replies[i].poststat.ia_ctime;
-                                cmp_ctime_nsec =
-                                        replies[i].poststat.ia_ctime_nsec;
-                                fav_child = i;
-                        }
-                }
-        }
-        return fav_child;
+afr_sh_fav_by_ctime(xlator_t *this, struct afr_reply *replies, inode_t *inode)
+{
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint32_t cmp_ctime = 0;
+    uint32_t cmp_ctime_nsec = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s ctime = %" PRId64
+                         ", ctime_nsec = %d for "
+                         "gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_ctime,
+                         replies[i].poststat.ia_ctime_nsec,
+                         uuid_utoa(inode->gfid));
+            if (replies[i].poststat.ia_ctime > cmp_ctime) {
+                cmp_ctime = replies[i].poststat.ia_ctime;
+                cmp_ctime_nsec = replies[i].poststat.ia_ctime_nsec;
+                fav_child = i;
+            } else if ((replies[i].poststat.ia_ctime == cmp_ctime) &&
+                       (replies[i].poststat.ia_ctime_nsec > cmp_ctime_nsec)) {
+                cmp_ctime = replies[i].poststat.ia_ctime;
+                cmp_ctime_nsec = replies[i].poststat.ia_ctime_nsec;
+                fav_child = i;
+            }
+        }
+    }
+    return fav_child;
 }
 
 /*
- * afr_sh_fav_by_size: Choose favorite child by size.
+ * afr_sh_fav_by_size: Choose favorite child by size
+ * when not all files are of zero size.
  */
 int
-afr_sh_fav_by_size (xlator_t *this, struct afr_reply *replies, inode_t *inode)
-{
-        afr_private_t *priv;
-        int fav_child = -1;
-        int i = 0;
-        uint64_t cmp_sz = 0;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (replies[i].valid == 1) {
-                        gf_msg_debug (this->name, 0, "Child:%s "
-                                "file size = %lu for gfid %s",
-                                priv->children[i]->name,
-                                replies[i].poststat.ia_size,
-                                uuid_utoa (inode->gfid));
-                        if (replies[i].poststat.ia_size > cmp_sz) {
-                                cmp_sz = replies[i].poststat.ia_size;
-                                fav_child = i;
-                        }
-                }
-        }
-        return fav_child;
+afr_sh_fav_by_size(xlator_t *this, struct afr_reply *replies, inode_t *inode)
+{
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint64_t cmp_sz = 0;
+
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid) {
+            continue;
+        }
+        gf_msg_debug(this->name, 0,
+                     "Child:%s file size = %" PRIu64 " for gfid %s",
+                     priv->children[i]->name, replies[i].poststat.ia_size,
+                     uuid_utoa(inode->gfid));
+        if (replies[i].poststat.ia_type == IA_IFDIR) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+                   "Cannot perform selfheal on %s. "
+                   "Size policy is not applicable to directories.",
+                   uuid_utoa(inode->gfid));
+            break;
+        }
+        if (replies[i].poststat.ia_size > cmp_sz) {
+            cmp_sz = replies[i].poststat.ia_size;
+            fav_child = i;
+        } else if (replies[i].poststat.ia_size == cmp_sz) {
+            fav_child = -1;
+        }
+    }
+    if (fav_child == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "No bigger file");
+    }
+    return fav_child;
 }
 
 int
-afr_sh_get_fav_by_policy (xlator_t *this, struct afr_reply *replies,
-                          inode_t *inode, char **policy_str)
+afr_sh_get_fav_by_policy(xlator_t *this, struct afr_reply *replies,
+                         inode_t *inode, char **policy_str)
 {
-        afr_private_t *priv = NULL;
-        int fav_child = -1;
+    afr_private_t *priv = NULL;
+    int fav_child = -1;
 
-        priv = this->private;
-        if (!afr_can_decide_split_brain_source_sinks (replies,
-                                                      priv->child_count)) {
-                return -1;
-        }
+    priv = this->private;
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        return -1;
+    }
 
-        switch (priv->fav_child_policy) {
+    switch (priv->fav_child_policy) {
         case AFR_FAV_CHILD_BY_SIZE:
-                fav_child = afr_sh_fav_by_size (this, replies, inode);
-                if (policy_str && fav_child >= 0) {
-                        *policy_str = "SIZE";
-                }
-                break;
+            fav_child = afr_sh_fav_by_size(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "SIZE";
+            }
+            break;
         case AFR_FAV_CHILD_BY_CTIME:
-                fav_child = afr_sh_fav_by_ctime (this, replies, inode);
-                if (policy_str && fav_child >= 0) {
-                        *policy_str = "CTIME";
-                }
-                break;
+            fav_child = afr_sh_fav_by_ctime(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "CTIME";
+            }
+            break;
         case AFR_FAV_CHILD_BY_MTIME:
-                fav_child = afr_sh_fav_by_mtime (this, replies, inode);
-                if (policy_str && fav_child >= 0) {
-                        *policy_str = "MTIME";
-                }
-                break;
+            fav_child = afr_sh_fav_by_mtime(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "MTIME";
+            }
+            break;
         case AFR_FAV_CHILD_BY_MAJORITY:
-                fav_child = afr_sh_fav_by_majority (this, replies, inode);
-                if (policy_str && fav_child >= 0) {
-                        *policy_str = "MAJORITY";
-                }
-                break;
+            fav_child = afr_sh_fav_by_majority(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "MAJORITY";
+            }
+            break;
         case AFR_FAV_CHILD_NONE:
         default:
-                break;
-        }
+            break;
+    }
 
-        return fav_child;
+    return fav_child;
 }
 
 int
-afr_mark_split_brain_source_sinks_by_policy (call_frame_t *frame,
-                                             xlator_t *this,
-                                             inode_t *inode,
-                                             unsigned char *sources,
-                                             unsigned char *sinks,
-                                             unsigned char *healed_sinks,
-                                             unsigned char *locked_on,
-                                             struct afr_reply *replies,
-                                             afr_transaction_type type)
-{
-        afr_private_t *priv = NULL;
-        int fav_child = -1;
-        char mtime_str[256];
-        char ctime_str[256];
-        char *policy_str = NULL;
-        struct tm *tm_ptr;
-        time_t time;
-
-        priv = this->private;
-
-        fav_child = afr_sh_get_fav_by_policy (this, replies, inode,
-                                              &policy_str);
-        if (fav_child > priv->child_count - 1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        AFR_MSG_SBRAIN_FAV_CHILD_POLICY, "Invalid child (%d) "
-                        "selected by policy %s.", fav_child, policy_str);
-        } else if (fav_child >= 0) {
-                time = replies[fav_child].poststat.ia_mtime;
-                tm_ptr = localtime (&time);
-                strftime (mtime_str, sizeof (mtime_str), "%Y-%m-%d %H:%M:%S",
-                          tm_ptr);
-                time = replies[fav_child].poststat.ia_ctime;
-                tm_ptr = localtime (&time);
-                strftime (ctime_str, sizeof (ctime_str), "%Y-%m-%d %H:%M:%S",
-                          tm_ptr);
-
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        AFR_MSG_SBRAIN_FAV_CHILD_POLICY, "Source %s "
-                        "selected as authentic to resolve conflicting "
-                         "data in file (gfid:%s) by %s (%lu bytes @ %s mtime, "
-                         "%s ctime).",
-                         priv->children[fav_child]->name,
-                         uuid_utoa (inode->gfid),
-                         policy_str,
-                         replies[fav_child].poststat.ia_size,
-                         mtime_str,
-                         ctime_str);
-
-                sources[fav_child] = 1;
-                sinks[fav_child] = 0;
-                healed_sinks[fav_child] = 0;
-        }
-        return fav_child;
+afr_mark_split_brain_source_sinks_by_policy(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type)
+{
+    afr_private_t *priv = NULL;
+    int fav_child = -1;
+    char mtime_str[256];
+    char ctime_str[256];
+    char *policy_str = NULL;
+    struct tm *tm_ptr;
+    time_t time;
+
+    priv = this->private;
+
+    fav_child = afr_sh_get_fav_by_policy(this, replies, inode, &policy_str);
+    if (fav_child == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "No child selected by favorite-child policy.");
+    } else if (fav_child > priv->child_count - 1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "Invalid child (%d) "
+               "selected by policy %s.",
+               fav_child, policy_str);
+    } else if (fav_child >= 0) {
+        time = replies[fav_child].poststat.ia_mtime;
+        tm_ptr = localtime(&time);
+        strftime(mtime_str, sizeof(mtime_str), "%Y-%m-%d %H:%M:%S", tm_ptr);
+        time = replies[fav_child].poststat.ia_ctime;
+        tm_ptr = localtime(&time);
+        strftime(ctime_str, sizeof(ctime_str), "%Y-%m-%d %H:%M:%S", tm_ptr);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "Source %s selected as authentic to resolve conflicting data "
+               "in file (gfid:%s) by %s (%" PRIu64
+               " bytes @ %s mtime, %s "
+               "ctime).",
+               priv->children[fav_child]->name, uuid_utoa(inode->gfid),
+               policy_str, replies[fav_child].poststat.ia_size, mtime_str,
+               ctime_str);
+
+        sources[fav_child] = 1;
+        sinks[fav_child] = 0;
+        healed_sinks[fav_child] = 0;
+    }
+    return fav_child;
 }
 
-int
-afr_mark_source_sinks_if_file_empty (xlator_t *this, unsigned char *sources,
-                                     unsigned char *sinks,
-                                     unsigned char *healed_sinks,
-                                     unsigned char *locked_on,
-                                     struct afr_reply *replies,
-                                     afr_transaction_type type)
-{
-        int source = -1;
-        int i = 0;
-        afr_private_t *priv = this->private;
-        struct iatt stbuf = {0, };
-
-        if ((AFR_COUNT (locked_on, priv->child_count) < priv->child_count) ||
-            (afr_success_count(replies, priv->child_count) < priv->child_count))
-                return -1;
+gf_boolean_t
+afr_is_file_empty_on_all_children(afr_private_t *priv,
+                                  struct afr_reply *replies)
+{
+    int i = 0;
 
-        if (type == AFR_DATA_TRANSACTION) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (replies[i].poststat.ia_size != 0)
-                                return -1;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if ((!replies[i].valid) || (replies[i].op_ret != 0) ||
+            (replies[i].poststat.ia_size != 0))
+            return _gf_false;
+    }
 
-                goto mark;
-        }
+    return _gf_true;
+}
 
-        /*For AFR_METADATA_TRANSACTION, metadata must be same on all bricks.*/
-        stbuf = replies[0].poststat;
-        for (i = 1; i < priv->child_count; i++) {
-                if ((!IA_EQUAL (stbuf, replies[i].poststat, type)) ||
-                    (!IA_EQUAL (stbuf, replies[i].poststat, uid)) ||
-                    (!IA_EQUAL (stbuf, replies[i].poststat, gid)) ||
-                    (!IA_EQUAL (stbuf, replies[i].poststat, prot)))
-                        return -1;
-        }
-        for (i = 1; i < priv->child_count; i++) {
-                if (!afr_xattrs_are_equal (replies[0].xdata,
-                                           replies[i].xdata))
-                        return -1;
-        }
+int
+afr_mark_source_sinks_if_file_empty(xlator_t *this, unsigned char *sources,
+                                    unsigned char *sinks,
+                                    unsigned char *healed_sinks,
+                                    unsigned char *locked_on,
+                                    struct afr_reply *replies,
+                                    afr_transaction_type type)
+{
+    int source = -1;
+    int i = 0;
+    afr_private_t *priv = this->private;
+    struct iatt stbuf = {
+        0,
+    };
+
+    if ((AFR_COUNT(locked_on, priv->child_count) < priv->child_count) ||
+        (afr_success_count(replies, priv->child_count) < priv->child_count))
+        return -1;
+
+    if (type == AFR_DATA_TRANSACTION) {
+        if (!afr_is_file_empty_on_all_children(priv, replies))
+            return -1;
+        goto mark;
+    }
+
+    /*For AFR_METADATA_TRANSACTION, metadata must be same on all bricks.*/
+    stbuf = replies[0].poststat;
+    for (i = 1; i < priv->child_count; i++) {
+        if ((!IA_EQUAL(stbuf, replies[i].poststat, type)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, uid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, gid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, prot)))
+            return -1;
+    }
+    for (i = 1; i < priv->child_count; i++) {
+        if (!afr_xattrs_are_equal(replies[0].xdata, replies[i].xdata))
+            return -1;
+    }
 
 mark:
-        /* data/metadata is same on all bricks. Pick one of them as source. Rest
-         * are sinks.*/
-        for (i = 0 ; i < priv->child_count; i++) {
-                if (source == -1) {
-                        source = i;
-                        sources[i] = 1;
-                        sinks[i] = 0;
-                        healed_sinks[i] = 0;
-                        continue;
-                }
-                sources[i] = 0;
-                sinks[i] = 1;
-                healed_sinks[i] = 1;
+    /* data/metadata is same on all bricks. Pick one of them as source. Rest
+     * are sinks.*/
+    for (i = 0; i < priv->child_count; i++) {
+        if (source == -1) {
+            source = i;
+            sources[i] = 1;
+            sinks[i] = 0;
+            healed_sinks[i] = 0;
+            continue;
         }
+        sources[i] = 0;
+        sinks[i] = 1;
+        healed_sinks[i] = 1;
+    }
 
-        return source;
+    return source;
 }
 
 /* Return a source depending on the type of heal_op, and set sources[source],
@@ -1324,171 +1389,156 @@ mark:
  * sinks[node] are 1. This should be the case if the file is in split-brain.
  */
 int
-afr_mark_split_brain_source_sinks (call_frame_t *frame, xlator_t *this,
-                                   inode_t *inode,
-                                   unsigned char *sources,
-                                   unsigned char *sinks,
-                                   unsigned char *healed_sinks,
-                                   unsigned char *locked_on,
-                                   struct afr_reply *replies,
-                                   afr_transaction_type type)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        dict_t *xdata_req = NULL;
-        int heal_op = -1;
-        int ret = -1;
-        int source = -1;
-
-        local = frame->local;
-        priv = this->private;
-        xdata_req = local->xdata_req;
-
-        source = afr_mark_source_sinks_if_file_empty (this, sources, sinks,
-                                                      healed_sinks, locked_on,
-                                                      replies, type);
-        if (source >= 0)
-                return source;
-
-        ret = dict_get_int32 (xdata_req, "heal-op", &heal_op);
-        if (ret)
-                goto autoheal;
-
-        source = afr_mark_split_brain_source_sinks_by_heal_op (frame, this,
-                                                            sources, sinks,
-                                                            healed_sinks,
-                                                            locked_on, replies,
-                                                            type, heal_op);
+afr_mark_split_brain_source_sinks(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata_req = NULL;
+    int heal_op = -1;
+    int ret = -1;
+    int source = -1;
+
+    local = frame->local;
+    priv = this->private;
+    xdata_req = local->xdata_req;
+
+    source = afr_mark_source_sinks_if_file_empty(
+        this, sources, sinks, healed_sinks, locked_on, replies, type);
+    if (source >= 0)
         return source;
 
+    ret = dict_get_int32_sizen(xdata_req, "heal-op", &heal_op);
+    if (ret)
+        goto autoheal;
+
+    source = afr_mark_split_brain_source_sinks_by_heal_op(
+        frame, this, sources, sinks, healed_sinks, locked_on, replies, type,
+        heal_op);
+    return source;
+
 autoheal:
-        /* Automatically heal if fav_child_policy is set. */
-        if (priv->fav_child_policy != AFR_FAV_CHILD_NONE) {
-                source = afr_mark_split_brain_source_sinks_by_policy (frame,
-                                                                      this,
-                                                                      inode,
-                                                                      sources,
-                                                                      sinks,
-                                                                   healed_sinks,
-                                                                      locked_on,
-                                                                      replies,
-                                                                      type);
-                if (source != -1) {
-                        ret = dict_set_int32 (xdata_req, "fav-child-policy", 1);
-                        if (ret)
-                                return -1;
-                }
+    /* Automatically heal if fav_child_policy is set. */
+    if (priv->fav_child_policy != AFR_FAV_CHILD_NONE) {
+        source = afr_mark_split_brain_source_sinks_by_policy(
+            frame, this, inode, sources, sinks, healed_sinks, locked_on,
+            replies, type);
+        if (source != -1) {
+            ret = dict_set_int32_sizen(xdata_req, "fav-child-policy", 1);
+            if (ret)
+                return -1;
         }
+    }
 
-        return source;
+    return source;
 }
 
 int
-_afr_fav_child_reset_sink_xattrs (call_frame_t *frame, xlator_t *this,
-                                  inode_t *inode, int source,
-                                  unsigned char *healed_sinks,
-                                  unsigned char *undid_pending,
-                                  afr_transaction_type type,
-                                  unsigned char *locked_on,
-                                  struct afr_reply *replies)
+_afr_fav_child_reset_sink_xattrs(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, int source,
+                                 unsigned char *healed_sinks,
+                                 unsigned char *undid_pending,
+                                 afr_transaction_type type,
+                                 unsigned char *locked_on,
+                                 struct afr_reply *replies)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int *input_dirty = NULL;
-        int **input_matrix = NULL;
-	int *output_dirty = NULL;
-	int **output_matrix = NULL;
-        dict_t *xattr = NULL;
-        dict_t *xdata = NULL;
-        int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int *input_dirty = NULL;
+    int **input_matrix = NULL;
+    int *output_dirty = NULL;
+    int **output_matrix = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
 
-        priv = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        if (!dict_get (local->xdata_req, "fav-child-policy"))
-                return 0;
+    if (!dict_get_sizen(local->xdata_req, "fav-child-policy"))
+        return 0;
 
-        xdata = dict_new();
-        if (!xdata)
-                return -1;
+    xdata = dict_new();
+    if (!xdata)
+        return -1;
 
-        input_dirty = alloca0 (priv->child_count * sizeof (int));
-	input_matrix = ALLOC_MATRIX (priv->child_count, int);
-	output_dirty = alloca0 (priv->child_count * sizeof (int));
-	output_matrix = ALLOC_MATRIX (priv->child_count, int);
+    input_dirty = alloca0(priv->child_count * sizeof(int));
+    input_matrix = ALLOC_MATRIX(priv->child_count, int);
+    output_dirty = alloca0(priv->child_count * sizeof(int));
+    output_matrix = ALLOC_MATRIX(priv->child_count, int);
 
-        afr_selfheal_extract_xattr (this, replies, type, input_dirty,
-                                    input_matrix);
+    afr_selfheal_extract_xattr(this, replies, type, input_dirty, input_matrix);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (i == source || !healed_sinks[i])
-                        continue;
-                output_dirty[i] = -input_dirty[i];
-                output_matrix[i][source] = -input_matrix[i][source];
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source || !healed_sinks[i])
+            continue;
+        output_dirty[i] = -input_dirty[i];
+        output_matrix[i][source] = -input_matrix[i][source];
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!healed_sinks[i] || !locked_on[i])
-                        continue;
-                xattr = afr_selfheal_output_xattr (this, _gf_false, type,
-                                                   output_dirty, output_matrix,
-                                                   i, NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i] || !locked_on[i])
+            continue;
+        xattr = afr_selfheal_output_xattr(this, _gf_false, type, output_dirty,
+                                          output_matrix, i, NULL);
 
-                afr_selfheal_post_op (frame, this, inode, i, xattr, xdata);
+        afr_selfheal_post_op(frame, this, inode, i, xattr, xdata);
 
-                undid_pending[i] = 1;
-                dict_unref (xattr);
-        }
+        undid_pending[i] = 1;
+        dict_unref(xattr);
+    }
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 gf_boolean_t
-afr_does_witness_exist (xlator_t *this, uint64_t *witness)
+afr_does_witness_exist(xlator_t *this, uint64_t *witness)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (witness[i])
-                        return _gf_true;
-        }
-        return _gf_false;
+    for (i = 0; i < priv->child_count; i++) {
+        if (witness[i])
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
 unsigned int
-afr_get_quorum_count (afr_private_t *priv)
+afr_get_quorum_count(afr_private_t *priv)
 {
-        if (priv->quorum_count == AFR_QUORUM_AUTO) {
-                return priv->child_count/2 + 1;
-        } else {
-                return priv->quorum_count;
-        }
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        return priv->child_count / 2 + 1;
+    } else {
+        return priv->quorum_count;
+    }
 }
 
 void
-afr_selfheal_post_op_failure_accounting (afr_private_t *priv, char *accused,
-                                         unsigned char *sources,
-                                         unsigned char *locked_on)
+afr_selfheal_post_op_failure_accounting(afr_private_t *priv, char *accused,
+                                        unsigned char *sources,
+                                        unsigned char *locked_on)
 {
-        int i = 0;
-        unsigned int  quorum_count = 0;
+    int i = 0;
+    unsigned int quorum_count = 0;
 
-        if (AFR_COUNT (sources, priv->child_count) != 0)
-                return;
+    if (AFR_COUNT(sources, priv->child_count) != 0)
+        return;
 
-        quorum_count = afr_get_quorum_count (priv);
-        for (i = 0; i < priv->child_count; i++) {
-                if ((accused[i] < quorum_count) && locked_on[i]) {
-                        sources[i] = 1;
-                }
+    quorum_count = afr_get_quorum_count(priv);
+    for (i = 0; i < priv->child_count; i++) {
+        if ((accused[i] < quorum_count) && locked_on[i]) {
+            sources[i] = 1;
         }
-        return;
+    }
+    return;
 }
 
 /*
@@ -1511,700 +1561,711 @@ afr_selfheal_post_op_failure_accounting (afr_private_t *priv, char *accused,
  */
 
 int
-afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
-                             struct afr_reply *replies,
-                             afr_transaction_type type,
-                             unsigned char *locked_on, unsigned char *sources,
-                             unsigned char *sinks, uint64_t *witness,
-                             gf_boolean_t *pflag)
-{
-        afr_private_t *priv = NULL;
-        int i = 0;
-        int j = 0;
-        int *dirty = NULL; /* Denotes if dirty xattr is set */
-        int **matrix = NULL;/* Changelog matrix */
-        char *accused = NULL;/* Accused others without any self-accusal */
-        char *pending = NULL;/* Have pending operations on others */
-        char *self_accused = NULL; /* Accused itself */
-        int min_participants = -1;
-
-	priv = this->private;
-
-	dirty = alloca0 (priv->child_count * sizeof (int));
-	accused = alloca0 (priv->child_count);
-        pending = alloca0 (priv->child_count);
-        self_accused = alloca0 (priv->child_count);
-	matrix = ALLOC_MATRIX(priv->child_count, int);
-        memset (witness, 0, sizeof (*witness) * priv->child_count);
-
-	/* First construct the pending matrix for further analysis */
-	afr_selfheal_extract_xattr (this, replies, type, dirty, matrix);
-
-        if (pflag) {
-                for (i = 0; i < priv->child_count; i++) {
-                        for (j = 0; j < priv->child_count; j++)
-                                if (matrix[i][j])
-                                        *pflag = _gf_true;
-                        if (*pflag)
-                                break;
-                }
+afr_selfheal_find_direction(call_frame_t *frame, xlator_t *this,
+                            struct afr_reply *replies,
+                            afr_transaction_type type, unsigned char *locked_on,
+                            unsigned char *sources, unsigned char *sinks,
+                            uint64_t *witness, unsigned char *pflag)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int j = 0;
+    int *dirty = NULL;         /* Denotes if dirty xattr is set */
+    int **matrix = NULL;       /* Changelog matrix */
+    char *accused = NULL;      /* Accused others without any self-accusal */
+    char *pending = NULL;      /* Have pending operations on others */
+    char *self_accused = NULL; /* Accused itself */
+
+    priv = this->private;
+
+    dirty = alloca0(priv->child_count * sizeof(int));
+    accused = alloca0(priv->child_count);
+    pending = alloca0(priv->child_count);
+    self_accused = alloca0(priv->child_count);
+    matrix = ALLOC_MATRIX(priv->child_count, int);
+    memset(witness, 0, sizeof(*witness) * priv->child_count);
+
+    /* First construct the pending matrix for further analysis */
+    afr_selfheal_extract_xattr(this, replies, type, dirty, matrix);
+
+    if (pflag) {
+        for (i = 0; i < priv->child_count; i++) {
+            for (j = 0; j < priv->child_count; j++)
+                if (matrix[i][j])
+                    *pflag |= PFLAG_PENDING;
+            if (*pflag)
+                break;
         }
+    }
 
-        if (type == AFR_DATA_TRANSACTION) {
-                min_participants = priv->child_count;
-        } else {
-                min_participants = AFR_SH_MIN_PARTICIPANTS;
-        }
-        if (afr_success_count (replies,
-                               priv->child_count) < min_participants) {
-                /* Treat this just like locks not being acquired */
-                return -ENOTCONN;
+    if (afr_success_count(replies, priv->child_count) < priv->child_count) {
+        /* Treat this just like locks not being acquired */
+        return -ENOTCONN;
+    }
+
+    /* short list all self-accused */
+    for (i = 0; i < priv->child_count; i++) {
+        if (matrix[i][i])
+            self_accused[i] = 1;
+    }
+
+    /* Next short list all accused to exclude them from being sources */
+    /* Self-accused can't accuse others as they are FOOLs */
+    for (i = 0; i < priv->child_count; i++) {
+        for (j = 0; j < priv->child_count; j++) {
+            if (matrix[i][j]) {
+                if (!self_accused[i])
+                    accused[j] += 1;
+                if (i != j)
+                    pending[i] += 1;
+            }
         }
+    }
 
-        /* short list all self-accused */
+    /* Short list all non-accused as sources */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!accused[i] && locked_on[i])
+            sources[i] = 1;
+        else
+            sources[i] = 0;
+    }
+
+    /* Everyone accused by non-self-accused sources are sinks */
+    memset(sinks, 0, priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (self_accused[i])
+            continue;
+        for (j = 0; j < priv->child_count; j++) {
+            if (matrix[i][j])
+                sinks[j] = 1;
+        }
+    }
+
+    /* For breaking ties provide with number of fops they witnessed */
+
+    /*
+     * count the pending fops witnessed from itself to others when it is
+     * self-accused
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!self_accused[i])
+            continue;
+        for (j = 0; j < priv->child_count; j++) {
+            if (i == j)
+                continue;
+            witness[i] += matrix[i][j];
+        }
+    }
+
+    if (type == AFR_DATA_TRANSACTION || type == AFR_METADATA_TRANSACTION)
+        afr_selfheal_post_op_failure_accounting(priv, accused, sources,
+                                                locked_on);
+
+    /* If no sources, all locked nodes are sinks - split brain */
+    if (AFR_COUNT(sources, priv->child_count) == 0) {
         for (i = 0; i < priv->child_count; i++) {
-                if (matrix[i][i])
-                        self_accused[i] = 1;
+            if (locked_on[i])
+                sinks[i] = 1;
         }
+        if (pflag)
+            *pflag |= PFLAG_SBRAIN;
+    }
 
-	/* Next short list all accused to exclude them from being sources */
-        /* Self-accused can't accuse others as they are FOOLs */
-	for (i = 0; i < priv->child_count; i++) {
-		for (j = 0; j < priv->child_count; j++) {
-                        if (matrix[i][j]) {
-                                if (!self_accused[i])
-                                        accused[j] += 1;
-                                if (i != j)
-                                        pending[i] += 1;
-                         }
-		}
-	}
-
-	/* Short list all non-accused as sources */
-	memset (sources, 0, priv->child_count);
-	for (i = 0; i < priv->child_count; i++) {
-		if (!accused[i] && locked_on[i])
-			sources[i] = 1;
-	}
-
-        /* Everyone accused by non-self-accused sources are sinks */
-        memset (sinks, 0, priv->child_count);
+    /* One more class of witness similar to dirty in v2 is where no pending
+     * exists but we have self-accusing markers. This can happen in afr-v1
+     * if the brick crashes just after doing xattrop on self but
+     * before xattrop on the other xattrs on the brick in pre-op. */
+    if (AFR_COUNT(pending, priv->child_count) == 0) {
         for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (self_accused[i])
-                        continue;
-                for (j = 0; j < priv->child_count; j++) {
-                        if (matrix[i][j])
-                                sinks[j] = 1;
-                }
+            if (self_accused[i])
+                witness[i] += matrix[i][i];
         }
-
-        /* For breaking ties provide with number of fops they witnessed */
-
-        /*
-         * count the pending fops witnessed from itself to others when it is
-         * self-accused
+    } else {
+        /* In afr-v1 if a file is self-accused and has pending
+         * operations on others then it is similar to 'dirty' in afr-v2.
+         * Consider such cases as witness.
          */
         for (i = 0; i < priv->child_count; i++) {
-                if (!self_accused[i])
-                        continue;
-                for (j = 0; j < priv->child_count; j++) {
-                        if (i == j)
-                                continue;
-                        witness[i] += matrix[i][j];
-                }
+            if (self_accused[i] && pending[i])
+                witness[i] += matrix[i][i];
         }
+    }
 
-        if (type == AFR_DATA_TRANSACTION)
-                afr_selfheal_post_op_failure_accounting (priv, accused,
-                                                         sources, locked_on);
-
-         /* If no sources, all locked nodes are sinks - split brain */
-         if (AFR_COUNT (sources, priv->child_count) == 0) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (locked_on[i])
-                                sinks[i] = 1;
-                }
-        }
+    /* count the number of dirty fops witnessed */
+    for (i = 0; i < priv->child_count; i++)
+        witness[i] += dirty[i];
 
-        /* One more class of witness similar to dirty in v2 is where no pending
-         * exists but we have self-accusing markers. This can happen in afr-v1
-         * if the brick crashes just after doing xattrop on self but
-         * before xattrop on the other xattrs on the brick in pre-op. */
-        if (AFR_COUNT (pending, priv->child_count) == 0) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (self_accused[i])
-                                witness[i] += matrix[i][i];
-                }
-        } else {
-                /* In afr-v1 if a file is self-accused and has pending
-                 * operations on others then it is similar to 'dirty' in afr-v2.
-                 * Consider such cases as witness.
-                 */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (self_accused[i] && pending[i])
-                                witness[i] += matrix[i][i];
-                }
-        }
-
-
-        /* count the number of dirty fops witnessed */
-        for (i = 0; i < priv->child_count; i++)
-                witness[i] += dirty[i];
-
-	return 0;
+    return 0;
 }
 
 void
-afr_log_selfheal (uuid_t gfid, xlator_t *this, int ret, char *type,
-                  int source, unsigned char *sources,
-                  unsigned char *healed_sinks)
-{
-        char *status = NULL;
-        char *sinks_str = NULL;
-        char *p = NULL;
-        char *sources_str = NULL;
-        char *q = NULL;
-        afr_private_t *priv = NULL;
-        gf_loglevel_t loglevel = GF_LOG_NONE;
-        int i = 0;
-
-        priv = this->private;
-        sinks_str = alloca0 (priv->child_count * 8);
-        p = sinks_str;
-        sources_str = alloca0 (priv->child_count * 8);
-        q = sources_str;
-        for (i = 0; i < priv->child_count; i++) {
-                if (healed_sinks[i])
-                        p += sprintf (p, "%d ", i);
-                if (sources[i]) {
-                        if (source == i) {
-                                q += sprintf (q, "[%d] ", i);
-                        } else {
-                                q += sprintf (q, "%d ", i);
-                        }
-                }
-        }
-
-        if (ret < 0) {
-                status = "Failed";
-                loglevel = GF_LOG_DEBUG;
-        } else {
-                status = "Completed";
-                loglevel = GF_LOG_INFO;
-        }
-
-        gf_msg (this->name, loglevel, 0,
-                AFR_MSG_SELF_HEAL_INFO, "%s %s selfheal on %s. "
-                "sources=%s sinks=%s", status, type, uuid_utoa (gfid),
-                sources_str, sinks_str);
+afr_log_selfheal(uuid_t gfid, xlator_t *this, int ret, char *type, int source,
+                 unsigned char *sources, unsigned char *healed_sinks)
+{
+    char *status = NULL;
+    char *sinks_str = NULL;
+    char *p = NULL;
+    char *sources_str = NULL;
+    char *q = NULL;
+    afr_private_t *priv = NULL;
+    gf_loglevel_t loglevel = GF_LOG_NONE;
+    int i = 0;
+
+    priv = this->private;
+    sinks_str = alloca0(priv->child_count * 8);
+    p = sinks_str;
+    sources_str = alloca0(priv->child_count * 8);
+    q = sources_str;
+    for (i = 0; i < priv->child_count; i++) {
+        if (healed_sinks[i])
+            p += sprintf(p, "%d ", i);
+        if (sources[i]) {
+            if (source == i) {
+                q += sprintf(q, "[%d] ", i);
+            } else {
+                q += sprintf(q, "%d ", i);
+            }
+        }
+    }
+
+    if (ret < 0) {
+        status = "Failed";
+        loglevel = GF_LOG_DEBUG;
+    } else {
+        status = "Completed";
+        loglevel = GF_LOG_INFO;
+    }
+
+    gf_msg(this->name, loglevel, 0, AFR_MSG_SELF_HEAL_INFO,
+           "%s %s selfheal on %s. "
+           "sources=%s sinks=%s",
+           status, type, uuid_utoa(gfid), sources_str, sinks_str);
 }
 
 int
-afr_selfheal_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int op_ret, int op_errno, inode_t *inode,
-			   struct iatt *buf, dict_t *xdata, struct iatt *parbuf)
-{
-	afr_local_t *local = NULL;
-	int i = -1;
-        GF_UNUSED int ret = -1;
-	int8_t need_heal = 1;
-
-	local = frame->local;
-	i = (long) cookie;
-
-	local->replies[i].valid = 1;
-	local->replies[i].op_ret = op_ret;
-	local->replies[i].op_errno = op_errno;
-	if (buf)
-		local->replies[i].poststat = *buf;
-	if (parbuf)
-		local->replies[i].postparent = *parbuf;
-	if (xdata) {
-		local->replies[i].xdata = dict_ref (xdata);
-                ret = dict_get_int8 (xdata, "link-count", &need_heal);
-                local->replies[i].need_heal = need_heal;
-        } else {
-                local->replies[i].need_heal = need_heal;
-        }
-
-	syncbarrier_wake (&local->barrier);
+afr_selfheal_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xdata, struct iatt *parbuf)
+{
+    afr_local_t *local = NULL;
+    int i = -1;
+    GF_UNUSED int ret = -1;
+    int8_t need_heal = 1;
 
-	return 0;
-}
+    local = frame->local;
+    i = (long)cookie;
 
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (buf)
+        local->replies[i].poststat = *buf;
+    if (parbuf)
+        local->replies[i].postparent = *parbuf;
+    if (xdata) {
+        local->replies[i].xdata = dict_ref(xdata);
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+    }
 
-inode_t *
-afr_selfheal_unlocked_lookup_on (call_frame_t *frame, inode_t *parent,
-				 const char *name, struct afr_reply *replies,
-				 unsigned char *lookup_on, dict_t *xattr)
-{
-	loc_t loc = {0, };
-	dict_t *xattr_req = NULL;
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	inode_t *inode = NULL;
+    local->replies[i].need_heal = need_heal;
+    syncbarrier_wake(&local->barrier);
 
-	local = frame->local;
-	priv = frame->this->private;
+    return 0;
+}
 
-	xattr_req = dict_new ();
-	if (!xattr_req)
-		return NULL;
+inode_t *
+afr_selfheal_unlocked_lookup_on(call_frame_t *frame, inode_t *parent,
+                                const char *name, struct afr_reply *replies,
+                                unsigned char *lookup_on, dict_t *xattr)
+{
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr_req = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+
+    local = frame->local;
+    priv = frame->this->private;
+
+    xattr_req = dict_new();
+    if (!xattr_req)
+        return NULL;
 
-        if (xattr)
-                dict_copy (xattr, xattr_req);
+    if (xattr)
+        dict_copy(xattr, xattr_req);
 
-	if (afr_xattr_req_prepare (frame->this, xattr_req) != 0) {
-		dict_unref (xattr_req);
-		return NULL;
-	}
+    if (afr_xattr_req_prepare(frame->this, xattr_req) != 0) {
+        dict_unref(xattr_req);
+        return NULL;
+    }
 
-	inode = inode_new (parent->table);
-	if (!inode) {
-		dict_unref (xattr_req);
-		return NULL;
-	}
+    inode = inode_new(parent->table);
+    if (!inode) {
+        dict_unref(xattr_req);
+        return NULL;
+    }
 
-	loc.parent = inode_ref (parent);
-	gf_uuid_copy (loc.pargfid, parent->gfid);
-	loc.name = name;
-	loc.inode = inode_ref (inode);
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = name;
+    loc.inode = inode_ref(inode);
 
-	AFR_ONLIST (lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
-		    xattr_req);
+    AFR_ONLIST(lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xattr_req);
 
-	afr_replies_copy (replies, local->replies, priv->child_count);
+    afr_replies_copy(replies, local->replies, priv->child_count);
 
-	loc_wipe (&loc);
-	dict_unref (xattr_req);
+    loc_wipe(&loc);
+    dict_unref(xattr_req);
 
-	return inode;
+    return inode;
 }
 
-int
-afr_selfheal_unlocked_discover_on (call_frame_t *frame, inode_t *inode,
-				   uuid_t gfid, struct afr_reply *replies,
-				   unsigned char *discover_on)
+static int
+afr_set_multi_dom_lock_count_request(xlator_t *this, dict_t *dict)
 {
-	loc_t loc = {0, };
-	dict_t *xattr_req = NULL;
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
+    int ret = 0;
+    afr_private_t *priv = NULL;
+    char *key1 = NULL;
+    char *key2 = NULL;
 
-	local = frame->local;
-	priv = frame->this->private;
+    priv = this->private;
+    key1 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(this->name));
+    key2 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(priv->sh_domain));
 
-	xattr_req = dict_new ();
-	if (!xattr_req)
-		return -ENOMEM;
+    ret = dict_set_uint32(dict, GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS, 1);
+    if (ret)
+        return ret;
 
-	if (afr_xattr_req_prepare (frame->this, xattr_req) != 0) {
-		dict_unref (xattr_req);
-		return -ENOMEM;
-	}
+    sprintf(key1, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, this->name);
+    ret = dict_set_uint32(dict, key1, 1);
+    if (ret)
+        return ret;
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, gfid);
+    sprintf(key2, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, priv->sh_domain);
+    ret = dict_set_uint32(dict, key2, 1);
+    if (ret)
+        return ret;
+
+    return 0;
+}
+
+int
+afr_selfheal_unlocked_discover_on(call_frame_t *frame, inode_t *inode,
+                                  uuid_t gfid, struct afr_reply *replies,
+                                  unsigned char *discover_on, dict_t *dict)
+{
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr_req = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = frame->this->private;
+
+    xattr_req = dict_new();
+    if (!xattr_req)
+        return -ENOMEM;
+    if (dict)
+        dict_copy(dict, xattr_req);
+
+    if (afr_xattr_req_prepare(frame->this, xattr_req) != 0) {
+        dict_unref(xattr_req);
+        return -ENOMEM;
+    }
+
+    if (afr_set_multi_dom_lock_count_request(frame->this, xattr_req)) {
+        dict_unref(xattr_req);
+        return -1;
+    }
 
-	AFR_ONLIST (discover_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
-		    xattr_req);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, gfid);
 
-	afr_replies_copy (replies, local->replies, priv->child_count);
+    AFR_ONLIST(discover_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xattr_req);
 
-	loc_wipe (&loc);
-	dict_unref (xattr_req);
+    afr_replies_copy(replies, local->replies, priv->child_count);
 
-	return 0;
+    loc_wipe(&loc);
+    dict_unref(xattr_req);
+
+    return 0;
 }
 
 int
-afr_selfheal_unlocked_discover (call_frame_t *frame, inode_t *inode,
-				uuid_t gfid, struct afr_reply *replies)
+afr_selfheal_unlocked_discover(call_frame_t *frame, inode_t *inode, uuid_t gfid,
+                               struct afr_reply *replies)
 {
-	afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    dict_t *dict = NULL;
+
+    local = frame->local;
 
-	priv = frame->this->private;
+    if (local->xattr_req)
+        dict = local->xattr_req;
 
-	return afr_selfheal_unlocked_discover_on (frame, inode, gfid, replies,
-						  priv->child_up);
+    return afr_selfheal_unlocked_discover_on(frame, inode, gfid, replies,
+                                             local->child_up, dict);
 }
 
 unsigned int
-afr_success_count (struct afr_reply *replies, unsigned int count)
+afr_success_count(struct afr_reply *replies, unsigned int count)
 {
-        int     i = 0;
-        unsigned int success = 0;
+    int i = 0;
+    unsigned int success = 0;
 
-        for (i = 0; i < count; i++)
-                if (replies[i].valid && replies[i].op_ret == 0)
-                        success++;
-        return success;
+    for (i = 0; i < count; i++)
+        if (replies[i].valid && replies[i].op_ret == 0)
+            success++;
+    return success;
 }
 
 int
-afr_selfheal_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno, dict_t *xdata)
+afr_selfheal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	int i = 0;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-	local = frame->local;
-	i = (long) cookie;
+    local = frame->local;
+    i = (long)cookie;
 
-	local->replies[i].valid = 1;
-	local->replies[i].op_ret = op_ret;
-	local->replies[i].op_errno = op_errno;
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
 
-	syncbarrier_wake (&local->barrier);
+    syncbarrier_wake(&local->barrier);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_locked_fill (call_frame_t *frame, xlator_t *this,
-                 unsigned char *locked_on)
+afr_locked_fill(call_frame_t *frame, xlator_t *this, unsigned char *locked_on)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	int count = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int count = 0;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->replies[i].valid && local->replies[i].op_ret == 0) {
-			locked_on[i] = 1;
-			count++;
-		} else {
-			locked_on[i] = 0;
-		}
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].op_ret == 0) {
+            locked_on[i] = 1;
+            count++;
+        } else {
+            locked_on[i] = 0;
+        }
+    }
 
-	return count;
+    return count;
 }
 
-
 int
-afr_selfheal_tryinodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			 char *dom, off_t off, size_t size,
-			 unsigned char *locked_on)
+afr_selfheal_tryinodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, off_t off, size_t size,
+                        unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	struct gf_flock flock = {0, };
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	flock.l_type = F_WRLCK;
-	flock.l_start = off;
-	flock.l_len = size;
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, inodelk, dom,
-		    &loc, F_SETLK, &flock, NULL);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
-
 int
-afr_selfheal_inodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      char *dom, off_t off, size_t size,
-		      unsigned char *locked_on)
+afr_selfheal_inodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, off_t off, size_t size,
+                     unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	struct gf_flock flock = {0, };
-	afr_local_t *local = NULL;
-	int i = 0;
-	afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-	priv = this->private;
-	local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	flock.l_type = F_WRLCK;
-	flock.l_start = off;
-	flock.l_len = size;
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, inodelk, dom,
-		    &loc, F_SETLK, &flock, NULL);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->replies[i].op_ret == -1 &&
-		    local->replies[i].op_errno == EAGAIN) {
-			afr_locked_fill (frame, this, locked_on);
-			afr_selfheal_uninodelk (frame, this, inode, dom, off,
-						size, locked_on);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == -1 &&
+            local->replies[i].op_errno == EAGAIN) {
+            afr_locked_fill(frame, this, locked_on);
+            afr_selfheal_uninodelk(frame, this, inode, dom, off, size,
+                                   locked_on);
 
-			AFR_SEQ (frame, afr_selfheal_lock_cbk, inodelk, dom,
-				 &loc, F_SETLKW, &flock, NULL);
-			break;
-		}
-	}
+            AFR_SEQ(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLKW,
+                    &flock, NULL);
+            break;
+        }
+    }
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
 static void
-afr_get_lock_and_eagain_counts (afr_private_t *priv, struct afr_reply *replies,
-                                int *lock_count, int *eagain_count)
-{
-	int i = 0;
-
-	for (i = 0; i < priv->child_count; i++) {
-	        if (!replies[i].valid)
-	                continue;
-	        if (replies[i].op_ret == 0) {
-	                (*lock_count)++;
-                } else if (replies[i].op_ret == -1 &&
-		         replies[i].op_errno == EAGAIN) {
-		        (*eagain_count)++;
-                }
-	}
+afr_get_lock_and_eagain_counts(afr_private_t *priv, struct afr_reply *replies,
+                               int *lock_count, int *eagain_count)
+{
+    int i = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret == 0) {
+            (*lock_count)++;
+        } else if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
+            (*eagain_count)++;
+        }
+    }
 }
 
 /*Do blocking locks if number of locks acquired is majority and there were some
  * EAGAINs. Useful for odd-way replication*/
 int
-afr_selfheal_tie_breaker_inodelk (call_frame_t *frame, xlator_t *this,
-                                  inode_t *inode, char *dom, off_t off,
-                                  size_t size, unsigned char *locked_on)
+afr_selfheal_tie_breaker_inodelk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, off_t off,
+                                 size_t size, unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	struct gf_flock flock = {0, };
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int lock_count = 0;
-	int eagain_count = 0;
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int lock_count = 0;
+    int eagain_count = 0;
 
-	priv = this->private;
-	local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	flock.l_type = F_WRLCK;
-	flock.l_start = off;
-	flock.l_len = size;
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, inodelk, dom,
-		   &loc, F_SETLK, &flock, NULL);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
 
-        afr_get_lock_and_eagain_counts (priv, local->replies, &lock_count,
-                                        &eagain_count);
+    afr_get_lock_and_eagain_counts(priv, local->replies, &lock_count,
+                                   &eagain_count);
 
-	if (lock_count > priv->child_count/2 && eagain_count) {
-                afr_locked_fill (frame, this, locked_on);
-                afr_selfheal_uninodelk (frame, this, inode, dom, off,
-                                        size, locked_on);
+    if (lock_count > priv->child_count / 2 && eagain_count) {
+        afr_locked_fill(frame, this, locked_on);
+        afr_selfheal_uninodelk(frame, this, inode, dom, off, size, locked_on);
 
-                AFR_SEQ (frame, afr_selfheal_lock_cbk, inodelk, dom,
-                         &loc, F_SETLKW, &flock, NULL);
-        }
+        AFR_SEQ(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLKW,
+                &flock, NULL);
+    }
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
 int
-afr_selfheal_uninodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			char *dom, off_t off, size_t size,
-			const unsigned char *locked_on)
+afr_selfheal_uninodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, off_t off, size_t size,
+                       const unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	struct gf_flock flock = {0, };
-
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	flock.l_type = F_UNLCK;
-	flock.l_start = off;
-	flock.l_len = size;
+    flock.l_type = F_UNLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-	AFR_ONLIST (locked_on, frame, afr_selfheal_lock_cbk, inodelk,
-		    dom, &loc, F_SETLK, &flock, NULL);
+    AFR_ONLIST(locked_on, frame, afr_selfheal_lock_cbk, inodelk, dom, &loc,
+               F_SETLK, &flock, NULL);
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_selfheal_tryentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			 char *dom, const char *name, unsigned char *locked_on)
+afr_selfheal_tryentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, const char *name, unsigned char *locked_on)
 {
-	loc_t loc = {0,};
+    loc_t loc = {
+        0,
+    };
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, entrylk, dom,
-		   &loc, name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
-
 int
-afr_selfheal_entrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      char *dom, const char *name, unsigned char *locked_on)
+afr_selfheal_entrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, const char *name, unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	afr_local_t *local = NULL;
-	int i = 0;
-	afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-	priv = this->private;
-	local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, entrylk, dom, &loc,
-		   name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->replies[i].op_ret == -1 &&
-		    local->replies[i].op_errno == EAGAIN) {
-			afr_locked_fill (frame, this, locked_on);
-			afr_selfheal_unentrylk (frame, this, inode, dom, name,
-						locked_on, NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == -1 &&
+            local->replies[i].op_errno == EAGAIN) {
+            afr_locked_fill(frame, this, locked_on);
+            afr_selfheal_unentrylk(frame, this, inode, dom, name, locked_on,
+                                   NULL);
 
-			AFR_SEQ (frame, afr_selfheal_lock_cbk, entrylk, dom,
-				 &loc, name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-			break;
-		}
-	}
+            AFR_SEQ(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+                    ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+            break;
+        }
+    }
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
 int
-afr_selfheal_tie_breaker_entrylk (call_frame_t *frame, xlator_t *this,
-                                  inode_t *inode, char *dom, const char *name,
-                                  unsigned char *locked_on)
+afr_selfheal_tie_breaker_entrylk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, const char *name,
+                                 unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int lock_count = 0;
-	int eagain_count = 0;
+    loc_t loc = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int lock_count = 0;
+    int eagain_count = 0;
 
-	priv = this->private;
-	local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, entrylk, dom, &loc,
-		   name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-        afr_get_lock_and_eagain_counts (priv, local->replies, &lock_count,
-                                        &eagain_count);
+    afr_get_lock_and_eagain_counts(priv, local->replies, &lock_count,
+                                   &eagain_count);
 
-	if (lock_count > priv->child_count/2 && eagain_count) {
-                afr_locked_fill (frame, this, locked_on);
-                afr_selfheal_unentrylk (frame, this, inode, dom, name,
-                                        locked_on, NULL);
+    if (lock_count > priv->child_count / 2 && eagain_count) {
+        afr_locked_fill(frame, this, locked_on);
+        afr_selfheal_unentrylk(frame, this, inode, dom, name, locked_on, NULL);
 
-                AFR_SEQ (frame, afr_selfheal_lock_cbk, entrylk, dom,
-                         &loc, name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-	}
+        AFR_SEQ(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+                ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+    }
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
-
 int
-afr_selfheal_unentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			char *dom, const char *name, unsigned char *locked_on,
-                        dict_t *xdata)
-{
-	loc_t loc = {0,};
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-	AFR_ONLIST (locked_on, frame, afr_selfheal_lock_cbk, entrylk,
-		    dom, &loc, name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
-
-	loc_wipe (&loc);
-
-	return 0;
-}
-
-
-gf_boolean_t
-afr_is_pending_set (xlator_t *this, dict_t *xdata, int type)
+afr_selfheal_unentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, const char *name, unsigned char *locked_on,
+                       dict_t *xdata)
 {
-	int idx = -1;
-	afr_private_t *priv = NULL;
-	void *pending_raw = NULL;
-	int *pending_int = NULL;
-	int i = 0;
-
-	priv = this->private;
-	idx = afr_index_for_transaction_type (type);
+    loc_t loc = {
+        0,
+    };
 
-	if (dict_get_ptr (xdata, AFR_DIRTY, &pending_raw) == 0) {
-		if (pending_raw) {
-			pending_int = pending_raw;
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-			if (ntoh32 (pending_int[idx]))
-				return _gf_true;
-		}
-	}
+    AFR_ONLIST(locked_on, frame, afr_selfheal_lock_cbk, entrylk, dom, &loc,
+               name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (dict_get_ptr (xdata, priv->pending_key[i],
-				  &pending_raw))
-			continue;
-		if (!pending_raw)
-			continue;
-		pending_int = pending_raw;
+    loc_wipe(&loc);
 
-		if (ntoh32 (pending_int[idx]))
-			return _gf_true;
-	}
-
-	return _gf_false;
+    return 0;
 }
 
-
 gf_boolean_t
-afr_is_data_set (xlator_t *this, dict_t *xdata)
+afr_is_data_set(xlator_t *this, dict_t *xdata)
 {
-	return afr_is_pending_set (this, xdata, AFR_DATA_TRANSACTION);
+    return afr_is_pending_set(this, xdata, AFR_DATA_TRANSACTION);
 }
 
 gf_boolean_t
-afr_is_metadata_set (xlator_t *this, dict_t *xdata)
+afr_is_metadata_set(xlator_t *this, dict_t *xdata)
 {
-	return afr_is_pending_set (this, xdata, AFR_METADATA_TRANSACTION);
+    return afr_is_pending_set(this, xdata, AFR_METADATA_TRANSACTION);
 }
 
 gf_boolean_t
-afr_is_entry_set (xlator_t *this, dict_t *xdata)
+afr_is_entry_set(xlator_t *this, dict_t *xdata)
 {
-	return afr_is_pending_set (this, xdata, AFR_ENTRY_TRANSACTION);
+    return afr_is_pending_set(this, xdata, AFR_ENTRY_TRANSACTION);
 }
 
 /*
@@ -2217,317 +2278,310 @@ afr_is_entry_set (xlator_t *this, dict_t *xdata)
  */
 
 int
-afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this,
-			       uuid_t gfid, inode_t **link_inode,
-			       gf_boolean_t *data_selfheal,
-			       gf_boolean_t *metadata_selfheal,
-			       gf_boolean_t *entry_selfheal)
-{
-	afr_private_t *priv = NULL;
-        inode_t *inode = NULL;
-	int i = 0;
-	int valid_cnt = 0;
-	struct iatt first = {0, };
-        int first_idx = 0;
-	struct afr_reply *replies = NULL;
-	int ret = -1;
-
-	priv = this->private;
-
-        inode = afr_inode_find (this, gfid);
-        if (!inode)
-                goto out;
+afr_selfheal_unlocked_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
+                              inode_t **link_inode, gf_boolean_t *data_selfheal,
+                              gf_boolean_t *metadata_selfheal,
+                              gf_boolean_t *entry_selfheal,
+                              struct afr_reply *replies_dst)
+{
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    int i = 0;
+    int valid_cnt = 0;
+    struct iatt first = {
+        0,
+    };
+    int first_idx = 0;
+    struct afr_reply *replies = NULL;
+    int ret = -1;
+
+    priv = this->private;
+
+    inode = afr_inode_find(this, gfid);
+    if (!inode)
+        goto out;
 
-	replies = alloca0 (sizeof (*replies) * priv->child_count);
+    replies = alloca0(sizeof(*replies) * priv->child_count);
 
-	ret = afr_selfheal_unlocked_discover (frame, inode, gfid, replies);
-	if (ret)
-                goto out;
+    ret = afr_selfheal_unlocked_discover(frame, inode, gfid, replies);
+    if (ret)
+        goto out;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-		if (replies[i].op_ret == -1)
-			continue;
-
-                /* The data segment of the changelog can be non-zero to indicate
-                 * the directory needs a full heal. So the check below ensures
-                 * it's not a directory before setting the data_selfheal boolean.
-                 */
-		if (data_selfheal && !IA_ISDIR (replies[i].poststat.ia_type) &&
-                    afr_is_data_set (this, replies[i].xdata))
-			*data_selfheal = _gf_true;
-
-		if (metadata_selfheal &&
-                    afr_is_metadata_set (this, replies[i].xdata))
-			*metadata_selfheal = _gf_true;
-
-		if (entry_selfheal && afr_is_entry_set (this, replies[i].xdata))
-			*entry_selfheal = _gf_true;
-
-		valid_cnt++;
-		if (valid_cnt == 1) {
-			first = replies[i].poststat;
-                        first_idx = i;
-			continue;
-		}
-
-		if (!IA_EQUAL (first, replies[i].poststat, type)) {
-		        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN,
-				"TYPE mismatch %d vs %d on %s for gfid:%s",
-				(int) first.ia_type,
-				(int) replies[i].poststat.ia_type,
-				priv->children[i]->name,
-				uuid_utoa (replies[i].poststat.ia_gfid));
-                        gf_event (EVENT_AFR_SPLIT_BRAIN, "subvol=%s;"
-                                "type=file;gfid=%s;"
-                                "ia_type-%d=%s;ia_type-%d=%s",
-                                this->name,
-                                uuid_utoa (replies[i].poststat.ia_gfid),
-                                first_idx,
-                                gf_inode_type_to_str (first.ia_type), i,
-                            gf_inode_type_to_str (replies[i].poststat.ia_type));
-                        ret = -EIO;
-                        goto out;
-		}
-
-		if (!IA_EQUAL (first, replies[i].poststat, uid)) {
-		        gf_msg_debug (this->name, 0,
-				      "UID mismatch "
-                                      "%d vs %d on %s for gfid:%s",
-				      (int) first.ia_uid,
-				      (int) replies[i].poststat.ia_uid,
-				      priv->children[i]->name,
-				      uuid_utoa (replies[i].poststat.ia_gfid));
-
-                        if (metadata_selfheal)
-                                *metadata_selfheal = _gf_true;
-		}
-
-		if (!IA_EQUAL (first, replies[i].poststat, gid)) {
-		        gf_msg_debug (this->name, 0,
-				      "GID mismatch "
-                                      "%d vs %d on %s for gfid:%s",
-				      (int) first.ia_uid,
-				      (int) replies[i].poststat.ia_uid,
-				      priv->children[i]->name,
-				      uuid_utoa (replies[i].poststat.ia_gfid));
-
-                        if (metadata_selfheal)
-                                *metadata_selfheal = _gf_true;
-		}
-
-		if (!IA_EQUAL (first, replies[i].poststat, prot)) {
-		        gf_msg_debug (this->name, 0,
-			              "MODE mismatch "
-                                      "%d vs %d on %s for gfid:%s",
-				      (int) st_mode_from_ia (first.ia_prot, 0),
-				      (int) st_mode_from_ia
-                                      (replies[i].poststat.ia_prot, 0),
-				      priv->children[i]->name,
-				      uuid_utoa (replies[i].poststat.ia_gfid));
-
-                        if (metadata_selfheal)
-                                *metadata_selfheal = _gf_true;
-		}
-
-		if (IA_ISREG(first.ia_type) &&
-		    !IA_EQUAL (first, replies[i].poststat, size)) {
-		        gf_msg_debug (this->name, 0,
-                                      "SIZE mismatch "
-                                      "%lld vs %lld on %s for gfid:%s",
-                                      (long long) first.ia_size,
-                                      (long long) replies[i].poststat.ia_size,
-                                      priv->children[i]->name,
-                                      uuid_utoa (replies[i].poststat.ia_gfid));
-
-                        if (data_selfheal)
-                                *data_selfheal = _gf_true;
-		}
-	}
-
-	if (valid_cnt > 0 && link_inode) {
-		*link_inode = inode_link (inode, NULL, NULL, &first);
-                if (!*link_inode) {
-                        ret = -EINVAL;
-                        goto out;
-                }
-        } else if (valid_cnt < 2) {
-                ret = afr_check_stale_error (replies, priv);
-                goto out;
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret == -1)
+            continue;
 
-        ret = 0;
+        /* The data segment of the changelog can be non-zero to indicate
+         * the directory needs a full heal. So the check below ensures
+         * it's not a directory before setting the data_selfheal boolean.
+         */
+        if (data_selfheal && !IA_ISDIR(replies[i].poststat.ia_type) &&
+            afr_is_data_set(this, replies[i].xdata))
+            *data_selfheal = _gf_true;
+
+        if (metadata_selfheal && afr_is_metadata_set(this, replies[i].xdata))
+            *metadata_selfheal = _gf_true;
+
+        if (entry_selfheal && afr_is_entry_set(this, replies[i].xdata))
+            *entry_selfheal = _gf_true;
+
+        valid_cnt++;
+        if (valid_cnt == 1) {
+            first = replies[i].poststat;
+            first_idx = i;
+            continue;
+        }
+
+        if (!IA_EQUAL(first, replies[i].poststat, type)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                   "TYPE mismatch %d vs %d on %s for gfid:%s",
+                   (int)first.ia_type, (int)replies[i].poststat.ia_type,
+                   priv->children[i]->name,
+                   uuid_utoa(replies[i].poststat.ia_gfid));
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;"
+                     "type=file;gfid=%s;"
+                     "ia_type-%d=%s;ia_type-%d=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(replies[i].poststat.ia_gfid), first_idx,
+                     gf_inode_type_to_str(first.ia_type), i,
+                     gf_inode_type_to_str(replies[i].poststat.ia_type));
+            ret = -EIO;
+            goto out;
+        }
+
+        if (!IA_EQUAL(first, replies[i].poststat, uid)) {
+            gf_msg_debug(this->name, 0,
+                         "UID mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)first.ia_uid, (int)replies[i].poststat.ia_uid,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
+
+        if (!IA_EQUAL(first, replies[i].poststat, gid)) {
+            gf_msg_debug(this->name, 0,
+                         "GID mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)first.ia_uid, (int)replies[i].poststat.ia_uid,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
+
+        if (!IA_EQUAL(first, replies[i].poststat, prot)) {
+            gf_msg_debug(this->name, 0,
+                         "MODE mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)st_mode_from_ia(first.ia_prot, 0),
+                         (int)st_mode_from_ia(replies[i].poststat.ia_prot, 0),
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
+
+        if (IA_ISREG(first.ia_type) &&
+            !IA_EQUAL(first, replies[i].poststat, size)) {
+            gf_msg_debug(this->name, 0,
+                         "SIZE mismatch "
+                         "%lld vs %lld on %s for gfid:%s",
+                         (long long)first.ia_size,
+                         (long long)replies[i].poststat.ia_size,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (data_selfheal)
+                *data_selfheal = _gf_true;
+        }
+    }
+
+    if (valid_cnt > 0 && link_inode) {
+        *link_inode = inode_link(inode, NULL, NULL, &first);
+        if (!*link_inode) {
+            ret = -EINVAL;
+            goto out;
+        }
+    } else if (valid_cnt < 2) {
+        ret = afr_check_stale_error(replies, priv);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (inode)
-                inode_unref (inode);
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
+    if (replies && replies_dst)
+        afr_replies_copy(replies_dst, replies, priv->child_count);
+    if (inode)
+        inode_unref(inode);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
 
-	return ret;
+    return ret;
 }
 
-
 inode_t *
-afr_inode_find (xlator_t *this, uuid_t gfid)
+afr_inode_find(xlator_t *this, uuid_t gfid)
 {
-	inode_table_t *table = NULL;
-	inode_t *inode = NULL;
+    inode_table_t *table = NULL;
+    inode_t *inode = NULL;
 
-	table = this->itable;
-	if (!table)
-		return NULL;
+    table = this->itable;
+    if (!table)
+        return NULL;
 
-	inode = inode_find (table, gfid);
-	if (inode)
-		return inode;
+    inode = inode_find(table, gfid);
+    if (inode)
+        return inode;
 
-	inode = inode_new (table);
-	if (!inode)
-		return NULL;
+    inode = inode_new(table);
+    if (!inode)
+        return NULL;
 
-	gf_uuid_copy (inode->gfid, gfid);
+    gf_uuid_copy(inode->gfid, gfid);
 
-	return inode;
+    return inode;
 }
 
-
 call_frame_t *
-afr_frame_create (xlator_t *this, int32_t *op_errno)
+afr_frame_create(xlator_t *this, int32_t *op_errno)
 {
-	call_frame_t *frame    = NULL;
-	afr_local_t  *local    = NULL;
-	pid_t         pid      = GF_CLIENT_PID_SELF_HEALD;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    pid_t pid = GF_CLIENT_PID_SELF_HEALD;
 
-	frame = create_frame (this, this->ctx->pool);
-	if (!frame)
-		return NULL;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        return NULL;
+    }
 
-	local = AFR_FRAME_INIT (frame, (*op_errno));
-	if (!local) {
-		STACK_DESTROY (frame->root);
-		return NULL;
-	}
+    local = AFR_FRAME_INIT(frame, (*op_errno));
+    if (!local) {
+        STACK_DESTROY(frame->root);
+        return NULL;
+    }
 
-	syncopctx_setfspid (&pid);
+    syncopctx_setfspid(&pid);
 
-	frame->root->pid = pid;
+    frame->root->pid = pid;
 
-	afr_set_lk_owner (frame, this, frame->root);
+    afr_set_lk_owner(frame, this, frame->root);
 
-	return frame;
+    return frame;
 }
 
 int
-afr_selfheal_newentry_mark (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			    int source, struct afr_reply *replies,
-			    unsigned char *sources, unsigned char *newentry)
+afr_selfheal_newentry_mark(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, struct afr_reply *replies,
+                           unsigned char *sources, unsigned char *newentry)
 {
-	int ret = 0;
-	int i = 0;
-	afr_private_t *priv = NULL;
-	dict_t *xattr = NULL;
-	int **changelog = NULL;
+    int ret = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int **changelog = NULL;
 
-	priv = this->private;
+    priv = this->private;
 
-	gf_uuid_copy (inode->gfid, replies[source].poststat.ia_gfid);
+    gf_uuid_copy(inode->gfid, replies[source].poststat.ia_gfid);
 
-	xattr = dict_new();
-	if (!xattr)
-		return -ENOMEM;
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
 
-        changelog = afr_mark_pending_changelog (priv, newentry, xattr,
-                                            replies[source].poststat.ia_type);
+    changelog = afr_mark_pending_changelog(priv, newentry, xattr,
+                                           replies[source].poststat.ia_type);
 
-        if (!changelog) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    if (!changelog) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!sources[i])
-			continue;
-		ret |= afr_selfheal_post_op (frame, this, inode, i, xattr,
-                                             NULL);
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        ret |= afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
+    }
 out:
-        if (changelog)
-                afr_matrix_cleanup (changelog, priv->child_count);
-        if (xattr)
-                dict_unref (xattr);
-	return ret;
+    if (changelog)
+        afr_matrix_cleanup(changelog, priv->child_count);
+    if (xattr)
+        dict_unref(xattr);
+    return ret;
 }
 
 int
-afr_selfheal_do (call_frame_t *frame, xlator_t *this, uuid_t gfid)
-{
-	int           ret               = -1;
-        int           entry_ret         = 1;
-        int           metadata_ret      = 1;
-        int           data_ret          = 1;
-        int           or_ret            = 0;
-        inode_t      *inode             = NULL;
-        fd_t         *fd                = NULL;
-	gf_boolean_t  data_selfheal     = _gf_false;
-	gf_boolean_t  metadata_selfheal = _gf_false;
-	gf_boolean_t  entry_selfheal    = _gf_false;
-        afr_private_t *priv            = NULL;
-        gf_boolean_t dataheal_enabled   = _gf_false;
-
-        priv = this->private;
-
-        ret = gf_string2boolean (priv->data_self_heal, &dataheal_enabled);
-        if (ret)
-                goto out;
-
-	ret = afr_selfheal_unlocked_inspect (frame, this, gfid, &inode,
-					     &data_selfheal,
-					     &metadata_selfheal,
-					     &entry_selfheal);
-	if (ret)
-		goto out;
+afr_selfheal_do(call_frame_t *frame, xlator_t *this, uuid_t gfid)
+{
+    int ret = -1;
+    int entry_ret = 1;
+    int metadata_ret = 1;
+    int data_ret = 1;
+    int or_ret = 0;
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    gf_boolean_t data_selfheal = _gf_false;
+    gf_boolean_t metadata_selfheal = _gf_false;
+    gf_boolean_t entry_selfheal = _gf_false;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    ret = afr_selfheal_unlocked_inspect(frame, this, gfid, &inode,
+                                        &data_selfheal, &metadata_selfheal,
+                                        &entry_selfheal, NULL);
+    if (ret)
+        goto out;
 
-        if (!(data_selfheal || metadata_selfheal || entry_selfheal)) {
-                ret = 2;
-                goto out;
-        }
+    if (!(data_selfheal || metadata_selfheal || entry_selfheal)) {
+        ret = 2;
+        goto out;
+    }
 
-        if (inode->ia_type == IA_IFREG) {
-                ret = afr_selfheal_data_open (this, inode, &fd);
-                if (!fd) {
-                        ret = -EIO;
-                        goto out;
-                }
+    if (inode->ia_type == IA_IFREG) {
+        ret = afr_selfheal_data_open(this, inode, &fd);
+        if (!fd) {
+            ret = -EIO;
+            goto out;
         }
+    }
 
-	if (data_selfheal && dataheal_enabled)
-                data_ret = afr_selfheal_data (frame, this, fd);
+    if (data_selfheal && priv->data_self_heal)
+        data_ret = afr_selfheal_data(frame, this, fd);
 
-	if (metadata_selfheal && priv->metadata_self_heal)
-                metadata_ret = afr_selfheal_metadata (frame, this, inode);
+    if (metadata_selfheal && priv->metadata_self_heal)
+        metadata_ret = afr_selfheal_metadata(frame, this, inode);
 
-	if (entry_selfheal && priv->entry_self_heal)
-                entry_ret = afr_selfheal_entry (frame, this, inode);
+    if (entry_selfheal && priv->entry_self_heal)
+        entry_ret = afr_selfheal_entry(frame, this, inode);
 
-        or_ret = (data_ret | metadata_ret | entry_ret);
+    or_ret = (data_ret | metadata_ret | entry_ret);
 
-        if (data_ret == -EIO || metadata_ret == -EIO || entry_ret == -EIO)
-                ret = -EIO;
-        else if (data_ret == 1 && metadata_ret == 1 && entry_ret == 1)
-                ret = 1;
-        else if (or_ret < 0)
-                ret = or_ret;
-        else
-                ret = 0;
+    if (data_ret == -EIO || metadata_ret == -EIO || entry_ret == -EIO)
+        ret = -EIO;
+    else if (data_ret == 1 && metadata_ret == 1 && entry_ret == 1)
+        ret = 1;
+    else if (or_ret < 0)
+        ret = or_ret;
+    else
+        ret = 0;
 
 out:
-        if (inode)
-                inode_unref (inode);
-        if (fd)
-                fd_unref (fd);
-        return ret;
+    if (inode)
+        inode_unref(inode);
+    if (fd)
+        fd_unref(fd);
+    return ret;
 }
 /*
  * This is the entry point for healing a given GFID. The return values for this
@@ -2539,160 +2593,342 @@ out:
  */
 
 int
-afr_selfheal (xlator_t *this, uuid_t gfid)
+afr_selfheal(xlator_t *this, uuid_t gfid)
 {
-        int           ret   = -1;
-	call_frame_t *frame = NULL;
-        afr_local_t *local = NULL;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
 
-	frame = afr_frame_create (this, NULL);
-	if (!frame)
-		return ret;
+    frame = afr_frame_create(this, NULL);
+    if (!frame)
+        return ret;
 
-        local = frame->local;
-        local->xdata_req = dict_new();
+    local = frame->local;
+    local->xdata_req = dict_new();
 
-        ret = afr_selfheal_do (frame, this, gfid);
+    ret = afr_selfheal_do(frame, this, gfid);
 
-	if (frame)
-		AFR_STACK_DESTROY (frame);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
 
-	return ret;
+    return ret;
 }
 
-afr_local_t*
-__afr_dequeue_heals (afr_private_t *priv)
+afr_local_t *
+__afr_dequeue_heals(afr_private_t *priv)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-        if (list_empty (&priv->heal_waiting))
-                goto none;
-        if ((priv->background_self_heal_count > 0) &&
-            (priv->healers >= priv->background_self_heal_count))
-                goto none;
+    if (list_empty(&priv->heal_waiting))
+        goto none;
+    if ((priv->background_self_heal_count > 0) &&
+        (priv->healers >= priv->background_self_heal_count))
+        goto none;
 
-        local = list_entry (priv->heal_waiting.next, afr_local_t, healer);
-        priv->heal_waiters--;
-        GF_ASSERT (priv->heal_waiters >= 0);
-        list_del_init(&local->healer);
-        list_add(&local->healer, &priv->healing);
-        priv->healers++;
-        return local;
+    local = list_entry(priv->heal_waiting.next, afr_local_t, healer);
+    priv->heal_waiters--;
+    GF_ASSERT(priv->heal_waiters >= 0);
+    list_del_init(&local->healer);
+    list_add(&local->healer, &priv->healing);
+    priv->healers++;
+    return local;
 none:
-        gf_msg_debug (THIS->name, 0, "Nothing dequeued. "
-                      "Num healers: %d, Num Waiters: %d",
-                      priv->healers, priv->heal_waiters);
-        return NULL;
+    gf_msg_debug(THIS->name, 0,
+                 "Nothing dequeued. "
+                 "Num healers: %d, Num Waiters: %d",
+                 priv->healers, priv->heal_waiters);
+    return NULL;
 }
 
 int
-afr_refresh_selfheal_wrap (void *opaque)
+afr_refresh_selfheal_wrap(void *opaque)
 {
-        call_frame_t *heal_frame = opaque;
-        afr_local_t *local = heal_frame->local;
-        int ret = 0;
+    call_frame_t *heal_frame = opaque;
+    afr_local_t *local = heal_frame->local;
+    int ret = 0;
 
-        ret = afr_selfheal (heal_frame->this, local->refreshinode->gfid);
-        return ret;
+    ret = afr_selfheal(heal_frame->this, local->refreshinode->gfid);
+    return ret;
 }
 
 int
-afr_refresh_heal_done (int ret, call_frame_t *frame, void *opaque)
-{
-        call_frame_t *heal_frame = opaque;
-        xlator_t *this = heal_frame->this;
-        afr_private_t *priv = this->private;
-        afr_local_t *local = heal_frame->local;
-
-        LOCK (&priv->lock);
-        {
-                list_del_init(&local->healer);
-                priv->healers--;
-                GF_ASSERT (priv->healers >= 0);
-                local = __afr_dequeue_heals (priv);
-        }
-        UNLOCK (&priv->lock);
+afr_refresh_heal_done(int ret, call_frame_t *frame, void *opaque)
+{
+    call_frame_t *heal_frame = opaque;
+    xlator_t *this = heal_frame->this;
+    afr_private_t *priv = this->private;
+    afr_local_t *local = heal_frame->local;
 
-        AFR_STACK_DESTROY (heal_frame);
+    LOCK(&priv->lock);
+    {
+        list_del_init(&local->healer);
+        priv->healers--;
+        GF_ASSERT(priv->healers >= 0);
+        local = __afr_dequeue_heals(priv);
+    }
+    UNLOCK(&priv->lock);
 
-        if (local)
-                afr_heal_synctask (this, local);
-        return 0;
+    AFR_STACK_DESTROY(heal_frame);
 
+    if (local)
+        afr_heal_synctask(this, local);
+    return 0;
 }
 
 void
-afr_heal_synctask (xlator_t *this, afr_local_t *local)
+afr_heal_synctask(xlator_t *this, afr_local_t *local)
 {
-        int ret = 0;
-        call_frame_t *heal_frame = NULL;
+    int ret = 0;
+    call_frame_t *heal_frame = NULL;
 
-        heal_frame = local->heal_frame;
-        ret = synctask_new (this->ctx->env, afr_refresh_selfheal_wrap,
-                            afr_refresh_heal_done, heal_frame, heal_frame);
-        if (ret < 0)
-                /* Heal not launched. Will be queued when the next inode
-                 * refresh happens and shd hasn't healed it yet. */
-                afr_refresh_heal_done (ret, heal_frame, heal_frame);
+    heal_frame = local->heal_frame;
+    ret = synctask_new(this->ctx->env, afr_refresh_selfheal_wrap,
+                       afr_refresh_heal_done, heal_frame, heal_frame);
+    if (ret < 0)
+        /* Heal not launched. Will be queued when the next inode
+         * refresh happens and shd hasn't healed it yet. */
+        afr_refresh_heal_done(ret, heal_frame, heal_frame);
 }
 
 gf_boolean_t
-afr_throttled_selfheal (call_frame_t *frame, xlator_t *this)
-{
-        gf_boolean_t can_heal = _gf_true;
-        afr_private_t *priv = this->private;
-        afr_local_t *local = frame->local;
-
-        LOCK (&priv->lock);
-        {
-                if ((priv->background_self_heal_count > 0) &&
-                    (priv->heal_wait_qlen + priv->background_self_heal_count) >
-                    (priv->heal_waiters + priv->healers)) {
-                        list_add_tail(&local->healer, &priv->heal_waiting);
-                        priv->heal_waiters++;
-                        local = __afr_dequeue_heals (priv);
-                } else {
-                        can_heal = _gf_false;
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (can_heal) {
-                if (local)
-                        afr_heal_synctask (this, local);
-                else
-                        gf_msg_debug (this->name, 0, "Max number of heals are "
-                                      "pending, background self-heal rejected.");
+afr_throttled_selfheal(call_frame_t *frame, xlator_t *this)
+{
+    gf_boolean_t can_heal = _gf_true;
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+
+    LOCK(&priv->lock);
+    {
+        if ((priv->background_self_heal_count > 0) &&
+            (priv->heal_wait_qlen + priv->background_self_heal_count) >
+                (priv->heal_waiters + priv->healers)) {
+            list_add_tail(&local->healer, &priv->heal_waiting);
+            priv->heal_waiters++;
+            local = __afr_dequeue_heals(priv);
+        } else {
+            can_heal = _gf_false;
         }
+    }
+    UNLOCK(&priv->lock);
+
+    if (can_heal) {
+        if (local)
+            afr_heal_synctask(this, local);
+        else
+            gf_msg_debug(this->name, 0,
+                         "Max number of heals are "
+                         "pending, background self-heal rejected.");
+    }
 
-        return can_heal;
+    return can_heal;
 }
 
 int
-afr_choose_source_by_policy (afr_private_t *priv, unsigned char *sources,
-                             afr_transaction_type type)
+afr_choose_source_by_policy(afr_private_t *priv, unsigned char *sources,
+                            afr_transaction_type type)
 {
-        int source = -1;
-        int i      = 0;
+    int source = -1;
+    int i = 0;
 
-        /* Give preference to local child to save on bandwidth */
-        for (i = 0; i < priv->child_count; i++) {
-                if (priv->local[i] && sources[i]) {
-                        if ((type == AFR_DATA_TRANSACTION) &&
-                            AFR_IS_ARBITER_BRICK (priv, i))
-                                continue;
+    /* Give preference to local child to save on bandwidth */
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->local[i] && sources[i]) {
+            if ((type == AFR_DATA_TRANSACTION) && AFR_IS_ARBITER_BRICK(priv, i))
+                continue;
 
-                        source = i;
-                        goto out;
-                }
+            source = i;
+            goto out;
         }
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i]) {
-                        source = i;
-                        goto out;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i]) {
+            source = i;
+            goto out;
         }
+    }
 out:
-        return source;
+    return source;
+}
+
+static int
+afr_anon_inode_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *buf, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->replies[i].poststat = *buf;
+        local->replies[i].preparent = *preparent;
+        local->replies[i].postparent = *postparent;
+    }
+    if (xdata) {
+        local->replies[i].xdata = dict_ref(xdata);
+    }
+
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
+
+int
+afr_anon_inode_create(xlator_t *this, int child, inode_t **linked_inode)
+{
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
+    unsigned char *mkdir_on = alloca0(priv->child_count);
+    unsigned char *lookup_on = alloca0(priv->child_count);
+    loc_t loc = {0};
+    int32_t op_errno = 0;
+    int32_t child_op_errno = 0;
+    struct iatt iatt = {0};
+    dict_t *xdata = NULL;
+    uuid_t anon_inode_gfid = {0};
+    int mkdir_count = 0;
+    int i = 0;
+
+    /*Try to mkdir everywhere and return success if the dir exists on 'child'
+     */
+
+    if (!priv->use_anon_inode) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    frame = afr_frame_create(this, &op_errno);
+    if (op_errno) {
+        goto out;
+    }
+    local = frame->local;
+    if (!local->child_up[child]) {
+        /*Other bricks may need mkdir so don't error out yet*/
+        child_op_errno = ENOTCONN;
+    }
+    gf_uuid_parse(priv->anon_gfid_str, anon_inode_gfid);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
+
+        if (priv->anon_inode[i]) {
+            mkdir_on[i] = 0;
+        } else {
+            mkdir_on[i] = 1;
+            mkdir_count++;
+        }
+    }
+
+    if (mkdir_count == 0) {
+        *linked_inode = inode_find(this->itable, anon_inode_gfid);
+        if (*linked_inode) {
+            op_errno = 0;
+            goto out;
+        }
+    }
+
+    loc.parent = inode_ref(this->itable->root);
+    loc.name = priv->anon_inode_name;
+    loc.inode = inode_new(this->itable);
+    if (!loc.inode) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_errno = -dict_set_gfuuid(xdata, "gfid-req", anon_inode_gfid, _gf_true);
+    if (op_errno) {
+        goto out;
+    }
+
+    if (mkdir_count == 0) {
+        memcpy(lookup_on, local->child_up, priv->child_count);
+        goto lookup;
+    }
+
+    AFR_ONLIST(mkdir_on, frame, afr_anon_inode_mkdir_cbk, mkdir, &loc, 0755, 0,
+               xdata);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!mkdir_on[i]) {
+            continue;
+        }
+
+        if (local->replies[i].op_ret == 0) {
+            priv->anon_inode[i] = 1;
+            iatt = local->replies[i].poststat;
+        } else if (local->replies[i].op_ret < 0 &&
+                   local->replies[i].op_errno == EEXIST) {
+            lookup_on[i] = 1;
+        } else if (i == child) {
+            child_op_errno = local->replies[i].op_errno;
+        }
+    }
+
+    if (AFR_COUNT(lookup_on, priv->child_count) == 0) {
+        goto link;
+    }
+
+lookup:
+    AFR_ONLIST(lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xdata);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!lookup_on[i]) {
+            continue;
+        }
+
+        if (local->replies[i].op_ret == 0) {
+            if (gf_uuid_compare(anon_inode_gfid,
+                                local->replies[i].poststat.ia_gfid) == 0) {
+                priv->anon_inode[i] = 1;
+                iatt = local->replies[i].poststat;
+            } else {
+                if (i == child)
+                    child_op_errno = EINVAL;
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_DATA,
+                       "%s has gfid: %s", priv->anon_inode_name,
+                       uuid_utoa(local->replies[i].poststat.ia_gfid));
+            }
+        } else if (i == child) {
+            child_op_errno = local->replies[i].op_errno;
+        }
+    }
+link:
+    if (!gf_uuid_is_null(iatt.ia_gfid)) {
+        *linked_inode = inode_link(loc.inode, loc.parent, loc.name, &iatt);
+        if (*linked_inode) {
+            op_errno = 0;
+            inode_lookup(*linked_inode);
+        } else {
+            op_errno = ENOMEM;
+        }
+        goto out;
+    }
+
+out:
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
+    /*child_op_errno takes precedence*/
+    if (child_op_errno == 0) {
+        child_op_errno = op_errno;
+    }
+
+    if (child_op_errno && *linked_inode) {
+        inode_unref(*linked_inode);
+        *linked_inode = NULL;
+    }
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return -child_op_errno;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 40dee7a7d6c..37bcc2b3f9e 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -8,605 +8,592 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "afr.h"
 #include "afr-self-heal.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "protocol-common.h"
 #include "afr-messages.h"
-#include "events.h"
-
-enum {
-	AFR_SELFHEAL_DATA_FULL = 0,
-	AFR_SELFHEAL_DATA_DIFF,
-};
-
+#include <glusterfs/events.h>
 
 #define HAS_HOLES(i) ((i->ia_blocks * 512) < (i->ia_size))
 static int
-__checksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int op_ret, int op_errno, uint32_t weak, uint8_t *strong,
-		dict_t *xdata)
+__checksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, uint32_t weak, uint8_t *strong, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-        struct afr_reply *replies = NULL;
-	int i = (long) cookie;
-
-	local = frame->local;
-        replies = local->replies;
-
-	replies[i].valid = 1;
-	replies[i].op_ret = op_ret;
-	replies[i].op_errno = op_errno;
-        if (xdata) {
-                replies[i].buf_has_zeroes = dict_get_str_boolean (xdata,
-                                                   "buf-has-zeroes", _gf_false);
-                replies[i].fips_mode_rchecksum = dict_get_str_boolean (xdata,
-                                              "fips-mode-rchecksum", _gf_false);
-        }
-	if (strong) {
-                if (replies[i].fips_mode_rchecksum) {
-                        memcpy (local->replies[i].checksum, strong,
-                                SHA256_DIGEST_LENGTH);
-                } else {
-                        memcpy (local->replies[i].checksum, strong,
-                                MD5_DIGEST_LENGTH);
-                }
+    afr_local_t *local = NULL;
+    struct afr_reply *replies = NULL;
+    int i = (long)cookie;
+
+    local = frame->local;
+    replies = local->replies;
+
+    replies[i].valid = 1;
+    replies[i].op_ret = op_ret;
+    replies[i].op_errno = op_errno;
+    if (xdata) {
+        replies[i].buf_has_zeroes = dict_get_str_boolean(
+            xdata, "buf-has-zeroes", _gf_false);
+        replies[i].fips_mode_rchecksum = dict_get_str_boolean(
+            xdata, "fips-mode-rchecksum", _gf_false);
+    }
+    if (strong) {
+        if (replies[i].fips_mode_rchecksum) {
+            memcpy(local->replies[i].checksum, strong, SHA256_DIGEST_LENGTH);
+        } else {
+            memcpy(local->replies[i].checksum, strong, MD5_DIGEST_LENGTH);
         }
+    }
 
-	syncbarrier_wake (&local->barrier);
-	return 0;
+    syncbarrier_wake(&local->barrier);
+    return 0;
 }
 
 static gf_boolean_t
-__afr_can_skip_data_block_heal (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                                int source, unsigned char *healed_sinks,
-				off_t offset, size_t size,
-                                struct iatt *poststat)
+__afr_can_skip_data_block_heal(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                               int source, unsigned char *healed_sinks,
+                               off_t offset, size_t size, struct iatt *poststat)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	unsigned char *wind_subvols = NULL;
-        gf_boolean_t checksum_match = _gf_true;
-        struct afr_reply *replies = NULL;
-        dict_t *xdata = NULL;
-	int i = 0;
-
-	priv = this->private;
-	local = frame->local;
-        replies = local->replies;
-
-        xdata = dict_new();
-        if (!xdata)
-                goto out;
-        if (dict_set_int32 (xdata, "check-zero-filled", 1)) {
-                dict_unref (xdata);
-                goto out;
-        }
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    unsigned char *wind_subvols = NULL;
+    gf_boolean_t checksum_match = _gf_true;
+    struct afr_reply *replies = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+    replies = local->replies;
+
+    xdata = dict_new();
+    if (!xdata)
+        goto out;
+    if (dict_set_int32_sizen(xdata, "check-zero-filled", 1)) {
+        dict_unref(xdata);
+        goto out;
+    }
+
+    wind_subvols = alloca0(priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source || healed_sinks[i])
+            wind_subvols[i] = 1;
+    }
+
+    AFR_ONLIST(wind_subvols, frame, __checksum_cbk, rchecksum, fd, offset, size,
+               xdata);
+    if (xdata)
+        dict_unref(xdata);
+
+    if (!replies[source].valid || replies[source].op_ret != 0)
+        return _gf_false;
 
-	wind_subvols = alloca0 (priv->child_count);
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == source || healed_sinks[i])
-			wind_subvols[i] = 1;
-	}
-
-	AFR_ONLIST (wind_subvols, frame, __checksum_cbk, rchecksum, fd,
-		    offset, size, xdata);
-        if (xdata)
-                dict_unref (xdata);
-
-	if (!replies[source].valid || replies[source].op_ret != 0)
-		return _gf_false;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == source)
-			continue;
-                if (replies[i].valid) {
-                        if (memcmp (replies[source].checksum,
-                                    replies[i].checksum,
-                                    replies[source].fips_mode_rchecksum ?
-                                    SHA256_DIGEST_LENGTH : MD5_DIGEST_LENGTH)) {
-                                checksum_match = _gf_false;
-                                break;
-                        }
-                }
-	}
-
-        if (checksum_match) {
-                if (HAS_HOLES (poststat))
-                        return _gf_true;
-
-                /* For non-sparse files, we might be better off writing the
-                 * zeroes to sinks to avoid mismatch of disk-usage in bricks. */
-                if (local->replies[source].buf_has_zeroes)
-                        return _gf_false;
-                else
-                        return _gf_true;
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source)
+            continue;
+        if (replies[i].valid) {
+            if (memcmp(replies[source].checksum, replies[i].checksum,
+                       replies[source].fips_mode_rchecksum
+                           ? SHA256_DIGEST_LENGTH
+                           : MD5_DIGEST_LENGTH)) {
+                checksum_match = _gf_false;
+                break;
+            }
         }
+    }
+
+    if (checksum_match) {
+        if (HAS_HOLES(poststat))
+            return _gf_true;
+
+        /* For non-sparse files, we might be better off writing the
+         * zeroes to sinks to avoid mismatch of disk-usage in bricks. */
+        if (local->replies[source].buf_has_zeroes)
+            return _gf_false;
+        else
+            return _gf_true;
+    }
 out:
-        return _gf_false;
+    return _gf_false;
 }
 
-
 static gf_boolean_t
-__afr_is_sink_zero_filled (xlator_t *this, fd_t *fd, size_t size,
-                           off_t offset, int sink)
+__afr_is_sink_zero_filled(xlator_t *this, fd_t *fd, size_t size, off_t offset,
+                          int sink)
 {
-        afr_private_t *priv   = NULL;
-        struct iobref *iobref = NULL;
-        struct iovec  *iovec  = NULL;
-        int            count  = 0;
-        int            ret    = 0;
-        gf_boolean_t   zero_filled   = _gf_false;
-
-        priv = this->private;
-        ret = syncop_readv (priv->children[sink], fd, size, offset, 0, &iovec,
-                            &count, &iobref, NULL, NULL, NULL);
-        if (ret < 0)
-                goto out;
-        ret = iov_0filled (iovec, count);
-        if (!ret)
-                zero_filled = _gf_true;
+    afr_private_t *priv = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec *iovec = NULL;
+    int count = 0;
+    int ret = 0;
+    gf_boolean_t zero_filled = _gf_false;
+
+    priv = this->private;
+    ret = syncop_readv(priv->children[sink], fd, size, offset, 0, &iovec,
+                       &count, &iobref, NULL, NULL, NULL);
+    if (ret < 0)
+        goto out;
+    ret = iov_0filled(iovec, count);
+    if (!ret)
+        zero_filled = _gf_true;
 out:
-        if (iovec)
-                GF_FREE (iovec);
-        if (iobref)
-                iobref_unref (iobref);
-        return zero_filled;
+    if (iovec)
+        GF_FREE(iovec);
+    if (iobref)
+        iobref_unref(iobref);
+    return zero_filled;
 }
 
 static int
-__afr_selfheal_data_read_write (call_frame_t *frame, xlator_t *this, fd_t *fd,
-				int source, unsigned char *healed_sinks,
-				off_t offset, size_t size,
-				struct afr_reply *replies, int type)
+__afr_selfheal_data_read_write(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                               int source, unsigned char *healed_sinks,
+                               off_t offset, size_t size,
+                               struct afr_reply *replies, int type)
 {
-	struct iovec *iovec = NULL;
-	int count = 0;
-	struct iobref *iobref = NULL;
-	int ret = 0;
-	int i = 0;
-	afr_private_t *priv = NULL;
-
-	priv = this->private;
-
-	ret = syncop_readv (priv->children[source], fd, size, offset, 0,
-                           &iovec, &count, &iobref, NULL, NULL, NULL);
-	if (ret <= 0)
-		return ret;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!healed_sinks[i])
-			continue;
-
-		/*
-		 * TODO: Use fiemap() and discard() to heal holes
-		 * in the future.
-		 *
-		 * For now,
-		 *
-		 * - if the source had any holes at all,
-		 * AND
-		 * - if we are writing past the original file size
-		 *   of the sink
-		 * AND
-		 * - is NOT the last block of the source file. if
-		 *   the block contains EOF, it has to be written
-		 *   in order to set the file size even if the
-		 *   last block is 0-filled.
-		 * AND
-		 * - if the read buffer is filled with only 0's
-		 *
-		 * then, skip writing to this source. We don't depend
-		 * on the write to happen to update the size as we
-		 * have performed an ftruncate() upfront anyways.
-		 */
-#define is_last_block(o,b,s) ((s >= o) && (s <= (o + b)))
-		if (HAS_HOLES ((&replies[source].poststat)) &&
-		    offset >= replies[i].poststat.ia_size &&
-		    !is_last_block (offset, size,
-				    replies[source].poststat.ia_size) &&
-		    (iov_0filled (iovec, count) == 0))
-			continue;
-
-                /* Avoid filling up sparse regions of the sink with 0-filled
-                 * writes.*/
-                if (type == AFR_SELFHEAL_DATA_FULL &&
-                    HAS_HOLES ((&replies[source].poststat)) &&
-                    ((offset + size) <= replies[i].poststat.ia_size) &&
-                    (iov_0filled (iovec, count) == 0) &&
-                    __afr_is_sink_zero_filled (this, fd, size, offset, i)) {
-                                continue;
-                }
-
-		ret = syncop_writev (priv->children[i], fd, iovec, count,
-                                    offset, iobref, 0, NULL, NULL, NULL, NULL);
-		if (ret != iov_length (iovec, count)) {
-			/* write() failed on this sink. unset the corresponding
-			   member in sinks[] (which is healed_sinks[] in the
-			   caller) so that this server does NOT get considered
-			   as successfully healed.
-			*/
-			healed_sinks[i] = 0;
-		}
-	}
-        if (iovec)
-                GF_FREE (iovec);
-	if (iobref)
-		iobref_unref (iobref);
-
-	return ret;
+    struct iovec *iovec = NULL;
+    int count = 0;
+    struct iobref *iobref = NULL;
+    int ret = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    ret = syncop_readv(priv->children[source], fd, size, offset, 0, &iovec,
+                       &count, &iobref, NULL, NULL, NULL);
+    if (ret <= 0)
+        return ret;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i])
+            continue;
+
+            /*
+             * TODO: Use fiemap() and discard() to heal holes
+             * in the future.
+             *
+             * For now,
+             *
+             * - if the source had any holes at all,
+             * AND
+             * - if we are writing past the original file size
+             *   of the sink
+             * AND
+             * - is NOT the last block of the source file. if
+             *   the block contains EOF, it has to be written
+             *   in order to set the file size even if the
+             *   last block is 0-filled.
+             * AND
+             * - if the read buffer is filled with only 0's
+             *
+             * then, skip writing to this source. We don't depend
+             * on the write to happen to update the size as we
+             * have performed an ftruncate() upfront anyways.
+             */
+#define is_last_block(o, b, s) ((s >= o) && (s <= (o + b)))
+        if (HAS_HOLES((&replies[source].poststat)) &&
+            offset >= replies[i].poststat.ia_size &&
+            !is_last_block(offset, size, replies[source].poststat.ia_size) &&
+            (iov_0filled(iovec, count) == 0))
+            continue;
+
+        /* Avoid filling up sparse regions of the sink with 0-filled
+         * writes.*/
+        if (type == AFR_SELFHEAL_DATA_FULL &&
+            HAS_HOLES((&replies[source].poststat)) &&
+            ((offset + size) <= replies[i].poststat.ia_size) &&
+            (iov_0filled(iovec, count) == 0) &&
+            __afr_is_sink_zero_filled(this, fd, size, offset, i)) {
+            continue;
+        }
+
+        ret = syncop_writev(priv->children[i], fd, iovec, count, offset, iobref,
+                            0, NULL, NULL, NULL, NULL);
+        if (ret != iov_length(iovec, count)) {
+            /* write() failed on this sink. unset the corresponding
+               member in sinks[] (which is healed_sinks[] in the
+               caller) so that this server does NOT get considered
+               as successfully healed.
+            */
+            healed_sinks[i] = 0;
+        }
+    }
+    if (iovec)
+        GF_FREE(iovec);
+    if (iobref)
+        iobref_unref(iobref);
+
+    return ret;
 }
 
-static int
-afr_selfheal_data_block (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			 int source, unsigned char *healed_sinks, off_t offset,
-			 size_t size, int type, struct afr_reply *replies)
+static gf_boolean_t
+afr_source_sinks_locked(xlator_t *this, unsigned char *locked_on, int source,
+                        unsigned char *healed_sinks)
 {
-	int ret = -1;
-	int sink_count = 0;
-	afr_private_t *priv = NULL;
-	unsigned char *data_lock = NULL;
-
-	priv = this->private;
-	sink_count = AFR_COUNT (healed_sinks, priv->child_count);
-	data_lock = alloca0 (priv->child_count);
-
-	ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name,
-				    offset, size, data_lock);
-	{
-		if (ret < sink_count) {
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		if (type == AFR_SELFHEAL_DATA_DIFF &&
-		    __afr_can_skip_data_block_heal (frame, this, fd, source,
-					            healed_sinks, offset, size,
-                                                   &replies[source].poststat)) {
-			ret = 0;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_data_read_write (frame, this, fd, source,
-						      healed_sinks, offset, size,
-						      replies, type);
-	}
-unlock:
-	afr_selfheal_uninodelk (frame, this, fd->inode, this->name,
-				offset, size, data_lock);
-	return ret;
-}
+    afr_private_t *priv = this->private;
+    int i = 0;
 
+    if (!locked_on[source])
+        return _gf_false;
 
+    for (i = 0; i < priv->child_count; i++) {
+        if (healed_sinks[i] && locked_on[i])
+            return _gf_true;
+    }
 
-static int
-afr_selfheal_data_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			 unsigned char *healed_sinks)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-        if (!priv->ensure_durability)
-                return 0;
-
-	AFR_ONLIST (healed_sinks, frame, afr_sh_generic_fop_cbk, fsync, fd, 0,
-                    NULL);
-
-	for (i = 0; i < priv->child_count; i++)
-		if (healed_sinks[i] && local->replies[i].op_ret != 0)
-			/* fsync() failed. Do NOT consider this server
-			   as successfully healed. Mark it so.
-			*/
-			healed_sinks[i] = 0;
-	return 0;
+    return _gf_false;
 }
 
 static int
-afr_data_self_heal_type_get (afr_private_t *priv, unsigned char *healed_sinks,
-                             int source, struct afr_reply *replies)
+afr_selfheal_data_block(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        int source, unsigned char *healed_sinks, off_t offset,
+                        size_t size, int type, struct afr_reply *replies)
 {
-        int type = AFR_SELFHEAL_DATA_FULL;
-        int i = 0;
-
-        if (priv->data_self_heal_algorithm == NULL) {
-                type = AFR_SELFHEAL_DATA_FULL;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!healed_sinks[i] && i != source)
-                                continue;
-                        if (replies[i].poststat.ia_size) {
-                                type = AFR_SELFHEAL_DATA_DIFF;
-                                break;
-                        }
-                }
-        } else if (strcmp (priv->data_self_heal_algorithm, "full") == 0) {
-                type = AFR_SELFHEAL_DATA_FULL;
-        } else if (strcmp (priv->data_self_heal_algorithm, "diff") == 0) {
-                type = AFR_SELFHEAL_DATA_DIFF;
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    unsigned char *data_lock = NULL;
+
+    priv = this->private;
+    data_lock = alloca0(priv->child_count);
+
+    ret = afr_selfheal_inodelk(frame, this, fd->inode, this->name, offset, size,
+                               data_lock);
+    {
+        if (!afr_source_sinks_locked(this, data_lock, source, healed_sinks)) {
+            ret = -ENOTCONN;
+            goto unlock;
         }
-        return type;
-}
 
-static int
-afr_selfheal_data_do (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		      int source, unsigned char *healed_sinks,
-		      struct afr_reply *replies)
-{
-	afr_private_t *priv = NULL;
-	off_t off = 0;
-	size_t block = 0;
-	int type = AFR_SELFHEAL_DATA_FULL;
-	int ret = -1;
-	call_frame_t *iter_frame = NULL;
-        unsigned char arbiter_sink_status = 0;
-
-	priv = this->private;
-        if (priv->arbiter_count) {
-                arbiter_sink_status = healed_sinks[ARBITER_BRICK_INDEX];
-                healed_sinks[ARBITER_BRICK_INDEX] = 0;
+        if (type == AFR_SELFHEAL_DATA_DIFF &&
+            __afr_can_skip_data_block_heal(frame, this, fd, source,
+                                           healed_sinks, offset, size,
+                                           &replies[source].poststat)) {
+            ret = 0;
+            goto unlock;
         }
 
-        block = 128 * 1024 * priv->data_self_heal_window_size;
-
-        type = afr_data_self_heal_type_get (priv, healed_sinks, source,
-                                            replies);
+        ret = __afr_selfheal_data_read_write(
+            frame, this, fd, source, healed_sinks, offset, size, replies, type);
+    }
+unlock:
+    afr_selfheal_uninodelk(frame, this, fd->inode, this->name, offset, size,
+                           data_lock);
+    return ret;
+}
 
-	iter_frame = afr_copy_frame (frame);
-	if (!iter_frame) {
-                ret = -ENOMEM;
-                goto out;
-        }
+static int
+afr_selfheal_data_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        unsigned char *healed_sinks)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-	for (off = 0; off < replies[source].poststat.ia_size; off += block) {
-                if (AFR_COUNT (healed_sinks, priv->child_count) == 0) {
-                        ret = -ENOTCONN;
-                        goto out;
-                }
+    local = frame->local;
+    priv = this->private;
 
-		ret = afr_selfheal_data_block (iter_frame, this, fd, source,
-					       healed_sinks, off, block, type,
-					       replies);
-		if (ret < 0)
-			goto out;
+    if (!priv->ensure_durability)
+        return 0;
 
-		AFR_STACK_RESET (iter_frame);
-		if (iter_frame->local == NULL) {
-		        ret = -ENOTCONN;
-		        goto out;
-                }
-	}
+    AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, fsync, fd, 0, NULL);
 
-	ret = afr_selfheal_data_fsync (frame, this, fd, healed_sinks);
+    for (i = 0; i < priv->child_count; i++)
+        if (healed_sinks[i] && local->replies[i].op_ret != 0)
+            /* fsync() failed. Do NOT consider this server
+               as successfully healed. Mark it so.
+            */
+            healed_sinks[i] = 0;
+    return 0;
+}
 
-out:
-        if (arbiter_sink_status)
-                healed_sinks[ARBITER_BRICK_INDEX] = arbiter_sink_status;
+static int
+afr_data_self_heal_type_get(afr_private_t *priv, unsigned char *healed_sinks,
+                            int source, struct afr_reply *replies)
+{
+    int type = AFR_SELFHEAL_DATA_FULL;
+    int i = 0;
 
-	if (iter_frame)
-		AFR_STACK_DESTROY (iter_frame);
-	return ret;
+    if (priv->data_self_heal_algorithm == AFR_SELFHEAL_DATA_DYNAMIC) {
+        type = AFR_SELFHEAL_DATA_FULL;
+        for (i = 0; i < priv->child_count; i++) {
+            if (!healed_sinks[i] && i != source)
+                continue;
+            if (replies[i].poststat.ia_size) {
+                type = AFR_SELFHEAL_DATA_DIFF;
+                break;
+            }
+        }
+    } else {
+        type = priv->data_self_heal_algorithm;
+    }
+    return type;
 }
 
-
 static int
-__afr_selfheal_truncate_sinks (call_frame_t *frame, xlator_t *this,
-			       fd_t *fd, unsigned char *healed_sinks,
-			       uint64_t size)
+afr_selfheal_data_do(call_frame_t *frame, xlator_t *this, fd_t *fd, int source,
+                     unsigned char *healed_sinks, struct afr_reply *replies)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-        unsigned char arbiter_sink_status = 0;
-	int i = 0;
+    afr_private_t *priv = NULL;
+    off_t off = 0;
+    size_t block = 0;
+    int type = AFR_SELFHEAL_DATA_FULL;
+    int ret = -1;
+    call_frame_t *iter_frame = NULL;
+    unsigned char arbiter_sink_status = 0;
+
+    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+           "performing data selfheal on %s", uuid_utoa(fd->inode->gfid));
+
+    priv = this->private;
+    if (priv->arbiter_count) {
+        arbiter_sink_status = healed_sinks[ARBITER_BRICK_INDEX];
+        healed_sinks[ARBITER_BRICK_INDEX] = 0;
+    }
+
+    block = 128 * 1024 * priv->data_self_heal_window_size;
+
+    type = afr_data_self_heal_type_get(priv, healed_sinks, source, replies);
+
+    iter_frame = afr_copy_frame(frame);
+    if (!iter_frame) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    for (off = 0; off < replies[source].poststat.ia_size; off += block) {
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            ret = -ENOTCONN;
+            goto out;
+        }
 
-	local = frame->local;
-	priv = this->private;
+        ret = afr_selfheal_data_block(iter_frame, this, fd, source,
+                                      healed_sinks, off, block, type, replies);
+        if (ret < 0)
+            goto out;
 
-        if (priv->arbiter_count) {
-                arbiter_sink_status = healed_sinks[ARBITER_BRICK_INDEX];
-                healed_sinks[ARBITER_BRICK_INDEX] = 0;
+        AFR_STACK_RESET(iter_frame);
+        if (iter_frame->local == NULL) {
+            ret = -ENOTCONN;
+            goto out;
         }
+    }
 
-	AFR_ONLIST (healed_sinks, frame, afr_sh_generic_fop_cbk, ftruncate, fd,
-                    size, NULL);
+    ret = afr_selfheal_data_fsync(frame, this, fd, healed_sinks);
 
-	for (i = 0; i < priv->child_count; i++)
-		if (healed_sinks[i] && local->replies[i].op_ret == -1)
-			/* truncate() failed. Do NOT consider this server
-			   as successfully healed. Mark it so.
-			*/
-			healed_sinks[i] = 0;
+out:
+    if (arbiter_sink_status)
+        healed_sinks[ARBITER_BRICK_INDEX] = arbiter_sink_status;
 
-        if (arbiter_sink_status)
-                healed_sinks[ARBITER_BRICK_INDEX] = arbiter_sink_status;
-	return 0;
+    if (iter_frame)
+        AFR_STACK_DESTROY(iter_frame);
+    return ret;
+}
+
+static int
+__afr_selfheal_truncate_sinks(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              unsigned char *healed_sinks, uint64_t size)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    /* This will send truncate on the arbiter brick as well if it is marked as
+     * sink. If changelog is enabled on the volume it captures truncate as a
+     * data transactions on the arbiter brick. This will help geo-rep to
+     * properly sync the data from master to slave if arbiter is the ACTIVE
+     * brick during syncing and which had got some entries healed for data as
+     * part of self heal.
+     */
+    AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, ftruncate, fd, size,
+               NULL);
+
+    for (i = 0; i < priv->child_count; i++)
+        if (healed_sinks[i] && local->replies[i].op_ret == -1)
+            /* truncate() failed. Do NOT consider this server
+               as successfully healed. Mark it so.
+            */
+            healed_sinks[i] = 0;
+
+    return 0;
 }
 
 gf_boolean_t
-afr_has_source_witnesses (xlator_t *this, unsigned char *sources,
-                          uint64_t *witness)
+afr_has_source_witnesses(xlator_t *this, unsigned char *sources,
+                         uint64_t *witness)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i] && witness[i])
-                        return _gf_true;
-        }
-        return _gf_false;
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i] && witness[i])
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
 static gf_boolean_t
-afr_does_size_mismatch (xlator_t *this, unsigned char *sources,
-                        struct afr_reply *replies)
+afr_does_size_mismatch(xlator_t *this, unsigned char *sources,
+                       struct afr_reply *replies)
 {
-        int     i = 0;
-        afr_private_t *priv = NULL;
-        struct iatt *min = NULL;
-        struct iatt *max = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    struct iatt *min = NULL;
+    struct iatt *max = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid)
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                if (replies[i].op_ret < 0)
-                        continue;
+        if (replies[i].op_ret < 0)
+            continue;
 
-                if (!sources[i])
-                        continue;
+        if (!sources[i])
+            continue;
 
-                if (AFR_IS_ARBITER_BRICK (priv, i) &&
-                    (replies[i].poststat.ia_size == 0))
-                        continue;
+        if (AFR_IS_ARBITER_BRICK(priv, i) && (replies[i].poststat.ia_size == 0))
+            continue;
 
-                if (!min)
-                        min = &replies[i].poststat;
+        if (!min)
+            min = &replies[i].poststat;
 
-                if (!max)
-                        max = &replies[i].poststat;
+        if (!max)
+            max = &replies[i].poststat;
 
-                if (min->ia_size > replies[i].poststat.ia_size)
-                        min = &replies[i].poststat;
+        if (min->ia_size > replies[i].poststat.ia_size)
+            min = &replies[i].poststat;
 
-                if (max->ia_size < replies[i].poststat.ia_size)
-                        max = &replies[i].poststat;
-        }
+        if (max->ia_size < replies[i].poststat.ia_size)
+            max = &replies[i].poststat;
+    }
 
-        if (min && max) {
-                if (min->ia_size != max->ia_size)
-                        return _gf_true;
-        }
+    if (min && max) {
+        if (min->ia_size != max->ia_size)
+            return _gf_true;
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static void
-afr_mark_biggest_witness_as_source (xlator_t *this, unsigned char *sources,
-                                    uint64_t *witness)
+afr_mark_biggest_witness_as_source(xlator_t *this, unsigned char *sources,
+                                   uint64_t *witness)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
-        uint64_t biggest_witness = 0;
-
-        priv = this->private;
-        /* Find source with biggest witness count */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (biggest_witness < witness[i])
-                        biggest_witness = witness[i];
-        }
-
-        /* Mark files with less witness count as not source */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (witness[i] < biggest_witness)
-                        sources[i] = 0;
-        }
-
-        return;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint64_t biggest_witness = 0;
+
+    priv = this->private;
+    /* Find source with biggest witness count */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (biggest_witness < witness[i])
+            biggest_witness = witness[i];
+    }
+
+    /* Mark files with less witness count as not source */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (witness[i] < biggest_witness)
+            sources[i] = 0;
+    }
+
+    return;
 }
 
 /* This is a tie breaker function. Only one source be assigned here */
 static void
-afr_mark_newest_file_as_source (xlator_t *this, unsigned char *sources,
-                                struct afr_reply *replies)
+afr_mark_newest_file_as_source(xlator_t *this, unsigned char *sources,
+                               struct afr_reply *replies)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
-        int source = -1;
-        uint32_t max_ctime = 0;
-
-        priv = this->private;
-        /* Find source with latest ctime */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-
-                if (max_ctime <= replies[i].poststat.ia_ctime) {
-                        source = i;
-                        max_ctime = replies[i].poststat.ia_ctime;
-                }
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    uint32_t max_ctime = 0;
+
+    priv = this->private;
+    /* Find source with latest ctime */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+
+        if (max_ctime <= replies[i].poststat.ia_ctime) {
+            source = i;
+            max_ctime = replies[i].poststat.ia_ctime;
         }
+    }
 
-        /* Only mark one of the files as source to break ties */
-        memset (sources, 0, sizeof (*sources) * priv->child_count);
-        sources[source] = 1;
+    /* Only mark one of the files as source to break ties */
+    memset(sources, 0, sizeof(*sources) * priv->child_count);
+    sources[source] = 1;
 }
 
 static int
-__afr_selfheal_data_finalize_source (call_frame_t *frame, xlator_t *this,
-                                     inode_t *inode,
-                                     unsigned char *sources,
-                                     unsigned char *sinks,
-				     unsigned char *healed_sinks,
-				     unsigned char *locked_on,
-				     unsigned char *undid_pending,
-				     struct afr_reply *replies,
-                                     uint64_t *witness)
+__afr_selfheal_data_finalize_source(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    unsigned char *undid_pending, struct afr_reply *replies, uint64_t *witness)
 {
-	afr_private_t *priv = NULL;
-	int source = -1;
-	int sources_count = 0;
-	priv = this->private;
-
-	sources_count = AFR_COUNT (sources, priv->child_count);
-
-	if ((AFR_CMP (locked_on, healed_sinks, priv->child_count) == 0)
-            || !sources_count) {
-		/* split brain */
-                source = afr_mark_split_brain_source_sinks (frame, this, inode,
-                                                            sources, sinks,
-                                                            healed_sinks,
-                                                            locked_on, replies,
-                                                          AFR_DATA_TRANSACTION);
-                if (source < 0) {
-                        gf_event (EVENT_AFR_SPLIT_BRAIN, "subvol=%s;type=data;"
-                                 "file=%s", this->name, uuid_utoa(inode->gfid));
-                        return -EIO;
-                }
-
-                _afr_fav_child_reset_sink_xattrs (frame, this, inode, source,
-                                                 healed_sinks, undid_pending,
-                                                 AFR_DATA_TRANSACTION,
-                                                 locked_on, replies);
-                goto out;
-	}
-
-        /* No split brain at this point. If we were called from
-         * afr_heal_splitbrain_file(), abort.*/
-        if (afr_dict_contains_heal_op(frame))
-                return -EIO;
-
-        /* If there are no witnesses/size-mismatches on sources we are done*/
-        if (!afr_does_size_mismatch (this, sources, replies) &&
-            !afr_has_source_witnesses (this, sources, witness))
-                goto out;
-
-        afr_mark_largest_file_as_source (this, sources, replies);
-        afr_mark_biggest_witness_as_source (this, sources, witness);
-        afr_mark_newest_file_as_source (this, sources, replies);
-        if (priv->arbiter_count)
-                /* Choose non-arbiter brick as source for empty files. */
-                afr_mark_source_sinks_if_file_empty (this, sources, sinks,
-                                                     healed_sinks, locked_on,
-                                                     replies,
-                                                     AFR_DATA_TRANSACTION);
+    afr_private_t *priv = NULL;
+    int source = -1;
+    int sources_count = 0;
+    priv = this->private;
+
+    sources_count = AFR_COUNT(sources, priv->child_count);
+
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count) {
+        /* split brain */
+        source = afr_mark_split_brain_source_sinks(
+            frame, this, inode, sources, sinks, healed_sinks, locked_on,
+            replies, AFR_DATA_TRANSACTION);
+        if (source < 0) {
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;type=data;"
+                     "file=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(inode->gfid));
+            return -EIO;
+        }
+
+        _afr_fav_child_reset_sink_xattrs(
+            frame, this, inode, source, healed_sinks, undid_pending,
+            AFR_DATA_TRANSACTION, locked_on, replies);
+        goto out;
+    }
+
+    /* No split brain at this point. If we were called from
+     * afr_heal_splitbrain_file(), abort.*/
+    if (afr_dict_contains_heal_op(frame))
+        return -EIO;
+
+    /* If there are no witnesses/size-mismatches on sources we are done*/
+    if (!afr_does_size_mismatch(this, sources, replies) &&
+        !afr_has_source_witnesses(this, sources, witness))
+        goto out;
+
+    afr_mark_largest_file_as_source(this, sources, replies);
+    afr_mark_biggest_witness_as_source(this, sources, witness);
+    afr_mark_newest_file_as_source(this, sources, replies);
+    if (priv->arbiter_count)
+        /* Choose non-arbiter brick as source for empty files. */
+        afr_mark_source_sinks_if_file_empty(this, sources, sinks, healed_sinks,
+                                            locked_on, replies,
+                                            AFR_DATA_TRANSACTION);
 
 out:
-        afr_mark_active_sinks (this, sources, locked_on, healed_sinks);
-        source = afr_choose_source_by_policy (priv, sources,
-                                              AFR_DATA_TRANSACTION);
+    afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+    source = afr_choose_source_by_policy(priv, sources, AFR_DATA_TRANSACTION);
 
-	return source;
+    return source;
 }
 
 /*
@@ -619,290 +606,286 @@ out:
  * for self-healing, or -1 if no healing is necessary/split brain.
  */
 int
-__afr_selfheal_data_prepare (call_frame_t *frame, xlator_t *this,
-                             inode_t *inode, unsigned char *locked_on,
-                             unsigned char *sources, unsigned char *sinks,
-                             unsigned char *healed_sinks,
-                             unsigned char *undid_pending,
-			     struct afr_reply *replies, gf_boolean_t *pflag)
+__afr_selfheal_data_prepare(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                            unsigned char *locked_on, unsigned char *sources,
+                            unsigned char *sinks, unsigned char *healed_sinks,
+                            unsigned char *undid_pending,
+                            struct afr_reply *replies, unsigned char *pflag)
 {
-	int ret = -1;
-	int source = -1;
-	afr_private_t *priv = NULL;
-        uint64_t *witness = NULL;
-
-	priv = this->private;
-
-	ret = afr_selfheal_unlocked_discover (frame, inode, inode->gfid,
-					      replies);
-
-        if (ret)
-                return ret;
-
-        witness = alloca0(priv->child_count * sizeof (*witness));
-	ret = afr_selfheal_find_direction (frame, this, replies,
-					   AFR_DATA_TRANSACTION,
-					   locked_on, sources, sinks, witness,
-                                           pflag);
-	if (ret)
-		return ret;
-
-        /* Initialize the healed_sinks[] array optimistically to
-           the intersection of to-be-healed (i.e sinks[]) and
-           the list of servers which are up (i.e locked_on[]).
-           As we encounter failures in the healing process, we
-           will unmark the respective servers in the healed_sinks[]
-           array.
-        */
-        AFR_INTERSECT (healed_sinks, sinks, locked_on, priv->child_count);
-
-	source = __afr_selfheal_data_finalize_source (frame, this, inode,
-                                                      sources, sinks,
-                                                      healed_sinks,
-                                                      locked_on, undid_pending,
-                                                      replies, witness);
-	if (source < 0)
-		return -EIO;
-
-	return source;
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
+
+    if (ret)
+        return ret;
+
+    witness = alloca0(priv->child_count * sizeof(*witness));
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_DATA_TRANSACTION, locked_on, sources,
+                                      sinks, witness, pflag);
+    if (ret)
+        return ret;
+
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
+
+    source = __afr_selfheal_data_finalize_source(
+        frame, this, inode, sources, sinks, healed_sinks, locked_on,
+        undid_pending, replies, witness);
+    if (source < 0)
+        return -EIO;
+
+    return source;
 }
 
-
 static int
-__afr_selfheal_data (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		     unsigned char *locked_on)
+__afr_selfheal_data(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    unsigned char *locked_on)
 {
-	afr_private_t *priv = NULL;
-	int ret = -1;
-	unsigned char *sources = NULL;
-	unsigned char *sinks = NULL;
-	unsigned char *data_lock = NULL;
-	unsigned char *healed_sinks = NULL;
-	unsigned char *undid_pending = NULL;
-	struct afr_reply *locked_replies = NULL;
-	int source = -1;
-        gf_boolean_t did_sh = _gf_true;
-        gf_boolean_t is_arbiter_the_only_sink = _gf_false;
-
-	priv = this->private;
-
-	sources = alloca0 (priv->child_count);
-	sinks = alloca0 (priv->child_count);
-	healed_sinks = alloca0 (priv->child_count);
-	data_lock = alloca0 (priv->child_count);
-        undid_pending = alloca0 (priv->child_count);
-
-	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-	ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name, 0, 0,
-				    data_lock);
-	{
-		if (ret < priv->child_count) {
-                        gf_msg_debug (this->name, 0, "%s: Skipping "
-                                      "self-heal as only %d number "
-                                      "of subvolumes "
-                                      "could be locked",
-                                      uuid_utoa (fd->inode->gfid),
-                                      ret);
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_data_prepare (frame, this, fd->inode,
-                                                   data_lock, sources, sinks,
-                                                   healed_sinks, undid_pending,
-                                                   locked_replies, NULL);
-		if (ret < 0)
-			goto unlock;
-
-                if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
-                        did_sh = _gf_false;
-                        goto unlock;
-                }
-
-		source = ret;
-
-                if (AFR_IS_ARBITER_BRICK(priv, source)) {
-                        did_sh = _gf_false;
-                        goto unlock;
-                }
-
-                if (priv->arbiter_count &&
-                    AFR_COUNT (healed_sinks, priv->child_count) == 1 &&
-                    healed_sinks[ARBITER_BRICK_INDEX]) {
-                        is_arbiter_the_only_sink = _gf_true;
-                        goto restore_time;
-                }
-
-		ret = __afr_selfheal_truncate_sinks (frame, this, fd, healed_sinks,
-						     locked_replies[source].poststat.ia_size);
-		if (ret < 0)
-			goto unlock;
-
-		ret = 0;
-
-	}
-unlock:
-	afr_selfheal_uninodelk (frame, this, fd->inode, this->name, 0, 0,
-				data_lock);
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *data_lock = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    int source = -1;
+    gf_boolean_t did_sh = _gf_true;
+    gf_boolean_t is_arbiter_the_only_sink = _gf_false;
+    gf_boolean_t empty_file = _gf_false;
+
+    priv = this->private;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    data_lock = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = afr_selfheal_inodelk(frame, this, fd->inode, this->name, 0, 0,
+                               data_lock);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "self-heal as only %d number "
+                         "of subvolumes "
+                         "could be locked",
+                         uuid_utoa(fd->inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_data_prepare(frame, this, fd->inode, data_lock,
+                                          sources, sinks, healed_sinks,
+                                          undid_pending, locked_replies, NULL);
+        if (ret < 0)
+            goto unlock;
+
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            did_sh = _gf_false;
+            goto unlock;
+        }
+
+        source = ret;
+
+        if (AFR_IS_ARBITER_BRICK(priv, source)) {
+            empty_file = afr_is_file_empty_on_all_children(priv,
+                                                           locked_replies);
+            if (empty_file)
+                goto restore_time;
+
+            did_sh = _gf_false;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_truncate_sinks(
+            frame, this, fd, healed_sinks,
+            locked_replies[source].poststat.ia_size);
         if (ret < 0)
-		goto out;
+            goto unlock;
 
-        if (!did_sh)
-                goto out;
+        if (priv->arbiter_count &&
+            AFR_COUNT(healed_sinks, priv->child_count) == 1 &&
+            healed_sinks[ARBITER_BRICK_INDEX]) {
+            is_arbiter_the_only_sink = _gf_true;
+            goto restore_time;
+        }
+        ret = 0;
+    }
+unlock:
+    afr_selfheal_uninodelk(frame, this, fd->inode, this->name, 0, 0, data_lock);
+    if (ret < 0)
+        goto out;
 
-	ret = afr_selfheal_data_do (frame, this, fd, source, healed_sinks,
-				    locked_replies);
-	if (ret)
-                goto out;
+    if (!did_sh)
+        goto out;
+
+    ret = afr_selfheal_data_do(frame, this, fd, source, healed_sinks,
+                               locked_replies);
+    if (ret)
+        goto out;
 restore_time:
-	afr_selfheal_restore_time (frame, this, fd->inode, source,
-					healed_sinks, locked_replies);
-
-        if (!is_arbiter_the_only_sink) {
-                ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name,
-                                            0, 0, data_lock);
-                if (ret < priv->child_count) {
-                        ret = -ENOTCONN;
-                        did_sh = _gf_false;
-                        goto skip_undo_pending;
-                }
+    afr_selfheal_restore_time(frame, this, fd->inode, source, healed_sinks,
+                              locked_replies);
+
+    if (!is_arbiter_the_only_sink && !empty_file) {
+        ret = afr_selfheal_inodelk(frame, this, fd->inode, this->name, 0, 0,
+                                   data_lock);
+        if (ret < priv->child_count) {
+            ret = -ENOTCONN;
+            did_sh = _gf_false;
+            goto skip_undo_pending;
         }
-        ret = afr_selfheal_undo_pending (frame, this, fd->inode,
-                                         sources, sinks, healed_sinks,
-                                         undid_pending, AFR_DATA_TRANSACTION,
-                                         locked_replies, data_lock);
+    }
+    ret = afr_selfheal_undo_pending(
+        frame, this, fd->inode, sources, sinks, healed_sinks, undid_pending,
+        AFR_DATA_TRANSACTION, locked_replies, data_lock);
 skip_undo_pending:
-	afr_selfheal_uninodelk (frame, this, fd->inode, this->name, 0, 0,
-				data_lock);
+    afr_selfheal_uninodelk(frame, this, fd->inode, this->name, 0, 0, data_lock);
 out:
 
-        if (did_sh)
-                afr_log_selfheal (fd->inode->gfid, this, ret, "data", source,
-                                  sources, healed_sinks);
-        else
-                ret = 1;
+    if (did_sh)
+        afr_log_selfheal(fd->inode->gfid, this, ret, "data", source, sources,
+                         healed_sinks);
+    else
+        ret = 1;
 
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
+    if (locked_replies)
+        afr_replies_wipe(locked_replies, priv->child_count);
 
-	return ret;
+    return ret;
 }
 
 int
-afr_selfheal_data_open_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            fd_t *fd, dict_t *xdata)
+afr_selfheal_data_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, fd_t *fd,
+                           dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int         i      = (long) cookie;
+    afr_local_t *local = NULL;
+    int i = (long)cookie;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->replies[i].valid = 1;
-        local->replies[i].op_ret = op_ret;
-        local->replies[i].op_errno = op_errno;
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
 
-        syncbarrier_wake (&local->barrier);
+    syncbarrier_wake(&local->barrier);
 
-        return 0;
+    return 0;
 }
 
 int
-afr_selfheal_data_open (xlator_t *this, inode_t *inode, fd_t **fd)
+afr_selfheal_data_open(xlator_t *this, inode_t *inode, fd_t **fd)
 {
-        int           ret      = 0;
-        fd_t          *fd_tmp  = NULL;
-        loc_t         loc      = {0,};
-        call_frame_t  *frame   = NULL;
-        afr_local_t   *local   = NULL;
-        afr_private_t *priv    = NULL;
-        int           i        = 0;
-
-        priv = this->private;
-
-	fd_tmp = fd_create (inode, 0);
-	if (!fd_tmp)
-		return -ENOMEM;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-        frame = afr_frame_create (this, &ret);
-        if (!frame) {
-                ret = -ret;
-                fd_unref (fd_tmp);
-                goto out;
+    int ret = 0;
+    fd_t *fd_tmp = NULL;
+    loc_t loc = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+
+    fd_tmp = fd_create(inode, 0);
+    if (!fd_tmp)
+        return -ENOMEM;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        fd_unref(fd_tmp);
+        goto out;
+    }
+    local = frame->local;
+
+    AFR_ONLIST(local->child_up, frame, afr_selfheal_data_open_cbk, open, &loc,
+               O_RDWR | O_LARGEFILE, fd_tmp, NULL);
+
+    ret = -ENOTCONN;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+
+        if (local->replies[i].op_ret < 0) {
+            ret = -local->replies[i].op_errno;
+            continue;
         }
-        local = frame->local;
 
-        AFR_ONLIST (local->child_up, frame, afr_selfheal_data_open_cbk, open,
-                    &loc, O_RDWR|O_LARGEFILE, fd_tmp, NULL);
+        ret = 0;
+        break;
+    }
 
-        ret = -ENOTCONN;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->replies[i].valid)
-                        continue;
-
-                if (local->replies[i].op_ret < 0) {
-                        ret = -local->replies[i].op_errno;
-                        continue;
-                }
+    if (ret < 0) {
+        fd_unref(fd_tmp);
+        goto out;
+    } else {
+        fd_bind(fd_tmp);
+    }
 
-                ret = 0;
-                break;
-        }
-
-	if (ret < 0) {
-		fd_unref (fd_tmp);
-                goto out;
-	} else {
-		fd_bind (fd_tmp);
-	}
-
-        *fd = fd_tmp;
+    *fd = fd_tmp;
 out:
-        loc_wipe (&loc);
-        if (frame)
-                AFR_STACK_DESTROY (frame);
-	return ret;
+    loc_wipe(&loc);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-afr_selfheal_data (call_frame_t *frame, xlator_t *this, fd_t *fd)
+afr_selfheal_data(call_frame_t *frame, xlator_t *this, fd_t *fd)
 {
-	afr_private_t *priv = NULL;
-	unsigned char *locked_on = NULL;
-	int ret = 0;
-        inode_t *inode = fd->inode;
-
-	priv = this->private;
-
-	locked_on = alloca0 (priv->child_count);
-
-	ret = afr_selfheal_tie_breaker_inodelk (frame, this, inode,
-	                                        priv->sh_domain, 0, 0,
-				                locked_on);
-	{
-		if (ret < priv->child_count) {
-                        gf_msg_debug (this->name, 0, "%s: Skipping "
-                                      "self-heal as only %d number of "
-                                      "subvolumes could be locked",
-                                      uuid_utoa (fd->inode->gfid),
-                                ret);
-			/* Either less than two subvols available, or another
-			   selfheal (from another server) is in progress. Skip
-			   for now in any case there isn't anything to do.
-			*/
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_data (frame, this, fd, locked_on);
-	}
+    afr_private_t *priv = NULL;
+    unsigned char *locked_on = NULL;
+    int ret = 0;
+    inode_t *inode = fd->inode;
+
+    priv = this->private;
+
+    locked_on = alloca0(priv->child_count);
+
+    ret = afr_selfheal_tie_breaker_inodelk(frame, this, inode, priv->sh_domain,
+                                           0, 0, locked_on);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "self-heal as only %d number of "
+                         "subvolumes could be locked",
+                         uuid_utoa(fd->inode->gfid), ret);
+            /* Either less than two subvols available, or another
+               selfheal (from another server) is in progress. Skip
+               for now in any case there isn't anything to do.
+            */
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_data(frame, this, fd, locked_on);
+    }
 unlock:
-	afr_selfheal_uninodelk (frame, this, inode, priv->sh_domain, 0, 0,
-	                        locked_on);
+    afr_selfheal_uninodelk(frame, this, inode, priv->sh_domain, 0, 0,
+                           locked_on);
 
-	return ret;
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index b601040504e..64893f441e3 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -8,1115 +8,1269 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "afr.h"
 #include "afr-self-heal.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "afr-transaction.h"
 #include "afr-messages.h"
-#include "syncop-utils.h"
-#include "events.h"
+#include <glusterfs/syncop-utils.h>
+#include <glusterfs/events.h>
 
-static int
-afr_selfheal_entry_delete (xlator_t *this, inode_t *dir, const char *name,
-                           inode_t *inode, int child, struct afr_reply *replies)
+int
+afr_selfheal_entry_anon_inode(xlator_t *this, inode_t *dir, const char *name,
+                              inode_t *inode, int child,
+                              struct afr_reply *replies,
+                              gf_boolean_t *anon_inode)
 {
-	afr_private_t *priv = NULL;
-	xlator_t *subvol = NULL;
-	int ret = 0;
-	loc_t loc = {0, };
-	char g[64];
-
-	priv = this->private;
-
-	subvol = priv->children[child];
-
-	loc.parent = inode_ref (dir);
-	gf_uuid_copy (loc.pargfid, dir->gfid);
-	loc.name = name;
-	loc.inode = inode_ref (inode);
-
-	if (replies[child].valid && replies[child].op_ret == 0) {
-		switch (replies[child].poststat.ia_type) {
-		case IA_IFDIR:
-		        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_EXPUNGING_FILE_OR_DIR,
-				"expunging dir %s/%s (%s) on %s",
-				uuid_utoa (dir->gfid), name,
-				uuid_utoa_r (replies[child].poststat.ia_gfid, g),
-				subvol->name);
-			ret = syncop_rmdir (subvol, &loc, 1, NULL, NULL);
-			break;
-		default:
-		        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_EXPUNGING_FILE_OR_DIR,
-				"expunging file %s/%s (%s) on %s",
-				uuid_utoa (dir->gfid), name,
-				uuid_utoa_r (replies[child].poststat.ia_gfid, g),
-				subvol->name);
-			ret = syncop_unlink (subvol, &loc, NULL, NULL);
-			break;
-		}
-	}
-
-	loc_wipe (&loc);
-
-	return ret;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+    int i = 0;
+    char g[64] = {0};
+    unsigned char *lookup_success = NULL;
+    call_frame_t *frame = NULL;
+    loc_t loc2 = {
+        0,
+    };
+    loc_t loc = {
+        0,
+    };
+
+    priv = this->private;
+    subvol = priv->children[child];
+    lookup_success = alloca0(priv->child_count);
+    uuid_utoa_r(replies[child].poststat.ia_gfid, g);
+    loc.inode = inode_new(inode->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    if (replies[child].poststat.ia_type == IA_IFDIR) {
+        /* This directory may have sub-directory hierarchy which may need to
+         * be preserved for subsequent heals. So unconditionally move the
+         * directory to anonymous-inode directory*/
+        *anon_inode = _gf_true;
+        goto anon_inode;
+    }
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+    local = frame->local;
+    gf_uuid_copy(loc.gfid, replies[child].poststat.ia_gfid);
+    AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == 0) {
+            lookup_success[i] = 1;
+        } else if (local->replies[i].op_errno != ENOENT &&
+                   local->replies[i].op_errno != ESTALE) {
+            ret = -local->replies[i].op_errno;
+        }
+    }
+
+    if (priv->quorum_count) {
+        if (afr_has_quorum(lookup_success, this, NULL)) {
+            *anon_inode = _gf_true;
+        }
+    } else if (AFR_COUNT(lookup_success, priv->child_count) > 1) {
+        *anon_inode = _gf_true;
+    } else if (ret) {
+        goto out;
+    }
+
+anon_inode:
+    if (!*anon_inode) {
+        ret = 0;
+        goto out;
+    }
+
+    loc.parent = inode_ref(dir);
+    gf_uuid_copy(loc.pargfid, dir->gfid);
+    loc.name = name;
+
+    ret = afr_anon_inode_create(this, child, &loc2.parent);
+    if (ret < 0)
+        goto out;
+
+    loc2.name = g;
+    ret = syncop_rename(subvol, &loc, &loc2, NULL, NULL);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+               "Rename to %s dir %s/%s (%s) on %s failed",
+               priv->anon_inode_name, uuid_utoa(dir->gfid), name, g,
+               subvol->name);
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+               "Rename to %s dir %s/%s (%s) on %s successful",
+               priv->anon_inode_name, uuid_utoa(dir->gfid), name, g,
+               subvol->name);
+    }
+
+out:
+    loc_wipe(&loc);
+    loc_wipe(&loc2);
+    if (frame) {
+        AFR_STACK_DESTROY(frame);
+    }
+
+    return ret;
 }
 
 int
-afr_selfheal_recreate_entry (call_frame_t *frame, int dst, int source,
-                             unsigned char *sources, inode_t *dir,
-                             const char *name, inode_t *inode,
-                             struct afr_reply *replies)
+afr_selfheal_entry_delete(xlator_t *this, inode_t *dir, const char *name,
+                          inode_t *inode, int child, struct afr_reply *replies)
 {
-	int ret = 0;
-	loc_t loc = {0,};
-	loc_t srcloc = {0,};
-        xlator_t *this = frame->this;
-	afr_private_t *priv = NULL;
-	dict_t *xdata = NULL;
-	struct iatt *iatt = NULL;
-	char *linkname = NULL;
-	mode_t mode = 0;
-	struct iatt newent = {0,};
-        unsigned char *newentry = NULL;
-
-        priv = this->private;
-	iatt = &replies[source].poststat;
-        if (iatt->ia_type == IA_INVAL || gf_uuid_is_null (iatt->ia_gfid)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, AFR_MSG_SELF_HEAL_FAILED,
-                        "Invalid ia_type (%d) or gfid(%s). source brick=%d, "
-                        "pargfid=%s, name=%s", iatt->ia_type,
-                        uuid_utoa(iatt->ia_gfid), source,
-                        uuid_utoa(dir->gfid), name);
-                ret = -EINVAL;
-                goto out;
-        }
+    char g[64] = {0};
+    afr_private_t *priv = NULL;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    gf_boolean_t anon_inode = _gf_false;
+
+    priv = this->private;
+    subvol = priv->children[child];
+
+    if ((!replies[child].valid) || (replies[child].op_ret < 0)) {
+        /*Nothing to do*/
+        ret = 0;
+        goto out;
+    }
+
+    if (priv->use_anon_inode) {
+        ret = afr_selfheal_entry_anon_inode(this, dir, name, inode, child,
+                                            replies, &anon_inode);
+        if (ret < 0 || anon_inode)
+            goto out;
+    }
+
+    loc.parent = inode_ref(dir);
+    loc.inode = inode_new(inode->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    loc.name = name;
+    switch (replies[child].poststat.ia_type) {
+        case IA_IFDIR:
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+                   "expunging dir %s/%s (%s) on %s", uuid_utoa(dir->gfid), name,
+                   uuid_utoa_r(replies[child].poststat.ia_gfid, g),
+                   subvol->name);
+            ret = syncop_rmdir(subvol, &loc, 1, NULL, NULL);
+            break;
+        default:
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+                   "expunging file %s/%s (%s) on %s", uuid_utoa(dir->gfid),
+                   name, uuid_utoa_r(replies[child].poststat.ia_gfid, g),
+                   subvol->name);
+            ret = syncop_unlink(subvol, &loc, NULL, NULL);
+            break;
+    }
 
-	xdata = dict_new();
-	if (!xdata)
-		return -ENOMEM;
-        newentry = alloca0 (priv->child_count);
-	loc.parent = inode_ref (dir);
-	gf_uuid_copy (loc.pargfid, dir->gfid);
-	loc.name = name;
-	loc.inode = inode_ref (inode);
-
-	ret = afr_selfheal_entry_delete (this, dir, name, inode, dst, replies);
-	if (ret)
-		goto out;
-
-	ret = dict_set_gfuuid (xdata, "gfid-req",
-                               replies[source].poststat.ia_gfid, true);
-	if (ret)
-		goto out;
-
-	srcloc.inode = inode_ref (inode);
-	gf_uuid_copy (srcloc.gfid, iatt->ia_gfid);
-        if (iatt->ia_type != IA_IFDIR)
-                ret = syncop_lookup (priv->children[dst], &srcloc, 0, 0, 0, 0);
-        if (iatt->ia_type == IA_IFDIR || ret == -ENOENT || ret == -ESTALE) {
-                newentry[dst] = 1;
-                ret = afr_selfheal_newentry_mark (frame, this, inode, source,
-                                                  replies, sources, newentry);
-                if (ret)
-                        goto out;
-        }
+out:
+    loc_wipe(&loc);
+    return ret;
+}
 
-	mode = st_mode_from_ia (iatt->ia_prot, iatt->ia_type);
-
-	switch (iatt->ia_type) {
-	case IA_IFDIR:
-		ret = syncop_mkdir (priv->children[dst], &loc, mode, 0,
-                                    xdata, NULL);
-		break;
-	case IA_IFLNK:
-		if (!newentry[dst]) {
-			ret = syncop_link (priv->children[dst], &srcloc, &loc,
-					   &newent, NULL, NULL);
-		} else {
-			ret = syncop_readlink (priv->children[source], &srcloc,
-					       &linkname, 4096, NULL, NULL);
-			if (ret <= 0)
-				goto out;
-			ret = syncop_symlink (priv->children[dst], &loc,
-                                              linkname, NULL, xdata, NULL);
-                }
-		break;
-	default:
-		ret = dict_set_int32 (xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
-		if (ret)
-			goto out;
-		ret = syncop_mknod (priv->children[dst], &loc, mode,
-                    makedev (ia_major(iatt->ia_rdev), ia_minor (iatt->ia_rdev)),
-                    &newent, xdata, NULL);
-		break;
-	}
+int
+afr_selfheal_recreate_entry(call_frame_t *frame, int dst, int source,
+                            unsigned char *sources, inode_t *dir,
+                            const char *name, inode_t *inode,
+                            struct afr_reply *replies)
+{
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    loc_t srcloc = {
+        0,
+    };
+    loc_t anonloc = {
+        0,
+    };
+    xlator_t *this = frame->this;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    struct iatt *iatt = NULL;
+    char *linkname = NULL;
+    mode_t mode = 0;
+    struct iatt newent = {
+        0,
+    };
+    unsigned char *newentry = NULL;
+    char iatt_uuid_str[64] = {0};
+    char dir_uuid_str[64] = {0};
+
+    priv = this->private;
+    iatt = &replies[source].poststat;
+    uuid_utoa_r(iatt->ia_gfid, iatt_uuid_str);
+    if (iatt->ia_type == IA_INVAL || gf_uuid_is_null(iatt->ia_gfid)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SELF_HEAL_FAILED,
+               "Invalid ia_type (%d) or gfid(%s). source brick=%d, "
+               "pargfid=%s, name=%s",
+               iatt->ia_type, iatt_uuid_str, source,
+               uuid_utoa_r(dir->gfid, dir_uuid_str), name);
+        ret = -EINVAL;
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata)
+        return -ENOMEM;
+    newentry = alloca0(priv->child_count);
+    loc.parent = inode_ref(dir);
+    gf_uuid_copy(loc.pargfid, dir->gfid);
+    loc.name = name;
+    loc.inode = inode_ref(inode);
+
+    ret = afr_selfheal_entry_delete(this, dir, name, inode, dst, replies);
+    if (ret)
+        goto out;
+
+    ret = dict_set_gfuuid(xdata, "gfid-req", replies[source].poststat.ia_gfid,
+                          true);
+    if (ret)
+        goto out;
+
+    srcloc.inode = inode_ref(inode);
+    gf_uuid_copy(srcloc.gfid, iatt->ia_gfid);
+    ret = syncop_lookup(priv->children[dst], &srcloc, 0, 0, 0, 0);
+    if (ret == -ENOENT || ret == -ESTALE) {
+        newentry[dst] = 1;
+        ret = afr_selfheal_newentry_mark(frame, this, inode, source, replies,
+                                         sources, newentry);
+        if (ret)
+            goto out;
+    } else if (ret == 0 && iatt->ia_type == IA_IFDIR && priv->use_anon_inode) {
+        // Try rename from hidden directory
+        ret = afr_anon_inode_create(this, dst, &anonloc.parent);
+        if (ret < 0)
+            goto out;
+        anonloc.inode = inode_ref(inode);
+        anonloc.name = iatt_uuid_str;
+        ret = syncop_rename(priv->children[dst], &anonloc, &loc, NULL, NULL);
+        if (ret == -ENOENT || ret == -ESTALE)
+            ret = -1; /*This sets 'mismatch' to true*/
+        goto out;
+    }
+
+    mode = st_mode_from_ia(iatt->ia_prot, iatt->ia_type);
+
+    switch (iatt->ia_type) {
+        case IA_IFDIR:
+            ret = syncop_mkdir(priv->children[dst], &loc, mode, 0, xdata, NULL);
+            break;
+        case IA_IFLNK:
+            if (!newentry[dst]) {
+                ret = syncop_link(priv->children[dst], &srcloc, &loc, &newent,
+                                  NULL, NULL);
+            } else {
+                ret = syncop_readlink(priv->children[source], &srcloc,
+                                      &linkname, 4096, NULL, NULL);
+                if (ret <= 0)
+                    goto out;
+                ret = syncop_symlink(priv->children[dst], &loc, linkname, NULL,
+                                     xdata, NULL);
+            }
+            break;
+        default:
+            ret = dict_set_int32_sizen(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+            if (ret)
+                goto out;
+            ret = syncop_mknod(
+                priv->children[dst], &loc, mode,
+                makedev(ia_major(iatt->ia_rdev), ia_minor(iatt->ia_rdev)),
+                &newent, xdata, NULL);
+            break;
+    }
 
 out:
-	if (xdata)
-		dict_unref (xdata);
-	GF_FREE (linkname);
-	loc_wipe (&loc);
-	loc_wipe (&srcloc);
-	return ret;
+    if (xdata)
+        dict_unref(xdata);
+    GF_FREE(linkname);
+    loc_wipe(&loc);
+    loc_wipe(&srcloc);
+    loc_wipe(&anonloc);
+    return ret;
 }
 
 static int
-__afr_selfheal_heal_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			    char *name, inode_t *inode, int source,
-			    unsigned char *sources, unsigned char *healed_sinks,
-			    unsigned char *locked_on, struct afr_reply *replies)
+__afr_selfheal_heal_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                           char *name, inode_t *inode, int source,
+                           unsigned char *sources, unsigned char *healed_sinks,
+                           unsigned char *locked_on, struct afr_reply *replies)
 {
-	int ret = 0;
-	afr_private_t *priv = NULL;
-	int i = 0;
+    int ret = 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-	priv = this->private;
+    priv = this->private;
 
-	if (!replies[source].valid)
-		return -EIO;
+    if (!replies[source].valid)
+        return -EIO;
 
-        /* Skip healing this entry if the last lookup on it failed for reasons
-         * other than ENOENT.
-         */
-        if ((replies[source].op_ret < 0) &&
-            (replies[source].op_errno != ENOENT))
-                return -replies[source].op_errno;
-
-        if (replies[source].op_ret == 0) {
-                ret = afr_lookup_and_heal_gfid (this, fd->inode, name,
-                                                inode, replies, source,
-                                             &replies[source].poststat.ia_gfid);
-                if (ret)
-                        return ret;
+    /* Skip healing this entry if the last lookup on it failed for reasons
+     * other than ENOENT.
+     */
+    if ((replies[source].op_ret < 0) && (replies[source].op_errno != ENOENT))
+        return -replies[source].op_errno;
+
+    if (replies[source].op_ret == 0) {
+        ret = afr_lookup_and_heal_gfid(this, fd->inode, name, inode, replies,
+                                       source, sources,
+                                       &replies[source].poststat.ia_gfid, NULL);
+        if (ret)
+            return ret;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i])
+            continue;
+        if (replies[source].op_ret == -1 &&
+            replies[source].op_errno == ENOENT) {
+            ret = afr_selfheal_entry_delete(this, fd->inode, name, inode, i,
+                                            replies);
+        } else {
+            if (!gf_uuid_compare(replies[i].poststat.ia_gfid,
+                                 replies[source].poststat.ia_gfid))
+                continue;
+
+            ret = afr_selfheal_recreate_entry(frame, i, source, sources,
+                                              fd->inode, name, inode, replies);
         }
+        if (ret < 0)
+            break;
+    }
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!healed_sinks[i])
-			continue;
-		if (replies[source].op_ret == -1 &&
-		    replies[source].op_errno == ENOENT) {
-			ret = afr_selfheal_entry_delete (this, fd->inode, name,
-                                                         inode, i, replies);
-                } else {
-			if (!gf_uuid_compare (replies[i].poststat.ia_gfid,
-					   replies[source].poststat.ia_gfid))
-				continue;
-
-			ret = afr_selfheal_recreate_entry (frame, i, source,
-                                                           sources, fd->inode,
-                                                           name, inode,
-							   replies);
-                }
-		if (ret < 0)
-			break;
-	}
-
-	return ret;
+    return ret;
 }
 
 static int
-afr_selfheal_detect_gfid_and_type_mismatch (xlator_t *this,
-                                            struct afr_reply *replies,
-                                            inode_t *inode,
-                                            uuid_t pargfid,
-                                            char *bname, int src_idx,
-                                            unsigned char *locked_on,
-                                            int *src)
+afr_selfheal_detect_gfid_and_type_mismatch(xlator_t *this,
+                                           struct afr_reply *replies,
+                                           inode_t *inode, uuid_t pargfid,
+                                           char *bname, int src_idx,
+                                           unsigned char *locked_on, int *src)
 {
-        int             i        = 0;
-        int             ret      = -1;
-        afr_private_t  *priv     = NULL;
-        void *gfid               = NULL;
-        ia_type_t ia_type        = IA_INVAL;
+    int i = 0;
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    void *gfid = NULL;
+    ia_type_t ia_type = IA_INVAL;
 
-        priv = this->private;
-        gfid = &replies[src_idx].poststat.ia_gfid;
-        ia_type = replies[src_idx].poststat.ia_type;
+    priv = this->private;
+    gfid = &replies[src_idx].poststat.ia_gfid;
+    ia_type = replies[src_idx].poststat.ia_type;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (i == src_idx)
-                        continue;
-
-                if (!replies[i].valid)
-                        continue;
-
-                if (replies[i].op_ret != 0)
-                        continue;
-
-                if (gf_uuid_compare (gfid, replies[i].poststat.ia_gfid) &&
-                    (ia_type == replies[i].poststat.ia_type)) {
-                        ret = afr_gfid_split_brain_source (this, replies, inode,
-                                                           pargfid, bname,
-                                                           src_idx, i,
-                                                           locked_on, src,
-                                                           NULL);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        AFR_MSG_SPLIT_BRAIN,
-                                        "Skipping conservative merge on the "
-                                        "file.");
-                        return ret;
-                }
-
-                if (ia_type != replies[i].poststat.ia_type) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN, "Type mismatch detected "
-                                "for <gfid:%s>/%s>, %s on %s and %s on %s. "
-                                "Skipping conservative merge on the file.",
-                                uuid_utoa (pargfid), bname,
-                                gf_inode_type_to_str (replies[i].poststat.ia_type),
-                                priv->children[i]->name,
-                                gf_inode_type_to_str (replies[src_idx].poststat.ia_type),
-                                priv->children[src_idx]->name);
-                        gf_event (EVENT_AFR_SPLIT_BRAIN, "subvol=%s;type=file;"
-                                  "file=<gfid:%s>/%s>;count=2;child-%d=%s;type-"
-                                  "%d=%s;child-%d=%s;type-%d=%s",
-                                  this->name, uuid_utoa (pargfid), bname, i,
-                                  priv->children[i]->name, i,
-                                  gf_inode_type_to_str(replies[i].poststat.ia_type),
-                                  src_idx, priv->children[src_idx]->name, src_idx,
-                                  gf_inode_type_to_str(replies[src_idx].poststat.ia_type));
-                        return -1;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == src_idx)
+            continue;
+
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret != 0)
+            continue;
+
+        if (gf_uuid_is_null(replies[i].poststat.ia_gfid))
+            continue;
+
+        if (replies[i].poststat.ia_type == IA_INVAL)
+            continue;
+
+        if (ia_type == IA_INVAL || gf_uuid_is_null(gfid)) {
+            src_idx = i;
+            ia_type = replies[src_idx].poststat.ia_type;
+            gfid = &replies[src_idx].poststat.ia_gfid;
+            continue;
         }
 
-        return 0;
+        if (gf_uuid_compare(gfid, replies[i].poststat.ia_gfid) &&
+            (ia_type == replies[i].poststat.ia_type)) {
+            ret = afr_gfid_split_brain_source(this, replies, inode, pargfid,
+                                              bname, src_idx, i, locked_on, src,
+                                              NULL);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       "Skipping conservative merge on the "
+                       "file.");
+            return ret;
+        }
+
+        if (ia_type != replies[i].poststat.ia_type) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                   "Type mismatch detected "
+                   "for <gfid:%s>/%s>, %s on %s and %s on %s. "
+                   "Skipping conservative merge on the file.",
+                   uuid_utoa(pargfid), bname,
+                   gf_inode_type_to_str(replies[i].poststat.ia_type),
+                   priv->children[i]->name,
+                   gf_inode_type_to_str(replies[src_idx].poststat.ia_type),
+                   priv->children[src_idx]->name);
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;type=file;"
+                     "file=<gfid:%s>/%s>;count=2;child-%d=%s;type-"
+                     "%d=%s;child-%d=%s;type-%d=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(pargfid), bname, i, priv->children[i]->name, i,
+                     gf_inode_type_to_str(replies[i].poststat.ia_type), src_idx,
+                     priv->children[src_idx]->name, src_idx,
+                     gf_inode_type_to_str(replies[src_idx].poststat.ia_type));
+            return -1;
+        }
+    }
+
+    return 0;
 }
 
 static int
-__afr_selfheal_merge_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			     char *name, inode_t *inode, unsigned char *sources,
-			     unsigned char *healed_sinks, unsigned char *locked_on,
-			     struct afr_reply *replies)
+__afr_selfheal_merge_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                            char *name, inode_t *inode, unsigned char *sources,
+                            unsigned char *healed_sinks,
+                            unsigned char *locked_on, struct afr_reply *replies)
 {
-        int             ret       = 0;
-        int             i         = 0;
-        int             source    = -1;
-        int             src       = -1;
-        afr_private_t  *priv      = NULL;
+    int ret = 0;
+    int i = 0;
+    int source = -1;
+    int src = -1;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            source = i;
+            break;
+        }
+    }
+
+    if (source == -1) {
+        /* entry got deleted in the mean time? */
+        return 0;
+    }
 
-	priv = this->private;
+    /* Set all the sources as 1, otheriwse newentry_mark won't be set */
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            sources[i] = 1;
+        }
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-		if (replies[i].valid && replies[i].op_ret == 0) {
-			source = i;
-			break;
-		}
-	}
-
-	if (source == -1) {
-		/* entry got deleted in the mean time? */
-		return 0;
-	}
-
-        /* Set all the sources as 1, otheriwse newentry_mark won't be set */
-	for (i = 0; i < priv->child_count; i++) {
-		if (replies[i].valid && replies[i].op_ret == 0) {
-			sources[i] = 1;
-		}
-	}
-
-        ret = afr_lookup_and_heal_gfid (this, fd->inode, name, inode, replies,
-                                        source,
-                                        &replies[source].poststat.ia_gfid);
-        if (ret)
-                return ret;
+    ret = afr_lookup_and_heal_gfid(this, fd->inode, name, inode, replies,
+                                   source, sources,
+                                   &replies[source].poststat.ia_gfid, NULL);
+    if (ret)
+        return ret;
 
-        /* In case of type mismatch / unable to resolve gfid mismatch on the
-         * entry, return -1.*/
-        ret = afr_selfheal_detect_gfid_and_type_mismatch (this, replies, inode,
-                                                          fd->inode->gfid,
-                                                          name, source,
-                                                          locked_on, &src);
+    /* In case of type mismatch / unable to resolve gfid mismatch on the
+     * entry, return -1.*/
+    ret = afr_selfheal_detect_gfid_and_type_mismatch(
+        this, replies, inode, fd->inode->gfid, name, source, locked_on, &src);
 
-        if (ret < 0)
-                return ret;
-        if (src != -1) {
-                source = src;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (i != src && replies[i].valid &&
-                            gf_uuid_compare (replies[src].poststat.ia_gfid,
-                            replies[i].poststat.ia_gfid)) {
-                                sources[i] = 0;
-                        }
-                }
+    if (ret < 0)
+        return ret;
+    if (src != -1) {
+        source = src;
+        for (i = 0; i < priv->child_count; i++) {
+            if (i != src && replies[i].valid &&
+                gf_uuid_compare(replies[src].poststat.ia_gfid,
+                                replies[i].poststat.ia_gfid)) {
+                sources[i] = 0;
+            }
         }
+    }
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == source || !healed_sinks[i])
-			continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source || !healed_sinks[i])
+            continue;
 
-                if (src != -1) {
-                        if (!gf_uuid_compare (replies[src].poststat.ia_gfid,
-                                              replies[i].poststat.ia_gfid))
-                                continue;
-                } else if (replies[i].op_errno != ENOENT) {
-			continue;
-                }
+        if (src != -1) {
+            if (!gf_uuid_compare(replies[src].poststat.ia_gfid,
+                                 replies[i].poststat.ia_gfid))
+                continue;
+        } else if (replies[i].op_errno != ENOENT) {
+            continue;
+        }
 
-		ret |= afr_selfheal_recreate_entry (frame, i, source, sources,
-                                                    fd->inode, name, inode,
-                                                    replies);
-	}
+        ret |= afr_selfheal_recreate_entry(frame, i, source, sources, fd->inode,
+                                           name, inode, replies);
+    }
 
-	return ret;
+    return ret;
 }
 
-
 static int
-__afr_selfheal_entry_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                             char *name, inode_t *inode, int source,
-                             unsigned char *sources, unsigned char *healed_sinks,
-                             unsigned char *locked_on,
-                             struct afr_reply *replies)
+__afr_selfheal_entry_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                            char *name, inode_t *inode, int source,
+                            unsigned char *sources, unsigned char *healed_sinks,
+                            unsigned char *locked_on, struct afr_reply *replies)
 {
-	int ret = -1;
-
-	if (source < 0)
-		ret = __afr_selfheal_merge_dirent (frame, this, fd, name, inode,
-						   sources, healed_sinks,
-						   locked_on, replies);
-	else
-		ret = __afr_selfheal_heal_dirent (frame, this, fd, name, inode,
-						  source, sources, healed_sinks,
-						  locked_on, replies);
-	return ret;
+    int ret = -1;
+
+    if (source < 0)
+        ret = __afr_selfheal_merge_dirent(frame, this, fd, name, inode, sources,
+                                          healed_sinks, locked_on, replies);
+    else
+        ret = __afr_selfheal_heal_dirent(frame, this, fd, name, inode, source,
+                                         sources, healed_sinks, locked_on,
+                                         replies);
+    return ret;
 }
 
 static gf_boolean_t
-is_full_heal_marker_present (xlator_t *this, dict_t *xdata, int idx)
+is_full_heal_marker_present(xlator_t *this, dict_t *xdata, int idx)
 {
-        int                  i           = 0;
-        int                  pending[3]  = {0,};
-        void                *pending_raw = NULL;
-        afr_private_t       *priv        = NULL;
+    int i = 0;
+    int pending[3] = {
+        0,
+    };
+    void *pending_raw = NULL;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!xdata)
-                return _gf_false;
+    if (!xdata)
+        return _gf_false;
 
-        /* Iterate over each of the priv->pending_keys[] elements and then
-         * see if any of them have data segment non-zero. If they do, return
-         * true. Else return false.
-         */
-        for (i = 0; i < priv->child_count; i++) {
-                if (dict_get_ptr (xdata, priv->pending_key[i], &pending_raw))
-                        continue;
+    /* Iterate over each of the priv->pending_keys[] elements and then
+     * see if any of them have data segment non-zero. If they do, return
+     * true. Else return false.
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
+            continue;
 
-                if (!pending_raw)
-                        continue;
+        if (!pending_raw)
+            continue;
 
-                memcpy (pending, pending_raw, sizeof (pending));
-                if (ntoh32 (pending[idx]))
-                        return _gf_true;
-        }
+        memcpy(pending, pending_raw, sizeof(pending));
+        if (ntoh32(pending[idx]))
+            return _gf_true;
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static gf_boolean_t
-afr_need_full_heal (xlator_t *this, struct afr_reply *replies, int source,
-                    unsigned char *healed_sinks, afr_transaction_type type)
+afr_need_full_heal(xlator_t *this, struct afr_reply *replies, int source,
+                   unsigned char *healed_sinks, afr_transaction_type type)
 {
-        int                i     = 0;
-        int                idx   = 0;
-        afr_private_t     *priv  = NULL;
+    int i = 0;
+    int idx = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->esh_granular)
-                return _gf_true;
+    if (!priv->esh_granular)
+        return _gf_true;
 
-        if (type != AFR_ENTRY_TRANSACTION)
-                return _gf_true;
+    if (type != AFR_ENTRY_TRANSACTION)
+        return _gf_true;
 
-        priv = this->private;
-        idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
+    priv = this->private;
+    idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
 
-        /* If there is a clear source, check whether the full-heal-indicator
-         * is present in its xdata. Otherwise, we need to examine all the
-         * participating bricks and then figure if *even* one of them has a
-         * full-heal-indicator.
-         */
+    /* If there is a clear source, check whether the full-heal-indicator
+     * is present in its xdata. Otherwise, we need to examine all the
+     * participating bricks and then figure if *even* one of them has a
+     * full-heal-indicator.
+     */
 
-        if (source != -1) {
-                if (is_full_heal_marker_present (this, replies[source].xdata,
-                                                 idx))
-                        return _gf_true;
-        }
+    if (source != -1) {
+        if (is_full_heal_marker_present(this, replies[source].xdata, idx))
+            return _gf_true;
+    }
 
-        /* else ..*/
+    /* else ..*/
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!healed_sinks[i])
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i])
+            continue;
 
-                if (is_full_heal_marker_present (this, replies[i].xdata, idx))
-                        return _gf_true;
-        }
+        if (is_full_heal_marker_present(this, replies[i].xdata, idx))
+            return _gf_true;
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static int
-__afr_selfheal_entry_finalize_source (xlator_t *this, unsigned char *sources,
-				      unsigned char *healed_sinks,
-                                      unsigned char *locked_on,
-                                      struct afr_reply *replies,
-                                      uint64_t *witness)
+__afr_selfheal_entry_finalize_source(xlator_t *this, unsigned char *sources,
+                                     unsigned char *healed_sinks,
+                                     unsigned char *locked_on,
+                                     struct afr_reply *replies,
+                                     uint64_t *witness)
 {
-	afr_private_t *priv = NULL;
-	int source = -1;
-	int sources_count = 0;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    int sources_count = 0;
+    int i = 0;
 
-	priv = this->private;
+    priv = this->private;
 
-	sources_count = AFR_COUNT (sources, priv->child_count);
+    sources_count = AFR_COUNT(sources, priv->child_count);
 
-	if ((AFR_CMP (locked_on, healed_sinks, priv->child_count) == 0)
-            || !sources_count || afr_does_witness_exist (this, witness)) {
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count || afr_does_witness_exist(this, witness)) {
+        memset(sources, 0, sizeof(*sources) * priv->child_count);
+        afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+        return -1;
+    }
 
-                memset (sources, 0, sizeof (*sources) * priv->child_count);
-                afr_mark_active_sinks (this, sources, locked_on, healed_sinks);
-		return -1;
-	}
+    source = afr_choose_source_by_policy(priv, sources, AFR_ENTRY_TRANSACTION);
+
+    /*If the selected source does not blame any other brick, then mark
+     * everything as sink to trigger conservative merge.
+     */
+    if (source != -1 && !AFR_COUNT(healed_sinks, priv->child_count)) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (locked_on[i]) {
+                sources[i] = 0;
+                healed_sinks[i] = 1;
+            }
+        }
+        return -1;
+    }
 
-        source = afr_choose_source_by_policy (priv, sources,
-                                              AFR_ENTRY_TRANSACTION);
-	return source;
+    return source;
 }
 
 int
-__afr_selfheal_entry_prepare (call_frame_t *frame, xlator_t *this,
-                              inode_t *inode, unsigned char *locked_on,
-                              unsigned char *sources, unsigned char *sinks,
-                              unsigned char *healed_sinks,
-			      struct afr_reply *replies, int *source_p,
-                              gf_boolean_t *pflag)
+__afr_selfheal_entry_prepare(call_frame_t *frame, xlator_t *this,
+                             inode_t *inode, unsigned char *locked_on,
+                             unsigned char *sources, unsigned char *sinks,
+                             unsigned char *healed_sinks,
+                             struct afr_reply *replies, int *source_p,
+                             unsigned char *pflag)
 {
-	int ret = -1;
-	int source = -1;
-	afr_private_t *priv = NULL;
-        uint64_t *witness = NULL;
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    uint64_t *witness = NULL;
 
-	priv = this->private;
+    priv = this->private;
 
-	ret = afr_selfheal_unlocked_discover (frame, inode, inode->gfid,
-					      replies);
-        if (ret)
-                return ret;
-
-        witness = alloca0 (sizeof (*witness) * priv->child_count);
-	ret = afr_selfheal_find_direction (frame, this, replies,
-					   AFR_ENTRY_TRANSACTION,
-					   locked_on, sources, sinks, witness,
-                                           pflag);
-	if (ret)
-		return ret;
-
-        /* Initialize the healed_sinks[] array optimistically to
-           the intersection of to-be-healed (i.e sinks[]) and
-           the list of servers which are up (i.e locked_on[]).
-
-           As we encounter failures in the healing process, we
-           will unmark the respective servers in the healed_sinks[]
-           array.
-        */
-        AFR_INTERSECT (healed_sinks, sinks, locked_on, priv->child_count);
-
-	source = __afr_selfheal_entry_finalize_source (this, sources,
-                                                       healed_sinks,
-						       locked_on, replies,
-                                                       witness);
-
-	if (source < 0) {
-		/* If source is < 0 (typically split-brain), we perform a
-		   conservative merge of entries rather than erroring out */
-	}
-	*source_p = source;
-
-	return ret;
-}
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
+    if (ret)
+        return ret;
 
-static int
-afr_selfheal_entry_dirent (call_frame_t *frame, xlator_t *this,
-                           fd_t *fd, char *name, inode_t *parent_idx_inode,
-                           xlator_t *subvol, gf_boolean_t full_crawl)
-{
-        int                ret          = 0;
-        int                source       = -1;
-        unsigned char     *locked_on    = NULL;
-        unsigned char     *sources      = NULL;
-        unsigned char     *sinks        = NULL;
-        unsigned char     *healed_sinks = NULL;
-        inode_t           *inode        = NULL;
-        struct afr_reply  *replies      = NULL;
-        struct afr_reply  *par_replies  = NULL;
-        afr_private_t     *priv         = NULL;
-        dict_t            *xattr        = NULL;
-
-	priv = this->private;
-
-        xattr = dict_new ();
-        if (!xattr)
-                return -ENOMEM;
-        ret = dict_set_int32 (xattr, GF_GFIDLESS_LOOKUP, 1);
-        if (ret) {
-                dict_unref (xattr);
-                return -1;
-        }
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_ENTRY_TRANSACTION, locked_on, sources,
+                                      sinks, witness, pflag);
+    if (ret)
+        return ret;
 
-        sources = alloca0 (priv->child_count);
-        sinks   = alloca0 (priv->child_count);
-        healed_sinks = alloca0 (priv->child_count);
-	locked_on = alloca0 (priv->child_count);
-
-	replies = alloca0 (priv->child_count * sizeof(*replies));
-	par_replies = alloca0 (priv->child_count * sizeof(*par_replies));
-
-        ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name, NULL,
-                                    locked_on);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-                        gf_msg_debug (this->name, 0, "%s: Skipping "
-                                      "entry self-heal as only %d sub-volumes "
-                                      " could be locked in %s domain",
-                                      uuid_utoa (fd->inode->gfid),
-                                      ret, this->name);
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-                ret = __afr_selfheal_entry_prepare (frame, this, fd->inode,
-                                                    locked_on,
-                                                    sources, sinks,
-                                                    healed_sinks, par_replies,
-                                                    &source, NULL);
-                if (ret < 0)
-                        goto unlock;
-
-		inode = afr_selfheal_unlocked_lookup_on (frame, fd->inode, name,
-							 replies, locked_on,
-                                                         xattr);
-		if (!inode) {
-			ret = -ENOMEM;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_entry_dirent (frame, this, fd, name, inode,
-						   source, sources, healed_sinks,
-						   locked_on, replies);
-
-                if ((ret == 0) && (priv->esh_granular) && parent_idx_inode) {
-                        ret = afr_shd_index_purge (subvol, parent_idx_inode,
-                                                   name, inode->ia_type);
-                        /* Why is ret force-set to 0? We do not care about
-                         * index purge failing for full heal as it is quite
-                         * possible during replace-brick that not all files
-                         * and directories have their name indices present in
-                         * entry-changes/.
-                         */
-                        ret = 0;
-                }
-	}
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
 
-unlock:
-        afr_selfheal_unentrylk (frame, this, fd->inode, this->name, NULL,
-                                locked_on, NULL);
-	if (inode)
-		inode_unref (inode);
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
-        if (par_replies)
-                afr_replies_wipe (par_replies, priv->child_count);
-        if (xattr)
-                dict_unref (xattr);
-
-	return ret;
-}
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
 
+    source = __afr_selfheal_entry_finalize_source(this, sources, healed_sinks,
+                                                  locked_on, replies, witness);
 
-static inode_t *
-afr_shd_entry_changes_index_inode (xlator_t *this, xlator_t *subvol,
-                                   uuid_t pargfid)
+    if (source < 0) {
+        /* If source is < 0 (typically split-brain), we perform a
+           conservative merge of entries rather than erroring out */
+    }
+    *source_p = source;
+
+    return ret;
+}
+
+static int
+afr_selfheal_entry_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                          char *name, inode_t *parent_idx_inode,
+                          xlator_t *subvol, gf_boolean_t full_crawl)
 {
-        int             ret         = -1;
-        void           *index_gfid  = NULL;
-        loc_t           rootloc     = {0,};
-        loc_t           loc         = {0,};
-        dict_t         *xattr       = NULL;
-        inode_t        *inode       = NULL;
-        struct iatt     iatt        = {0,};
-
-        rootloc.inode = inode_ref (this->itable->root);
-        gf_uuid_copy (rootloc.gfid, rootloc.inode->gfid);
-
-        ret = syncop_getxattr (subvol, &rootloc, &xattr,
-                               GF_XATTROP_ENTRY_CHANGES_GFID, NULL, NULL);
-        if (ret || !xattr) {
-                errno = -ret;
-                goto out;
+    int ret = 0;
+    int source = -1;
+    unsigned char *locked_on = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *healed_sinks = NULL;
+    inode_t *inode = NULL;
+    struct afr_reply *replies = NULL;
+    struct afr_reply *par_replies = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+
+    priv = this->private;
+
+    if (afr_is_private_directory(priv, fd->inode->gfid, name,
+                                 GF_CLIENT_PID_SELF_HEALD)) {
+        return 0;
+    }
+
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
+    ret = dict_set_int32_sizen(xattr, GF_GFIDLESS_LOOKUP, 1);
+    if (ret) {
+        dict_unref(xattr);
+        return -1;
+    }
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    locked_on = alloca0(priv->child_count);
+
+    replies = alloca0(priv->child_count * sizeof(*replies));
+    par_replies = alloca0(priv->child_count * sizeof(*par_replies));
+
+    ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
+                               locked_on);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "entry self-heal as only %d sub-volumes "
+                         " could be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret, this->name);
+            ret = -ENOTCONN;
+            goto unlock;
         }
 
-        ret = dict_get_ptr (xattr, GF_XATTROP_ENTRY_CHANGES_GFID, &index_gfid);
-        if (ret) {
-                errno = EINVAL;
-                goto out;
-        }
+        ret = __afr_selfheal_entry_prepare(frame, this, fd->inode, locked_on,
+                                           sources, sinks, healed_sinks,
+                                           par_replies, &source, NULL);
+        if (ret < 0)
+            goto unlock;
 
-        loc.inode = inode_new (this->itable);
-        if (!loc.inode) {
-                errno = ENOMEM;
-                goto out;
+        inode = afr_selfheal_unlocked_lookup_on(frame, fd->inode, name, replies,
+                                                locked_on, xattr);
+        if (!inode) {
+            ret = -ENOMEM;
+            goto unlock;
         }
 
-        gf_uuid_copy (loc.pargfid, index_gfid);
-        loc.name = gf_strdup (uuid_utoa (pargfid));
-
-        ret = syncop_lookup (subvol, &loc, &iatt, NULL, NULL, NULL);
-        if (ret < 0) {
-                errno = -ret;
-                goto out;
+        ret = __afr_selfheal_entry_dirent(frame, this, fd, name, inode, source,
+                                          sources, healed_sinks, locked_on,
+                                          replies);
+
+        if ((ret == 0) && (priv->esh_granular) && parent_idx_inode) {
+            ret = afr_shd_entry_purge(subvol, parent_idx_inode, name,
+                                      inode->ia_type);
+            /* Why is ret force-set to 0? We do not care about
+             * index purge failing for full heal as it is quite
+             * possible during replace-brick that not all files
+             * and directories have their name indices present in
+             * entry-changes/.
+             */
+            ret = 0;
         }
+    }
+
+unlock:
+    afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL, locked_on,
+                           NULL);
+    if (inode)
+        inode_unref(inode);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    if (par_replies)
+        afr_replies_wipe(par_replies, priv->child_count);
+    if (xattr)
+        dict_unref(xattr);
+
+    return ret;
+}
 
-        inode = inode_link (loc.inode, NULL, NULL, &iatt);
+static inode_t *
+afr_shd_entry_changes_index_inode(xlator_t *this, xlator_t *subvol,
+                                  uuid_t pargfid)
+{
+    int ret = -1;
+    void *index_gfid = NULL;
+    loc_t rootloc = {
+        0,
+    };
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
+
+    ret = syncop_getxattr(subvol, &rootloc, &xattr,
+                          GF_XATTROP_ENTRY_CHANGES_GFID, NULL, NULL);
+    if (ret || !xattr) {
+        errno = -ret;
+        goto out;
+    }
+
+    ret = dict_get_ptr(xattr, GF_XATTROP_ENTRY_CHANGES_GFID, &index_gfid);
+    if (ret) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    loc.inode = inode_new(this->itable);
+    if (!loc.inode) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_copy(loc.pargfid, index_gfid);
+    loc.name = gf_strdup(uuid_utoa(pargfid));
+
+    ret = syncop_lookup(subvol, &loc, &iatt, NULL, NULL, NULL);
+    if (ret < 0) {
+        errno = -ret;
+        goto out;
+    }
+
+    inode = inode_link(loc.inode, NULL, NULL, &iatt);
 
 out:
-        if (xattr)
-                dict_unref (xattr);
-        loc_wipe (&rootloc);
-        GF_FREE ((char *)loc.name);
-        loc_wipe (&loc);
+    if (xattr)
+        dict_unref(xattr);
+    loc_wipe(&rootloc);
+    GF_FREE((char *)loc.name);
+    loc_wipe(&loc);
 
-        return inode;
+    return inode;
 }
 
 static int
-afr_selfheal_entry_do_subvol (call_frame_t *frame, xlator_t *this,
-                              fd_t *fd, int child)
+afr_selfheal_entry_do_subvol(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                             int child)
 {
-	int ret = 0;
-	gf_dirent_t entries;
-	gf_dirent_t *entry = NULL;
-	off_t offset = 0;
-	call_frame_t *iter_frame = NULL;
-	xlator_t *subvol = NULL;
-	afr_private_t *priv = NULL;
-        gf_boolean_t mismatch = _gf_false;
-        afr_local_t *local = NULL;
-        loc_t loc = {0,};
-
-	priv = this->private;
-	subvol = priv->children[child];
-
-	INIT_LIST_HEAD (&entries.list);
-
-        local = frame->local;
-
-	iter_frame = afr_copy_frame (frame);
-	if (!iter_frame)
-		return -ENOMEM;
-
-        loc.inode = afr_shd_entry_changes_index_inode (this, subvol,
-                                                       fd->inode->gfid);
-
-	while ((ret = syncop_readdir (subvol, fd, 131072, offset, &entries,
-                                      NULL, NULL))) {
-		if (ret > 0)
-			ret = 0;
-		list_for_each_entry (entry, &entries.list, list) {
-			offset = entry->d_off;
-
-			if (!strcmp (entry->d_name, ".") ||
-			    !strcmp (entry->d_name, ".."))
-				continue;
-
-			if (__is_root_gfid (fd->inode->gfid) &&
-			    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR))
-				continue;
-
-			ret = afr_selfheal_entry_dirent (iter_frame, this, fd,
-                                                         entry->d_name,
-                                                         loc.inode, subvol,
-                                                        local->need_full_crawl);
-			AFR_STACK_RESET (iter_frame);
-			if (iter_frame->local == NULL) {
-                                ret = -ENOTCONN;
-                                break;
-                        }
-
-                        if (ret == -1) {
-                                /* gfid or type mismatch. */
-                                mismatch = _gf_true;
-                                ret = 0;
-                        }
-			if (ret)
-				break;
-		}
-
-		gf_dirent_free (&entries);
-		if (ret)
-			break;
-	}
-
-        loc_wipe (&loc);
-
-	AFR_STACK_DESTROY (iter_frame);
-        if (mismatch == _gf_true)
-                /* undo pending will be skipped */
-                ret = -1;
-	return ret;
-}
+    int ret = 0;
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    off_t offset = 0;
+    call_frame_t *iter_frame = NULL;
+    xlator_t *subvol = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t mismatch = _gf_false;
+    afr_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    priv = this->private;
+    subvol = priv->children[child];
+
+    INIT_LIST_HEAD(&entries.list);
+
+    local = frame->local;
+
+    iter_frame = afr_copy_frame(frame);
+    if (!iter_frame)
+        return -ENOMEM;
+
+    loc.inode = afr_shd_entry_changes_index_inode(this, subvol,
+                                                  fd->inode->gfid);
+
+    while ((ret = syncop_readdir(subvol, fd, 131072, offset, &entries, NULL,
+                                 NULL))) {
+        if (ret > 0)
+            ret = 0;
+        list_for_each_entry(entry, &entries.list, list)
+        {
+            offset = entry->d_off;
 
-static int
-afr_selfheal_entry_granular_dirent (xlator_t *subvol, gf_dirent_t *entry,
-                                    loc_t *parent, void *data)
-{
-        int                      ret  = 0;
-        loc_t                    loc  = {0,};
-        struct iatt              iatt = {0,};
-        afr_granular_esh_args_t *args = data;
-
-        /* Look up the actual inode associated with entry. If the lookup returns
-         * ESTALE or ENOENT, then it means we have a stale index. Remove it.
-         * This is analogous to the check in afr_shd_index_heal() except that
-         * here it is achieved through LOOKUP and in afr_shd_index_heal() through
-         * a GETXATTR.
-         */
+            if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+                continue;
+
+            ret = afr_selfheal_entry_dirent(iter_frame, this, fd, entry->d_name,
+                                            loc.inode, subvol,
+                                            local->need_full_crawl);
+            AFR_STACK_RESET(iter_frame);
+            if (iter_frame->local == NULL) {
+                ret = -ENOTCONN;
+                break;
+            }
 
-        loc.inode = inode_new (args->xl->itable);
-        loc.parent = inode_ref (args->heal_fd->inode);
-        gf_uuid_copy (loc.pargfid, loc.parent->gfid);
-        loc.name = entry->d_name;
-
-        ret = syncop_lookup (args->xl, &loc, &iatt, NULL, NULL, NULL);
-        if ((ret == -ENOENT) || (ret == -ESTALE)) {
-                /* The name indices under the pgfid index dir are guaranteed
-                 * to be regular files. Hence the hardcoding.
-                 */
-                afr_shd_index_purge (subvol, parent->inode, entry->d_name,
-                                     IA_IFREG);
+            if (ret == -1) {
+                /* gfid or type mismatch. */
+                mismatch = _gf_true;
                 ret = 0;
-                goto out;
+            }
+            if (ret)
+                break;
         }
-        /* TBD: afr_shd_zero_xattrop? */
 
-        ret = afr_selfheal_entry_dirent (args->frame, args->xl, args->heal_fd,
-                                         entry->d_name, parent->inode, subvol,
-                                         _gf_false);
-        AFR_STACK_RESET (args->frame);
-        if (args->frame->local == NULL)
-                ret = -ENOTCONN;
+        gf_dirent_free(&entries);
+        if (ret)
+            break;
+    }
 
-        if (ret == -1)
-                args->mismatch = _gf_true;
+    loc_wipe(&loc);
 
-out:
-        loc_wipe (&loc);
-        return 0;
+    AFR_STACK_DESTROY(iter_frame);
+    if (mismatch == _gf_true)
+        /* undo pending will be skipped */
+        ret = -1;
+    return ret;
 }
 
 static int
-afr_selfheal_entry_granular (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                             int subvol_idx, gf_boolean_t is_src)
+afr_selfheal_entry_granular_dirent(xlator_t *subvol, gf_dirent_t *entry,
+                                   loc_t *parent, void *data)
 {
-        int                         ret    = 0;
-        loc_t                       loc    = {0,};
-        xlator_t                   *subvol = NULL;
-        afr_private_t              *priv   = NULL;
-        afr_granular_esh_args_t     args   = {0,};
-
-        priv = this->private;
-        subvol = priv->children[subvol_idx];
-
-        args.frame = afr_copy_frame (frame);
-        args.xl = this;
-        /* args.heal_fd represents the fd associated with the original directory
-         * on which entry heal is being attempted.
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    afr_granular_esh_args_t *args = data;
+
+    /* Look up the actual inode associated with entry. If the lookup returns
+     * ESTALE or ENOENT, then it means we have a stale index. Remove it.
+     * This is analogous to the check in afr_shd_index_heal() except that
+     * here it is achieved through LOOKUP and in afr_shd_index_heal() through
+     * a GETXATTR.
+     */
+
+    loc.inode = inode_new(args->xl->itable);
+    loc.parent = inode_ref(args->heal_fd->inode);
+    gf_uuid_copy(loc.pargfid, loc.parent->gfid);
+    loc.name = entry->d_name;
+
+    ret = syncop_lookup(args->xl, &loc, &iatt, NULL, NULL, NULL);
+    if ((ret == -ENOENT) || (ret == -ESTALE)) {
+        /* The name indices under the pgfid index dir are guaranteed
+         * to be regular files. Hence the hardcoding.
          */
-        args.heal_fd = fd;
+        afr_shd_entry_purge(subvol, parent->inode, entry->d_name, IA_IFREG);
+        ret = 0;
+        goto out;
+    }
+    /* TBD: afr_shd_zero_xattrop? */
+
+    ret = afr_selfheal_entry_dirent(args->frame, args->xl, args->heal_fd,
+                                    entry->d_name, parent->inode, subvol,
+                                    _gf_false);
+    AFR_STACK_RESET(args->frame);
+    if (args->frame->local == NULL)
+        ret = -ENOTCONN;
+
+    if (ret == -1)
+        args->mismatch = _gf_true;
 
-        /* @subvol here represents the subvolume of AFR where
-         * indices/entry-changes/<pargfid> will be processed
-         */
-        loc.inode = afr_shd_entry_changes_index_inode (this, subvol,
-                                                       fd->inode->gfid);
-        if (!loc.inode) {
-                /* If granular heal failed on the sink (as it might sometimes
-                 * because it is the src that would mostly contain the granular
-                 * changelogs and the sink's entry-changes would be empty),
-                 * do not treat heal as failure.
-                 */
-                if (is_src)
-                        return -errno;
-                else
-                        return 0;
-        }
+out:
+    loc_wipe(&loc);
+    return 0;
+}
 
-        ret = syncop_dir_scan (subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
-                               &args, afr_selfheal_entry_granular_dirent);
+static int
+afr_selfheal_entry_granular(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                            int subvol_idx, gf_boolean_t is_src)
+{
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    afr_private_t *priv = NULL;
+    afr_granular_esh_args_t args = {
+        0,
+    };
+
+    priv = this->private;
+    subvol = priv->children[subvol_idx];
+
+    args.frame = afr_copy_frame(frame);
+    if (!args.frame)
+        goto out;
+    args.xl = this;
+    /* args.heal_fd represents the fd associated with the original directory
+     * on which entry heal is being attempted.
+     */
+    args.heal_fd = fd;
+
+    /* @subvol here represents the subvolume of AFR where
+     * indices/entry-changes/<pargfid> will be processed
+     */
+    loc.inode = afr_shd_entry_changes_index_inode(this, subvol,
+                                                  fd->inode->gfid);
+    if (!loc.inode) {
+        /* If granular heal failed on the sink (as it might sometimes
+         * because it is the src that would mostly contain the granular
+         * changelogs and the sink's entry-changes would be empty),
+         * do not treat heal as failure.
+         */
+        if (is_src)
+            ret = -errno;
+        else
+            ret = 0;
+        goto out;
+    }
 
-        loc_wipe (&loc);
+    ret = syncop_dir_scan(subvol, &loc, GF_CLIENT_PID_SELF_HEALD, &args,
+                          afr_selfheal_entry_granular_dirent);
 
-        if (args.mismatch == _gf_true)
-                ret = -1;
+    loc_wipe(&loc);
 
-        return ret;
+    if (args.mismatch == _gf_true)
+        ret = -1;
+out:
+    if (args.frame)
+        AFR_STACK_DESTROY(args.frame);
+    return ret;
 }
 
 static int
-afr_selfheal_entry_do (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		       int source, unsigned char *sources,
-		       unsigned char *healed_sinks)
+afr_selfheal_entry_do(call_frame_t *frame, xlator_t *this, fd_t *fd, int source,
+                      unsigned char *sources, unsigned char *healed_sinks)
 {
-	int i                   = 0;
-	int ret                 = 0;
-        gf_boolean_t   mismatch = _gf_false;
-        afr_local_t   *local    = NULL;
-	afr_private_t *priv     = NULL;
-
-	priv = this->private;
-        local = frame->local;
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                AFR_MSG_SELF_HEAL_INFO, "performing entry selfheal on %s",
-		uuid_utoa (fd->inode->gfid));
-
-	for (i = 0; i < priv->child_count; i++) {
-                /* Expunge */
-		if (!healed_sinks[i])
-			continue;
-
-                if (!local->need_full_crawl)
-                /* Why call afr_selfheal_entry_granular() on a "healed sink",
-                 * given that it is the source that contains the granular
-                 * indices?
-                 * If the index for this directory is non-existent or empty on
-                 * this subvol (=> clear sink), then it will return early
-                 * without failure status.
-                 * If the index is non-empty and it is yet a 'healed sink', then
-                 * it is due to a split-brain in which case we anyway need to
-                 * crawl the indices/entry-changes/pargfid directory.
-                 */
-                        ret = afr_selfheal_entry_granular (frame, this, fd, i,
-                                                           _gf_false);
-                else
-                        ret = afr_selfheal_entry_do_subvol (frame, this, fd, i);
-
-                if (ret == -1) {
-                        /* gfid or type mismatch. */
-                        mismatch = _gf_true;
-                        ret = 0;
-                }
-                if (ret)
-                        break;
-	}
-
-        if (!ret && source != -1) {
-                /* Impunge */
-                if (local->need_full_crawl)
-                        ret = afr_selfheal_entry_do_subvol (frame, this, fd,
-                                                            source);
-                else
-                        ret = afr_selfheal_entry_granular (frame, this, fd,
-                                                           source, _gf_true);
+    int i = 0;
+    int ret = 0;
+    gf_boolean_t mismatch = _gf_false;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+           "performing entry selfheal on %s", uuid_utoa(fd->inode->gfid));
+
+    for (i = 0; i < priv->child_count; i++) {
+        /* Expunge */
+        if (!healed_sinks[i])
+            continue;
+
+        if (!local->need_full_crawl)
+            /* Why call afr_selfheal_entry_granular() on a "healed sink",
+             * given that it is the source that contains the granular
+             * indices?
+             * If the index for this directory is non-existent or empty on
+             * this subvol (=> clear sink), then it will return early
+             * without failure status.
+             * If the index is non-empty and it is yet a 'healed sink', then
+             * it is due to a split-brain in which case we anyway need to
+             * crawl the indices/entry-changes/pargfid directory.
+             */
+            ret = afr_selfheal_entry_granular(frame, this, fd, i, _gf_false);
+        else
+            ret = afr_selfheal_entry_do_subvol(frame, this, fd, i);
+
+        if (ret == -1) {
+            /* gfid or type mismatch. */
+            mismatch = _gf_true;
+            ret = 0;
         }
+        if (ret)
+            break;
+    }
 
-        if (mismatch == _gf_true)
-                /* undo pending will be skipped */
-                ret = -1;
-	return ret;
+    if (!ret && source != -1) {
+        /* Impunge */
+        if (local->need_full_crawl)
+            ret = afr_selfheal_entry_do_subvol(frame, this, fd, source);
+        else
+            ret = afr_selfheal_entry_granular(frame, this, fd, source,
+                                              _gf_true);
+    }
+
+    if (mismatch == _gf_true)
+        /* undo pending will be skipped */
+        ret = -1;
+    return ret;
 }
 
 static int
-__afr_selfheal_entry (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		      unsigned char *locked_on)
+__afr_selfheal_entry(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     unsigned char *locked_on)
 {
-	int                     ret                   = -1;
-	int                     source                = -1;
-	unsigned char          *sources               = NULL;
-	unsigned char          *sinks                 = NULL;
-	unsigned char          *data_lock             = NULL;
-        unsigned char          *postop_lock           = NULL;
-	unsigned char          *healed_sinks          = NULL;
-	unsigned char          *undid_pending         = NULL;
-	struct afr_reply       *locked_replies        = NULL;
-        afr_local_t            *local                 = NULL;
-	afr_private_t          *priv                  = NULL;
-        gf_boolean_t            did_sh                = _gf_true;
-
-	priv = this->private;
-        local = frame->local;
-
-	sources = alloca0 (priv->child_count);
-	sinks = alloca0 (priv->child_count);
-	healed_sinks = alloca0 (priv->child_count);
-	undid_pending = alloca0 (priv->child_count);
-	data_lock = alloca0 (priv->child_count);
-        postop_lock = alloca0 (priv->child_count);
-
-	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-	ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name, NULL,
-				    data_lock);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-                        gf_msg_debug (this->name, 0, "%s: Skipping "
-                                      "entry self-heal as only %d sub-volumes could "
-                                      "be locked in %s domain",
-                                      uuid_utoa (fd->inode->gfid), ret,
-                                      this->name);
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_entry_prepare (frame, this, fd->inode,
-                                                    data_lock, sources, sinks,
-                                                    healed_sinks,
-						    locked_replies, &source,
-                                                    NULL);
-                if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
-                        did_sh = _gf_false;
-                        goto unlock;
-                }
-
-                local->need_full_crawl = afr_need_full_heal (this,
-                                                             locked_replies,
-                                                             source,
-                                                             healed_sinks,
-                                                         AFR_ENTRY_TRANSACTION);
-	}
-unlock:
-	afr_selfheal_unentrylk (frame, this, fd->inode, this->name, NULL,
-				data_lock, NULL);
-	if (ret < 0)
-		goto out;
+    int ret = -1;
+    int source = -1;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *data_lock = NULL;
+    unsigned char *postop_lock = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t did_sh = _gf_true;
+
+    priv = this->private;
+    local = frame->local;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+    data_lock = alloca0(priv->child_count);
+    postop_lock = alloca0(priv->child_count);
+
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
+                               data_lock);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "entry self-heal as only %d sub-volumes could "
+                         "be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret, this->name);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
 
-        if (!did_sh)
-                goto out;
+        ret = __afr_selfheal_entry_prepare(frame, this, fd->inode, data_lock,
+                                           sources, sinks, healed_sinks,
+                                           locked_replies, &source, NULL);
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            did_sh = _gf_false;
+            goto unlock;
+        }
 
-	ret = afr_selfheal_entry_do (frame, this, fd, source, sources,
-                                     healed_sinks);
-	if (ret)
-		goto out;
-
-        /* Take entrylks in xlator domain before doing post-op (undo-pending) in
-         * entry self-heal. This is to prevent a parallel name self-heal on
-         * an entry under @fd->inode from reading pending xattrs while it is
-         * being modified by SHD after entry sh below, given that
-         * name self-heal takes locks ONLY in xlator domain and is free to read
-         * pending changelog in the absence of the following locking.
-         */
-        ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name, NULL,
-                                    postop_lock);
-        {
-                if (AFR_CMP (data_lock, postop_lock, priv->child_count) != 0) {
-                        gf_msg_debug (this->name, 0, "%s: Skipping "
-                                      "post-op after entry self-heal as %d "
-                                      "sub-volumes, as opposed to %d, "
-                                      "could be locked in %s domain",
-                                      uuid_utoa (fd->inode->gfid),
-                                      ret, AFR_COUNT (data_lock,
-                                      priv->child_count), this->name);
-                        ret = -ENOTCONN;
-                        goto postop_unlock;
-                }
-
-                ret = afr_selfheal_undo_pending (frame, this, fd->inode,
-                                                 sources, sinks, healed_sinks,
-                                                 undid_pending,
-                                                 AFR_ENTRY_TRANSACTION,
-                                                 locked_replies, postop_lock);
+        local->need_full_crawl = afr_need_full_heal(
+            this, locked_replies, source, healed_sinks, AFR_ENTRY_TRANSACTION);
+    }
+unlock:
+    afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL, data_lock,
+                           NULL);
+    if (ret < 0)
+        goto out;
+
+    if (!did_sh)
+        goto out;
+
+    ret = afr_selfheal_entry_do(frame, this, fd, source, sources, healed_sinks);
+    if (ret)
+        goto out;
+
+    /* Take entrylks in xlator domain before doing post-op (undo-pending) in
+     * entry self-heal. This is to prevent a parallel name self-heal on
+     * an entry under @fd->inode from reading pending xattrs while it is
+     * being modified by SHD after entry sh below, given that
+     * name self-heal takes locks ONLY in xlator domain and is free to read
+     * pending changelog in the absence of the following locking.
+     */
+    ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
+                               postop_lock);
+    {
+        if (AFR_CMP(data_lock, postop_lock, priv->child_count) != 0) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "post-op after entry self-heal as %d "
+                         "sub-volumes, as opposed to %d, "
+                         "could be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret,
+                         AFR_COUNT(data_lock, priv->child_count), this->name);
+            ret = -ENOTCONN;
+            goto postop_unlock;
         }
+
+        afr_selfheal_restore_time(frame, this, fd->inode, source, healed_sinks,
+                                  locked_replies);
+        ret = afr_selfheal_undo_pending(
+            frame, this, fd->inode, sources, sinks, healed_sinks, undid_pending,
+            AFR_ENTRY_TRANSACTION, locked_replies, postop_lock);
+    }
 postop_unlock:
-        afr_selfheal_unentrylk (frame, this, fd->inode, this->name, NULL,
-                                postop_lock, NULL);
+    afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL,
+                           postop_lock, NULL);
 out:
-        if (did_sh)
-                afr_log_selfheal (fd->inode->gfid, this, ret, "entry", source,
-                                  sources, healed_sinks);
-        else
-                ret = 1;
-
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-	return ret;
+    if (did_sh)
+        afr_log_selfheal(fd->inode->gfid, this, ret, "entry", source, sources,
+                         healed_sinks);
+    else
+        ret = 1;
+
+    if (locked_replies)
+        afr_replies_wipe(locked_replies, priv->child_count);
+    return ret;
 }
 
-
 static fd_t *
-afr_selfheal_data_opendir (xlator_t *this, inode_t *inode)
+afr_selfheal_data_opendir(xlator_t *this, inode_t *inode)
 {
-	loc_t loc = {0,};
-	int ret = 0;
-	fd_t *fd = NULL;
-
-	fd = fd_create (inode, 0);
-	if (!fd)
-		return NULL;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-	ret = syncop_opendir (this, &loc, fd, NULL, NULL);
-	if (ret) {
-		fd_unref (fd);
-		fd = NULL;
-	} else {
-		fd_bind (fd);
-	}
-
-	loc_wipe (&loc);
-        return fd;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
+    fd_t *fd = NULL;
+
+    fd = fd_create(inode, 0);
+    if (!fd)
+        return NULL;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = syncop_opendir(this, &loc, fd, NULL, NULL);
+    if (ret) {
+        fd_unref(fd);
+        fd = NULL;
+    } else {
+        fd_bind(fd);
+    }
+
+    loc_wipe(&loc);
+    return fd;
 }
 
-
 int
-afr_selfheal_entry (call_frame_t *frame, xlator_t *this, inode_t *inode)
+afr_selfheal_entry(call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-	afr_private_t *priv = NULL;
-	unsigned char *locked_on = NULL;
-	fd_t *fd = NULL;
-	int ret = 0;
-
-	priv = this->private;
-
-	fd = afr_selfheal_data_opendir (this, inode);
-	if (!fd)
-		return -EIO;
-
-	locked_on = alloca0 (priv->child_count);
-
-	ret = afr_selfheal_tie_breaker_entrylk (frame, this, inode,
-	                                        priv->sh_domain, NULL,
-                                                locked_on);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-                        gf_msg_debug (this->name, 0, "%s: Skipping "
-                                      "entry self-heal as only %d sub-volumes could "
-                                      "be locked in %s domain",
-                                      uuid_utoa (fd->inode->gfid), ret,
-                                      priv->sh_domain);
-			/* Either less than two subvols available, or another
-			   selfheal (from another server) is in progress. Skip
-			   for now in any case there isn't anything to do.
-			*/
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-                ret = __afr_selfheal_entry (frame, this, fd, locked_on);
-	}
+    afr_private_t *priv = NULL;
+    unsigned char *locked_on = NULL;
+    fd_t *fd = NULL;
+    int ret = 0;
+
+    priv = this->private;
+
+    fd = afr_selfheal_data_opendir(this, inode);
+    if (!fd)
+        return -EIO;
+
+    locked_on = alloca0(priv->child_count);
+
+    ret = afr_selfheal_tie_breaker_entrylk(frame, this, inode, priv->sh_domain,
+                                           NULL, locked_on);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "entry self-heal as only %d sub-volumes could "
+                         "be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret, priv->sh_domain);
+            /* Either less than two subvols available, or another
+               selfheal (from another server) is in progress. Skip
+               for now in any case there isn't anything to do.
+            */
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_entry(frame, this, fd, locked_on);
+    }
 unlock:
-	afr_selfheal_unentrylk (frame, this, inode, priv->sh_domain, NULL,
-                                locked_on, NULL);
+    afr_selfheal_unentrylk(frame, this, inode, priv->sh_domain, NULL, locked_on,
+                           NULL);
 
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
 
-	return ret;
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index 199f8961480..03f43bad16e 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -8,109 +8,108 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "afr.h"
 #include "afr-self-heal.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "protocol-common.h"
-#include "events.h"
+#include <glusterfs/events.h>
 
-#define AFR_HEAL_ATTR (GF_SET_ATTR_UID|GF_SET_ATTR_GID|GF_SET_ATTR_MODE)
+#define AFR_HEAL_ATTR (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE)
 
 static gf_boolean_t
-_afr_ignorable_key_match (dict_t *d, char *k, data_t *val, void *mdata)
+_afr_ignorable_key_match(dict_t *d, char *k, data_t *val, void *mdata)
 {
-        return afr_is_xattr_ignorable (k);
+    return afr_is_xattr_ignorable(k);
 }
 
 void
-afr_delete_ignorable_xattrs (dict_t *xattr)
+afr_delete_ignorable_xattrs(dict_t *xattr)
 {
-        dict_foreach_match (xattr, _afr_ignorable_key_match, NULL,
-                            dict_remove_foreach_fn, NULL);
+    dict_foreach_match(xattr, _afr_ignorable_key_match, NULL,
+                       dict_remove_foreach_fn, NULL);
 }
 
 int
-__afr_selfheal_metadata_do (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                            int source, unsigned char *healed_sinks,
-                            struct afr_reply *locked_replies)
+__afr_selfheal_metadata_do(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, unsigned char *healed_sinks,
+                           struct afr_reply *locked_replies)
 {
-	int ret = -1;
-	loc_t loc = {0,};
-	dict_t *xattr = NULL;
-	dict_t *old_xattr = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-
-	priv = this->private;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                AFR_MSG_SELF_HEAL_INFO, "performing metadata selfheal on %s",
-		uuid_utoa (inode->gfid));
-
-	ret = syncop_getxattr (priv->children[source], &loc, &xattr, NULL,
-                               NULL, NULL);
-	if (ret < 0) {
-		ret = -EIO;
-                goto out;
-	}
-
-	afr_delete_ignorable_xattrs (xattr);
-
-	for (i = 0; i < priv->child_count; i++) {
-                if (old_xattr) {
-                        dict_unref (old_xattr);
-                        old_xattr = NULL;
-                }
-
-		if (!healed_sinks[i])
-			continue;
-
-		ret = syncop_setattr (priv->children[i], &loc,
-				      &locked_replies[source].poststat,
-				      AFR_HEAL_ATTR, NULL, NULL, NULL, NULL);
-		if (ret)
-			healed_sinks[i] = 0;
-
-		ret = syncop_getxattr (priv->children[i], &loc, &old_xattr, 0,
-                                       NULL, NULL);
-		if (old_xattr) {
-			afr_delete_ignorable_xattrs (old_xattr);
-			ret = syncop_removexattr (priv->children[i], &loc, "",
-						  old_xattr, NULL);
-                        if (ret)
-                                healed_sinks[i] = 0;
-		}
-
-		ret = syncop_setxattr (priv->children[i], &loc, xattr, 0, NULL,
-                                       NULL);
-		if (ret)
-			healed_sinks[i] = 0;
-	}
-        ret = 0;
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    dict_t *old_xattr = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+           "performing metadata selfheal on %s", uuid_utoa(inode->gfid));
+
+    ret = syncop_getxattr(priv->children[source], &loc, &xattr, NULL, NULL,
+                          NULL);
+    if (ret < 0) {
+        ret = -EIO;
+        goto out;
+    }
+
+    afr_delete_ignorable_xattrs(xattr);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (old_xattr) {
+            dict_unref(old_xattr);
+            old_xattr = NULL;
+        }
+
+        if (!healed_sinks[i])
+            continue;
+
+        ret = syncop_setattr(priv->children[i], &loc,
+                             &locked_replies[source].poststat, AFR_HEAL_ATTR,
+                             NULL, NULL, NULL, NULL);
+        if (ret)
+            healed_sinks[i] = 0;
+
+        ret = syncop_getxattr(priv->children[i], &loc, &old_xattr, 0, NULL,
+                              NULL);
+        if (old_xattr) {
+            afr_delete_ignorable_xattrs(old_xattr);
+            ret = syncop_removexattr(priv->children[i], &loc, "", old_xattr,
+                                     NULL);
+            if (ret)
+                healed_sinks[i] = 0;
+        }
+
+        ret = syncop_setxattr(priv->children[i], &loc, xattr, 0, NULL, NULL);
+        if (ret)
+            healed_sinks[i] = 0;
+    }
+    ret = 0;
 
 out:
-	loc_wipe (&loc);
-	if (xattr)
-		dict_unref (xattr);
-        if (old_xattr)
-                dict_unref (old_xattr);
+    loc_wipe(&loc);
+    if (xattr)
+        dict_unref(xattr);
+    if (old_xattr)
+        dict_unref(old_xattr);
 
-	return ret;
+    return ret;
 }
 
 static uint64_t
 mtime_ns(struct iatt *ia)
 {
-        uint64_t ret;
+    uint64_t ret;
 
-        ret = (((uint64_t)(ia->ia_mtime)) * 1000000000)
-            + (uint64_t)(ia->ia_mtime_nsec);
+    ret = (((uint64_t)(ia->ia_mtime)) * 1000000000) +
+          (uint64_t)(ia->ia_mtime_nsec);
 
-        return ret;
+    return ret;
 }
 
 /*
@@ -123,382 +122,425 @@ mtime_ns(struct iatt *ia)
  * the source with the most recent modification date.
  */
 static int
-afr_dirtime_splitbrain_source (call_frame_t *frame, xlator_t *this,
-                               struct afr_reply *replies,
-                               unsigned char *locked_on)
+afr_dirtime_splitbrain_source(call_frame_t *frame, xlator_t *this,
+                              struct afr_reply *replies,
+                              unsigned char *locked_on)
 {
-        afr_private_t *priv  = NULL;
-        int            source = -1;
-        struct iatt    source_ia;
-        struct iatt    child_ia;
-        uint64_t       mtime = 0;
-        int            i;
-        int            ret   = -1;
-
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!locked_on[i])
-                        continue;
-
-                if (!replies[i].valid)
-                        continue;
-
-                if (replies[i].op_ret != 0)
-                        continue;
-
-                if (mtime_ns(&replies[i].poststat) <= mtime)
-                        continue;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    struct iatt source_ia;
+    struct iatt child_ia;
+    uint64_t mtime = 0;
+    int i;
+    int ret = -1;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!locked_on[i])
+            continue;
+
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret != 0)
+            continue;
+
+        if (mtime_ns(&replies[i].poststat) <= mtime)
+            continue;
+
+        mtime = mtime_ns(&replies[i].poststat);
+        source = i;
+    }
+
+    if (source == -1)
+        goto out;
+
+    source_ia = replies[source].poststat;
+    if (source_ia.ia_type != IA_IFDIR)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source)
+            continue;
+
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret != 0)
+            continue;
+
+        child_ia = replies[i].poststat;
+
+        if (!IA_EQUAL(source_ia, child_ia, gfid) ||
+            !IA_EQUAL(source_ia, child_ia, type) ||
+            !IA_EQUAL(source_ia, child_ia, prot) ||
+            !IA_EQUAL(source_ia, child_ia, uid) ||
+            !IA_EQUAL(source_ia, child_ia, gid) ||
+            !afr_xattrs_are_equal(replies[source].xdata, replies[i].xdata))
+            goto out;
+    }
+
+    /*
+     * Metadata split brain is just about [amc]time
+     * We return our source.
+     */
+    ret = source;
+out:
+    return ret;
+}
 
-                mtime = mtime_ns(&replies[i].poststat);
-                source = i;
+static int
+__afr_selfheal_metadata_mark_pending_xattrs(call_frame_t *frame, xlator_t *this,
+                                            inode_t *inode,
+                                            struct afr_reply *replies,
+                                            unsigned char *sources)
+{
+    int ret = 0;
+    int i = 0;
+    int m_idx = 0;
+    afr_private_t *priv = NULL;
+    int raw[AFR_NUM_CHANGE_LOGS] = {0};
+    dict_t *xattr = NULL;
+
+    priv = this->private;
+    m_idx = afr_index_for_transaction_type(AFR_METADATA_TRANSACTION);
+    raw[m_idx] = 1;
+
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i])
+            continue;
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], raw,
+                                  sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret) {
+            ret = -1;
+            goto out;
         }
-
-        if (source == -1)
-                goto out;
-
-        source_ia = replies[source].poststat;
-        if (source_ia.ia_type != IA_IFDIR)
-                goto out;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (i == source)
-                        continue;
-
-                if (!replies[i].valid)
-                        continue;
-
-                if (replies[i].op_ret != 0)
-                        continue;
-
-                child_ia = replies[i].poststat;
-
-                if (!IA_EQUAL(source_ia, child_ia, gfid) ||
-                    !IA_EQUAL(source_ia, child_ia, type) ||
-                    !IA_EQUAL(source_ia, child_ia, prot) ||
-                    !IA_EQUAL(source_ia, child_ia, uid) ||
-                    !IA_EQUAL(source_ia, child_ia, gid) ||
-                    !afr_xattrs_are_equal (replies[source].xdata,
-                                           replies[i].xdata))
-                        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        ret = afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, -ret, AFR_MSG_SELF_HEAL_INFO,
+                   "Failed to set pending metadata xattr on child %d for %s", i,
+                   uuid_utoa(inode->gfid));
+            goto out;
         }
+    }
+
+    afr_replies_wipe(replies, priv->child_count);
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
 
-        /*
-         * Metadata split brain is just about [amc]time
-         * We return our source.
-         */
-        ret = source;
 out:
-        return ret;
+    if (xattr)
+        dict_unref(xattr);
+    return ret;
 }
 
-
 /*
  * Look for mismatching uid/gid or mode or user xattrs even if
  * AFR xattrs don't say so, and pick one arbitrarily as winner. */
 
 static int
-__afr_selfheal_metadata_finalize_source (call_frame_t *frame, xlator_t *this,
-                                         inode_t *inode,
-                                         unsigned char *sources,
-                                         unsigned char *sinks,
-					 unsigned char *healed_sinks,
-					 unsigned char *undid_pending,
-					 unsigned char *locked_on,
-					 struct afr_reply *replies)
+__afr_selfheal_metadata_finalize_source(call_frame_t *frame, xlator_t *this,
+                                        inode_t *inode, unsigned char *sources,
+                                        unsigned char *sinks,
+                                        unsigned char *healed_sinks,
+                                        unsigned char *undid_pending,
+                                        unsigned char *locked_on,
+                                        struct afr_reply *replies)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-	struct iatt srcstat = {0, };
-	int source = -1;
-	int sources_count = 0;
-
-	priv = this->private;
-
-	sources_count = AFR_COUNT (sources, priv->child_count);
-
-	if ((AFR_CMP (locked_on, healed_sinks, priv->child_count) == 0)
-	    || !sources_count) {
-
-                source = afr_mark_split_brain_source_sinks (frame, this, inode,
-                                                            sources, sinks,
-                                                            healed_sinks,
-                                                            locked_on, replies,
-                                                      AFR_METADATA_TRANSACTION);
-                if (source >= 0) {
-                        _afr_fav_child_reset_sink_xattrs (frame, this, inode,
-                                                         source, healed_sinks,
-                                                         undid_pending,
-                                                       AFR_METADATA_TRANSACTION,
-                                                         locked_on, replies);
-                        goto out;
-                }
-
-		/* If this is a directory mtime/ctime only split brain
-		   use the most recent */
-		source = afr_dirtime_splitbrain_source (frame, this,
-							replies, locked_on);
-		if (source != -1) {
-		        gf_msg (this->name, GF_LOG_INFO, 0,
-                                AFR_MSG_SPLIT_BRAIN, "clear time "
-				"split brain on %s",
-				 uuid_utoa (replies[source].poststat.ia_gfid));
-			sources[source] = 1;
-			healed_sinks[source] = 0;
-                        goto out;
-		}
-
-		if (!priv->metadata_splitbrain_forced_heal) {
-                        gf_event (EVENT_AFR_SPLIT_BRAIN, "subvol=%s;"
-                                  "type=metadata;file=%s",
-                                  this->name, uuid_utoa(inode->gfid));
-			return -EIO;
-		}
-
-		/* Metadata split brain, select one subvol
-		   arbitrarily */
-		for (i = 0; i < priv->child_count; i++) {
-			if (locked_on[i] && healed_sinks[i]) {
-				sources[i] = 1;
-				healed_sinks[i] = 0;
-				break;
-			}
-		}
-	}
-
-        /* No split brain at this point. If we were called from
-         * afr_heal_splitbrain_file(), abort.*/
-        if (afr_dict_contains_heal_op(frame))
-                return -EIO;
-
-        source = afr_choose_source_by_policy (priv, sources,
-                                              AFR_METADATA_TRANSACTION);
-        srcstat = replies[source].poststat;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!sources[i] || i == source)
-			continue;
-		if (!IA_EQUAL (srcstat, replies[i].poststat, type) ||
-		    !IA_EQUAL (srcstat, replies[i].poststat, uid) ||
-		    !IA_EQUAL (srcstat, replies[i].poststat, gid) ||
-		    !IA_EQUAL (srcstat, replies[i].poststat, prot)) {
-                        gf_msg_debug (this->name, 0, "%s: iatt mismatch "
-                                      "for source(%d) vs (%d)",
-                                      uuid_utoa
-                                      (replies[source].poststat.ia_gfid),
-                                      source, i);
-			sources[i] = 0;
-			healed_sinks[i] = 1;
-		}
-	}
-
-        for (i =0; i < priv->child_count; i++) {
-		if (!sources[i] || i == source)
-			continue;
-                if (!afr_xattrs_are_equal (replies[source].xdata,
-                                           replies[i].xdata)) {
-                        gf_msg_debug (this->name, 0, "%s: xattr mismatch "
-                                      "for source(%d) vs (%d)",
-                                      uuid_utoa
-                                      (replies[source].poststat.ia_gfid),
-                                      source, i);
-                        sources[i] = 0;
-                        healed_sinks[i] = 1;
-                }
+    int i = 0;
+    afr_private_t *priv = NULL;
+    struct iatt srcstat = {
+        0,
+    };
+    int source = -1;
+    int sources_count = 0;
+    int ret = 0;
+
+    priv = this->private;
+
+    sources_count = AFR_COUNT(sources, priv->child_count);
+
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count) {
+        source = afr_mark_split_brain_source_sinks(
+            frame, this, inode, sources, sinks, healed_sinks, locked_on,
+            replies, AFR_METADATA_TRANSACTION);
+        if (source >= 0) {
+            _afr_fav_child_reset_sink_xattrs(
+                frame, this, inode, source, healed_sinks, undid_pending,
+                AFR_METADATA_TRANSACTION, locked_on, replies);
+            goto out;
+        }
+
+        /* If this is a directory mtime/ctime only split brain
+           use the most recent */
+        source = afr_dirtime_splitbrain_source(frame, this, replies, locked_on);
+        if (source != -1) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SPLIT_BRAIN,
+                   "clear time "
+                   "split brain on %s",
+                   uuid_utoa(replies[source].poststat.ia_gfid));
+            sources[source] = 1;
+            healed_sinks[source] = 0;
+            goto out;
+        }
+
+        if (!priv->metadata_splitbrain_forced_heal) {
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;"
+                     "type=metadata;file=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(inode->gfid));
+            return -EIO;
         }
 
+        /* Metadata split brain, select one subvol
+           arbitrarily */
+        for (i = 0; i < priv->child_count; i++) {
+            if (locked_on[i] && healed_sinks[i]) {
+                sources[i] = 1;
+                healed_sinks[i] = 0;
+                break;
+            }
+        }
+    }
+
+    /* No split brain at this point. If we were called from
+     * afr_heal_splitbrain_file(), abort.*/
+    if (afr_dict_contains_heal_op(frame))
+        return -EIO;
+
+    source = afr_choose_source_by_policy(priv, sources,
+                                         AFR_METADATA_TRANSACTION);
+    srcstat = replies[source].poststat;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i] || i == source)
+            continue;
+        if (!IA_EQUAL(srcstat, replies[i].poststat, type) ||
+            !IA_EQUAL(srcstat, replies[i].poststat, uid) ||
+            !IA_EQUAL(srcstat, replies[i].poststat, gid) ||
+            !IA_EQUAL(srcstat, replies[i].poststat, prot)) {
+            gf_msg_debug(this->name, 0,
+                         "%s: iatt mismatch "
+                         "for source(%d) vs (%d)",
+                         uuid_utoa(replies[source].poststat.ia_gfid), source,
+                         i);
+            sources[i] = 0;
+            healed_sinks[i] = 1;
+        }
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i] || i == source)
+            continue;
+        if (!afr_xattrs_are_equal(replies[source].xdata, replies[i].xdata)) {
+            gf_msg_debug(this->name, 0,
+                         "%s: xattr mismatch "
+                         "for source(%d) vs (%d)",
+                         uuid_utoa(replies[source].poststat.ia_gfid), source,
+                         i);
+            sources[i] = 0;
+            healed_sinks[i] = 1;
+        }
+    }
+    if ((sources_count == priv->child_count) && (source > -1) &&
+        (AFR_COUNT(healed_sinks, priv->child_count) != 0)) {
+        ret = __afr_selfheal_metadata_mark_pending_xattrs(frame, this, inode,
+                                                          replies, sources);
+        if (ret < 0)
+            return ret;
+    }
 out:
-        afr_mark_active_sinks (this, sources, locked_on, healed_sinks);
-	return source;
+    afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+    return source;
 }
 
-
 int
-__afr_selfheal_metadata_prepare (call_frame_t *frame, xlator_t *this, inode_t *inode,
-				 unsigned char *locked_on, unsigned char *sources,
-				 unsigned char *sinks, unsigned char *healed_sinks,
-                                 unsigned char *undid_pending,
-				 struct afr_reply *replies, gf_boolean_t *pflag)
+__afr_selfheal_metadata_prepare(call_frame_t *frame, xlator_t *this,
+                                inode_t *inode, unsigned char *locked_on,
+                                unsigned char *sources, unsigned char *sinks,
+                                unsigned char *healed_sinks,
+                                unsigned char *undid_pending,
+                                struct afr_reply *replies, unsigned char *pflag)
 {
-	int ret = -1;
-	int source = -1;
-	afr_private_t *priv = NULL;
-	int i = 0;
-        uint64_t *witness = NULL;
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    uint64_t *witness = NULL;
 
-	priv = this->private;
+    priv = this->private;
 
-	ret = afr_selfheal_unlocked_discover (frame, inode, inode->gfid,
-					      replies);
-        if (ret)
-                return ret;
-
-        witness = alloca0 (sizeof (*witness) * priv->child_count);
-        ret = afr_selfheal_find_direction (frame, this, replies,
-					   AFR_METADATA_TRANSACTION,
-					   locked_on, sources, sinks, witness,
-                                           pflag);
-	if (ret)
-		return ret;
-
-        /* Initialize the healed_sinks[] array optimistically to
-           the intersection of to-be-healed (i.e sinks[]) and
-           the list of servers which are up (i.e locked_on[]).
-
-           As we encounter failures in the healing process, we
-           will unmark the respective servers in the healed_sinks[]
-           array.
-        */
-        AFR_INTERSECT (healed_sinks, sinks, locked_on, priv->child_count);
-
-        /* If any source has witness, pick first
-         * witness source and make everybody else sinks */
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i] && witness[i]) {
-                        source = i;
-                        break;
-                }
-        }
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
+    if (ret)
+        return ret;
 
-        if (source != -1) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (i != source && sources[i]) {
-                                sources[i] = 0;
-                                healed_sinks[i] = 1;
-                        }
-                }
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_METADATA_TRANSACTION, locked_on,
+                                      sources, sinks, witness, pflag);
+    if (ret)
+        return ret;
+
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
+
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
+
+    /* If any source has witness, pick first
+     * witness source and make everybody else sinks */
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i] && witness[i]) {
+            source = i;
+            break;
         }
+    }
 
-	source = __afr_selfheal_metadata_finalize_source (frame, this, inode,
-                                                          sources, sinks,
-                                                          healed_sinks,
-                                                          undid_pending,
-                                                          locked_on, replies);
+    if (source != -1) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (i != source && sources[i]) {
+                sources[i] = 0;
+                healed_sinks[i] = 1;
+            }
+        }
+    }
 
-	if (source < 0)
-		return -EIO;
+    source = __afr_selfheal_metadata_finalize_source(
+        frame, this, inode, sources, sinks, healed_sinks, undid_pending,
+        locked_on, replies);
 
-	return source;
-}
+    if (source < 0)
+        return -EIO;
 
-int
-afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode)
-{
-	afr_private_t *priv = NULL;
-	int ret = -1;
-	unsigned char *sources = NULL;
-	unsigned char *sinks = NULL;
-	unsigned char *data_lock = NULL;
-	unsigned char *healed_sinks = NULL;
-	unsigned char *undid_pending = NULL;
-	struct afr_reply *locked_replies = NULL;
-        gf_boolean_t did_sh = _gf_true;
-	int source = -1;
-
-	priv = this->private;
-
-	sources = alloca0 (priv->child_count);
-	sinks = alloca0 (priv->child_count);
-	healed_sinks = alloca0 (priv->child_count);
-	undid_pending = alloca0 (priv->child_count);
-	data_lock = alloca0 (priv->child_count);
-
-	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-	ret = afr_selfheal_inodelk (frame, this, inode, this->name,
-				    LLONG_MAX - 1, 0, data_lock);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_metadata_prepare (frame, this, inode,
-                                                       data_lock, sources,
-                                                       sinks, healed_sinks,
-                                                       undid_pending,
-						       locked_replies, NULL);
-		if (ret < 0)
-			goto unlock;
-
-		source = ret;
-
-                if (AFR_COUNT (healed_sinks, priv->child_count) == 0) {
-                        did_sh = _gf_false;
-                        goto unlock;
-                }
-
-                ret = __afr_selfheal_metadata_do (frame, this, inode, source,
-                                                  healed_sinks, locked_replies);
-                if (ret)
-                        goto unlock;
-
-                /* Restore atime/mtime for files that don't need data heal as
-                 * restoring timestamps happens only as a part of data-heal.
-                 */
-                if (!IA_ISREG (locked_replies[source].poststat.ia_type))
-                        afr_selfheal_restore_time (frame, this, inode, source,
-                                                  healed_sinks, locked_replies);
-
-                ret = afr_selfheal_undo_pending (frame, this, inode, sources,
-                                                 sinks, healed_sinks,
-                                                 undid_pending,
-                                                 AFR_METADATA_TRANSACTION,
-                                                 locked_replies, data_lock);
-	}
-unlock:
-	afr_selfheal_uninodelk (frame, this, inode, this->name,
-				LLONG_MAX -1, 0, data_lock);
-
-        if (did_sh)
-                afr_log_selfheal (inode->gfid, this, ret, "metadata", source,
-                                  sources, healed_sinks);
-        else
-                ret = 1;
-
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-	return ret;
+    return source;
 }
 
 int
-afr_selfheal_metadata_by_stbuf (xlator_t *this, struct iatt *stbuf)
+afr_selfheal_metadata(call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        inode_t      *inode      = NULL;
-        inode_t      *link_inode = NULL;
-        call_frame_t *frame      = NULL;
-        int          ret         = 0;
-
-        if (gf_uuid_is_null (stbuf->ia_gfid)) {
-                ret = -EINVAL;
-                goto out;
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *data_lock = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    gf_boolean_t did_sh = _gf_true;
+    int source = -1;
+
+    priv = this->private;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+    data_lock = alloca0(priv->child_count);
+
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = afr_selfheal_inodelk(frame, this, inode, this->name, LLONG_MAX - 1, 0,
+                               data_lock);
+    {
+        if (ret < priv->child_count) {
+            ret = -ENOTCONN;
+            goto unlock;
         }
 
-        inode = inode_new (this->itable);
-        if (!inode) {
-                ret = -ENOMEM;
-                goto out;
-        }
+        ret = __afr_selfheal_metadata_prepare(
+            frame, this, inode, data_lock, sources, sinks, healed_sinks,
+            undid_pending, locked_replies, NULL);
+        if (ret < 0)
+            goto unlock;
 
-        link_inode = inode_link (inode, NULL, NULL, stbuf);
-        if (!link_inode) {
-                ret = -ENOMEM;
-                goto out;
-        }
+        source = ret;
 
-        frame = afr_frame_create (this, &ret);
-        if (!frame) {
-                ret = -ret;
-                goto out;
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            did_sh = _gf_false;
+            goto unlock;
         }
 
-        ret = afr_selfheal_metadata (frame, this, link_inode);
+        ret = __afr_selfheal_metadata_do(frame, this, inode, source,
+                                         healed_sinks, locked_replies);
+        if (ret)
+            goto unlock;
+
+        afr_selfheal_restore_time(frame, this, inode, source, healed_sinks,
+                                  locked_replies);
+
+        ret = afr_selfheal_undo_pending(
+            frame, this, inode, sources, sinks, healed_sinks, undid_pending,
+            AFR_METADATA_TRANSACTION, locked_replies, data_lock);
+    }
+unlock:
+    afr_selfheal_uninodelk(frame, this, inode, this->name, LLONG_MAX - 1, 0,
+                           data_lock);
+
+    if (did_sh)
+        afr_log_selfheal(inode->gfid, this, ret, "metadata", source, sources,
+                         healed_sinks);
+    else
+        ret = 1;
+
+    if (locked_replies)
+        afr_replies_wipe(locked_replies, priv->child_count);
+    return ret;
+}
+
+int
+afr_selfheal_metadata_by_stbuf(xlator_t *this, struct iatt *stbuf)
+{
+    inode_t *inode = NULL;
+    inode_t *link_inode = NULL;
+    call_frame_t *frame = NULL;
+    int ret = 0;
+
+    if (gf_uuid_is_null(stbuf->ia_gfid)) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    inode = inode_new(this->itable);
+    if (!inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    link_inode = inode_link(inode, NULL, NULL, stbuf);
+    if (!link_inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    ret = afr_selfheal_metadata(frame, this, link_inode);
 out:
-        if (inode)
-                inode_unref (inode);
-        if (link_inode)
-                inode_unref (link_inode);
-        if (frame)
-                AFR_STACK_DESTROY (frame);
-        return ret;
+    if (inode)
+        inode_unref(inode);
+    if (link_inode)
+        inode_unref(link_inode);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-name.c b/xlators/cluster/afr/src/afr-self-heal-name.c
index 556d14baf82..834aac86d48 100644
--- a/xlators/cluster/afr/src/afr-self-heal-name.c
+++ b/xlators/cluster/afr/src/afr-self-heal-name.c
@@ -8,690 +8,609 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include "events.h"
+#include <glusterfs/events.h>
 #include "afr.h"
 #include "afr-self-heal.h"
 #include "afr-messages.h"
 
 int
-__afr_selfheal_assign_gfid (xlator_t *this, inode_t *parent, uuid_t pargfid,
-                            const char *bname, inode_t *inode,
-                            struct afr_reply *replies, void *gfid,
-                            unsigned char *locked_on, int source,
-                            gf_boolean_t is_gfid_absent)
+__afr_selfheal_assign_gfid(xlator_t *this, inode_t *parent, uuid_t pargfid,
+                           const char *bname, inode_t *inode,
+                           struct afr_reply *replies, void *gfid,
+                           unsigned char *locked_on, int source,
+                           unsigned char *sources, gf_boolean_t is_gfid_absent,
+                           int *gfid_idx)
 {
-	int             ret          = 0;
-        int             up_count     = 0;
-        int             locked_count = 0;
-	afr_private_t  *priv         = NULL;
+    int ret = 0;
+    int up_count = 0;
+    int locked_count = 0;
+    afr_private_t *priv = NULL;
 
-	priv = this->private;
+    priv = this->private;
 
-	gf_uuid_copy (parent->gfid, pargfid);
+    gf_uuid_copy(parent->gfid, pargfid);
 
-        if (is_gfid_absent) {
-                /* Ensure all children of AFR are up before performing gfid heal, to
-                 * guard against the possibility of gfid split brain. */
+    if (is_gfid_absent) {
+        /* Ensure all children of AFR are up before performing gfid heal, to
+         * guard against the possibility of gfid split brain. */
 
-                up_count = AFR_COUNT (priv->child_up, priv->child_count);
-                if (up_count != priv->child_count) {
-                        ret = -EIO;
-                        goto out;
-                }
+        up_count = AFR_COUNT(priv->child_up, priv->child_count);
+        if (up_count != priv->child_count) {
+            ret = -EIO;
+            goto out;
+        }
 
-                locked_count = AFR_COUNT (locked_on, priv->child_count);
-                if (locked_count != priv->child_count) {
-                        ret = -EIO;
-                        goto out;
-                }
+        locked_count = AFR_COUNT(locked_on, priv->child_count);
+        if (locked_count != priv->child_count) {
+            ret = -EIO;
+            goto out;
         }
+    }
 
-        afr_lookup_and_heal_gfid (this, parent, bname, inode, replies, source,
-                                  gfid);
+    ret = afr_lookup_and_heal_gfid(this, parent, bname, inode, replies, source,
+                                   sources, gfid, gfid_idx);
 
 out:
-	return ret;
+    return ret;
 }
 
 int
-__afr_selfheal_name_impunge (call_frame_t *frame, xlator_t *this,
-                             inode_t *parent, uuid_t pargfid,
-                             const char *bname, inode_t *inode,
-                             struct afr_reply *replies, int gfid_idx)
+__afr_selfheal_name_impunge(call_frame_t *frame, xlator_t *this,
+                            inode_t *parent, uuid_t pargfid, const char *bname,
+                            inode_t *inode, struct afr_reply *replies,
+                            int gfid_idx)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-        int ret = 0;
-        unsigned char *sources = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int ret = 0;
+    unsigned char *sources = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-	sources = alloca0 (priv->child_count);
+    sources = alloca0(priv->child_count);
 
-	gf_uuid_copy (parent->gfid, pargfid);
+    gf_uuid_copy(parent->gfid, pargfid);
 
-	for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret != 0)
-			continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
 
-		if (gf_uuid_compare (replies[i].poststat.ia_gfid,
-				  replies[gfid_idx].poststat.ia_gfid) == 0) {
-                        sources[i] = 1;
-			continue;
-                }
+        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
+                            replies[gfid_idx].poststat.ia_gfid) == 0) {
+            sources[i] = 1;
+            continue;
         }
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i])
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i])
+            continue;
 
-		ret |= afr_selfheal_recreate_entry (frame, i, gfid_idx, sources,
-                                                    parent, bname, inode,
-                                                    replies);
-	}
+        ret |= afr_selfheal_recreate_entry(frame, i, gfid_idx, sources, parent,
+                                           bname, inode, replies);
+    }
 
-	return ret;
+    return ret;
 }
 
-
 int
-__afr_selfheal_name_expunge (xlator_t *this, inode_t *parent, uuid_t pargfid,
-                             const char *bname, inode_t *inode,
-			     struct afr_reply *replies)
-{
-	loc_t loc = {0, };
-	int i = 0;
-	afr_private_t *priv = NULL;
-	char g[64];
-	int ret = 0;
-
-	priv = this->private;
-
-	loc.parent = inode_ref (parent);
-	gf_uuid_copy (loc.pargfid, pargfid);
-	loc.name = bname;
-	loc.inode = inode_ref (inode);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-
-		if (replies[i].op_ret)
-			continue;
-
-		switch (replies[i].poststat.ia_type) {
-		case IA_IFDIR:
-		        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_EXPUNGING_FILE_OR_DIR,
-			        "expunging dir %s/%s (%s) on %s",
-			        uuid_utoa (pargfid), bname,
-			        uuid_utoa_r (replies[i].poststat.ia_gfid, g),
-			        priv->children[i]->name);
-
-			ret |= syncop_rmdir (priv->children[i], &loc, 1, NULL,
-                                             NULL);
-			break;
-		default:
-		        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_EXPUNGING_FILE_OR_DIR,
-		                "expunging file %s/%s (%s) on %s",
-			        uuid_utoa (pargfid), bname,
-			        uuid_utoa_r (replies[i].poststat.ia_gfid, g),
-			        priv->children[i]->name);
-
-			ret |= syncop_unlink (priv->children[i], &loc, NULL,
-                                              NULL);
-			break;
-		}
-	}
-
-	loc_wipe (&loc);
-
-	return ret;
-
-}
-
-/* This function is to be called after ensuring that there is no gfid mismatch
- * for the inode across multiple sources
- */
-static int
-afr_selfheal_gfid_idx_get (xlator_t *this, struct afr_reply *replies,
-                           unsigned char *sources)
+__afr_selfheal_name_expunge(xlator_t *this, inode_t *parent, uuid_t pargfid,
+                            const char *bname, inode_t *inode,
+                            struct afr_reply *replies)
 {
-        int             i        =  0;
-        int             gfid_idx = -1;
-        afr_private_t  *priv     =  NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int ret = 0;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret != 0)
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                if (!sources[i])
-                        continue;
+        if (replies[i].op_ret)
+            continue;
 
-                if (gf_uuid_is_null (replies[i].poststat.ia_gfid))
-                        continue;
+        ret |= afr_selfheal_entry_delete(this, parent, bname, inode, i,
+                                         replies);
+    }
 
-                gfid_idx = i;
-                break;
-        }
-        return gfid_idx;
+    return ret;
 }
 
 static gf_boolean_t
-afr_selfheal_name_need_heal_check (xlator_t *this, struct afr_reply *replies)
+afr_selfheal_name_need_heal_check(xlator_t *this, struct afr_reply *replies)
 {
-        int             i           = 0;
-	int             first_idx   = -1;
-        gf_boolean_t    need_heal   = _gf_false;
-        afr_private_t  *priv        = NULL;
-
-        priv = this->private;
+    int i = 0;
+    int first_idx = -1;
+    gf_boolean_t need_heal = _gf_false;
+    afr_private_t *priv = NULL;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
+    priv = this->private;
 
-                if ((replies[i].op_ret == -1) &&
-                    (replies[i].op_errno == ENODATA))
-                        need_heal = _gf_true;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-		if (first_idx == -1) {
-			first_idx = i;
-			continue;
-		}
+        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA))
+            need_heal = _gf_true;
 
-		if (replies[i].op_ret != replies[first_idx].op_ret)
-			need_heal = _gf_true;
+        if (first_idx == -1) {
+            first_idx = i;
+            continue;
+        }
 
-		if (gf_uuid_compare (replies[i].poststat.ia_gfid,
-				  replies[first_idx].poststat.ia_gfid))
-			need_heal = _gf_true;
+        if (replies[i].op_ret != replies[first_idx].op_ret)
+            need_heal = _gf_true;
 
-                if ((replies[i].op_ret == 0) &&
-                    (gf_uuid_is_null(replies[i].poststat.ia_gfid)))
-                        need_heal = _gf_true;
+        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
+                            replies[first_idx].poststat.ia_gfid))
+            need_heal = _gf_true;
 
-	}
+        if ((replies[i].op_ret == 0) &&
+            (gf_uuid_is_null(replies[i].poststat.ia_gfid)))
+            need_heal = _gf_true;
+    }
 
-        return need_heal;
+    return need_heal;
 }
 
 static int
-afr_selfheal_name_type_mismatch_check (xlator_t *this, struct afr_reply *replies,
-                                       int source, unsigned char *sources,
-                                       uuid_t pargfid, const char *bname)
+afr_selfheal_name_type_mismatch_check(xlator_t *this, struct afr_reply *replies,
+                                      int source, unsigned char *sources,
+                                      uuid_t pargfid, const char *bname)
 {
-        int             i           = 0;
-        int             type_idx    = -1;
-        ia_type_t       inode_type  = IA_INVAL;
-        ia_type_t       inode_type1 = IA_INVAL;
-        afr_private_t  *priv        = NULL;
+    int i = 0;
+    int type_idx = -1;
+    ia_type_t inode_type = IA_INVAL;
+    ia_type_t inode_type1 = IA_INVAL;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret != 0)
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
 
-                if (replies[i].poststat.ia_type == IA_INVAL)
-                        continue;
+        if (replies[i].poststat.ia_type == IA_INVAL)
+            continue;
 
-                if (inode_type == IA_INVAL) {
-                        inode_type = replies[i].poststat.ia_type;
-                        type_idx = i;
-                        continue;
-                }
-                inode_type1 = replies[i].poststat.ia_type;
-                if (sources[i] || source == -1) {
-                        if ((sources[type_idx] || source == -1) &&
-                            (inode_type != inode_type1)) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        AFR_MSG_SPLIT_BRAIN,
-                                        "Type mismatch for <gfid:%s>/%s: "
-                                        "%s on %s and %s on %s",
-                                        uuid_utoa(pargfid), bname,
-                                        gf_inode_type_to_str (inode_type1),
-                                        priv->children[i]->name,
-                                        gf_inode_type_to_str (inode_type),
-                                        priv->children[type_idx]->name);
-                                gf_event (EVENT_AFR_SPLIT_BRAIN,
-                                         "subvol=%s;type=file;"
-                                         "file=<gfid:%s>/%s;count=2;"
-                                         "child-%d=%s;type-%d=%s;child-%d=%s;"
-                                         "type-%d=%s", this->name,
-                                         uuid_utoa (pargfid), bname, i,
-                                         priv->children[i]->name, i,
-                                         gf_inode_type_to_str (inode_type1),
-                                         type_idx,
-                                         priv->children[type_idx]->name,
-                                         type_idx,
-                                         gf_inode_type_to_str (inode_type));
-                                return -EIO;
-                        }
-                        inode_type = replies[i].poststat.ia_type;
-                        type_idx = i;
-                }
+        if (inode_type == IA_INVAL) {
+            inode_type = replies[i].poststat.ia_type;
+            type_idx = i;
+            continue;
         }
-        return 0;
+        inode_type1 = replies[i].poststat.ia_type;
+        if (sources[i] || source == -1) {
+            if ((sources[type_idx] || source == -1) &&
+                (inode_type != inode_type1)) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN,
+                       "Type mismatch for <gfid:%s>/%s: "
+                       "%s on %s and %s on %s",
+                       uuid_utoa(pargfid), bname,
+                       gf_inode_type_to_str(inode_type1),
+                       priv->children[i]->name,
+                       gf_inode_type_to_str(inode_type),
+                       priv->children[type_idx]->name);
+                gf_event(EVENT_AFR_SPLIT_BRAIN,
+                         "client-pid=%d;"
+                         "subvol=%s;type=file;"
+                         "file=<gfid:%s>/%s;count=2;"
+                         "child-%d=%s;type-%d=%s;child-%d=%s;"
+                         "type-%d=%s",
+                         this->ctx->cmd_args.client_pid, this->name,
+                         uuid_utoa(pargfid), bname, i, priv->children[i]->name,
+                         i, gf_inode_type_to_str(inode_type1), type_idx,
+                         priv->children[type_idx]->name, type_idx,
+                         gf_inode_type_to_str(inode_type));
+                return -EIO;
+            }
+            inode_type = replies[i].poststat.ia_type;
+            type_idx = i;
+        }
+    }
+    return 0;
 }
 
 static int
-afr_selfheal_name_gfid_mismatch_check (xlator_t *this, struct afr_reply *replies,
-                                       int source, unsigned char *sources,
-                                       int *gfid_idx, uuid_t pargfid,
-                                       const char *bname, inode_t *inode,
-                                       unsigned char *locked_on, dict_t *xdata)
+afr_selfheal_name_gfid_mismatch_check(xlator_t *this, struct afr_reply *replies,
+                                      int source, unsigned char *sources,
+                                      int *gfid_idx, uuid_t pargfid,
+                                      const char *bname, inode_t *inode,
+                                      unsigned char *locked_on, dict_t *xdata)
 {
-        int             i             = 0;
-	int             gfid_idx_iter = -1;
-        int             ret           = -1;
-        void           *gfid          = NULL;
-        void           *gfid1         = NULL;
-        afr_private_t  *priv          = NULL;
-
-        priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret != 0)
-                        continue;
-
-		if (gf_uuid_is_null (replies[i].poststat.ia_gfid))
-			continue;
-
-		if (!gfid) {
-			gfid = &replies[i].poststat.ia_gfid;
-			gfid_idx_iter = i;
-			continue;
-		}
-
-                gfid1 = &replies[i].poststat.ia_gfid;
-		if (sources[i] || source == -1) {
-			if ((sources[gfid_idx_iter] || source == -1) &&
-			    gf_uuid_compare (gfid, gfid1)) {
-                                ret = afr_gfid_split_brain_source (this,
-                                                                   replies,
-                                                                   inode,
-                                                                   pargfid,
-                                                                   bname,
-                                                                   gfid_idx_iter,
-                                                                   i, locked_on,
-                                                                   gfid_idx,
-                                                                   xdata);
-                                if (!ret && *gfid_idx >= 0) {
-                                        ret = dict_set_str (xdata,
-                                                             "gfid-heal-msg",
-                                                             "GFID split-brain "
-                                                             "resolved");
-                                        if (ret)
-                                                gf_msg (this->name,
-                                                        GF_LOG_ERROR, 0,
-                                                        AFR_MSG_DICT_SET_FAILED,
-                                                        "Error setting gfid-"
-                                                        "heal-msg dict");
-                                }
-                                return ret;
-			}
-                        gfid = &replies[i].poststat.ia_gfid;
-			gfid_idx_iter = i;
-		}
-	}
-
-        *gfid_idx = gfid_idx_iter;
-        return 0;
+    int i = 0;
+    int gfid_idx_iter = -1;
+    int ret = -1;
+    void *gfid = NULL;
+    void *gfid1 = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
+
+        if (gf_uuid_is_null(replies[i].poststat.ia_gfid))
+            continue;
+
+        if (!gfid) {
+            gfid = &replies[i].poststat.ia_gfid;
+            gfid_idx_iter = i;
+            continue;
+        }
+
+        gfid1 = &replies[i].poststat.ia_gfid;
+        if (sources[i] || source == -1) {
+            if ((sources[gfid_idx_iter] || source == -1) &&
+                gf_uuid_compare(gfid, gfid1)) {
+                ret = afr_gfid_split_brain_source(this, replies, inode, pargfid,
+                                                  bname, gfid_idx_iter, i,
+                                                  locked_on, gfid_idx, xdata);
+                if (!ret && *gfid_idx >= 0) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   "GFID split-brain resolved");
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error setting gfid-"
+                               "heal-msg dict");
+                }
+                return ret;
+            }
+            gfid = &replies[i].poststat.ia_gfid;
+            gfid_idx_iter = i;
+        }
+    }
+
+    *gfid_idx = gfid_idx_iter;
+    return 0;
 }
 
 static gf_boolean_t
-afr_selfheal_name_source_empty_check (xlator_t *this, struct afr_reply *replies,
-                                      unsigned char *sources, int source)
+afr_selfheal_name_source_empty_check(xlator_t *this, struct afr_reply *replies,
+                                     unsigned char *sources, int source)
 {
-	int             i               = 0;
-	afr_private_t  *priv            = NULL;
-	gf_boolean_t    source_is_empty = _gf_true;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    gf_boolean_t source_is_empty = _gf_true;
 
-	priv = this->private;
+    priv = this->private;
 
-        if (source == -1) {
-                source_is_empty = _gf_false;
-                goto out;
-        }
+    if (source == -1) {
+        source_is_empty = _gf_false;
+        goto out;
+    }
 
-	for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
 
-                if (replies[i].op_ret == -1 && replies[i].op_errno == ENOENT)
-                        continue;
+        if (replies[i].op_ret == -1 && replies[i].op_errno == ENOENT)
+            continue;
 
-                source_is_empty = _gf_false;
-                break;
-	}
+        source_is_empty = _gf_false;
+        break;
+    }
 out:
-        return source_is_empty;
+    return source_is_empty;
 }
 
 int
-__afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent,
-                        uuid_t pargfid, const char *bname, inode_t *inode,
-                        unsigned char *sources, unsigned char *sinks,
-			unsigned char *healed_sinks, int source,
-			unsigned char *locked_on, struct afr_reply *replies,
-                        void *gfid_req, dict_t *xdata)
+__afr_selfheal_name_do(call_frame_t *frame, xlator_t *this, inode_t *parent,
+                       uuid_t pargfid, const char *bname, inode_t *inode,
+                       unsigned char *sources, unsigned char *sinks,
+                       unsigned char *healed_sinks, int source,
+                       unsigned char *locked_on, struct afr_reply *replies,
+                       void *gfid_req, dict_t *xdata)
 {
-	int             gfid_idx        = -1;
-        int             ret             = -1;
-	void           *gfid            = NULL;
-	gf_boolean_t    source_is_empty = _gf_true;
-	gf_boolean_t    need_heal       = _gf_false;
-        gf_boolean_t    is_gfid_absent  = _gf_false;
-
-        need_heal = afr_selfheal_name_need_heal_check (this, replies);
-	if (!need_heal)
-		return 0;
-
-        source_is_empty = afr_selfheal_name_source_empty_check (this, replies,
-                                                                sources,
-                                                                source);
-	if (source_is_empty) {
-		ret = __afr_selfheal_name_expunge (this, parent, pargfid,
-						    bname, inode, replies);
-                if (ret == -EIO)
-                        ret = -1;
-                return ret;
-        }
-
-        ret = afr_selfheal_name_type_mismatch_check (this, replies, source,
-                                                     sources, pargfid, bname);
-        if (ret)
-                return ret;
+    int gfid_idx = -1;
+    int ret = -1;
+    void *gfid = NULL;
+    gf_boolean_t source_is_empty = _gf_true;
+    gf_boolean_t need_heal = _gf_false;
+    gf_boolean_t is_gfid_absent = _gf_false;
+
+    need_heal = afr_selfheal_name_need_heal_check(this, replies);
+    if (!need_heal)
+        return 0;
 
-        ret = afr_selfheal_name_gfid_mismatch_check (this, replies, source,
-                                                     sources, &gfid_idx,
-                                                     pargfid, bname, inode,
-                                                     locked_on, xdata);
-        if (ret)
-                return ret;
+    source_is_empty = afr_selfheal_name_source_empty_check(this, replies,
+                                                           sources, source);
+    if (source_is_empty) {
+        ret = __afr_selfheal_name_expunge(this, parent, pargfid, bname, inode,
+                                          replies);
+        if (ret == -EIO)
+            ret = -1;
+        return ret;
+    }
 
-	if (gfid_idx == -1) {
-                if (!gfid_req || gf_uuid_is_null (gfid_req))
-                        return -1;
-                gfid = gfid_req;
-        } else {
-                gfid = &replies[gfid_idx].poststat.ia_gfid;
-        }
+    ret = afr_selfheal_name_type_mismatch_check(this, replies, source, sources,
+                                                pargfid, bname);
+    if (ret)
+        return ret;
 
-        is_gfid_absent = (gfid_idx == -1) ? _gf_true : _gf_false;
-	ret = __afr_selfheal_assign_gfid (this, parent, pargfid, bname, inode,
-                                          replies, gfid, locked_on, source,
-                                          is_gfid_absent);
-        if (ret)
-                return ret;
+    ret = afr_selfheal_name_gfid_mismatch_check(this, replies, source, sources,
+                                                &gfid_idx, pargfid, bname,
+                                                inode, locked_on, xdata);
+    if (ret)
+        return ret;
 
-        if (gfid_idx == -1) {
-                gfid_idx = afr_selfheal_gfid_idx_get (this, replies, sources);
-                if (gfid_idx == -1)
-                        return -1;
-        }
+    if (gfid_idx == -1) {
+        if (!gfid_req || gf_uuid_is_null(gfid_req))
+            return -1;
+        gfid = gfid_req;
+    } else {
+        gfid = &replies[gfid_idx].poststat.ia_gfid;
+        if (source == -1)
+            /* Either entry split-brain or dirty xattrs are present on parent.*/
+            source = gfid_idx;
+    }
+
+    is_gfid_absent = (gfid_idx == -1) ? _gf_true : _gf_false;
+    ret = __afr_selfheal_assign_gfid(this, parent, pargfid, bname, inode,
+                                     replies, gfid, locked_on, source, sources,
+                                     is_gfid_absent, &gfid_idx);
+    if (ret || (gfid_idx < 0))
+        return ret;
 
-	ret = __afr_selfheal_name_impunge (frame, this, parent, pargfid,
-                                           bname, inode, replies, gfid_idx);
-        if (ret == -EIO)
-                ret = -1;
+    ret = __afr_selfheal_name_impunge(frame, this, parent, pargfid, bname,
+                                      inode, replies, gfid_idx);
+    if (ret == -EIO)
+        ret = -1;
 
-        return ret;
+    return ret;
 }
 
-
 int
-__afr_selfheal_name_finalize_source (xlator_t *this, unsigned char *sources,
-				     unsigned char *healed_sinks,
-				     unsigned char *locked_on,
-                                     uint64_t *witness)
+__afr_selfheal_name_finalize_source(xlator_t *this, unsigned char *sources,
+                                    unsigned char *healed_sinks,
+                                    unsigned char *locked_on, uint64_t *witness)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-	int source = -1;
-	int sources_count = 0;
-
-	priv = this->private;
-
-	sources_count = AFR_COUNT (sources, priv->child_count);
-
-	if ((AFR_CMP (locked_on, healed_sinks, priv->child_count) == 0)
-            || !sources_count || afr_does_witness_exist (this, witness)) {
-                memset (sources, 0, sizeof (*sources) * priv->child_count);
-                afr_mark_active_sinks (this, sources, locked_on, healed_sinks);
-		return -1;
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sources[i]) {
-			source = i;
-			break;
-		}
-	}
-
-	return source;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    int sources_count = 0;
+
+    priv = this->private;
+
+    sources_count = AFR_COUNT(sources, priv->child_count);
+
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count || afr_does_witness_exist(this, witness)) {
+        memset(sources, 0, sizeof(*sources) * priv->child_count);
+        afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+        return -1;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i]) {
+            source = i;
+            break;
+        }
+    }
+
+    return source;
 }
 
 int
-__afr_selfheal_name_prepare (call_frame_t *frame, xlator_t *this, inode_t *parent,
-			     uuid_t pargfid, unsigned char *locked_on,
-			     unsigned char *sources, unsigned char *sinks,
-			     unsigned char *healed_sinks, int *source_p)
+__afr_selfheal_name_prepare(call_frame_t *frame, xlator_t *this,
+                            inode_t *parent, uuid_t pargfid,
+                            unsigned char *locked_on, unsigned char *sources,
+                            unsigned char *sinks, unsigned char *healed_sinks,
+                            int *source_p)
 {
-	int ret = -1;
-	int source = -1;
-	afr_private_t *priv = NULL;
-        struct afr_reply *replies = NULL;
-        uint64_t *witness = NULL;
-
-	priv = this->private;
-
-        replies = alloca0 (priv->child_count * sizeof(*replies));
-
-	ret = afr_selfheal_unlocked_discover (frame, parent, pargfid, replies);
-	if (ret)
-		goto out;
-
-        witness = alloca0 (sizeof (*witness) * priv->child_count);
-	ret = afr_selfheal_find_direction (frame, this, replies,
-					   AFR_ENTRY_TRANSACTION,
-					   locked_on, sources, sinks, witness,
-                                           NULL);
-	if (ret)
-		goto out;
-
-        /* Initialize the healed_sinks[] array optimistically to
-           the intersection of to-be-healed (i.e sinks[]) and
-           the list of servers which are up (i.e locked_on[]).
-
-           As we encounter failures in the healing process, we
-           will unmark the respective servers in the healed_sinks[]
-           array.
-        */
-        AFR_INTERSECT (healed_sinks, sinks, locked_on, priv->child_count);
-
-	source = __afr_selfheal_name_finalize_source (this, sources,
-                                                      healed_sinks,
-						      locked_on, witness);
-	if (source < 0) {
-		/* If source is < 0 (typically split-brain), we perform a
-		   conservative merge of entries rather than erroring out */
-	}
-	*source_p = source;
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+
+    replies = alloca0(priv->child_count * sizeof(*replies));
+
+    ret = afr_selfheal_unlocked_discover(frame, parent, pargfid, replies);
+    if (ret)
+        goto out;
+
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_ENTRY_TRANSACTION, locked_on, sources,
+                                      sinks, witness, NULL);
+    if (ret)
+        goto out;
+
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
+
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
+
+    source = __afr_selfheal_name_finalize_source(this, sources, healed_sinks,
+                                                 locked_on, witness);
+    if (source < 0) {
+        /* If source is < 0 (typically split-brain), we perform a
+           conservative merge of entries rather than erroring out */
+    }
+    *source_p = source;
 
 out:
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
 
-	return ret;
+    return ret;
 }
 
-
 int
-afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent,
-		      uuid_t pargfid, const char *bname, void *gfid_req,
-                      dict_t *xdata)
+afr_selfheal_name_do(call_frame_t *frame, xlator_t *this, inode_t *parent,
+                     uuid_t pargfid, const char *bname, void *gfid_req,
+                     dict_t *xdata)
 {
-	afr_private_t *priv = NULL;
-	unsigned char *sources = NULL;
-	unsigned char *sinks = NULL;
-	unsigned char *healed_sinks = NULL;
-	unsigned char *locked_on = NULL;
-	int source = -1;
-        struct afr_reply *replies = NULL;
-	int ret = -1;
-	inode_t *inode = NULL;
-        dict_t *xattr = NULL;
-
-        xattr = dict_new ();
-        if (!xattr)
-                return -ENOMEM;
-
-        ret = dict_set_int32 (xattr, GF_GFIDLESS_LOOKUP, 1);
-        if (ret) {
-                dict_unref (xattr);
-                return -1;
+    afr_private_t *priv = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *locked_on = NULL;
+    int source = -1;
+    struct afr_reply *replies = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
+    dict_t *xattr = NULL;
+
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
+
+    ret = dict_set_int32_sizen(xattr, GF_GFIDLESS_LOOKUP, 1);
+    if (ret) {
+        dict_unref(xattr);
+        return -1;
+    }
+
+    priv = this->private;
+
+    locked_on = alloca0(priv->child_count);
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+
+    replies = alloca0(priv->child_count * sizeof(*replies));
+
+    ret = afr_selfheal_entrylk(frame, this, parent, this->name, bname,
+                               locked_on);
+    {
+        if (ret < priv->child_count) {
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_name_prepare(frame, this, parent, pargfid,
+                                          locked_on, sources, sinks,
+                                          healed_sinks, &source);
+        if (ret)
+            goto unlock;
+
+        inode = afr_selfheal_unlocked_lookup_on(frame, parent, bname, replies,
+                                                locked_on, xattr);
+        if (!inode) {
+            ret = -ENOMEM;
+            goto unlock;
         }
 
-	priv = this->private;
-
-	locked_on = alloca0 (priv->child_count);
-	sources = alloca0 (priv->child_count);
-	sinks = alloca0 (priv->child_count);
-	healed_sinks = alloca0 (priv->child_count);
-
-	replies = alloca0 (priv->child_count * sizeof(*replies));
-
-	ret = afr_selfheal_entrylk (frame, this, parent, this->name, bname,
-				    locked_on);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_name_prepare (frame, this, parent, pargfid,
-						   locked_on, sources, sinks,
-						   healed_sinks, &source);
-		if (ret)
-			goto unlock;
-
-                inode = afr_selfheal_unlocked_lookup_on (frame, parent, bname,
-						         replies, locked_on,
-                                                         xattr);
-		if (!inode) {
-			ret = -ENOMEM;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_name_do (frame, this, parent, pargfid,
-                                              bname, inode, sources, sinks,
-                                              healed_sinks, source, locked_on,
-                                              replies, gfid_req, xdata);
-	}
+        ret = __afr_selfheal_name_do(frame, this, parent, pargfid, bname, inode,
+                                     sources, sinks, healed_sinks, source,
+                                     locked_on, replies, gfid_req, xdata);
+    }
 unlock:
-	afr_selfheal_unentrylk (frame, this, parent, this->name, bname,
-				locked_on, NULL);
-	if (inode)
-		inode_unref (inode);
+    afr_selfheal_unentrylk(frame, this, parent, this->name, bname, locked_on,
+                           NULL);
+    if (inode)
+        inode_unref(inode);
 
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
-        if (xattr)
-                dict_unref (xattr);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    if (xattr)
+        dict_unref(xattr);
 
-	return ret;
+    return ret;
 }
 
-
 int
-afr_selfheal_name_unlocked_inspect (call_frame_t *frame, xlator_t *this,
-				    inode_t *parent, uuid_t pargfid,
-				    const char *bname, gf_boolean_t *need_heal)
+afr_selfheal_name_unlocked_inspect(call_frame_t *frame, xlator_t *this,
+                                   inode_t *parent, uuid_t pargfid,
+                                   const char *bname, gf_boolean_t *need_heal)
 {
-	afr_private_t *priv = NULL;
-	int i = 0;
-	struct afr_reply *replies = NULL;
-	inode_t *inode = NULL;
-	int first_idx = -1;
-
-	priv = this->private;
-
-	replies = alloca0 (sizeof (*replies) * priv->child_count);
-
-	inode = afr_selfheal_unlocked_lookup_on (frame, parent, bname,
-						 replies, priv->child_up, NULL);
-	if (!inode)
-		return -ENOMEM;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-
-                if ((replies[i].op_ret == -1) &&
-                    (replies[i].op_errno == ENODATA))
-                        *need_heal = _gf_true;
-
-		if (first_idx == -1) {
-			first_idx = i;
-			continue;
-		}
-
-		if (replies[i].op_ret != replies[first_idx].op_ret)
-			*need_heal = _gf_true;
-
-		if (gf_uuid_compare (replies[i].poststat.ia_gfid,
-				  replies[first_idx].poststat.ia_gfid))
-			*need_heal = _gf_true;
-	}
-
-	if (inode)
-		inode_unref (inode);
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
-	return 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    struct afr_reply *replies = NULL;
+    inode_t *inode = NULL;
+    int first_idx = -1;
+    afr_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    replies = alloca0(sizeof(*replies) * priv->child_count);
+
+    inode = afr_selfheal_unlocked_lookup_on(frame, parent, bname, replies,
+                                            local->child_up, NULL);
+    if (!inode)
+        return -ENOMEM;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA)) {
+            *need_heal = _gf_true;
+            break;
+        }
+
+        if (first_idx == -1) {
+            first_idx = i;
+            continue;
+        }
+
+        if (replies[i].op_ret != replies[first_idx].op_ret) {
+            *need_heal = _gf_true;
+            break;
+        }
+
+        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
+                            replies[first_idx].poststat.ia_gfid)) {
+            *need_heal = _gf_true;
+            break;
+        }
+    }
+
+    if (inode)
+        inode_unref(inode);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    return 0;
 }
 
 int
-afr_selfheal_name (xlator_t *this, uuid_t pargfid, const char *bname,
-                   void *gfid_req, dict_t *xdata)
+afr_selfheal_name(xlator_t *this, uuid_t pargfid, const char *bname,
+                  void *gfid_req, dict_t *xdata)
 {
-	inode_t *parent = NULL;
-	call_frame_t *frame = NULL;
-	int ret = -1;
-	gf_boolean_t need_heal = _gf_false;
-
-	parent = afr_inode_find (this, pargfid);
-	if (!parent)
-		goto out;
-
-	frame = afr_frame_create (this, NULL);
-	if (!frame)
-		goto out;
-
-	ret = afr_selfheal_name_unlocked_inspect (frame, this, parent, pargfid,
-						  bname, &need_heal);
-	if (ret)
-		goto out;
-
-	if (need_heal) {
-		ret = afr_selfheal_name_do (frame, this, parent, pargfid, bname,
-                                            gfid_req, xdata);
-                if (ret)
-                        goto out;
-        }
+    inode_t *parent = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -1;
+    gf_boolean_t need_heal = _gf_false;
+
+    parent = afr_inode_find(this, pargfid);
+    if (!parent)
+        goto out;
+
+    frame = afr_frame_create(this, NULL);
+    if (!frame)
+        goto out;
+
+    ret = afr_selfheal_name_unlocked_inspect(frame, this, parent, pargfid,
+                                             bname, &need_heal);
+    if (ret)
+        goto out;
+
+    if (need_heal) {
+        ret = afr_selfheal_name_do(frame, this, parent, pargfid, bname,
+                                   gfid_req, xdata);
+        if (ret)
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-	if (parent)
-		inode_unref (parent);
-	if (frame)
-		AFR_STACK_DESTROY (frame);
+    if (parent)
+        inode_unref(parent);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
 
-	return ret;
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index cd67d2a3192..48e6dbcfb18 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -8,265 +8,285 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _AFR_SELFHEAL_H
 #define _AFR_SELFHEAL_H
 
-#define AFR_SH_MIN_PARTICIPANTS 2
-
 /* Perform fop on all UP subvolumes and wait for all callbacks to return */
 
-#define AFR_ONALL(frame, rfn, fop, args ...) do {			\
-	afr_local_t *__local = frame->local;				\
-	afr_private_t *__priv = frame->this->private;			\
-        int __i = 0, __count = 0;                                       \
-        unsigned char *__child_up = NULL;                               \
-                                                                        \
-        __child_up = alloca0 (__priv->child_count);                     \
-        memcpy (__child_up, __priv->child_up,                           \
-                sizeof (*__child_up) * __priv->child_count);            \
-        __count = AFR_COUNT (__child_up, __priv->child_count);          \
-									\
-        __local->barrier.waitfor = __count;                             \
-	afr_local_replies_wipe (__local, __priv);			\
-									\
-	for (__i = 0; __i < __priv->child_count; __i++) {		\
-		if (!__child_up[__i])                                   \
-                        continue;			                \
-		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
-				   __priv->children[__i],		\
-				   __priv->children[__i]->fops->fop, args); \
-	}								\
-	syncbarrier_wait (&__local->barrier, __count);			\
-	} while (0)
-
+#define AFR_ONALL(frame, rfn, fop, args...)                                    \
+    do {                                                                       \
+        afr_local_t *__local = frame->local;                                   \
+        afr_private_t *__priv = frame->this->private;                          \
+        int __i = 0, __count = 0;                                              \
+        unsigned char *__child_up = alloca(__priv->child_count);               \
+                                                                               \
+        memcpy(__child_up, __priv->child_up,                                   \
+               sizeof(*__child_up) * __priv->child_count);                     \
+        __count = AFR_COUNT(__child_up, __priv->child_count);                  \
+                                                                               \
+        __local->barrier.waitfor = __count;                                    \
+        afr_local_replies_wipe(__local, __priv);                               \
+                                                                               \
+        for (__i = 0; __i < __priv->child_count; __i++) {                      \
+            if (!__child_up[__i])                                              \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, rfn, (void *)(long)__i,                   \
+                              __priv->children[__i],                           \
+                              __priv->children[__i]->fops->fop, args);         \
+        }                                                                      \
+        syncbarrier_wait(&__local->barrier, __count);                          \
+    } while (0)
 
 /* Perform fop on all subvolumes represented by list[] array and wait
    for all callbacks to return */
 
-#define AFR_ONLIST(list, frame, rfn, fop, args ...) do {		\
-	afr_local_t *__local = frame->local;				\
-	afr_private_t *__priv = frame->this->private;			\
-	int __i = 0;                                                    \
-        int __count = AFR_COUNT (list, __priv->child_count);            \
-									\
-        __local->barrier.waitfor = __count;                             \
-	afr_local_replies_wipe (__local, __priv);			\
-									\
-	for (__i = 0; __i < __priv->child_count; __i++) {		\
-		if (!list[__i]) continue;				\
-		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
-				   __priv->children[__i],		\
-				   __priv->children[__i]->fops->fop, args); \
-	}								\
-	syncbarrier_wait (&__local->barrier, __count);			\
-	} while (0)
-
-
-#define AFR_SEQ(frame, rfn, fop, args ...) do {				\
-	afr_local_t *__local = frame->local;				\
-	afr_private_t *__priv = frame->this->private;			\
-	int __i = 0;							\
-									\
-	afr_local_replies_wipe (__local, __priv);			\
-									\
-	for (__i = 0; __i < __priv->child_count; __i++) {		\
-		if (!__priv->child_up[__i]) continue;			\
-		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
-				   __priv->children[__i],		\
-				   __priv->children[__i]->fops->fop, args); \
-		syncbarrier_wait (&__local->barrier, 1);		\
-	}								\
-	} while (0)
-
-
-#define ALLOC_MATRIX(n, type) ({type **__ptr = NULL; \
-	int __i; \
-	__ptr = alloca0 (n * sizeof(type *)); \
-	for (__i = 0; __i < n; __i++) __ptr[__i] = alloca0 (n * sizeof(type)); \
-	__ptr;})
-
-
-#define IA_EQUAL(f,s,field) (memcmp (&(f.ia_##field), &(s.ia_##field), sizeof (s.ia_##field)) == 0)
-
-#define SBRAIN_HEAL_NO_GO_MSG "Failed to obtain replies from all bricks of "\
-                      "the replica (are they up?). Cannot resolve split-brain."
-int
-afr_selfheal (xlator_t *this, uuid_t gfid);
+#define AFR_ONLIST(list, frame, rfn, fop, args...)                             \
+    do {                                                                       \
+        afr_local_t *__local = frame->local;                                   \
+        afr_private_t *__priv = frame->this->private;                          \
+        int __i = 0;                                                           \
+        int __count = 0;                                                       \
+        unsigned char *__list = alloca(__priv->child_count);                   \
+                                                                               \
+        memcpy(__list, list, sizeof(*__list) * __priv->child_count);           \
+        __count = AFR_COUNT(__list, __priv->child_count);                      \
+        __local->barrier.waitfor = __count;                                    \
+        afr_local_replies_wipe(__local, __priv);                               \
+                                                                               \
+        for (__i = 0; __i < __priv->child_count; __i++) {                      \
+            if (!__list[__i])                                                  \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, rfn, (void *)(long)__i,                   \
+                              __priv->children[__i],                           \
+                              __priv->children[__i]->fops->fop, args);         \
+        }                                                                      \
+        syncbarrier_wait(&__local->barrier, __count);                          \
+    } while (0)
+
+#define AFR_SEQ(frame, rfn, fop, args...)                                      \
+    do {                                                                       \
+        afr_local_t *__local = frame->local;                                   \
+        afr_private_t *__priv = frame->this->private;                          \
+        int __i = 0;                                                           \
+                                                                               \
+        afr_local_replies_wipe(__local, __priv);                               \
+                                                                               \
+        for (__i = 0; __i < __priv->child_count; __i++) {                      \
+            if (!__priv->child_up[__i])                                        \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, rfn, (void *)(long)__i,                   \
+                              __priv->children[__i],                           \
+                              __priv->children[__i]->fops->fop, args);         \
+            syncbarrier_wait(&__local->barrier, 1);                            \
+        }                                                                      \
+    } while (0)
+
+#define ALLOC_MATRIX(n, type)                                                  \
+    ({                                                                         \
+        int __i;                                                               \
+        type **__ptr = alloca(n * sizeof(type *));                             \
+                                                                               \
+        for (__i = 0; __i < n; __i++)                                          \
+            __ptr[__i] = alloca0(n * sizeof(type));                            \
+        __ptr;                                                                 \
+    })
+
+#define IA_EQUAL(f, s, field)                                                  \
+    (memcmp(&(f.ia_##field), &(s.ia_##field), sizeof(s.ia_##field)) == 0)
+
+#define SBRAIN_HEAL_NO_GO_MSG                                                  \
+    "Failed to obtain replies from all bricks of "                             \
+    "the replica (are they up?). Cannot resolve split-brain."
+#define SFILE_NOT_IN_SPLIT_BRAIN "File not in split-brain"
+#define SNO_BIGGER_FILE "No bigger file"
+#define SNO_DIFF_IN_MTIME "No difference in mtime"
+#define SUSE_SOURCE_BRICK_TO_HEAL                                              \
+    "Use source-brick option to heal metadata"                                 \
+    " split-brain"
+#define SINVALID_BRICK_NAME "Invalid brick name"
+#define SBRICK_IS_NOT_UP "Brick is not up"
+#define SBRICK_NOT_CONNECTED "Brick is not connected"
+#define SLESS_THAN2_BRICKS_in_REP "< 2 bricks in replica are up"
+#define SBRICK_IS_REMOTE "Brick is remote"
+#define SSTARTED_SELF_HEAL "Started self-heal"
+#define SOP_NOT_SUPPORTED "Operation Not Supported"
+#define SFILE_NOT_UNDER_DATA                                                   \
+    "The file is not under data or metadata "                                  \
+    "split-brain"
+#define SFILE_NOT_IN_SPLIT_BRAIN "File not in split-brain"
+#define SALL_BRICKS_UP_TO_RESOLVE                                              \
+    "All the bricks should be up to resolve the"                               \
+    " gfid split brain"
+#define SERROR_GETTING_SRC_BRICK "Error getting the source brick"
+int
+afr_selfheal(xlator_t *this, uuid_t gfid);
 
 gf_boolean_t
-afr_throttled_selfheal (call_frame_t *frame, xlator_t *this);
+afr_throttled_selfheal(call_frame_t *frame, xlator_t *this);
 
 int
-afr_selfheal_name (xlator_t *this, uuid_t gfid, const char *name,
-                   void *gfid_req, dict_t *xdata);
+afr_selfheal_name(xlator_t *this, uuid_t gfid, const char *name, void *gfid_req,
+                  dict_t *xdata);
 
 int
-afr_selfheal_data (call_frame_t *frame, xlator_t *this, fd_t *fd);
+afr_selfheal_data(call_frame_t *frame, xlator_t *this, fd_t *fd);
 
 int
-afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode);
+afr_selfheal_metadata(call_frame_t *frame, xlator_t *this, inode_t *inode);
 
 int
-afr_selfheal_entry (call_frame_t *frame, xlator_t *this, inode_t *inode);
+afr_selfheal_entry(call_frame_t *frame, xlator_t *this, inode_t *inode);
 
 int
-afr_lookup_and_heal_gfid (xlator_t *this, inode_t *parent, const char *name,
-                          inode_t *inode, struct afr_reply *replies, int source,
-                          void *gfid);
+afr_lookup_and_heal_gfid(xlator_t *this, inode_t *parent, const char *name,
+                         inode_t *inode, struct afr_reply *replies, int source,
+                         unsigned char *sources, void *gfid, int *gfid_idx);
 
 int
-afr_selfheal_inodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      char *dom, off_t off, size_t size,
-		      unsigned char *locked_on);
+afr_selfheal_inodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, off_t off, size_t size,
+                     unsigned char *locked_on);
 
 int
-afr_selfheal_tryinodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			 char *dom, off_t off, size_t size,
-			 unsigned char *locked_on);
+afr_selfheal_tryinodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, off_t off, size_t size,
+                        unsigned char *locked_on);
 
 int
-afr_selfheal_tie_breaker_inodelk (call_frame_t *frame, xlator_t *this,
-                                  inode_t *inode, char *dom, off_t off,
-                                  size_t size, unsigned char *locked_on);
+afr_selfheal_tie_breaker_inodelk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, off_t off,
+                                 size_t size, unsigned char *locked_on);
 
 int
-afr_selfheal_uninodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			char *dom, off_t off, size_t size,
-			const unsigned char *locked_on);
+afr_selfheal_uninodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, off_t off, size_t size,
+                       const unsigned char *locked_on);
 
 int
-afr_selfheal_entrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      char *dom, const char *name, unsigned char *locked_on);
+afr_selfheal_entrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, const char *name, unsigned char *locked_on);
 
 int
-afr_selfheal_tryentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			 char *dom, const char *name, unsigned char *locked_on);
+afr_selfheal_tryentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, const char *name, unsigned char *locked_on);
 
 int
-afr_selfheal_tie_breaker_entrylk (call_frame_t *frame, xlator_t *this,
-                                  inode_t *inode, char *dom, const char *name,
-                                  unsigned char *locked_on);
+afr_selfheal_tie_breaker_entrylk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, const char *name,
+                                 unsigned char *locked_on);
 
 int
-afr_selfheal_unentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			char *dom, const char *name, unsigned char *locked_on,
-                        dict_t *xdata);
+afr_selfheal_unentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, const char *name, unsigned char *locked_on,
+                       dict_t *xdata);
 
 int
-afr_selfheal_unlocked_discover (call_frame_t *frame, inode_t *inode,
-				uuid_t gfid, struct afr_reply *replies);
+afr_selfheal_unlocked_discover(call_frame_t *frame, inode_t *inode, uuid_t gfid,
+                               struct afr_reply *replies);
 
 int
-afr_selfheal_unlocked_discover_on (call_frame_t *frame, inode_t *inode,
+afr_selfheal_unlocked_discover_on(call_frame_t *frame, inode_t *inode,
                                   uuid_t gfid, struct afr_reply *replies,
-                                   unsigned char *discover_on);
+                                  unsigned char *discover_on, dict_t *dict);
 inode_t *
-afr_selfheal_unlocked_lookup_on (call_frame_t *frame, inode_t *parent,
-				 const char *name, struct afr_reply *replies,
-				 unsigned char *lookup_on, dict_t *xattr);
+afr_selfheal_unlocked_lookup_on(call_frame_t *frame, inode_t *parent,
+                                const char *name, struct afr_reply *replies,
+                                unsigned char *lookup_on, dict_t *xattr);
 
 int
-afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
-                             struct afr_reply *replies,
-                             afr_transaction_type type,
-                             unsigned char *locked_on, unsigned char *sources,
-                             unsigned char *sinks, uint64_t *witness,
-                             gf_boolean_t *flag);
+afr_selfheal_find_direction(call_frame_t *frame, xlator_t *this,
+                            struct afr_reply *replies,
+                            afr_transaction_type type, unsigned char *locked_on,
+                            unsigned char *sources, unsigned char *sinks,
+                            uint64_t *witness, unsigned char *flag);
 int
-afr_selfheal_fill_matrix (xlator_t *this, int **matrix, int subvol, int idx,
-                          dict_t *xdata);
+afr_selfheal_fill_matrix(xlator_t *this, int **matrix, int subvol, int idx,
+                         dict_t *xdata);
 
 int
-afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
-			    afr_transaction_type type, int *dirty, int **matrix);
+afr_selfheal_extract_xattr(xlator_t *this, struct afr_reply *replies,
+                           afr_transaction_type type, int *dirty, int **matrix);
 
 int
-afr_sh_generic_fop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, struct iatt *pre,
-                        struct iatt *post, dict_t *xdata);
+afr_sh_generic_fop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *pre,
+                       struct iatt *post, dict_t *xdata);
 
 int
-afr_selfheal_restore_time (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                           int source, unsigned char *healed_sinks,
-                           struct afr_reply *replies);
+afr_selfheal_restore_time(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          int source, unsigned char *healed_sinks,
+                          struct afr_reply *replies);
 int
-afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			   unsigned char *sources, unsigned char *sinks,
-			   unsigned char *healed_sinks,
-                           unsigned char *undid_pending,
-                           afr_transaction_type type, struct afr_reply *replies,
-                           unsigned char *locked_on);
+afr_selfheal_undo_pending(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          unsigned char *sources, unsigned char *sinks,
+                          unsigned char *healed_sinks,
+                          unsigned char *undid_pending,
+                          afr_transaction_type type, struct afr_reply *replies,
+                          unsigned char *locked_on);
 
 int
-afr_selfheal_recreate_entry (call_frame_t *frame, int dst, int source,
-                             unsigned char *sources,
-                             inode_t *dir, const char *name, inode_t *inode,
-                             struct afr_reply *replies);
+afr_selfheal_recreate_entry(call_frame_t *frame, int dst, int source,
+                            unsigned char *sources, inode_t *dir,
+                            const char *name, inode_t *inode,
+                            struct afr_reply *replies);
 
 int
-afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      int subvol, dict_t *xattr, dict_t *xdata);
+afr_selfheal_post_op(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     int subvol, dict_t *xattr, dict_t *xdata);
 
 call_frame_t *
-afr_frame_create (xlator_t *this, int32_t *op_errno);
+afr_frame_create(xlator_t *this, int32_t *op_errno);
 
 inode_t *
-afr_inode_find (xlator_t *this, uuid_t gfid);
+afr_inode_find(xlator_t *this, uuid_t gfid);
 
 int
-afr_selfheal_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int op_ret, int op_errno, inode_t *inode,
-			   struct iatt *buf, dict_t *xdata,
-                           struct iatt *parbuf);
+afr_selfheal_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xdata, struct iatt *parbuf);
 void
-afr_reply_copy (struct afr_reply *dst, struct afr_reply *src);
+afr_reply_copy(struct afr_reply *dst, struct afr_reply *src);
 
 void
-afr_replies_copy (struct afr_reply *dst, struct afr_reply *src, int count);
+afr_replies_copy(struct afr_reply *dst, struct afr_reply *src, int count);
 
 int
-afr_selfheal_newentry_mark (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                            int source, struct afr_reply *replies,
-                            unsigned char *sources, unsigned char *newentry);
+afr_selfheal_newentry_mark(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, struct afr_reply *replies,
+                           unsigned char *sources, unsigned char *newentry);
 
 unsigned int
-afr_success_count (struct afr_reply *replies, unsigned int count);
+afr_success_count(struct afr_reply *replies, unsigned int count);
 
 void
-afr_log_selfheal (uuid_t gfid, xlator_t *this, int ret, char *type,
-                  int source, unsigned char *sources,
-                  unsigned char *healed_sinks);
+afr_log_selfheal(uuid_t gfid, xlator_t *this, int ret, char *type, int source,
+                 unsigned char *sources, unsigned char *healed_sinks);
 
 void
-afr_mark_largest_file_as_source (xlator_t *this, unsigned char *sources,
-                                 struct afr_reply *replies);
+afr_mark_largest_file_as_source(xlator_t *this, unsigned char *sources,
+                                struct afr_reply *replies);
 void
-afr_mark_active_sinks (xlator_t *this, unsigned char *sources,
-                       unsigned char *locked_on, unsigned char *sinks);
+afr_mark_active_sinks(xlator_t *this, unsigned char *sources,
+                      unsigned char *locked_on, unsigned char *sinks);
 
 gf_boolean_t
-afr_dict_contains_heal_op (call_frame_t *frame);
+afr_dict_contains_heal_op(call_frame_t *frame);
 
 gf_boolean_t
-afr_can_decide_split_brain_source_sinks (struct afr_reply *replies,
-                                         int child_count);
+afr_can_decide_split_brain_source_sinks(struct afr_reply *replies,
+                                        int child_count);
 int
-afr_mark_split_brain_source_sinks (call_frame_t *frame, xlator_t *this,
-                                   inode_t *inode,
-                                   unsigned char *sources,
-                                   unsigned char *sinks,
-                                   unsigned char *healed_sinks,
-                                   unsigned char *locked_on,
-                                   struct afr_reply *replies,
-                                   afr_transaction_type type);
+afr_mark_split_brain_source_sinks(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type);
 
 int
-afr_sh_get_fav_by_policy (xlator_t *this, struct afr_reply *replies,
-                          inode_t *inode, char **policy_str);
+afr_sh_get_fav_by_policy(xlator_t *this, struct afr_reply *replies,
+                         inode_t *inode, char **policy_str);
 
 int
-_afr_fav_child_reset_sink_xattrs (call_frame_t *frame, xlator_t *this,
+_afr_fav_child_reset_sink_xattrs(call_frame_t *frame, xlator_t *this,
                                  inode_t *inode, int source,
                                  unsigned char *healed_sinks,
                                  unsigned char *undid_pending,
@@ -275,82 +295,83 @@ _afr_fav_child_reset_sink_xattrs (call_frame_t *frame, xlator_t *this,
                                  struct afr_reply *replies);
 
 int
-afr_get_child_index_from_name (xlator_t *this, char *name);
+afr_get_child_index_from_name(xlator_t *this, char *name);
 
 gf_boolean_t
-afr_does_witness_exist (xlator_t *this, uint64_t *witness);
+afr_does_witness_exist(xlator_t *this, uint64_t *witness);
 
 int
-__afr_selfheal_data_prepare (call_frame_t *frame, xlator_t *this,
-                             inode_t *inode, unsigned char *locked_on,
-                             unsigned char *sources,
-                             unsigned char *sinks, unsigned char *healed_sinks,
-                             unsigned char *undid_pending,
-                             struct afr_reply *replies, gf_boolean_t *flag);
+__afr_selfheal_data_prepare(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                            unsigned char *locked_on, unsigned char *sources,
+                            unsigned char *sinks, unsigned char *healed_sinks,
+                            unsigned char *undid_pending,
+                            struct afr_reply *replies, unsigned char *flag);
 
 int
-__afr_selfheal_metadata_prepare (call_frame_t *frame, xlator_t *this,
-                                 inode_t *inode, unsigned char *locked_on,
-                                 unsigned char *sources,
-                                 unsigned char *sinks,
-                                 unsigned char *healed_sinks,
-                                 unsigned char *undid_pending,
-                                 struct afr_reply *replies,
-                                 gf_boolean_t *flag);
+__afr_selfheal_metadata_prepare(call_frame_t *frame, xlator_t *this,
+                                inode_t *inode, unsigned char *locked_on,
+                                unsigned char *sources, unsigned char *sinks,
+                                unsigned char *healed_sinks,
+                                unsigned char *undid_pending,
+                                struct afr_reply *replies, unsigned char *flag);
 int
-__afr_selfheal_entry_prepare (call_frame_t *frame, xlator_t *this,
-                              inode_t *inode, unsigned char *locked_on,
-                              unsigned char *sources,
-                              unsigned char *sinks,
-                              unsigned char *healed_sinks,
-                              struct afr_reply *replies, int *source_p,
-                              gf_boolean_t *flag);
+__afr_selfheal_entry_prepare(call_frame_t *frame, xlator_t *this,
+                             inode_t *inode, unsigned char *locked_on,
+                             unsigned char *sources, unsigned char *sinks,
+                             unsigned char *healed_sinks,
+                             struct afr_reply *replies, int *source_p,
+                             unsigned char *flag);
 
 int
-afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this,
-                               uuid_t gfid, inode_t **link_inode,
-                               gf_boolean_t *data_selfheal,
-                               gf_boolean_t *metadata_selfheal,
-                               gf_boolean_t *entry_selfheal);
+afr_selfheal_unlocked_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
+                              inode_t **link_inode, gf_boolean_t *data_selfheal,
+                              gf_boolean_t *metadata_selfheal,
+                              gf_boolean_t *entry_selfheal,
+                              struct afr_reply *replies);
 
 int
-afr_selfheal_do (call_frame_t *frame, xlator_t *this, uuid_t gfid);
+afr_selfheal_do(call_frame_t *frame, xlator_t *this, uuid_t gfid);
 
 int
-afr_selfheal_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno, dict_t *xdata);
+afr_selfheal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata);
 
 int
-afr_locked_fill (call_frame_t *frame, xlator_t *this,
-                 unsigned char *locked_on);
+afr_locked_fill(call_frame_t *frame, xlator_t *this, unsigned char *locked_on);
 int
-afr_choose_source_by_policy (afr_private_t *priv, unsigned char *sources,
-                             afr_transaction_type type);
+afr_choose_source_by_policy(afr_private_t *priv, unsigned char *sources,
+                            afr_transaction_type type);
 
 int
-afr_selfheal_metadata_by_stbuf (xlator_t *this, struct iatt *stbuf);
+afr_selfheal_metadata_by_stbuf(xlator_t *this, struct iatt *stbuf);
 
 int
-afr_sh_fav_by_size (xlator_t *this, struct afr_reply *replies,
-                    inode_t *inode);
+afr_sh_fav_by_size(xlator_t *this, struct afr_reply *replies, inode_t *inode);
 int
-afr_sh_fav_by_mtime (xlator_t *this, struct afr_reply *replies,
-                     inode_t *inode);
+afr_sh_fav_by_mtime(xlator_t *this, struct afr_reply *replies, inode_t *inode);
 int
-afr_sh_fav_by_ctime (xlator_t *this, struct afr_reply *replies,
-                     inode_t *inode);
+afr_sh_fav_by_ctime(xlator_t *this, struct afr_reply *replies, inode_t *inode);
 
 int
-afr_gfid_split_brain_source (xlator_t *this, struct afr_reply *replies,
-                             inode_t *inode, uuid_t pargfid, const char *bname,
-                             int src_idx, int child_idx,
-                             unsigned char *locked_on, int *src, dict_t *xdata);
+afr_gfid_split_brain_source(xlator_t *this, struct afr_reply *replies,
+                            inode_t *inode, uuid_t pargfid, const char *bname,
+                            int src_idx, int child_idx,
+                            unsigned char *locked_on, int *src, dict_t *xdata);
 int
-afr_mark_source_sinks_if_file_empty (xlator_t *this, unsigned char *sources,
-                                     unsigned char *sinks,
-                                     unsigned char *healed_sinks,
-                                     unsigned char *locked_on,
-                                     struct afr_reply *replies,
-                                     afr_transaction_type type);
+afr_mark_source_sinks_if_file_empty(xlator_t *this, unsigned char *sources,
+                                    unsigned char *sinks,
+                                    unsigned char *healed_sinks,
+                                    unsigned char *locked_on,
+                                    struct afr_reply *replies,
+                                    afr_transaction_type type);
+
+gf_boolean_t
+afr_is_file_empty_on_all_children(afr_private_t *priv,
+                                  struct afr_reply *replies);
 
+int
+afr_selfheal_entry_delete(xlator_t *this, inode_t *dir, const char *name,
+                          inode_t *inode, int child, struct afr_reply *replies);
+int
+afr_anon_inode_create(xlator_t *this, int child, inode_t **linked_inode);
 #endif /* !_AFR_SELFHEAL_H */
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
index 4667be63220..109fd4b7421 100644
--- a/xlators/cluster/afr/src/afr-self-heald.c
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -8,1387 +8,1709 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "afr.h"
 #include "afr-self-heal.h"
 #include "afr-self-heald.h"
 #include "protocol-common.h"
-#include "syncop-utils.h"
+#include <glusterfs/syncop-utils.h>
 #include "afr-messages.h"
-#include "byte-order.h"
-
-#define SHD_INODE_LRU_LIMIT          2048
-#define AFR_EH_SPLIT_BRAIN_LIMIT     1024
-#define AFR_STATISTICS_HISTORY_SIZE    50
-
-
-#define ASSERT_LOCAL(this, healer)				\
-	if (!afr_shd_is_subvol_local(this, healer->subvol)) {	\
-		healer->local = _gf_false;			\
-		if (safe_break (healer)) {			\
-			break;					\
-		} else {					\
-			continue;				\
-		}						\
-	} else {						\
-		healer->local = _gf_true;			\
-	}
-
-
-#define NTH_INDEX_HEALER(this, n) &((((afr_private_t *)this->private))->shd.index_healers[n])
-#define NTH_FULL_HEALER(this, n) &((((afr_private_t *)this->private))->shd.full_healers[n])
+#include <glusterfs/byte-order.h>
+
+#define AFR_EH_SPLIT_BRAIN_LIMIT 1024
+#define AFR_STATISTICS_HISTORY_SIZE 50
+
+#define ASSERT_LOCAL(this, healer)                                             \
+    if (!afr_shd_is_subvol_local(this, healer->subvol)) {                      \
+        healer->local = _gf_false;                                             \
+        if (safe_break(healer)) {                                              \
+            break;                                                             \
+        } else {                                                               \
+            continue;                                                          \
+        }                                                                      \
+    } else {                                                                   \
+        healer->local = _gf_true;                                              \
+    }
+
+#define NTH_INDEX_HEALER(this, n)                                              \
+    &((((afr_private_t *)this->private))->shd.index_healers[n])
+#define NTH_FULL_HEALER(this, n)                                               \
+    &((((afr_private_t *)this->private))->shd.full_healers[n])
 
 char *
-afr_subvol_name (xlator_t *this, int subvol)
+afr_subvol_name(xlator_t *this, int subvol)
 {
-	afr_private_t *priv = NULL;
+    afr_private_t *priv = NULL;
 
-	priv = this->private;
-	if (subvol < 0 || subvol > priv->child_count)
-		return NULL;
+    priv = this->private;
+    if (subvol < 0 || subvol > priv->child_count)
+        return NULL;
 
-	return priv->children[subvol]->name;
+    return priv->children[subvol]->name;
 }
 
-
 void
-afr_destroy_crawl_event_data (void *data)
+afr_destroy_crawl_event_data(void *data)
 {
-        return;
+    return;
 }
 
-
 void
-afr_destroy_shd_event_data (void *data)
+afr_destroy_shd_event_data(void *data)
 {
-	shd_event_t *shd_event = data;
-
-	if (!shd_event)
-		return;
-	GF_FREE (shd_event->path);
+    shd_event_t *shd_event = data;
 
+    if (!shd_event)
         return;
-}
+    GF_FREE(shd_event->path);
 
+    return;
+}
 
 gf_boolean_t
-afr_shd_is_subvol_local (xlator_t *this, int subvol)
+afr_shd_is_subvol_local(xlator_t *this, int subvol)
 {
-	afr_private_t *priv    = NULL;
-	gf_boolean_t  is_local = _gf_false;
-        loc_t         loc      = {0, };
-
-        loc.inode = this->itable->root;
-        gf_uuid_copy (loc.gfid, loc.inode->gfid);
-	priv = this->private;
-        syncop_is_subvol_local(priv->children[subvol], &loc, &is_local);
-        return is_local;
+    afr_private_t *priv = NULL;
+    gf_boolean_t is_local = _gf_false;
+    loc_t loc = {
+        0,
+    };
+
+    loc.inode = this->itable->root;
+    gf_uuid_copy(loc.gfid, loc.inode->gfid);
+    priv = this->private;
+    syncop_is_subvol_local(priv->children[subvol], &loc, &is_local);
+    return is_local;
 }
 
-
 int
-__afr_shd_healer_wait (struct subvol_healer *healer)
+__afr_shd_healer_wait(struct subvol_healer *healer)
 {
-	afr_private_t *priv = NULL;
-	struct timespec wait_till = {0, };
-	int ret = 0;
+    afr_private_t *priv = NULL;
+    struct timespec wait_till = {
+        0,
+    };
+    int ret = 0;
 
-	priv = healer->this->private;
+    priv = healer->this->private;
 
 disabled_loop:
-	wait_till.tv_sec = time (NULL) + priv->shd.timeout;
+    wait_till.tv_sec = gf_time() + priv->shd.timeout;
 
-	while (!healer->rerun) {
-		ret = pthread_cond_timedwait (&healer->cond,
-					      &healer->mutex,
-					      &wait_till);
-		if (ret == ETIMEDOUT)
-			break;
-	}
+    while (!healer->rerun) {
+        ret = pthread_cond_timedwait(&healer->cond, &healer->mutex, &wait_till);
+        if (ret == ETIMEDOUT)
+            break;
+    }
 
-	ret = healer->rerun;
-	healer->rerun = 0;
+    ret = healer->rerun;
+    healer->rerun = 0;
 
-	if (!priv->shd.enabled)
-		goto disabled_loop;
+    if (!priv->shd.enabled)
+        goto disabled_loop;
 
-	return ret;
+    return ret;
 }
 
-
 int
-afr_shd_healer_wait (struct subvol_healer *healer)
+afr_shd_healer_wait(struct subvol_healer *healer)
 {
-	int ret = 0;
+    int ret = 0;
 
-	pthread_mutex_lock (&healer->mutex);
-	{
-		ret = __afr_shd_healer_wait (healer);
-	}
-	pthread_mutex_unlock (&healer->mutex);
+    pthread_mutex_lock(&healer->mutex);
+    {
+        ret = __afr_shd_healer_wait(healer);
+    }
+    pthread_mutex_unlock(&healer->mutex);
 
-	return ret;
+    return ret;
 }
 
-
 gf_boolean_t
-safe_break (struct subvol_healer *healer)
+safe_break(struct subvol_healer *healer)
 {
-	gf_boolean_t ret = _gf_false;
+    gf_boolean_t ret = _gf_false;
 
-	pthread_mutex_lock (&healer->mutex);
-	{
-		if (healer->rerun)
-			goto unlock;
+    pthread_mutex_lock(&healer->mutex);
+    {
+        if (healer->rerun)
+            goto unlock;
 
-		healer->running = _gf_false;
-		ret = _gf_true;
-	}
+        healer->running = _gf_false;
+        ret = _gf_true;
+    }
 unlock:
-	pthread_mutex_unlock (&healer->mutex);
+    pthread_mutex_unlock(&healer->mutex);
 
-	return ret;
+    return ret;
 }
 
-
 inode_t *
-afr_shd_inode_find (xlator_t *this, xlator_t *subvol, uuid_t gfid)
+afr_shd_inode_find(xlator_t *this, xlator_t *subvol, uuid_t gfid)
 {
-        int          ret       = 0;
-        uint64_t     val       = IA_INVAL;
-        dict_t       *xdata    = NULL;
-        dict_t       *rsp_dict = NULL;
-        inode_t      *inode    = NULL;
-
-        xdata = dict_new ();
-        if (!xdata)
-                goto out;
-
-        ret = dict_set_int8 (xdata, GF_INDEX_IA_TYPE_GET_REQ, 1);
+    int ret = 0;
+    uint64_t val = IA_INVAL;
+    dict_t *xdata = NULL;
+    dict_t *rsp_dict = NULL;
+    inode_t *inode = NULL;
+
+    xdata = dict_new();
+    if (!xdata)
+        goto out;
+
+    ret = dict_set_int8(xdata, GF_INDEX_IA_TYPE_GET_REQ, 1);
+    if (ret)
+        goto out;
+
+    ret = syncop_inode_find(this, subvol, gfid, &inode, xdata, &rsp_dict);
+    if (ret < 0)
+        goto out;
+
+    if (rsp_dict) {
+        ret = dict_get_uint64(rsp_dict, GF_INDEX_IA_TYPE_GET_RSP, &val);
         if (ret)
-                goto out;
-
-	ret = syncop_inode_find (this, subvol, gfid, &inode,
-                                 xdata, &rsp_dict);
-	if (ret < 0)
-		goto out;
-
-        if (rsp_dict) {
-                ret = dict_get_uint64 (rsp_dict, GF_INDEX_IA_TYPE_GET_RSP,
-                                       &val);
-                if (ret)
-                        goto out;
-        }
-        ret = inode_ctx_set2 (inode, subvol, 0, &val);
+            goto out;
+    }
+    ret = inode_ctx_set2(inode, subvol, 0, &val);
 out:
-        if (ret && inode) {
-                inode_unref (inode);
-                inode = NULL;
-        }
-        if (xdata)
-                dict_unref (xdata);
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-	return inode;
+    if (ret && inode) {
+        inode_unref(inode);
+        inode = NULL;
+    }
+    if (xdata)
+        dict_unref(xdata);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    return inode;
 }
 
-inode_t*
-afr_shd_index_inode (xlator_t *this, xlator_t *subvol, char *vgfid)
+inode_t *
+afr_shd_index_inode(xlator_t *this, xlator_t *subvol, char *vgfid)
 {
-	loc_t rootloc = {0, };
-	inode_t *inode = NULL;
-	int ret = 0;
-	dict_t *xattr = NULL;
-	void *index_gfid = NULL;
+    loc_t rootloc = {
+        0,
+    };
+    inode_t *inode = NULL;
+    int ret = 0;
+    dict_t *xattr = NULL;
+    void *index_gfid = NULL;
 
-	rootloc.inode = inode_ref (this->itable->root);
-	gf_uuid_copy (rootloc.gfid, rootloc.inode->gfid);
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
 
-	ret = syncop_getxattr (subvol, &rootloc, &xattr,
-			       vgfid, NULL, NULL);
-	if (ret || !xattr) {
-		errno = -ret;
-		goto out;
-	}
+    ret = syncop_getxattr(subvol, &rootloc, &xattr, vgfid, NULL, NULL);
+    if (ret || !xattr) {
+        errno = -ret;
+        goto out;
+    }
 
-	ret = dict_get_ptr (xattr, vgfid, &index_gfid);
-	if (ret)
-		goto out;
+    ret = dict_get_ptr(xattr, vgfid, &index_gfid);
+    if (ret)
+        goto out;
 
-        gf_msg_debug (this->name, 0, "%s dir gfid for %s: %s",
-	              vgfid, subvol->name, uuid_utoa (index_gfid));
+    gf_msg_debug(this->name, 0, "%s dir gfid for %s: %s", vgfid, subvol->name,
+                 uuid_utoa(index_gfid));
 
-	inode = afr_shd_inode_find (this, subvol, index_gfid);
+    inode = afr_shd_inode_find(this, subvol, index_gfid);
 
 out:
-	loc_wipe (&rootloc);
+    loc_wipe(&rootloc);
 
-	if (xattr)
-		dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-	return inode;
+    return inode;
 }
 
 int
-afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name,
-                     ia_type_t type)
+afr_shd_entry_purge(xlator_t *subvol, inode_t *inode, char *name,
+                    ia_type_t type)
 {
-	int    ret = 0;
-	loc_t  loc = {0,};
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
 
-	loc.parent = inode_ref (inode);
-	loc.name = name;
+    loc.parent = inode_ref(inode);
+    loc.name = name;
 
-        if (IA_ISDIR (type))
-                ret = syncop_rmdir (subvol, &loc, 1, NULL, NULL);
-        else
-                ret = syncop_unlink (subvol, &loc, NULL, NULL);
+    if (IA_ISDIR(type))
+        ret = syncop_rmdir(subvol, &loc, 1, NULL, NULL);
+    else
+        ret = syncop_unlink(subvol, &loc, NULL, NULL);
 
-	loc_wipe (&loc);
-	return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
 void
-afr_shd_zero_xattrop (xlator_t *this, uuid_t gfid)
+afr_shd_zero_xattrop(xlator_t *this, uuid_t gfid)
 {
-
-        call_frame_t *frame = NULL;
-        inode_t *inode = NULL;
-        afr_private_t *priv = NULL;
-        dict_t  *xattr = NULL;
-        int ret = 0;
-        int i = 0;
-        int raw[AFR_NUM_CHANGE_LOGS] = {0};
-
-        priv = this->private;
-        frame = afr_frame_create (this, NULL);
-        if (!frame)
-                goto out;
-        inode = afr_inode_find (this, gfid);
-        if (!inode)
-                goto out;
-        xattr = dict_new();
-        if (!xattr)
-                goto out;
-        ret = dict_set_static_bin (xattr, AFR_DIRTY, raw,
-                                   sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    call_frame_t *frame = NULL;
+    inode_t *inode = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int ret = 0;
+    int i = 0;
+    int raw[AFR_NUM_CHANGE_LOGS] = {0};
+
+    priv = this->private;
+    frame = afr_frame_create(this, NULL);
+    if (!frame)
+        goto out;
+    inode = afr_inode_find(this, gfid);
+    if (!inode)
+        goto out;
+    xattr = dict_new();
+    if (!xattr)
+        goto out;
+    ret = dict_set_static_bin(xattr, AFR_DIRTY, raw,
+                              sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    if (ret)
+        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], raw,
+                                  sizeof(int) * AFR_NUM_CHANGE_LOGS);
         if (ret)
-                goto out;
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_set_static_bin (xattr, priv->pending_key[i], raw,
-                                           sizeof(int) * AFR_NUM_CHANGE_LOGS);
-                if (ret)
-                        goto out;
-        }
+            goto out;
+    }
 
-        /*Send xattrop to all bricks. Doing a lookup to see if bricks are up or
-        * has valid repies for this gfid seems a bit of an overkill.*/
-        for (i = 0; i < priv->child_count; i++)
-                afr_selfheal_post_op (frame, this, inode, i, xattr, NULL);
+    /*Send xattrop to all bricks. Doing a lookup to see if bricks are up or
+     * has valid repies for this gfid seems a bit of an overkill.*/
+    for (i = 0; i < priv->child_count; i++)
+        afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
 
 out:
-        if (frame)
-                AFR_STACK_DESTROY (frame);
-        if (inode)
-                inode_unref (inode);
-        if (xattr)
-                dict_unref (xattr);
-        return;
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    if (inode)
+        inode_unref(inode);
+    if (xattr)
+        dict_unref(xattr);
+    return;
 }
 
 int
-afr_shd_selfheal_name (struct subvol_healer *healer, int child, uuid_t parent,
-		       const char *bname)
+afr_shd_selfheal_name(struct subvol_healer *healer, int child, uuid_t parent,
+                      const char *bname)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = afr_selfheal_name (THIS, parent, bname, NULL, NULL);
+    ret = afr_selfheal_name(THIS, parent, bname, NULL, NULL);
 
-	return ret;
+    return ret;
 }
 
 int
-afr_shd_selfheal (struct subvol_healer *healer, int child, uuid_t gfid)
+afr_shd_selfheal(struct subvol_healer *healer, int child, uuid_t gfid)
 {
-	int ret = 0;
-	eh_t *eh = NULL;
-	afr_private_t *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	shd_event_t *shd_event = NULL;
-	char *path = NULL;
-	xlator_t *subvol = NULL;
-	xlator_t *this = NULL;
-	crawl_event_t *crawl_event = NULL;
-
-	this = healer->this;
-	priv = this->private;
-	shd = &priv->shd;
-	crawl_event = &healer->crawl_event;
-
-	subvol = priv->children[child];
-
-        //If this fails with ENOENT/ESTALE index is stale
-        ret = syncop_gfid_to_path (this->itable, subvol, gfid, &path);
-        if (ret < 0)
-                return ret;
-
-	ret = afr_selfheal (this, gfid);
-
-        LOCK (&priv->lock);
-        {
-                if (ret == -EIO) {
-                        eh = shd->split_brain;
-                        crawl_event->split_brain_count++;
-                } else if (ret < 0) {
-                        crawl_event->heal_failed_count++;
-                } else if (ret == 0) {
-                        crawl_event->healed_count++;
-                }
+    int ret = 0;
+    eh_t *eh = NULL;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    shd_event_t *shd_event = NULL;
+    char *path = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    crawl_event_t *crawl_event = NULL;
+
+    this = healer->this;
+    priv = this->private;
+    shd = &priv->shd;
+    crawl_event = &healer->crawl_event;
+
+    subvol = priv->children[child];
+
+    // If this fails with ENOENT/ESTALE index is stale
+    ret = syncop_gfid_to_path(this->itable, subvol, gfid, &path);
+    if (ret < 0)
+        return ret;
+
+    ret = afr_selfheal(this, gfid);
+
+    LOCK(&priv->lock);
+    {
+        if (ret == -EIO) {
+            eh = shd->split_brain;
+            crawl_event->split_brain_count++;
+        } else if (ret < 0) {
+            crawl_event->heal_failed_count++;
+        } else if (ret == 0) {
+            crawl_event->healed_count++;
         }
-        UNLOCK (&priv->lock);
+    }
+    UNLOCK(&priv->lock);
 
-	if (eh) {
-		shd_event = GF_CALLOC (1, sizeof(*shd_event),
-				       gf_afr_mt_shd_event_t);
-		if (!shd_event)
-                        goto out;
+    if (eh) {
+        shd_event = GF_CALLOC(1, sizeof(*shd_event), gf_afr_mt_shd_event_t);
+        if (!shd_event)
+            goto out;
 
-		shd_event->child = child;
-		shd_event->path = path;
+        shd_event->child = child;
+        shd_event->path = path;
 
-		if (eh_save_history (eh, shd_event) < 0)
-                        goto out;
+        if (eh_save_history(eh, shd_event) < 0)
+            goto out;
 
-                shd_event = NULL;
-                path = NULL;
-	}
+        shd_event = NULL;
+        path = NULL;
+    }
 out:
-        GF_FREE (shd_event);
-        GF_FREE (path);
-	return ret;
+    GF_FREE(shd_event);
+    GF_FREE(path);
+    return ret;
 }
 
-
 void
-afr_shd_sweep_prepare (struct subvol_healer *healer)
+afr_shd_sweep_prepare(struct subvol_healer *healer)
 {
-	crawl_event_t *event = NULL;
+    crawl_event_t *event = NULL;
 
-	event = &healer->crawl_event;
+    event = &healer->crawl_event;
 
-	event->healed_count = 0;
-	event->split_brain_count = 0;
-	event->heal_failed_count = 0;
+    event->healed_count = 0;
+    event->split_brain_count = 0;
+    event->heal_failed_count = 0;
 
-	time (&event->start_time);
-	event->end_time = 0;
+    event->start_time = gf_time();
+    event->end_time = 0;
+    _mask_cancellation();
 }
 
-
 void
-afr_shd_sweep_done (struct subvol_healer *healer)
+afr_shd_sweep_done(struct subvol_healer *healer)
 {
-	crawl_event_t *event = NULL;
-	crawl_event_t *history = NULL;
-	afr_self_heald_t *shd = NULL;
+    crawl_event_t *event = NULL;
+    crawl_event_t *history = NULL;
+    afr_self_heald_t *shd = NULL;
 
-	event = &healer->crawl_event;
-	shd = &(((afr_private_t *)healer->this->private)->shd);
+    event = &healer->crawl_event;
+    shd = &(((afr_private_t *)healer->this->private)->shd);
 
-	time (&event->end_time);
-	history = memdup (event, sizeof (*event));
-	event->start_time = 0;
+    event->end_time = gf_time();
+    history = gf_memdup(event, sizeof(*event));
+    event->start_time = 0;
 
-	if (!history)
-		return;
+    if (!history)
+        return;
 
-	if (eh_save_history (shd->statistics[healer->subvol], history) < 0)
-		GF_FREE (history);
+    if (eh_save_history(shd->statistics[healer->subvol], history) < 0)
+        GF_FREE(history);
+    _unmask_cancellation();
 }
 
 int
-afr_shd_index_heal (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                    void *data)
+afr_shd_index_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                   void *data)
 {
-        struct subvol_healer *healer = data;
-        afr_private_t        *priv   = NULL;
-        uuid_t               gfid    = {0};
-        int                  ret     = 0;
-        uint64_t             val     = IA_INVAL;
+    struct subvol_healer *healer = data;
+    afr_private_t *priv = NULL;
+    uuid_t gfid = {0};
+    int ret = 0;
+    uint64_t val = IA_INVAL;
 
-        priv = healer->this->private;
-        if (!priv->shd.enabled)
-                return -EBUSY;
+    priv = healer->this->private;
+    if (!priv->shd.enabled)
+        return -EBUSY;
 
-        gf_msg_debug (healer->this->name, 0, "got entry: %s from %s",
-                      entry->d_name, priv->children[healer->subvol]->name);
+    gf_msg_debug(healer->this->name, 0, "got entry: %s from %s", entry->d_name,
+                 priv->children[healer->subvol]->name);
 
-        ret = gf_uuid_parse (entry->d_name, gfid);
-        if (ret)
-                return 0;
+    ret = gf_uuid_parse(entry->d_name, gfid);
+    if (ret)
+        return 0;
 
-        inode_ctx_get2 (parent->inode, subvol, NULL, &val);
+    inode_ctx_get2(parent->inode, subvol, NULL, &val);
 
-        ret = afr_shd_selfheal (healer, healer->subvol, gfid);
+    ret = afr_shd_selfheal(healer, healer->subvol, gfid);
 
-        if (ret == -ENOENT || ret == -ESTALE)
-                afr_shd_index_purge (subvol, parent->inode, entry->d_name, val);
+    if (ret == -ENOENT || ret == -ESTALE)
+        afr_shd_entry_purge(subvol, parent->inode, entry->d_name, val);
 
-        if (ret == 2)
-                /* If bricks crashed in pre-op after creating indices/xattrop
-                 * link but before setting afr changelogs, we end up with stale
-                 * xattrop links but zero changelogs. Remove such entries by
-                 * sending a post-op with zero changelogs.
-                 */
-                afr_shd_zero_xattrop (healer->this, gfid);
+    if (ret == 2)
+        /* If bricks crashed in pre-op after creating indices/xattrop
+         * link but before setting afr changelogs, we end up with stale
+         * xattrop links but zero changelogs. Remove such entries by
+         * sending a post-op with zero changelogs.
+         */
+        afr_shd_zero_xattrop(healer->this, gfid);
 
-        return 0;
+    return 0;
 }
 
 int
-afr_shd_index_sweep (struct subvol_healer *healer, char *vgfid)
+afr_shd_index_sweep(struct subvol_healer *healer, char *vgfid)
 {
-	loc_t         loc     = {0};
-	afr_private_t *priv   = NULL;
-	int           ret     = 0;
-	xlator_t      *subvol = NULL;
-	dict_t        *xdata  = NULL;
-        call_frame_t  *frame  = NULL;
-
-	priv = healer->this->private;
-	subvol = priv->children[healer->subvol];
-
-        frame = afr_frame_create (healer->this, &ret);
-        if (!frame) {
-                ret = -ret;
-                goto out;
-        }
+    loc_t loc = {0};
+    afr_private_t *priv = NULL;
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    dict_t *xdata = NULL;
+    call_frame_t *frame = NULL;
+
+    priv = healer->this->private;
+    subvol = priv->children[healer->subvol];
+
+    frame = afr_frame_create(healer->this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    loc.inode = afr_shd_index_inode(healer->this, subvol, vgfid);
+    if (!loc.inode) {
+        gf_msg(healer->this->name, GF_LOG_WARNING, 0,
+               AFR_MSG_INDEX_DIR_GET_FAILED, "unable to get index-dir on %s",
+               subvol->name);
+        ret = -errno;
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata || dict_set_int32_sizen(xdata, "get-gfid-type", 1)) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_mt_dir_scan(frame, subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
+                             healer, afr_shd_index_heal, xdata,
+                             priv->shd.max_threads, priv->shd.wait_qlength);
+
+    if (ret == 0)
+        ret = healer->crawl_event.healed_count;
 
-	loc.inode = afr_shd_index_inode (healer->this, subvol, vgfid);
-	if (!loc.inode) {
-	        gf_msg (healer->this->name, GF_LOG_WARNING,
-                        0, AFR_MSG_INDEX_DIR_GET_FAILED,
-		        "unable to get index-dir on %s", subvol->name);
-		ret = -errno;
-	        goto out;
-	}
-
-        xdata = dict_new ();
-        if (!xdata || dict_set_int32 (xdata, "get-gfid-type", 1)) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = syncop_mt_dir_scan (frame, subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
-                                  healer, afr_shd_index_heal, xdata,
-                                 priv->shd.max_threads, priv->shd.wait_qlength);
+out:
+    loc_wipe(&loc);
 
-        if (ret == 0)
-                ret = healer->crawl_event.healed_count;
+    if (xdata)
+        dict_unref(xdata);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return ret;
+}
 
+int
+afr_shd_index_sweep_all(struct subvol_healer *healer)
+{
+    int ret = 0;
+    int count = 0;
+
+    ret = afr_shd_index_sweep(healer, GF_XATTROP_INDEX_GFID);
+    if (ret < 0)
+        goto out;
+    count = ret;
+
+    ret = afr_shd_index_sweep(healer, GF_XATTROP_DIRTY_GFID);
+    if (ret < 0)
+        goto out;
+    count += ret;
+
+    ret = afr_shd_index_sweep(healer, GF_XATTROP_ENTRY_CHANGES_GFID);
+    if (ret < 0)
+        goto out;
+    count += ret;
 out:
-        loc_wipe (&loc);
-
-        if (xdata)
-                dict_unref (xdata);
-        if (frame)
-                AFR_STACK_DESTROY (frame);
-	return ret;
+    if (ret < 0)
+        return ret;
+    else
+        return count;
 }
 
 int
-afr_shd_index_sweep_all (struct subvol_healer *healer)
+afr_shd_full_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                  void *data)
 {
-        int            ret    = 0;
-        int            count  = 0;
+    struct subvol_healer *healer = data;
+    xlator_t *this = healer->this;
+    afr_private_t *priv = NULL;
 
-        ret = afr_shd_index_sweep (healer, GF_XATTROP_INDEX_GFID);
-        if (ret < 0)
-                goto out;
-        count = ret;
+    priv = this->private;
 
-        ret = afr_shd_index_sweep (healer, GF_XATTROP_DIRTY_GFID);
-        if (ret < 0)
-                goto out;
-        count += ret;
+    if (this->cleanup_starting) {
+        return -ENOTCONN;
+    }
 
-        ret = afr_shd_index_sweep (healer, GF_XATTROP_ENTRY_CHANGES_GFID);
-        if (ret < 0)
-                goto out;
-        count += ret;
-out:
-        if (ret < 0)
-                return ret;
-        else
-                return count;
+    if (!priv->shd.enabled)
+        return -EBUSY;
+
+    afr_shd_selfheal_name(healer, healer->subvol, parent->inode->gfid,
+                          entry->d_name);
+
+    afr_shd_selfheal(healer, healer->subvol, entry->d_stat.ia_gfid);
+
+    return 0;
 }
 
 int
-afr_shd_full_heal (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                   void *data)
+afr_shd_full_sweep(struct subvol_healer *healer, inode_t *inode)
 {
-        struct subvol_healer *healer = data;
-        xlator_t             *this   = healer->this;
-        afr_private_t        *priv   = NULL;
+    afr_private_t *priv = NULL;
+    loc_t loc = {0};
 
-        priv = this->private;
-        if (!priv->shd.enabled)
-                return -EBUSY;
+    priv = healer->this->private;
+    loc.inode = inode;
+    return syncop_ftw(priv->children[healer->subvol], &loc,
+                      GF_CLIENT_PID_SELF_HEALD, healer, afr_shd_full_heal);
+}
 
-        afr_shd_selfheal_name (healer, healer->subvol,
-                               parent->inode->gfid, entry->d_name);
+int
+afr_shd_fill_ta_loc(xlator_t *this, loc_t *loc)
+{
+    afr_private_t *priv = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int ret = -1;
+
+    priv = this->private;
+    loc->parent = inode_ref(this->itable->root);
+    gf_uuid_copy(loc->pargfid, loc->parent->gfid);
+    loc->name = priv->pending_key[THIN_ARBITER_BRICK_INDEX];
+    loc->inode = inode_new(loc->parent->table);
+    GF_CHECK_ALLOC(loc->inode, ret, out);
+
+    if (!gf_uuid_is_null(priv->ta_gfid))
+        goto assign_gfid;
+
+    ret = syncop_lookup(priv->children[THIN_ARBITER_BRICK_INDEX], loc, &stbuf,
+                        0, 0, 0);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed lookup on file %s.", loc->name);
+        goto out;
+    }
+
+    gf_uuid_copy(priv->ta_gfid, stbuf.ia_gfid);
+
+assign_gfid:
+    gf_uuid_copy(loc->gfid, priv->ta_gfid);
+    ret = 0;
 
-        afr_shd_selfheal (healer, healer->subvol, entry->d_stat.ia_gfid);
+out:
+    if (ret)
+        loc_wipe(loc);
 
-        return 0;
+    return ret;
 }
 
 int
-afr_shd_full_sweep (struct subvol_healer *healer, inode_t *inode)
+_afr_shd_ta_get_xattrs(xlator_t *this, loc_t *loc, dict_t **xdata)
 {
-        afr_private_t *priv = NULL;
-        loc_t          loc  = {0};
-
-        priv = healer->this->private;
-        loc.inode = inode;
-        return syncop_ftw (priv->children[healer->subvol], &loc,
-                           GF_CLIENT_PID_SELF_HEALD, healer,
-                           afr_shd_full_heal);
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int raw[AFR_NUM_CHANGE_LOGS] = {
+        0,
+    };
+    int ret = -1;
+    int i = 0;
+
+    priv = this->private;
+
+    xattr = dict_new();
+    if (!xattr) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_GET_FAILED,
+               "Failed to create dict.");
+        goto out;
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], &raw,
+                                  AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret)
+            goto out;
+    }
+
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], loc,
+                         GF_XATTROP_ADD_ARRAY, xattr, NULL, xdata, NULL);
+    if (ret || !(*xdata)) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Xattrop failed on %s.", loc->name);
+    }
+
+out:
+    if (xattr)
+        dict_unref(xattr);
+
+    return ret;
 }
 
 void
-afr_shd_ta_set_xattrs (xlator_t *this, loc_t *loc, dict_t **xdata,
-                       int healer)
+afr_shd_ta_get_xattrs(xlator_t *this, loc_t *loc, struct subvol_healer *healer,
+                      dict_t **xdata)
 {
-        afr_private_t   *priv                          = NULL;
-        dict_t          *xattr                         = NULL;
-        struct gf_flock  flock                         = {0, };
-        gf_boolean_t     need_xattrop                  = _gf_false;
-        void            *pending_raw                   = NULL;
-        int             *raw                           = NULL;
-        int              pending[AFR_NUM_CHANGE_LOGS]  = {0,};
-        int              i                             = 0;
-        int              j                             = 0;
-        int              val                           = 0;
-        int              ret                           = 0;
-
-        priv = this->private;
-
-        xattr = dict_new ();
-        if (!xattr) {
-                goto out;
+    int ret = 0;
+
+    loc_wipe(loc);
+    if (afr_shd_fill_ta_loc(this, loc)) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc->name);
+        ret = -1;
+        goto out;
+    }
+
+    ret = afr_ta_post_op_lock(this, loc);
+    if (ret)
+        goto out;
+
+    ret = _afr_shd_ta_get_xattrs(this, loc, xdata);
+    if (ret) {
+        if (*xdata) {
+            dict_unref(*xdata);
+            *xdata = NULL;
         }
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                raw = GF_CALLOC (AFR_NUM_CHANGE_LOGS, sizeof(int),
-                                 gf_afr_mt_int32_t);
-                if (!raw) {
-                        goto out;
-                }
-
-                ret = dict_get_ptr (*xdata, priv->pending_key[i], &pending_raw);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                AFR_MSG_DICT_GET_FAILED, "Error getting value "
-                                "of pending key %s", priv->pending_key[i]);
-                        GF_FREE (raw);
-                        goto out;
-                }
+    afr_ta_post_op_unlock(this, loc);
 
-                memcpy (pending, pending_raw, sizeof (pending));
-                for (j = 0; j < AFR_NUM_CHANGE_LOGS; j++) {
-                        val = ntoh32 (pending[j]);
-                        if (val) {
-                                if (i == healer) {
-                                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                                AFR_MSG_THIN_ARB, "I am "
-                                                "not the good shd. Skipping. "
-                                                "SHD = %d.", healer);
-                                        GF_FREE (raw);
-                                        goto out;
-                                }
-                                need_xattrop = _gf_true;
-                                raw[j] = hton32 (-val);
-                        }
-                }
+out:
+    if (ret)
+        healer->rerun = 1;
+}
 
-                ret = dict_set_bin (xattr, priv->pending_key[i], raw,
-                                    AFR_NUM_CHANGE_LOGS * sizeof (int));
-                if (ret) {
-                        GF_FREE (raw);
-                        goto out;
-                }
+int
+afr_shd_ta_unset_xattrs(xlator_t *this, loc_t *loc, dict_t **xdata, int healer)
+{
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    gf_boolean_t need_xattrop = _gf_false;
+    void *pending_raw = NULL;
+    int *raw = NULL;
+    int pending[AFR_NUM_CHANGE_LOGS] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+    int val = 0;
+    int ret = -1;
+
+    priv = this->private;
+
+    xattr = dict_new();
+    if (!xattr) {
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        raw = GF_CALLOC(AFR_NUM_CHANGE_LOGS, sizeof(int), gf_afr_mt_int32_t);
+        if (!raw) {
+            goto out;
+        }
 
-                memset (pending, 0, sizeof (pending));
+        ret = dict_get_ptr(*xdata, priv->pending_key[i], &pending_raw);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                   "Error getting value "
+                   "of pending key %s",
+                   priv->pending_key[i]);
+            GF_FREE(raw);
+            goto out;
         }
 
-        if (!need_xattrop) {
-                goto out;
+        memcpy(pending, pending_raw, sizeof(pending));
+        for (j = 0; j < AFR_NUM_CHANGE_LOGS; j++) {
+            val = ntoh32(pending[j]);
+            if (val) {
+                if (i == healer) {
+                    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_THIN_ARB,
+                           "I am "
+                           "not the good shd. Skipping. "
+                           "SHD = %d.",
+                           healer);
+                    ret = 0;
+                    GF_FREE(raw);
+                    goto out;
+                }
+                need_xattrop = _gf_true;
+                raw[j] = hton32(-val);
+            }
         }
 
-        flock.l_type = F_WRLCK;
-        flock.l_start = 0;
-        flock.l_len = 0;
+        ret = dict_set_bin(xattr, priv->pending_key[i], raw,
+                           AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret) {
+            GF_FREE(raw);
+            goto out;
+        }
 
-        ret = syncop_inodelk (priv->children[THIN_ARBITER_BRICK_INDEX],
-                              THIN_ARBITER_DOM1, loc, F_SETLKW, &flock,
-                              NULL, NULL);
-        if (ret)
-                goto out;
+        if (need_xattrop)
+            break;
+    }
 
-        ret = syncop_xattrop (priv->children[THIN_ARBITER_BRICK_INDEX], loc,
-                              GF_XATTROP_ADD_ARRAY, xattr, NULL, NULL, NULL);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
-                        "Xattrop failed.");
+    if (!need_xattrop) {
+        ret = 0;
+        goto out;
+    }
 
-        flock.l_type = F_UNLCK;
-        syncop_inodelk (priv->children[THIN_ARBITER_BRICK_INDEX],
-                        THIN_ARBITER_DOM1, loc, F_SETLKW, &flock, NULL, NULL);
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], loc,
+                         GF_XATTROP_ADD_ARRAY, xattr, NULL, NULL, NULL);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Xattrop failed.");
 
 out:
-        if (xattr)
-                dict_unref (xattr);
-        return;
+    if (xattr)
+        dict_unref(xattr);
+
+    return ret;
 }
 
 void
-afr_shd_ta_get_xattrs (xlator_t *this, loc_t *loc, dict_t **xdata)
+afr_shd_ta_check_and_unset_xattrs(xlator_t *this, loc_t *loc,
+                                  struct subvol_healer *healer,
+                                  dict_t *pre_crawl_xdata)
 {
-	afr_private_t   *priv   = NULL;
-        dict_t          *xattr  = NULL;
-        struct iatt      stbuf  = {0,};
-        int             *raw    = NULL;
-	int              ret    = 0;
-        int              i      = 0;
-
-        priv = this->private;
-
-        loc->parent = inode_ref (this->itable->root);
-        gf_uuid_copy (loc->pargfid, loc->parent->gfid);
-        loc->name = priv->pending_key[THIN_ARBITER_BRICK_INDEX];
-        loc->inode = inode_new (loc->parent->table);
-        if (!loc->inode) {
-                goto out;
-        }
+    int ret_lock = 0;
+    int ret = 0;
+    dict_t *post_crawl_xdata = NULL;
 
-        ret = syncop_lookup (priv->children[THIN_ARBITER_BRICK_INDEX], loc,
-                             &stbuf, 0, 0, 0);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
-                        "Failed lookup on file %s.", loc->name);
-                goto out;
-        }
+    ret_lock = afr_ta_post_op_lock(this, loc);
+    if (ret_lock)
+        goto unref;
 
-        gf_uuid_copy (priv->ta_gfid, stbuf.ia_gfid);
-        gf_uuid_copy (loc->gfid, priv->ta_gfid);
+    ret = _afr_shd_ta_get_xattrs(this, loc, &post_crawl_xdata);
+    if (ret)
+        goto unref;
 
-        xattr = dict_new ();
-        if (!xattr) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_GET_FAILED,
-                        "Failed to create dict.");
-                goto out;
+    if (!are_dicts_equal(pre_crawl_xdata, post_crawl_xdata, NULL, NULL)) {
+        ret = -1;
+        goto unref;
+    }
+
+    ret = afr_shd_ta_unset_xattrs(this, loc, &post_crawl_xdata, healer->subvol);
+
+unref:
+    if (post_crawl_xdata) {
+        dict_unref(post_crawl_xdata);
+        post_crawl_xdata = NULL;
+    }
+
+    if (ret || ret_lock)
+        healer->rerun = 1;
+
+    if (!ret_lock)
+        afr_ta_post_op_unlock(this, loc);
+}
+
+gf_boolean_t
+afr_bricks_available_for_heal(afr_private_t *priv)
+{
+    int up_children = 0;
+
+    up_children = __afr_get_up_children_count(priv);
+    if (up_children < 2) {
+        return _gf_false;
+    }
+    return _gf_true;
+}
+
+static gf_boolean_t
+afr_shd_ta_needs_heal(xlator_t *this, struct subvol_healer *healer)
+{
+    dict_t *xdata = NULL;
+    afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    int i = 0;
+    gf_boolean_t need_heal = _gf_false;
+
+    priv = this->private;
+
+    ret = afr_shd_fill_ta_loc(this, &loc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        healer->rerun = 1;
+        goto out;
+    }
+
+    if (_afr_shd_ta_get_xattrs(this, &loc, &xdata)) {
+        healer->rerun = 1;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (afr_ta_dict_contains_pending_xattr(xdata, priv, i)) {
+            need_heal = _gf_true;
+            break;
         }
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                raw = GF_CALLOC (AFR_NUM_CHANGE_LOGS, sizeof(int),
-                                 gf_afr_mt_int32_t);
-                if (!raw) {
-                        goto out;
-                }
+out:
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
 
-                ret = dict_set_bin (xattr, priv->pending_key[i], raw,
-                                    AFR_NUM_CHANGE_LOGS * sizeof (int));
-                if (ret) {
-                        GF_FREE (raw);
-                        goto out;
+    return need_heal;
+}
+
+static int
+afr_shd_anon_inode_cleaner(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                           void *data)
+{
+    struct subvol_healer *healer = data;
+    afr_private_t *priv = healer->this->private;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+    loc_t loc = {0};
+    int count = 0;
+    int i = 0;
+    int op_errno = 0;
+    struct iatt *iatt = NULL;
+    gf_boolean_t multiple_links = _gf_false;
+    unsigned char *gfid_present = alloca0(priv->child_count);
+    unsigned char *entry_present = alloca0(priv->child_count);
+    char *type = "file";
+
+    frame = afr_frame_create(healer->this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+    local = frame->local;
+    if (AFR_COUNT(local->child_up, priv->child_count) != priv->child_count) {
+        gf_msg_debug(healer->this->name, 0,
+                     "Not all bricks are up. Skipping "
+                     "cleanup of %s on %s",
+                     entry->d_name, subvol->name);
+        ret = 0;
+        goto out;
+    }
+
+    loc.inode = inode_new(parent->inode->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    ret = gf_uuid_parse(entry->d_name, loc.gfid);
+    if (ret) {
+        ret = 0;
+        goto out;
+    }
+    AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == 0) {
+            count++;
+            gfid_present[i] = 1;
+            iatt = &local->replies[i].poststat;
+            if (iatt->ia_type == IA_IFDIR) {
+                type = "dir";
+            }
+
+            if (i == healer->subvol) {
+                if (local->replies[i].poststat.ia_nlink > 1) {
+                    multiple_links = _gf_true;
                 }
+            }
+        } else if (local->replies[i].op_errno != ENOENT &&
+                   local->replies[i].op_errno != ESTALE) {
+            /*We don't have complete view. Skip the entry*/
+            gf_msg_debug(healer->this->name, local->replies[i].op_errno,
+                         "Skipping cleanup of %s on %s", entry->d_name,
+                         subvol->name);
+            ret = 0;
+            goto out;
         }
-
-        ret = syncop_xattrop (priv->children[THIN_ARBITER_BRICK_INDEX],
-                              loc, GF_XATTROP_ADD_ARRAY, xattr, NULL, xdata,
-                              NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
-                        "Xattrop failed.");
+    }
+
+    /*Inode is deleted from subvol*/
+    if (count == 1 || (iatt->ia_type != IA_IFDIR && multiple_links)) {
+        gf_msg(healer->this->name, GF_LOG_WARNING, 0,
+               AFR_MSG_EXPUNGING_FILE_OR_DIR, "expunging %s %s/%s on %s", type,
+               priv->anon_inode_name, entry->d_name, subvol->name);
+        ret = afr_shd_entry_purge(subvol, parent->inode, entry->d_name,
+                                  iatt->ia_type);
+        if (ret == -ENOENT || ret == -ESTALE)
+            ret = 0;
+    } else if (count > 1) {
+        loc_wipe(&loc);
+        loc.parent = inode_ref(parent->inode);
+        loc.name = entry->d_name;
+        loc.inode = inode_new(parent->inode->table);
+        if (!loc.inode) {
+            ret = -ENOMEM;
+            goto out;
+        }
+        AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup,
+                   &loc, NULL);
+        count = 0;
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->replies[i].op_ret == 0) {
+                count++;
+                entry_present[i] = 1;
+                iatt = &local->replies[i].poststat;
+            } else if (local->replies[i].op_errno != ENOENT &&
+                       local->replies[i].op_errno != ESTALE) {
+                /*We don't have complete view. Skip the entry*/
+                gf_msg_debug(healer->this->name, local->replies[i].op_errno,
+                             "Skipping cleanup of %s on %s", entry->d_name,
+                             subvol->name);
+                ret = 0;
+                goto out;
+            }
+        }
+        for (i = 0; i < priv->child_count; i++) {
+            if (gfid_present[i] && !entry_present[i]) {
+                /*Entry is not anonymous on at least one subvol*/
+                gf_msg_debug(healer->this->name, 0,
+                             "Valid entry present on %s "
+                             "Skipping cleanup of %s on %s",
+                             priv->children[i]->name, entry->d_name,
+                             subvol->name);
+                ret = 0;
                 goto out;
+            }
+        }
+
+        gf_msg(healer->this->name, GF_LOG_WARNING, 0,
+               AFR_MSG_EXPUNGING_FILE_OR_DIR,
+               "expunging %s %s/%s on all subvols", type, priv->anon_inode_name,
+               entry->d_name);
+        ret = 0;
+        for (i = 0; i < priv->child_count; i++) {
+            op_errno = -afr_shd_entry_purge(priv->children[i], loc.parent,
+                                            entry->d_name, iatt->ia_type);
+            if (op_errno != ENOENT && op_errno != ESTALE) {
+                ret |= -op_errno;
+            }
         }
-        if (!(*xdata))
-                gf_msg (this->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_GET_FAILED,
-                        "Xdata response is empty.");
+    }
 
 out:
-        if (xattr)
-                dict_unref (xattr);
-        return;
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    loc_wipe(&loc);
+    return ret;
+}
+
+static void
+afr_cleanup_anon_inode_dir(struct subvol_healer *healer)
+{
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    afr_private_t *priv = healer->this->private;
+    loc_t loc = {0};
+
+    ret = afr_anon_inode_create(healer->this, healer->subvol, &loc.inode);
+    if (ret)
+        goto out;
+
+    frame = afr_frame_create(healer->this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    ret = syncop_mt_dir_scan(frame, priv->children[healer->subvol], &loc,
+                             GF_CLIENT_PID_SELF_HEALD, healer,
+                             afr_shd_anon_inode_cleaner, NULL,
+                             priv->shd.max_threads, priv->shd.wait_qlength);
+out:
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    loc_wipe(&loc);
+    return;
 }
 
 void *
-afr_shd_index_healer (void *data)
+afr_shd_index_healer(void *data)
 {
-	struct subvol_healer *healer = NULL;
-	xlator_t *this = NULL;
-	int ret = 0;
-	afr_private_t *priv = NULL;
-        dict_t *xdata = NULL;
-        loc_t loc = {0, };
-
-	healer = data;
-	THIS = this = healer->this;
-	priv = this->private;
-
-	for (;;) {
-		afr_shd_healer_wait (healer);
-
-		ASSERT_LOCAL(this, healer);
-		priv->local[healer->subvol] = healer->local;
-
-                if (priv->thin_arbiter_count) {
-                        loc_wipe (&loc);
-                        afr_shd_ta_get_xattrs (this, &loc, &xdata);
-                }
+    struct subvol_healer *healer = NULL;
+    xlator_t *this = NULL;
+    int ret = 0;
+    afr_private_t *priv = NULL;
+    dict_t *pre_crawl_xdata = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    healer = data;
+    THIS = this = healer->this;
+    priv = this->private;
+
+    for (;;) {
+        afr_shd_healer_wait(healer);
+
+        if (!afr_bricks_available_for_heal(priv))
+            continue;
+
+        ASSERT_LOCAL(this, healer);
+        priv->local[healer->subvol] = healer->local;
+
+        if (priv->thin_arbiter_count) {
+            if (afr_shd_ta_needs_heal(this, healer))
+                afr_shd_ta_get_xattrs(this, &loc, healer, &pre_crawl_xdata);
+        }
 
+        do {
+            gf_msg_debug(this->name, 0, "starting index sweep on subvol %s",
+                         afr_subvol_name(this, healer->subvol));
+
+            afr_shd_sweep_prepare(healer);
+
+            ret = afr_shd_index_sweep_all(healer);
+
+            afr_shd_sweep_done(healer);
+            /*
+              As long as at least one gfid was
+              healed, keep retrying. We may have
+              just healed a directory and thereby
+              created entries for other gfids which
+              could not be healed thus far.
+            */
+
+            gf_msg_debug(this->name, 0, "finished index sweep on subvol %s",
+                         afr_subvol_name(this, healer->subvol));
+            /*
+              Give a pause before retrying to avoid a busy loop
+              in case the only entry in index is because of
+              an ongoing I/O.
+            */
+            sleep(1);
+        } while (ret > 0);
+
+        if (ret == 0) {
+            afr_cleanup_anon_inode_dir(healer);
+        }
 
-		do {
-		        gf_msg_debug (this->name, 0,
-		                      "starting index sweep on subvol %s",
-			              afr_subvol_name (this, healer->subvol));
-
-			afr_shd_sweep_prepare (healer);
-
-                        ret = afr_shd_index_sweep_all (healer);
-
-			afr_shd_sweep_done (healer);
-			/*
-			  As long as at least one gfid was
-			  healed, keep retrying. We may have
-			  just healed a directory and thereby
-			  created entries for other gfids which
-			  could not be healed thus far.
-			*/
-
-		        gf_msg_debug (this->name, 0,
-			              "finished index sweep on subvol %s",
-			              afr_subvol_name (this, healer->subvol));
-			/*
-			  Give a pause before retrying to avoid a busy loop
-			  in case the only entry in index is because of
-			  an ongoing I/O.
-			*/
-			sleep (1);
-		} while (ret > 0);
-
-                if (xdata && !healer->crawl_event.heal_failed_count) {
-                        afr_shd_ta_set_xattrs (this, &loc, &xdata,
-                                               healer->subvol);
-                        dict_unref (xdata);
-                        xdata = NULL;
-                }
-	}
+        if (ret == 0 && pre_crawl_xdata &&
+            !healer->crawl_event.heal_failed_count) {
+            afr_shd_ta_check_and_unset_xattrs(this, &loc, healer,
+                                              pre_crawl_xdata);
+        }
 
-        loc_wipe (&loc);
+        if (pre_crawl_xdata) {
+            dict_unref(pre_crawl_xdata);
+            pre_crawl_xdata = NULL;
+        }
+    }
 
-	return NULL;
+    return NULL;
 }
 
-
 void *
-afr_shd_full_healer (void *data)
+afr_shd_full_healer(void *data)
 {
-	struct subvol_healer *healer = NULL;
-	xlator_t *this = NULL;
-	int run = 0;
+    struct subvol_healer *healer = NULL;
+    xlator_t *this = NULL;
+    int run = 0;
 
-	healer = data;
-	THIS = this = healer->this;
+    healer = data;
+    THIS = this = healer->this;
 
-	for (;;) {
-		pthread_mutex_lock (&healer->mutex);
-		{
-			run = __afr_shd_healer_wait (healer);
-			if (!run)
-				healer->running = _gf_false;
-		}
-		pthread_mutex_unlock (&healer->mutex);
+    for (;;) {
+        pthread_mutex_lock(&healer->mutex);
+        {
+            run = __afr_shd_healer_wait(healer);
+            if (!run)
+                healer->running = _gf_false;
+        }
+        pthread_mutex_unlock(&healer->mutex);
 
-		if (!run)
-			break;
+        if (!run)
+            break;
 
-		ASSERT_LOCAL(this, healer);
+        ASSERT_LOCAL(this, healer);
 
-	        gf_msg (this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
-		        "starting full sweep on subvol %s",
-		        afr_subvol_name (this, healer->subvol));
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+               "starting full sweep on subvol %s",
+               afr_subvol_name(this, healer->subvol));
 
-		afr_shd_sweep_prepare (healer);
+        afr_shd_sweep_prepare(healer);
 
-		afr_shd_full_sweep (healer, this->itable->root);
+        afr_shd_full_sweep(healer, this->itable->root);
 
-		afr_shd_sweep_done (healer);
+        afr_shd_sweep_done(healer);
 
-	        gf_msg (this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
-		        "finished full sweep on subvol %s",
-		        afr_subvol_name (this, healer->subvol));
-	}
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+               "finished full sweep on subvol %s",
+               afr_subvol_name(this, healer->subvol));
+    }
 
-	return NULL;
+    return NULL;
 }
 
-
 int
-afr_shd_healer_init (xlator_t *this, struct subvol_healer *healer)
+afr_shd_healer_init(xlator_t *this, struct subvol_healer *healer)
 {
-	int ret = 0;
+    int ret = 0;
 
-	ret = pthread_mutex_init (&healer->mutex, NULL);
-	if (ret)
-		goto out;
+    ret = pthread_mutex_init(&healer->mutex, NULL);
+    if (ret)
+        goto out;
 
-	ret = pthread_cond_init (&healer->cond, NULL);
-	if (ret)
-		goto out;
+    ret = pthread_cond_init(&healer->cond, NULL);
+    if (ret)
+        goto out;
 
-	healer->this = this;
-	healer->running = _gf_false;
-	healer->rerun = _gf_false;
-	healer->local = _gf_false;
+    healer->this = this;
+    healer->running = _gf_false;
+    healer->rerun = _gf_false;
+    healer->local = _gf_false;
 out:
-	return ret;
+    return ret;
 }
 
-
 int
-afr_shd_healer_spawn (xlator_t *this, struct subvol_healer *healer,
-		      void *(threadfn)(void *))
+afr_shd_healer_spawn(xlator_t *this, struct subvol_healer *healer,
+                     void *(threadfn)(void *))
 {
-	int ret = 0;
-
-	pthread_mutex_lock (&healer->mutex);
-	{
-		if (healer->running) {
-			pthread_cond_signal (&healer->cond);
-		} else {
-			ret = gf_thread_create (&healer->thread, NULL,
-						threadfn, healer, "shdheal");
-			if (ret)
-				goto unlock;
-			healer->running = 1;
-		}
-
-		healer->rerun = 1;
-	}
+    int ret = 0;
+
+    pthread_mutex_lock(&healer->mutex);
+    {
+        if (healer->running) {
+            pthread_cond_signal(&healer->cond);
+        } else {
+            ret = gf_thread_create(&healer->thread, NULL, threadfn, healer,
+                                   "shdheal");
+            if (ret)
+                goto unlock;
+            healer->running = 1;
+        }
+
+        healer->rerun = 1;
+    }
 unlock:
-	pthread_mutex_unlock (&healer->mutex);
+    pthread_mutex_unlock(&healer->mutex);
 
-	return ret;
+    return ret;
 }
 
-
 int
-afr_shd_full_healer_spawn (xlator_t *this, int subvol)
+afr_shd_full_healer_spawn(xlator_t *this, int subvol)
 {
-	return afr_shd_healer_spawn (this, NTH_FULL_HEALER (this, subvol),
-				     afr_shd_full_healer);
+    return afr_shd_healer_spawn(this, NTH_FULL_HEALER(this, subvol),
+                                afr_shd_full_healer);
 }
 
-
 int
-afr_shd_index_healer_spawn (xlator_t *this, int subvol)
+afr_shd_index_healer_spawn(xlator_t *this, int subvol)
 {
-	return afr_shd_healer_spawn (this, NTH_INDEX_HEALER (this, subvol),
-				     afr_shd_index_healer);
+    return afr_shd_healer_spawn(this, NTH_INDEX_HEALER(this, subvol),
+                                afr_shd_index_healer);
 }
 
-
 int
-afr_shd_dict_add_crawl_event (xlator_t *this, dict_t *output,
-			      crawl_event_t *crawl_event)
+afr_shd_dict_add_crawl_event(xlator_t *this, dict_t *output,
+                             crawl_event_t *crawl_event)
 {
-        int             ret = 0;
-        uint64_t        count = 0;
-        char            key[256] = {0};
-        int             xl_id = 0;
-        uint64_t        healed_count = 0;
-        uint64_t        split_brain_count = 0;
-        uint64_t        heal_failed_count = 0;
-        char            *start_time_str = 0;
-        char            *end_time_str = NULL;
-        char            *crawl_type = NULL;
-        int             progress = -1;
-	int             child = -1;
-
-	child = crawl_event->child;
-        healed_count = crawl_event->healed_count;
-        split_brain_count = crawl_event->split_brain_count;
-        heal_failed_count = crawl_event->heal_failed_count;
-        crawl_type = crawl_event->crawl_type;
-
-	if (!crawl_event->start_time)
-		goto out;
-
-        start_time_str = gf_strdup (ctime (&crawl_event->start_time));
-
-	if (crawl_event->end_time)
-		end_time_str = gf_strdup (ctime (&crawl_event->end_time));
-
-        ret = dict_get_int32 (output, this->name, &xl_id);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        AFR_MSG_DICT_GET_FAILED, "xl does not have id");
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
-        ret = dict_get_uint64 (output, key, &count);
-
-
-        snprintf (key, sizeof (key), "statistics_healed_cnt-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_uint64(output, key, healed_count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-		        "Could not add statistics_healed_count to output");
-                goto out;
-	}
-
-        snprintf (key, sizeof (key), "statistics_sb_cnt-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_uint64 (output, key, split_brain_count);
-	if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-		        "Could not add statistics_split_brain_count to output");
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "statistics_crawl_type-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_str (output, key, crawl_type);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-	                "Could not add statistics_crawl_type to output");
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "statistics_heal_failed_cnt-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_uint64 (output, key, heal_failed_count);
-	if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-	                "Could not add statistics_healed_failed_count to output");
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "statistics_strt_time-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_dynstr (output, key, start_time_str);
-	if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-		        "Could not add statistics_crawl_start_time to output");
-                goto out;
-        } else {
-		start_time_str = NULL;
-	}
-
-	if (!end_time_str)
-                progress = 1;
-        else
-                progress = 0;
-
-        snprintf (key, sizeof (key), "statistics_end_time-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        if (!end_time_str)
-                end_time_str = gf_strdup ("Could not determine the end time");
-        ret = dict_set_dynstr (output, key, end_time_str);
-	if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-		        "Could not add statistics_crawl_end_time to output");
-                goto out;
-        } else {
-		end_time_str = NULL;
-	}
-
-        snprintf (key, sizeof (key), "statistics_inprogress-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-
-        ret = dict_set_int32 (output, key, progress);
-	if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-		        "Could not add statistics_inprogress to output");
-                goto out;
-        }
-
-	snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
-	ret = dict_set_uint64 (output, key, count + 1);
-	if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-		        "Could not increment the counter.");
-                goto out;
-	}
+    int ret = 0;
+    uint64_t count = 0;
+    char key[128] = {0};
+    int keylen = 0;
+    char suffix[64] = {0};
+    int xl_id = 0;
+    uint64_t healed_count = 0;
+    uint64_t split_brain_count = 0;
+    uint64_t heal_failed_count = 0;
+    char *start_time_str = 0;
+    char *end_time_str = NULL;
+    char *crawl_type = NULL;
+    int progress = -1;
+    int child = -1;
+
+    child = crawl_event->child;
+    healed_count = crawl_event->healed_count;
+    split_brain_count = crawl_event->split_brain_count;
+    heal_failed_count = crawl_event->heal_failed_count;
+    crawl_type = crawl_event->crawl_type;
+
+    if (!crawl_event->start_time)
+        goto out;
+
+    start_time_str = gf_strdup(ctime(&crawl_event->start_time));
+
+    if (crawl_event->end_time)
+        end_time_str = gf_strdup(ctime(&crawl_event->end_time));
+
+    ret = dict_get_int32(output, this->name, &xl_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+               "xl does not have id");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics-%d-%d-count", xl_id, child);
+    ret = dict_get_uint64(output, key, &count);
+
+    snprintf(suffix, sizeof(suffix), "%d-%d-%" PRIu64, xl_id, child, count);
+    snprintf(key, sizeof(key), "statistics_healed_cnt-%s", suffix);
+    ret = dict_set_uint64(output, key, healed_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_healed_count to output");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics_sb_cnt-%s", suffix);
+    ret = dict_set_uint64(output, key, split_brain_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_split_brain_count to output");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "statistics_crawl_type-%s", suffix);
+    ret = dict_set_strn(output, key, keylen, crawl_type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_crawl_type to output");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics_heal_failed_cnt-%s", suffix);
+    ret = dict_set_uint64(output, key, heal_failed_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_healed_failed_count to output");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "statistics_strt_time-%s", suffix);
+    ret = dict_set_dynstrn(output, key, keylen, start_time_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_crawl_start_time to output");
+        goto out;
+    } else {
+        start_time_str = NULL;
+    }
+
+    if (!end_time_str)
+        progress = 1;
+    else
+        progress = 0;
+
+    keylen = snprintf(key, sizeof(key), "statistics_end_time-%s", suffix);
+    if (!end_time_str)
+        end_time_str = gf_strdup("Could not determine the end time");
+    ret = dict_set_dynstrn(output, key, keylen, end_time_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_crawl_end_time to output");
+        goto out;
+    } else {
+        end_time_str = NULL;
+    }
+
+    keylen = snprintf(key, sizeof(key), "statistics_inprogress-%s", suffix);
+
+    ret = dict_set_int32n(output, key, keylen, progress);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_inprogress to output");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics-%d-%d-count", xl_id, child);
+    ret = dict_set_uint64(output, key, count + 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not increment the counter.");
+        goto out;
+    }
 out:
-	GF_FREE (start_time_str);
-	GF_FREE (end_time_str);
-        return ret;
+    GF_FREE(start_time_str);
+    GF_FREE(end_time_str);
+    return ret;
 }
 
-
 int
-afr_shd_dict_add_path (xlator_t *this, dict_t *output, int child, char *path,
-		       struct timeval *tv)
+afr_shd_dict_add_path(xlator_t *this, dict_t *output, int child, char *path,
+                      struct timeval *tv)
 {
-        int             ret = -1;
-        uint64_t        count = 0;
-        char            key[256] = {0};
-        int             xl_id = 0;
-
-        ret = dict_get_int32 (output, this->name, &xl_id);
+    int ret = -1;
+    uint64_t count = 0;
+    char key[64] = {0};
+    int keylen = 0;
+    char xl_id_child_str[32] = {0};
+    int xl_id = 0;
+
+    ret = dict_get_int32(output, this->name, &xl_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+               "xl does not have id");
+        goto out;
+    }
+
+    snprintf(xl_id_child_str, sizeof(xl_id_child_str), "%d-%d", xl_id, child);
+    snprintf(key, sizeof(key), "%s-count", xl_id_child_str);
+    ret = dict_get_uint64(output, key, &count);
+
+    keylen = snprintf(key, sizeof(key), "%s-%" PRIu64, xl_id_child_str, count);
+    ret = dict_set_dynstrn(output, key, keylen, path);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Could not add to output", path);
+        goto out;
+    }
+
+    if (tv) {
+        snprintf(key, sizeof(key), "%s-%" PRIu64 "-time", xl_id_child_str,
+                 count);
+        ret = dict_set_uint32(output, key, tv->tv_sec);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        AFR_MSG_DICT_GET_FAILED, "xl does not have id");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "%s: Could not set time", path);
+            goto out;
         }
+    }
 
-        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
-        ret = dict_get_uint64 (output, key, &count);
-
-        snprintf (key, sizeof (key), "%d-%d-%"PRIu64, xl_id, child, count);
-	ret = dict_set_dynstr (output, key, path);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        AFR_MSG_DICT_SET_FAILED, "%s: Could not add to output",
-                        path);
-                goto out;
-        }
+    snprintf(key, sizeof(key), "%s-count", xl_id_child_str);
 
-	if (tv) {
-		snprintf (key, sizeof (key), "%d-%d-%"PRIu64"-time", xl_id,
-			  child, count);
-		ret = dict_set_uint32 (output, key, tv->tv_sec);
-		if (ret) {
-		        gf_msg (this->name, GF_LOG_ERROR,
-                                -ret, AFR_MSG_DICT_SET_FAILED,
-                                "%s: Could not set time",
-			        path);
-			goto out;
-		}
-	}
-
-        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
-
-        ret = dict_set_uint64 (output, key, count + 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        -ret, AFR_MSG_DICT_SET_FAILED,
-                        "Could not increment count");
-                goto out;
-        }
+    ret = dict_set_uint64(output, key, count + 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not increment count");
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-afr_add_shd_event (circular_buffer_t *cb, void *data)
+afr_add_shd_event(circular_buffer_t *cb, void *data)
 {
-	dict_t *output = NULL;
-	xlator_t *this = THIS;
-	afr_private_t *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	shd_event_t *shd_event = NULL;
-	char *path = NULL;
-
-	output = data;
-	priv = this->private;
-	shd = &priv->shd;
-	shd_event = cb->data;
-
-	if (!shd->index_healers[shd_event->child].local)
-		return 0;
-
-	path = gf_strdup (shd_event->path);
-	if (!path)
-		return -ENOMEM;
-
-	afr_shd_dict_add_path (this, output, shd_event->child, path,
-			       &cb->tv);
-	return 0;
+    dict_t *output = NULL;
+    xlator_t *this = THIS;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    shd_event_t *shd_event = NULL;
+    char *path = NULL;
+
+    output = data;
+    priv = this->private;
+    shd = &priv->shd;
+    shd_event = cb->data;
+
+    if (!shd->index_healers[shd_event->child].local)
+        return 0;
+
+    path = gf_strdup(shd_event->path);
+    if (!path)
+        return -ENOMEM;
+
+    afr_shd_dict_add_path(this, output, shd_event->child, path, &cb->tv);
+    return 0;
 }
 
 int
-afr_add_crawl_event (circular_buffer_t *cb, void *data)
+afr_add_crawl_event(circular_buffer_t *cb, void *data)
 {
-	dict_t *output = NULL;
-	xlator_t *this = THIS;
-	afr_private_t *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	crawl_event_t *crawl_event = NULL;
-
-	output = data;
-	priv = this->private;
-	shd = &priv->shd;
-	crawl_event = cb->data;
-
-	if (!shd->index_healers[crawl_event->child].local)
-		return 0;
+    dict_t *output = NULL;
+    xlator_t *this = THIS;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    crawl_event_t *crawl_event = NULL;
+
+    output = data;
+    priv = this->private;
+    shd = &priv->shd;
+    crawl_event = cb->data;
+
+    if (!shd->index_healers[crawl_event->child].local)
+        return 0;
 
-	afr_shd_dict_add_crawl_event (this, output, crawl_event);
+    afr_shd_dict_add_crawl_event(this, output, crawl_event);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_selfheal_daemon_init (xlator_t *this)
+afr_selfheal_daemon_init(xlator_t *this)
 {
-	afr_private_t *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	int ret = -1;
-	int i = 0;
-
-	priv = this->private;
-	shd = &priv->shd;
-
-	this->itable = inode_table_new (SHD_INODE_LRU_LIMIT, this);
-	if (!this->itable)
-		goto out;
-
-	shd->index_healers = GF_CALLOC (sizeof(*shd->index_healers),
-					priv->child_count,
-					gf_afr_mt_subvol_healer_t);
-	if (!shd->index_healers)
-		goto out;
-
-	for (i = 0; i < priv->child_count; i++) {
-		shd->index_healers[i].subvol = i;
-		ret = afr_shd_healer_init (this, &shd->index_healers[i]);
-		if (ret)
-			goto out;
-	}
-
-	shd->full_healers = GF_CALLOC (sizeof(*shd->full_healers),
-				       priv->child_count,
-				       gf_afr_mt_subvol_healer_t);
-	if (!shd->full_healers)
-		goto out;
-	for (i = 0; i < priv->child_count; i++) {
-		shd->full_healers[i].subvol = i;
-		ret = afr_shd_healer_init (this, &shd->full_healers[i]);
-		if (ret)
-			goto out;
-	}
-
-	shd->split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
-				   afr_destroy_shd_event_data);
-	if (!shd->split_brain)
-		goto out;
-
-        shd->statistics = GF_CALLOC (sizeof(eh_t *), priv->child_count,
-				     gf_common_mt_eh_t);
-        if (!shd->statistics)
-                goto out;
-
-        for (i = 0; i < priv->child_count ; i++) {
-                shd->statistics[i] = eh_new (AFR_STATISTICS_HISTORY_SIZE,
-					     _gf_false,
-					     afr_destroy_crawl_event_data);
-                if (!shd->statistics[i])
-                        goto out;
-		shd->full_healers[i].crawl_event.child = i;
-		shd->full_healers[i].crawl_event.crawl_type = "FULL";
-		shd->index_healers[i].crawl_event.child = i;
-		shd->index_healers[i].crawl_event.crawl_type = "INDEX";
-        }
-
-	ret = 0;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    int ret = -1;
+    int i = 0;
+
+    priv = this->private;
+    shd = &priv->shd;
+
+    shd->index_healers = GF_CALLOC(sizeof(*shd->index_healers),
+                                   priv->child_count,
+                                   gf_afr_mt_subvol_healer_t);
+    if (!shd->index_healers)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        shd->index_healers[i].subvol = i;
+        ret = afr_shd_healer_init(this, &shd->index_healers[i]);
+        if (ret)
+            goto out;
+    }
+
+    shd->full_healers = GF_CALLOC(sizeof(*shd->full_healers), priv->child_count,
+                                  gf_afr_mt_subvol_healer_t);
+    if (!shd->full_healers)
+        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        shd->full_healers[i].subvol = i;
+        ret = afr_shd_healer_init(this, &shd->full_healers[i]);
+        if (ret)
+            goto out;
+    }
+
+    shd->split_brain = eh_new(AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
+                              afr_destroy_shd_event_data);
+    if (!shd->split_brain)
+        goto out;
+
+    shd->statistics = GF_CALLOC(sizeof(eh_t *), priv->child_count,
+                                gf_common_mt_eh_t);
+    if (!shd->statistics)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        shd->statistics[i] = eh_new(AFR_STATISTICS_HISTORY_SIZE, _gf_false,
+                                    afr_destroy_crawl_event_data);
+        if (!shd->statistics[i])
+            goto out;
+        shd->full_healers[i].crawl_event.child = i;
+        shd->full_healers[i].crawl_event.crawl_type = "FULL";
+        shd->index_healers[i].crawl_event.child = i;
+        shd->index_healers[i].crawl_event.crawl_type = "INDEX";
+    }
+
+    ret = 0;
 out:
-	return ret;
+    return ret;
 }
 
-
-int
-afr_selfheal_childup (xlator_t *this, int subvol)
+void
+afr_selfheal_childup(xlator_t *this, afr_private_t *priv)
 {
-	afr_shd_index_healer_spawn (this, subvol);
+    int subvol = 0;
 
-	return 0;
-}
+    if (!priv->shd.iamshd)
+        return;
+    for (subvol = 0; subvol < priv->child_count; subvol++)
+        if (priv->child_up[subvol])
+            afr_shd_index_healer_spawn(this, subvol);
 
+    return;
+}
 
 int
-afr_shd_get_index_count (xlator_t *this, int i, uint64_t *count)
+afr_shd_get_index_count(xlator_t *this, int i, uint64_t *count)
 {
-	afr_private_t *priv = NULL;
-	xlator_t *subvol = NULL;
-	loc_t rootloc = {0, };
-	dict_t *xattr = NULL;
-	int ret = -1;
+    afr_private_t *priv = NULL;
+    xlator_t *subvol = NULL;
+    loc_t rootloc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int ret = -1;
 
-	priv = this->private;
-	subvol = priv->children[i];
+    priv = this->private;
+    subvol = priv->children[i];
 
-	rootloc.inode = inode_ref (this->itable->root);
-	gf_uuid_copy (rootloc.gfid, rootloc.inode->gfid);
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
 
-	ret = syncop_getxattr (subvol, &rootloc, &xattr,
-			       GF_XATTROP_INDEX_COUNT, NULL, NULL);
-	if (ret < 0)
-		goto out;
+    ret = syncop_getxattr(subvol, &rootloc, &xattr, GF_XATTROP_INDEX_COUNT,
+                          NULL, NULL);
+    if (ret < 0)
+        goto out;
 
-	ret = dict_get_uint64 (xattr, GF_XATTROP_INDEX_COUNT, count);
-	if (ret)
-		goto out;
+    ret = dict_get_uint64(xattr, GF_XATTROP_INDEX_COUNT, count);
+    if (ret)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (xattr)
-                dict_unref (xattr);
-	loc_wipe (&rootloc);
+    if (xattr)
+        dict_unref(xattr);
+    loc_wipe(&rootloc);
 
-	return ret;
+    return ret;
 }
 
-
 int
-afr_xl_op (xlator_t *this, dict_t *input, dict_t *output)
+afr_xl_op(xlator_t *this, dict_t *input, dict_t *output)
 {
-        gf_xl_afr_op_t   op = GF_SHD_OP_INVALID;
-        int              ret = 0;
-        int              xl_id = 0;
-	afr_private_t   *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	struct subvol_healer *healer = NULL;
-	int i = 0;
-	char key[64];
-	int op_ret = 0;
-	uint64_t cnt = 0;
-
-	priv = this->private;
-	shd = &priv->shd;
-
-        ret = dict_get_int32 (input, "xl-op", (int32_t*)&op);
-        if (ret)
-                goto out;
-        ret = dict_get_int32 (input, this->name, &xl_id);
-        if (ret)
-                goto out;
-        ret = dict_set_int32 (output, this->name, xl_id);
-        if (ret)
-                goto out;
-        switch (op) {
+    gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
+    int ret = 0;
+    int xl_id = 0;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    struct subvol_healer *healer = NULL;
+    int i = 0;
+    char key[64];
+    int keylen = 0;
+    int this_name_len = 0;
+    int op_ret = 0;
+    uint64_t cnt = 0;
+
+#define AFR_SET_DICT_AND_LOG(name, output, key, keylen, dict_str,              \
+                             dict_str_len)                                     \
+    {                                                                          \
+        int ret;                                                               \
+                                                                               \
+        ret = dict_set_nstrn(output, key, keylen, dict_str, dict_str_len);     \
+        if (ret) {                                                             \
+            gf_smsg(name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,         \
+                    "key=%s", key, "value=%s", dict_str, NULL);                \
+        }                                                                      \
+    }
+
+    priv = this->private;
+    shd = &priv->shd;
+
+    ret = dict_get_int32_sizen(input, "xl-op", (int32_t *)&op);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                "key=xl-op", NULL);
+        goto out;
+    }
+    this_name_len = strlen(this->name);
+    ret = dict_get_int32n(input, this->name, this_name_len, &xl_id);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                "key=%s", this->name, NULL);
+        goto out;
+    }
+    ret = dict_set_int32n(output, this->name, this_name_len, xl_id);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                "key=%s", this->name, NULL);
+        goto out;
+    }
+    switch (op) {
         case GF_SHD_OP_HEAL_INDEX:
-		op_ret = 0;
-
-		for (i = 0; i < priv->child_count; i++) {
-			healer = &shd->index_healers[i];
-			snprintf (key, sizeof (key), "%d-%d-status", xl_id, i);
-
-			if (!priv->child_up[i]) {
-				ret = dict_set_str (output, key,
-						    "Brick is not connected");
-                                op_ret = -1;
-			} else if (AFR_COUNT (priv->child_up,
-					      priv->child_count) < 2) {
-				ret = dict_set_str (output, key,
-						    "< 2 bricks in replica are up");
-                                op_ret = -1;
-			} else if (!afr_shd_is_subvol_local (this, healer->subvol)) {
-				ret = dict_set_str (output, key,
-						    "Brick is remote");
-			} else {
-				ret = dict_set_str (output, key,
-						    "Started self-heal");
-				afr_shd_index_healer_spawn (this, i);
-			}
-		}
-                break;
+            op_ret = 0;
+
+            for (i = 0; i < priv->child_count; i++) {
+                healer = &shd->index_healers[i];
+                keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
+
+                if (!priv->child_up[i]) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_NOT_CONNECTED,
+                                         SLEN(SBRICK_NOT_CONNECTED));
+                    op_ret = -1;
+                } else if (AFR_COUNT(priv->child_up, priv->child_count) < 2) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SLESS_THAN2_BRICKS_in_REP,
+                                         SLEN(SLESS_THAN2_BRICKS_in_REP));
+                    op_ret = -1;
+                } else if (!afr_shd_is_subvol_local(this, healer->subvol)) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_IS_REMOTE,
+                                         SLEN(SBRICK_IS_REMOTE));
+                } else {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SSTARTED_SELF_HEAL,
+                                         SLEN(SSTARTED_SELF_HEAL));
+
+                    ret = afr_shd_index_healer_spawn(this, i);
+
+                    if (ret) {
+                        gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                                AFR_MSG_HEALER_SPAWN_FAILED, NULL);
+                    }
+                }
+            }
+            break;
         case GF_SHD_OP_HEAL_FULL:
-		op_ret = -1;
-
-		for (i = 0; i < priv->child_count; i++) {
-			healer = &shd->full_healers[i];
-			snprintf (key, sizeof (key), "%d-%d-status", xl_id, i);
-
-			if (!priv->child_up[i]) {
-				ret = dict_set_str (output, key,
-						    "Brick is not connected");
-			} else if (AFR_COUNT (priv->child_up,
-					      priv->child_count) < 2) {
-				ret = dict_set_str (output, key,
-						    "< 2 bricks in replica are up");
-			} else if (!afr_shd_is_subvol_local (this, healer->subvol)) {
-				ret = dict_set_str (output, key,
-						    "Brick is remote");
-			} else {
-				ret = dict_set_str (output, key,
-						    "Started self-heal");
-				afr_shd_full_healer_spawn (this, i);
-				op_ret = 0;
-			}
-		}
-                break;
-        case GF_SHD_OP_INDEX_SUMMARY:
-                /* this case has been handled in glfs-heal.c */
-                break;
-        case GF_SHD_OP_HEALED_FILES:
-        case GF_SHD_OP_HEAL_FAILED_FILES:
-                for (i = 0; i < priv->child_count; i++) {
-                        snprintf (key, sizeof (key), "%d-%d-status", xl_id, i);
-                        ret = dict_set_str (output, key, "Operation Not "
-                                            "Supported");
+            op_ret = -1;
+
+            for (i = 0; i < priv->child_count; i++) {
+                healer = &shd->full_healers[i];
+                keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
+
+                if (!priv->child_up[i]) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_NOT_CONNECTED,
+                                         SLEN(SBRICK_NOT_CONNECTED));
+                } else if (AFR_COUNT(priv->child_up, priv->child_count) < 2) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SLESS_THAN2_BRICKS_in_REP,
+                                         SLEN(SLESS_THAN2_BRICKS_in_REP));
+                } else if (!afr_shd_is_subvol_local(this, healer->subvol)) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_IS_REMOTE,
+                                         SLEN(SBRICK_IS_REMOTE));
+                } else {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SSTARTED_SELF_HEAL,
+                                         SLEN(SSTARTED_SELF_HEAL));
+
+                    ret = afr_shd_full_healer_spawn(this, i);
+
+                    if (ret) {
+                        gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                                AFR_MSG_HEALER_SPAWN_FAILED, NULL);
+                    }
+                    op_ret = 0;
                 }
-                break;
+            }
+            break;
+        case GF_SHD_OP_INDEX_SUMMARY:
+            /* this case has been handled in glfs-heal.c */
+            break;
         case GF_SHD_OP_SPLIT_BRAIN_FILES:
-		eh_dump (shd->split_brain, output, afr_add_shd_event);
-                break;
+            eh_dump(shd->split_brain, output, afr_add_shd_event);
+            break;
         case GF_SHD_OP_STATISTICS:
-		for (i = 0; i < priv->child_count; i++) {
-			eh_dump (shd->statistics[i], output,
-				 afr_add_crawl_event);
-			afr_shd_dict_add_crawl_event (this, output,
-						      &shd->index_healers[i].crawl_event);
-			afr_shd_dict_add_crawl_event (this, output,
-						      &shd->full_healers[i].crawl_event);
-		}
-                break;
+            for (i = 0; i < priv->child_count; i++) {
+                eh_dump(shd->statistics[i], output, afr_add_crawl_event);
+                ret = afr_shd_dict_add_crawl_event(
+                    this, output, &shd->index_healers[i].crawl_event);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                            AFR_MSG_ADD_CRAWL_EVENT_FAILED, NULL);
+                }
+
+                ret = afr_shd_dict_add_crawl_event(
+                    this, output, &shd->full_healers[i].crawl_event);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                            AFR_MSG_ADD_CRAWL_EVENT_FAILED, NULL);
+                }
+            }
+            break;
         case GF_SHD_OP_STATISTICS_HEAL_COUNT:
         case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-		op_ret = -1;
-
-		for (i = 0; i < priv->child_count; i++) {
-			if (!priv->child_up[i]) {
-				snprintf (key, sizeof (key), "%d-%d-status",
-                                          xl_id, i);
-				ret = dict_set_str (output, key,
-						    "Brick is not connected");
-			} else {
-				snprintf (key, sizeof (key), "%d-%d-hardlinks",
-                                          xl_id, i);
-				ret = afr_shd_get_index_count (this, i, &cnt);
-				if (ret == 0) {
-					ret = dict_set_uint64 (output, key, cnt);
-				}
-				op_ret = 0;
-			}
-		}
-
-                break;
+            op_ret = -1;
+
+            for (i = 0; i < priv->child_count; i++) {
+                if (!priv->child_up[i]) {
+                    keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id,
+                                      i);
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_NOT_CONNECTED,
+                                         SLEN(SBRICK_NOT_CONNECTED));
+                } else {
+                    snprintf(key, sizeof(key), "%d-%d-hardlinks", xl_id, i);
+                    ret = afr_shd_get_index_count(this, i, &cnt);
+                    if (ret == 0) {
+                        ret = dict_set_uint64(output, key, cnt);
+                    }
+                    if (ret) {
+                        gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                                AFR_MSG_DICT_SET_FAILED, NULL);
+                    }
+                    op_ret = 0;
+                }
+            }
+
+            break;
 
         default:
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        AFR_MSG_INVALID_ARG, "Unknown set op %d", op);
-                break;
-        }
+            gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "op=%d",
+                    op, NULL);
+            break;
+    }
 out:
-        dict_del (output, this->name);
-        return op_ret;
+    dict_deln(output, this->name, this_name_len);
+    return op_ret;
+
+#undef AFR_SET_DICT_AND_LOG
 }
diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h
index 4ac1d32f58a..18db728ea7b 100644
--- a/xlators/cluster/afr/src/afr-self-heald.h
+++ b/xlators/cluster/afr/src/afr-self-heald.h
@@ -8,74 +8,68 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _AFR_SELF_HEALD_H
 #define _AFR_SELF_HEALD_H
 
 #include <pthread.h>
 
-
 typedef struct {
-	int child;
-	char *path;
+    char *path;
+    int child;
 } shd_event_t;
 
 typedef struct {
-	int      child;
-	uint64_t healed_count;
-        uint64_t split_brain_count;
-        uint64_t heal_failed_count;
-
-	/* If start_time is 0, it means crawler is not in progress
-	   and stats are not valid */
-	time_t   start_time;
-	/* If start_time is NOT 0 and end_time is 0, it means
-	   cralwer is in progress */
-        time_t   end_time;
-        char     *crawl_type;
+    uint64_t healed_count;
+    uint64_t split_brain_count;
+    uint64_t heal_failed_count;
+
+    /* If start_time is 0, it means crawler is not in progress
+       and stats are not valid */
+    time_t start_time;
+    /* If start_time is NOT 0 and end_time is 0, it means
+       cralwer is in progress */
+    time_t end_time;
+    char *crawl_type;
+    int child;
 } crawl_event_t;
 
 struct subvol_healer {
-	xlator_t        *this;
-	int              subvol;
-	gf_boolean_t     local;
-	gf_boolean_t     running;
-	gf_boolean_t     rerun;
-	crawl_event_t    crawl_event;
-	pthread_mutex_t  mutex;
-	pthread_cond_t   cond;
-	pthread_t        thread;
+    xlator_t *this;
+    crawl_event_t crawl_event;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_t thread;
+    int subvol;
+    gf_boolean_t local;
+    gf_boolean_t running;
+    gf_boolean_t rerun;
 };
 
 typedef struct {
-	gf_boolean_t            iamshd;
-	gf_boolean_t            enabled;
-	int                     timeout;
-	struct subvol_healer   *index_healers;
-	struct subvol_healer   *full_healers;
-
-        eh_t                    *split_brain;
-        eh_t                    **statistics;
-        uint32_t                max_threads;
-        uint32_t                wait_qlength;
-        uint32_t                halo_max_latency_msec;
+    struct subvol_healer *index_healers;
+    struct subvol_healer *full_healers;
+
+    eh_t *split_brain;
+    eh_t **statistics;
+    int timeout;
+    uint32_t max_threads;
+    uint32_t wait_qlength;
+    uint32_t halo_max_latency_msec;
+    gf_boolean_t iamshd;
+    gf_boolean_t enabled;
 } afr_self_heald_t;
 
-
-int
-afr_selfheal_childup (xlator_t *this, int subvol);
-
 int
-afr_selfheal_daemon_init (xlator_t *this);
+afr_selfheal_daemon_init(xlator_t *this);
 
 int
-afr_xl_op (xlator_t *this, dict_t *input, dict_t *output);
+afr_xl_op(xlator_t *this, dict_t *input, dict_t *output);
 
 int
-afr_shd_gfid_to_path (xlator_t *this, xlator_t *subvol, uuid_t gfid,
-                      char **path_p);
+afr_shd_gfid_to_path(xlator_t *this, xlator_t *subvol, uuid_t gfid,
+                     char **path_p);
 
 int
-afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name,
-                     ia_type_t type);
+afr_shd_entry_purge(xlator_t *subvol, inode_t *inode, char *name,
+                    ia_type_t type);
 #endif /* !_AFR_SELF_HEALD_H */
diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index b4d3062fc2b..a51f79b1f43 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -8,10 +8,10 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "dict.h"
-#include "byte-order.h"
-#include "common-utils.h"
-#include "timer.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/timer.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
@@ -21,57 +21,154 @@
 #include <signal.h>
 
 typedef enum {
-        AFR_TRANSACTION_PRE_OP,
-        AFR_TRANSACTION_POST_OP,
+    AFR_TRANSACTION_PRE_OP,
+    AFR_TRANSACTION_POST_OP,
 } afr_xattrop_type_t;
 
 static void
-afr_lock_resume_shared (struct list_head *list);
+afr_lock_resume_shared(struct list_head *list);
+
+static void
+afr_post_op_handle_success(call_frame_t *frame, xlator_t *this);
+
+static void
+afr_post_op_handle_failure(call_frame_t *frame, xlator_t *this, int op_errno);
 
 void
-__afr_transaction_wake_shared (afr_local_t *local, struct list_head *shared);
+__afr_transaction_wake_shared(afr_local_t *local, struct list_head *shared);
 
 void
-afr_changelog_post_op (call_frame_t *frame, xlator_t *this);
+afr_changelog_post_op_do(call_frame_t *frame, xlator_t *this);
 
 int
-afr_changelog_post_op_safe (call_frame_t *frame, xlator_t *this);
+afr_changelog_post_op_safe(call_frame_t *frame, xlator_t *this);
 
 gf_boolean_t
-afr_changelog_pre_op_uninherit (call_frame_t *frame, xlator_t *this);
+afr_changelog_pre_op_uninherit(call_frame_t *frame, xlator_t *this);
 
 gf_boolean_t
-afr_changelog_pre_op_update (call_frame_t *frame, xlator_t *this);
+afr_changelog_pre_op_update(call_frame_t *frame, xlator_t *this);
 
 int
-afr_changelog_call_count (afr_transaction_type type,
-                          unsigned char *pre_op_subvols,
-                          unsigned char *failed_subvols,
-                          unsigned int child_count);
+afr_changelog_call_count(afr_transaction_type type,
+                         unsigned char *pre_op_subvols,
+                         unsigned char *failed_subvols,
+                         unsigned int child_count);
 int
-afr_changelog_do (call_frame_t *frame, xlator_t *this, dict_t *xattr,
-		  afr_changelog_resume_t changelog_resume,
-                  afr_xattrop_type_t op);
+afr_changelog_do(call_frame_t *frame, xlator_t *this, dict_t *xattr,
+                 afr_changelog_resume_t changelog_resume,
+                 afr_xattrop_type_t op);
+
+static void
+afr_ta_decide_post_op_state(call_frame_t *frame, xlator_t *this);
+
+static int
+afr_ta_post_op_do(void *opaque);
+
+static int
+afr_ta_post_op_synctask(xlator_t *this, afr_local_t *local);
+
+static int
+afr_changelog_post_op_done(call_frame_t *frame, xlator_t *this);
+
+static void
+afr_changelog_post_op_fail(call_frame_t *frame, xlator_t *this, int op_errno);
 
 void
-afr_zero_fill_stat (afr_local_t *local)
-{
-        if (!local)
-                return;
-        if (local->transaction.type == AFR_DATA_TRANSACTION ||
-            local->transaction.type == AFR_METADATA_TRANSACTION) {
-                gf_zero_fill_stat (&local->cont.inode_wfop.prebuf);
-                gf_zero_fill_stat (&local->cont.inode_wfop.postbuf);
-        } else if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
-                   local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
-                gf_zero_fill_stat (&local->cont.dir_fop.buf);
-                gf_zero_fill_stat (&local->cont.dir_fop.preparent);
-                gf_zero_fill_stat (&local->cont.dir_fop.postparent);
-                if (local->transaction.type == AFR_ENTRY_TRANSACTION)
-                        return;
-                gf_zero_fill_stat (&local->cont.dir_fop.prenewparent);
-                gf_zero_fill_stat (&local->cont.dir_fop.postnewparent);
-        }
+afr_ta_locked_priv_invalidate(afr_private_t *priv)
+{
+    priv->ta_bad_child_index = AFR_CHILD_UNKNOWN;
+    priv->release_ta_notify_dom_lock = _gf_false;
+    priv->ta_notify_dom_lock_offset = 0;
+}
+
+static void
+afr_ta_process_waitq(xlator_t *this)
+{
+    afr_local_t *entry = NULL;
+    afr_private_t *priv = this->private;
+    struct list_head waitq = {
+        0,
+    };
+
+    INIT_LIST_HEAD(&waitq);
+    LOCK(&priv->lock);
+    list_splice_init(&priv->ta_waitq, &waitq);
+    UNLOCK(&priv->lock);
+    list_for_each_entry(entry, &waitq, ta_waitq)
+    {
+        afr_ta_decide_post_op_state(entry->transaction.frame, this);
+    }
+}
+
+int
+afr_ta_lock_release_done(int ret, call_frame_t *ta_frame, void *opaque)
+{
+    afr_ta_process_waitq(ta_frame->this);
+    STACK_DESTROY(ta_frame->root);
+    return 0;
+}
+
+int
+afr_release_notify_lock_for_ta(void *opaque)
+{
+    xlator_t *this = NULL;
+    afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = -1;
+
+    this = (xlator_t *)opaque;
+    priv = this->private;
+    ret = afr_fill_ta_loc(this, &loc, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate loc for thin-arbiter.");
+        goto out;
+    }
+    flock.l_type = F_UNLCK;
+    flock.l_start = priv->ta_notify_dom_lock_offset;
+    flock.l_len = 1;
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_NOTIFY, &loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to unlock AFR_TA_DOM_NOTIFY lock.");
+    }
+
+    LOCK(&priv->lock);
+    {
+        afr_ta_locked_priv_invalidate(priv);
+    }
+    UNLOCK(&priv->lock);
+out:
+    loc_wipe(&loc);
+    return ret;
+}
+
+void
+afr_zero_fill_stat(afr_local_t *local)
+{
+    if (!local)
+        return;
+    if (local->transaction.type == AFR_DATA_TRANSACTION ||
+        local->transaction.type == AFR_METADATA_TRANSACTION) {
+        gf_zero_fill_stat(&local->cont.inode_wfop.prebuf);
+        gf_zero_fill_stat(&local->cont.inode_wfop.postbuf);
+    } else if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
+               local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
+        gf_zero_fill_stat(&local->cont.dir_fop.buf);
+        gf_zero_fill_stat(&local->cont.dir_fop.preparent);
+        gf_zero_fill_stat(&local->cont.dir_fop.postparent);
+        if (local->transaction.type == AFR_ENTRY_TRANSACTION)
+            return;
+        gf_zero_fill_stat(&local->cont.dir_fop.prenewparent);
+        gf_zero_fill_stat(&local->cont.dir_fop.postnewparent);
+    }
 }
 
 /* In case of errors afr needs to choose which xdata from lower xlators it needs
@@ -79,2395 +176,2752 @@ afr_zero_fill_stat (afr_local_t *local)
  * any good subvols which failed. Give preference to errnos other than
  * ENOTCONN even if the child is source */
 void
-afr_pick_error_xdata (afr_local_t *local, afr_private_t *priv,
-                      inode_t *inode1, unsigned char *readable1,
-                      inode_t *inode2, unsigned char *readable2)
-{
-        int     s = -1;/*selection*/
-        int     i = 0;
-        unsigned char *readable = NULL;
+afr_pick_error_xdata(afr_local_t *local, afr_private_t *priv, inode_t *inode1,
+                     unsigned char *readable1, inode_t *inode2,
+                     unsigned char *readable2)
+{
+    int s = -1; /*selection*/
+    int i = 0;
+    unsigned char *readable = NULL;
+
+    if (local->xdata_rsp) {
+        dict_unref(local->xdata_rsp);
+        local->xdata_rsp = NULL;
+    }
+
+    readable = alloca0(priv->child_count * sizeof(*readable));
+    if (inode2 && readable2) { /*rename fop*/
+        AFR_INTERSECT(readable, readable1, readable2, priv->child_count);
+    } else {
+        memcpy(readable, readable1, sizeof(*readable) * priv->child_count);
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+
+        if (local->replies[i].op_ret >= 0)
+            continue;
+
+        if (local->replies[i].op_errno == ENOTCONN)
+            continue;
+
+        /*Order is important in the following condition*/
+        if ((s < 0) || (!readable[s] && readable[i]))
+            s = i;
+    }
+
+    if (s != -1 && local->replies[s].xdata) {
+        local->xdata_rsp = dict_ref(local->replies[s].xdata);
+    } else if (s == -1) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!local->replies[i].valid)
+                continue;
 
-        if (local->xdata_rsp) {
-                dict_unref (local->xdata_rsp);
-                local->xdata_rsp = NULL;
-        }
+            if (local->replies[i].op_ret >= 0)
+                continue;
 
-        readable = alloca0 (priv->child_count * sizeof (*readable));
-        if (inode2 && readable2) {/*rename fop*/
-                AFR_INTERSECT (readable, readable1, readable2,
-                               priv->child_count);
-        } else {
-                memcpy (readable, readable1,
-                        sizeof (*readable) * priv->child_count);
+            if (!local->replies[i].xdata)
+                continue;
+            local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            break;
         }
+    }
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->replies[i].valid)
-                        continue;
+gf_boolean_t
+afr_needs_changelog_update(afr_local_t *local)
+{
+    if (local->transaction.type == AFR_DATA_TRANSACTION)
+        return _gf_true;
+    if (!local->optimistic_change_log)
+        return _gf_true;
+    return _gf_false;
+}
 
-                if (local->replies[i].op_ret >= 0)
-                        continue;
+gf_boolean_t
+afr_changelog_has_quorum(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
+    unsigned char *success_children = NULL;
 
-                if (local->replies[i].op_errno == ENOTCONN)
-                        continue;
+    priv = this->private;
+    success_children = alloca0(priv->child_count);
 
-                /*Order is important in the following condition*/
-                if ((s < 0) || (!readable[s] && readable[i]))
-                        s = i;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.failed_subvols[i]) {
+            success_children[i] = 1;
         }
+    }
 
-        if (s != -1 && local->replies[s].xdata) {
-                local->xdata_rsp = dict_ref (local->replies[s].xdata);
-        } else if (s == -1) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!local->replies[i].valid)
-                                continue;
-
-                        if (local->replies[i].op_ret >= 0)
-                                continue;
+    if (afr_has_quorum(success_children, this, NULL)) {
+        return _gf_true;
+    }
 
-                        if (!local->replies[i].xdata)
-                                continue;
-                        local->xdata_rsp = dict_ref (local->replies[i].xdata);
-                        break;
-                }
-        }
+    return _gf_false;
 }
 
 gf_boolean_t
-afr_needs_changelog_update (afr_local_t *local)
+afr_is_write_subvol_valid(call_frame_t *frame, xlator_t *this)
 {
-        if (local->transaction.type == AFR_DATA_TRANSACTION)
-                return _gf_true;
-        if (!local->optimistic_change_log)
-                return _gf_true;
-        return _gf_false;
-}
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    uint64_t write_subvol = 0;
+    unsigned char *writable = NULL;
+    uint16_t datamap = 0;
 
-gf_boolean_t
-afr_changelog_has_quorum (afr_local_t *local, xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        int i = 0;
-        unsigned char *success_children = NULL;
+    local = frame->local;
+    priv = this->private;
+    writable = alloca0(priv->child_count);
 
-        priv = this->private;
-        success_children = alloca0 (priv->child_count);
+    write_subvol = afr_write_subvol_get(frame, this);
+    datamap = (write_subvol & 0x00000000ffff0000) >> 16;
+    for (i = 0; i < priv->child_count; i++) {
+        if (datamap & (1 << i))
+            writable[i] = 1;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->transaction.failed_subvols[i]) {
-                        success_children[i] = 1;
-                }
-        }
-
-        if (afr_has_quorum (success_children, this)) {
-                return _gf_true;
-        }
+        if (writable[i] && !local->transaction.failed_subvols[i])
+            return _gf_true;
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 int
-afr_transaction_fop (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        unsigned char *failed_subvols = NULL;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        failed_subvols = local->transaction.failed_subvols;
-        call_count = priv->child_count - AFR_COUNT (failed_subvols,
-                                                    priv->child_count);
-        /* Fail if pre-op did not succeed on quorum no. of bricks. */
-        if (!afr_changelog_has_quorum (local, this) || !call_count) {
-                local->op_ret = -1;
-                /* local->op_errno is already captured in changelog cbk. */
-                afr_transaction_resume (frame, this);
-                return 0;
-        }
-        local->call_count = call_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i] && !failed_subvols[i]) {
-			local->transaction.wind (frame, this, i);
+afr_transaction_fop(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = -1;
+    unsigned char *failed_subvols = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    failed_subvols = local->transaction.failed_subvols;
+    call_count = priv->child_count -
+                 AFR_COUNT(failed_subvols, priv->child_count);
+    /* Fail if pre-op did not succeed on quorum no. of bricks. */
+    if (!afr_changelog_has_quorum(local, this) || !call_count) {
+        local->op_ret = -1;
+        /* local->op_errno is already captured in changelog cbk. */
+        afr_transaction_resume(frame, this);
+        return 0;
+    }
 
-                        if (!--call_count)
-                                break;
-                }
+    /* Fail if at least one writeable brick isn't up.*/
+    if (local->transaction.type == AFR_DATA_TRANSACTION &&
+        !afr_is_write_subvol_valid(frame, this)) {
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        afr_transaction_resume(frame, this);
+        return 0;
+    }
+
+    local->call_count = call_count;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i] && !failed_subvols[i]) {
+            local->transaction.wind(frame, this, i);
+
+            if (!--call_count)
+                break;
         }
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-afr_transaction_done (call_frame_t *frame, xlator_t *this)
+afr_transaction_done(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local      = NULL;
-        afr_private_t *priv       = NULL;
-        gf_boolean_t  unwind      = _gf_false;
-        afr_lock_t    *lock       = NULL;
-        afr_local_t   *lock_local = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t unwind = _gf_false;
+    afr_lock_t *lock = NULL;
+    afr_local_t *lock_local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        if (priv->consistent_metadata) {
-                LOCK (&frame->lock);
-                {
-                        unwind = (local->transaction.main_frame != NULL);
-                }
-                UNLOCK (&frame->lock);
-                if (unwind)/*It definitely did post-op*/
-                        afr_zero_fill_stat (local);
+    if (priv->consistent_metadata) {
+        LOCK(&frame->lock);
+        {
+            unwind = (local->transaction.main_frame != NULL);
         }
+        UNLOCK(&frame->lock);
+        if (unwind) /*It definitely did post-op*/
+            afr_zero_fill_stat(local);
+    }
 
-        if (local->transaction.do_eager_unlock) {
-                lock = &local->inode_ctx->lock[local->transaction.type];
-                LOCK (&local->inode->lock);
-                {
-                        lock->acquired = _gf_false;
-                        lock->release = _gf_false;
-                        list_splice_init (&lock->frozen,
-                                          &lock->waiting);
-                        if (list_empty (&lock->waiting))
-                                goto unlock;
-                        lock_local = list_entry (lock->waiting.next,
-                                                 afr_local_t,
-                                                transaction.wait_list);
-                        list_del_init (&lock_local->transaction.wait_list);
-                        list_add (&lock_local->transaction.owner_list,
-                                  &lock->owners);
-                }
-unlock:
-                UNLOCK (&local->inode->lock);
-        }
-        if (lock_local) {
-                afr_lock (lock_local->transaction.frame,
-                          lock_local->transaction.frame->this);
-        }
-        local->transaction.unwind (frame, this);
+    if (local->transaction.do_eager_unlock) {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        LOCK(&local->inode->lock);
+        {
+            lock->acquired = _gf_false;
+            lock->release = _gf_false;
+            list_splice_init(&lock->frozen, &lock->waiting);
+            if (list_empty(&lock->waiting))
+                goto unlock;
+            lock_local = list_entry(lock->waiting.next, afr_local_t,
+                                    transaction.wait_list);
+            list_del_init(&lock_local->transaction.wait_list);
+            list_add(&lock_local->transaction.owner_list, &lock->owners);
+        }
+    unlock:
+        UNLOCK(&local->inode->lock);
+    }
+    if (lock_local) {
+        afr_lock(lock_local->transaction.frame,
+                 lock_local->transaction.frame->this);
+    }
+    local->transaction.unwind(frame, this);
 
-        AFR_STACK_DESTROY (frame);
+    GF_ASSERT(list_empty(&local->transaction.owner_list));
+    GF_ASSERT(list_empty(&local->transaction.wait_list));
+    AFR_STACK_DESTROY(frame);
 
-        return 0;
+    return 0;
 }
 
 static void
-afr_lock_fail_shared (afr_local_t *local, struct list_head *list)
-{
-        afr_local_t *each = NULL;
-
-        while (!list_empty(list)) {
-                each = list_entry (list->next, afr_local_t,
-                                   transaction.wait_list);
-                list_del_init(&each->transaction.wait_list);
-                each->op_ret = -1;
-                each->op_errno = local->op_errno;
-                afr_transaction_done (each->transaction.frame,
-                                      each->transaction.frame->this);
-        }
+afr_lock_fail_shared(afr_local_t *local, struct list_head *list)
+{
+    afr_local_t *each = NULL;
+
+    while (!list_empty(list)) {
+        each = list_entry(list->next, afr_local_t, transaction.wait_list);
+        list_del_init(&each->transaction.wait_list);
+        each->op_ret = -1;
+        each->op_errno = local->op_errno;
+        afr_transaction_done(each->transaction.frame,
+                             each->transaction.frame->this);
+    }
 }
 
 static void
-afr_handle_lock_acquire_failure (afr_local_t *local, gf_boolean_t locked)
+afr_handle_lock_acquire_failure(afr_local_t *local)
 {
-        struct list_head shared;
-        afr_lock_t *lock = NULL;
+    struct list_head shared;
+    afr_lock_t *lock = NULL;
 
-        if (!local->transaction.eager_lock_on)
-                goto out;
+    if (!local->transaction.eager_lock_on)
+        goto out;
 
-        lock = &local->inode_ctx->lock[local->transaction.type];
+    lock = &local->inode_ctx->lock[local->transaction.type];
 
-        INIT_LIST_HEAD (&shared);
-        LOCK (&local->inode->lock);
-        {
-                lock->release = _gf_true;
-                list_splice_init (&lock->waiting, &shared);
-        }
-        UNLOCK (&local->inode->lock);
+    INIT_LIST_HEAD(&shared);
+    LOCK(&local->inode->lock);
+    {
+        lock->release = _gf_true;
+        list_splice_init(&lock->waiting, &shared);
+    }
+    UNLOCK(&local->inode->lock);
 
-        afr_lock_fail_shared (local, &shared);
-        local->transaction.do_eager_unlock = _gf_true;
+    afr_lock_fail_shared(local, &shared);
+    local->transaction.do_eager_unlock = _gf_true;
 out:
-        if (locked) {
-                local->internal_lock.lock_cbk = afr_transaction_done;
-                afr_unlock (local->transaction.frame,
-                            local->transaction.frame->this);
-        } else {
-                afr_transaction_done (local->transaction.frame,
-                                      local->transaction.frame->this);
-        }
+    local->internal_lock.lock_cbk = afr_transaction_done;
+    afr_unlock(local->transaction.frame, local->transaction.frame->this);
 }
 
-call_frame_t*
-afr_transaction_detach_fop_frame (call_frame_t *frame)
+call_frame_t *
+afr_transaction_detach_fop_frame(call_frame_t *frame)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *fop_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *fop_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        afr_handle_inconsistent_fop (frame, &local->op_ret, &local->op_errno);
-        LOCK (&frame->lock);
-        {
-                fop_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+    afr_handle_inconsistent_fop(frame, &local->op_ret, &local->op_errno);
+    LOCK(&frame->lock);
+    {
+        fop_frame = local->transaction.main_frame;
+        local->transaction.main_frame = NULL;
+    }
+    UNLOCK(&frame->lock);
 
-        return fop_frame;
+    return fop_frame;
 }
 
-
 static void
-afr_save_lk_owner (call_frame_t *frame)
+afr_save_lk_owner(call_frame_t *frame)
 {
-        afr_local_t * local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->saved_lk_owner = frame->root->lk_owner;
+    local->saved_lk_owner = frame->root->lk_owner;
 }
 
-
 static void
-afr_restore_lk_owner (call_frame_t *frame)
+afr_restore_lk_owner(call_frame_t *frame)
 {
-        afr_local_t * local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        frame->root->lk_owner = local->saved_lk_owner;
+    frame->root->lk_owner = local->saved_lk_owner;
 }
 
 void
-__mark_all_success (call_frame_t *frame, xlator_t *this)
+__mark_all_success(call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	int i;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	for (i = 0; i < priv->child_count; i++) {
-		local->transaction.failed_subvols[i] = 0;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        local->transaction.failed_subvols[i] = 0;
+    }
 }
 
 void
-afr_compute_pre_op_sources (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        afr_transaction_type type = -1;
-        dict_t *xdata = NULL;
-        int **matrix = NULL;
-        int idx = -1;
-        int i = 0;
-        int j = 0;
-
-        priv = this->private;
-        local = frame->local;
-        type = local->transaction.type;
-        idx = afr_index_for_transaction_type (type);
-        matrix = ALLOC_MATRIX (priv->child_count, int);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->transaction.changelog_xdata[i])
-                        continue;
-                xdata = local->transaction.changelog_xdata[i];
-                afr_selfheal_fill_matrix (this, matrix, i, idx, xdata);
-        }
-
-        memset (local->transaction.pre_op_sources, 1, priv->child_count);
-
-        /*If lock or pre-op failed on a brick, it is not a source. */
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.failed_subvols[i])
-                        local->transaction.pre_op_sources[i] = 0;
-        }
-
-        /* If brick is blamed by others, it is not a source. */
-        for (i = 0; i < priv->child_count; i++)
-                for (j = 0; j < priv->child_count; j++)
-                        if (matrix[i][j] != 0)
-                                local->transaction.pre_op_sources[j] = 0;
+afr_compute_pre_op_sources(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_transaction_type type = -1;
+    dict_t *xdata = NULL;
+    int **matrix = NULL;
+    int idx = -1;
+    int i = 0;
+    int j = 0;
+
+    priv = this->private;
+    local = frame->local;
+    type = local->transaction.type;
+    idx = afr_index_for_transaction_type(type);
+    matrix = ALLOC_MATRIX(priv->child_count, int);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.changelog_xdata[i])
+            continue;
+        xdata = local->transaction.changelog_xdata[i];
+        afr_selfheal_fill_matrix(this, matrix, i, idx, xdata);
+    }
+
+    memset(local->transaction.pre_op_sources, 1, priv->child_count);
+
+    /*If lock or pre-op failed on a brick, it is not a source. */
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i])
+            local->transaction.pre_op_sources[i] = 0;
+    }
+
+    /* If brick is blamed by others, it is not a source. */
+    for (i = 0; i < priv->child_count; i++)
+        for (j = 0; j < priv->child_count; j++)
+            if (matrix[i][j] != 0)
+                local->transaction.pre_op_sources[j] = 0;
 }
 
-gf_boolean_t
-afr_has_arbiter_fop_cbk_quorum (call_frame_t *frame)
+void
+afr_txn_arbitrate_fop(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        xlator_t *this = NULL;
-        gf_boolean_t fop_failed = _gf_false;
-        unsigned char *pre_op_sources = NULL;
-        int i = 0;
-
-        local = frame->local;
-        this = frame->this;
-        priv  = this->private;
-        pre_op_sources = local->transaction.pre_op_sources;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int pre_op_sources_count = 0;
+    int i = 0;
 
-        /* If the fop failed on the brick, it is not a source. */
-        for (i = 0; i < priv->child_count; i++)
-                if (local->transaction.failed_subvols[i])
-                        pre_op_sources[i] = 0;
+    priv = this->private;
+    local = frame->local;
 
-        switch (AFR_COUNT (pre_op_sources, priv->child_count)) {
-        case 1:
-                if (pre_op_sources[ARBITER_BRICK_INDEX])
-                        fop_failed = _gf_true;
-                break;
-        case 0:
-                fop_failed = _gf_true;
-                break;
-        }
+    afr_compute_pre_op_sources(frame, this);
+    pre_op_sources_count = AFR_COUNT(local->transaction.pre_op_sources,
+                                     priv->child_count);
 
-        if (fop_failed)
-                return _gf_false;
+    /* If arbiter is the only source, do not proceed. */
+    if (pre_op_sources_count < 2 &&
+        local->transaction.pre_op_sources[ARBITER_BRICK_INDEX]) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        for (i = 0; i < priv->child_count; i++)
+            local->transaction.failed_subvols[i] = 1;
+    }
 
-        return _gf_true;
-}
+    afr_transaction_fop(frame, this);
 
-void
-afr_txn_arbitrate_fop (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int pre_op_sources_count = 0;
-        int i = 0;
-
-        priv = this->private;
-        local = frame->local;
-
-        afr_compute_pre_op_sources (frame, this);
-        pre_op_sources_count = AFR_COUNT (local->transaction.pre_op_sources,
-                                          priv->child_count);
-
-        /* If arbiter is the only source, do not proceed. */
-        if (pre_op_sources_count < 2 &&
-            local->transaction.pre_op_sources[ARBITER_BRICK_INDEX]) {
-                local->op_ret = -1;
-                local->op_errno =  ENOTCONN;
-                for (i = 0; i < priv->child_count; i++)
-                        local->transaction.failed_subvols[i] = 1;
-                afr_changelog_post_op (frame, this);/*uninherit should happen*/
-        } else {
-                afr_transaction_fop (frame, this);
-        }
-
-        return;
+    return;
 }
 
 int
-afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int           i      = 0;
-        int           ret    = 0;
-        int     failure_count = 0;
-        struct list_head shared;
-        afr_lock_t *lock = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        INIT_LIST_HEAD (&shared);
-        if (local->transaction.type == AFR_DATA_TRANSACTION &&
-            !local->transaction.inherited) {
-                ret = afr_write_subvol_set (frame, this);
-                if (ret) {
-                        /*act as if operation failed on all subvols*/
-                        local->op_ret = -1;
-                        local->op_errno = -ret;
-                        for (i = 0; i < priv->child_count; i++)
-                                local->transaction.failed_subvols[i] = 1;
-                }
-        }
-        /*  Perform fops with the lk-owner from top xlator.
-         *  Eg: lk-owner of posix-lk and flush should be same,
-         *  flush cant clear the  posix-lks without that lk-owner.
-         */
-        afr_save_lk_owner (frame);
-        frame->root->lk_owner =
-                local->transaction.main_frame->root->lk_owner;
-
-	if (local->pre_op_compat)
-		/* old mode, pre-op was done as afr_changelog_do()
-		   just now, before OP */
-		afr_changelog_pre_op_update (frame, this);
-
-        if (!local->transaction.eager_lock_on ||
-            local->transaction.inherited)
-                goto fop;
-        failure_count = AFR_COUNT (local->transaction.failed_subvols,
-                                   priv->child_count);
-        if (failure_count == priv->child_count) {
-                afr_handle_lock_acquire_failure (local, _gf_true);
-                return 0;
-        } else {
-                lock = &local->inode_ctx->lock[local->transaction.type];
-                LOCK (&local->inode->lock);
-                {
-                        lock->acquired = _gf_true;
-                        __afr_transaction_wake_shared (local, &shared);
-                }
-                UNLOCK (&local->inode->lock);
+afr_transaction_perform_fop(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int ret = 0;
+    int failure_count = 0;
+    struct list_head shared;
+    afr_lock_t *lock = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    INIT_LIST_HEAD(&shared);
+    if (local->transaction.type == AFR_DATA_TRANSACTION &&
+        !local->transaction.inherited) {
+        ret = afr_write_subvol_set(frame, this);
+        if (ret) {
+            /*act as if operation failed on all subvols*/
+            local->op_ret = -1;
+            local->op_errno = -ret;
+            for (i = 0; i < priv->child_count; i++)
+                local->transaction.failed_subvols[i] = 1;
+        }
+    }
+
+    if (local->pre_op_compat)
+        /* old mode, pre-op was done as afr_changelog_do()
+           just now, before OP */
+        afr_changelog_pre_op_update(frame, this);
+
+    if (!local->transaction.eager_lock_on || local->transaction.inherited)
+        goto fop;
+    failure_count = AFR_COUNT(local->transaction.failed_subvols,
+                              priv->child_count);
+    if (failure_count == priv->child_count) {
+        afr_handle_lock_acquire_failure(local);
+        return 0;
+    } else {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        LOCK(&local->inode->lock);
+        {
+            lock->acquired = _gf_true;
+            __afr_transaction_wake_shared(local, &shared);
         }
+        UNLOCK(&local->inode->lock);
+    }
 
 fop:
-        if (priv->arbiter_count == 1) {
-                afr_txn_arbitrate_fop (frame, this);
-        } else {
-                afr_transaction_fop (frame, this);
-        }
+    /*  Perform fops with the lk-owner from top xlator.
+     *  Eg: lk-owner of posix-lk and flush should be same,
+     *  flush cant clear the  posix-lks without that lk-owner.
+     */
+    afr_save_lk_owner(frame);
+    frame->root->lk_owner = local->transaction.main_frame->root->lk_owner;
 
-        afr_lock_resume_shared (&shared);
-	return 0;
+    if (priv->arbiter_count == 1) {
+        afr_txn_arbitrate_fop(frame, this);
+    } else {
+        afr_transaction_fop(frame, this);
+    }
+
+    afr_lock_resume_shared(&shared);
+    return 0;
 }
 
 int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int **pending)
+afr_set_pending_dict(afr_private_t *priv, dict_t *xattr, int **pending)
 {
-        int i = 0;
-        int ret = 0;
+    int i = 0;
+    int ret = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], pending[i],
+                                  AFR_NUM_CHANGE_LOGS * sizeof(int));
+        /* 3 = data+metadata+entry */
 
-                ret = dict_set_static_bin (xattr, priv->pending_key[i],
-					   pending[i],
-					   AFR_NUM_CHANGE_LOGS * sizeof (int));
-                /* 3 = data+metadata+entry */
+        if (ret)
+            break;
+    }
+
+    return ret;
+}
 
-                if (ret)
-                        break;
+static void
+afr_ta_dom_lock_check_and_release(afr_ta_fop_state_t fop_state, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    unsigned int inmem_count = 0;
+    unsigned int onwire_count = 0;
+    gf_boolean_t release = _gf_false;
+
+    LOCK(&priv->lock);
+    {
+        /*Once we get notify lock release upcall notification,
+         if any of the fop state counters are non-zero, we will
+         not release the lock.
+         */
+        onwire_count = priv->ta_on_wire_txn_count;
+        inmem_count = priv->ta_in_mem_txn_count;
+        switch (fop_state) {
+            case TA_GET_INFO_FROM_TA_FILE:
+                onwire_count = --priv->ta_on_wire_txn_count;
+                break;
+            case TA_INFO_IN_MEMORY_SUCCESS:
+            case TA_INFO_IN_MEMORY_FAILED:
+                inmem_count = --priv->ta_in_mem_txn_count;
+                break;
+            case TA_WAIT_FOR_NOTIFY_LOCK_REL:
+                GF_ASSERT(0);
+                break;
+            case TA_SUCCESS:
+                break;
         }
+        release = priv->release_ta_notify_dom_lock;
+    }
+    UNLOCK(&priv->lock);
 
-        return ret;
+    if (inmem_count != 0 || release == _gf_false || onwire_count != 0)
+        return;
+
+    afr_ta_lock_release_synctask(this);
 }
 
-/* {{{ pending */
+static void
+afr_ta_process_onwireq(afr_ta_fop_state_t fop_state, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *entry = NULL;
+    int bad_child = AFR_CHILD_UNKNOWN;
+
+    struct list_head onwireq = {
+        0,
+    };
+    INIT_LIST_HEAD(&onwireq);
+
+    LOCK(&priv->lock);
+    {
+        bad_child = priv->ta_bad_child_index;
+        if (bad_child == AFR_CHILD_UNKNOWN) {
+            /*The previous on-wire ta_post_op was a failure. Just dequeue
+             *one element to wind on-wire again. */
+            entry = list_entry(priv->ta_onwireq.next, afr_local_t, ta_onwireq);
+            list_del_init(&entry->ta_onwireq);
+        } else {
+            /* Prepare to process all fops based on bad_child_index. */
+            list_splice_init(&priv->ta_onwireq, &onwireq);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (entry) {
+        afr_ta_post_op_synctask(this, entry);
+        return;
+    } else {
+        while (!list_empty(&onwireq)) {
+            entry = list_entry(onwireq.next, afr_local_t, ta_onwireq);
+            list_del_init(&entry->ta_onwireq);
+            if (entry->ta_failed_subvol == bad_child) {
+                afr_post_op_handle_success(entry->transaction.frame, this);
+            } else {
+                afr_post_op_handle_failure(entry->transaction.frame, this, EIO);
+            }
+        }
+    }
+}
 
 int
-afr_changelog_post_op_done (call_frame_t *frame, xlator_t *this)
+afr_changelog_post_op_done(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-        afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    priv = this->private;
+    int_lock = &local->internal_lock;
 
-        /* Fail the FOP if post-op did not succeed on quorum no. of bricks. */
-        if (!afr_changelog_has_quorum (local, this)) {
-                local->op_ret = -1;
-                /*local->op_errno is already captured in changelog cbk*/
-        }
+    if (priv->thin_arbiter_count) {
+        /*fop should not come here with TA_WAIT_FOR_NOTIFY_LOCK_REL state */
+        afr_ta_dom_lock_check_and_release(local->fop_state, this);
+    }
 
-	if (local->transaction.resume_stub) {
-		call_resume (local->transaction.resume_stub);
-		local->transaction.resume_stub = NULL;
-	}
+    /* Fail the FOP if post-op did not succeed on quorum no. of bricks. */
+    if (!afr_changelog_has_quorum(local, this)) {
+        local->op_ret = -1;
+        /*local->op_errno is already captured in changelog cbk*/
+    }
 
-        int_lock->lock_cbk = afr_transaction_done;
-        afr_unlock (frame, this);
+    if (local->transaction.resume_stub) {
+        call_resume(local->transaction.resume_stub);
+        local->transaction.resume_stub = NULL;
+    }
 
-	return 0;
-}
+    int_lock->lock_cbk = afr_transaction_done;
+    afr_unlock(frame, this);
 
+    return 0;
+}
 
-unsigned char*
-afr_locked_nodes_get (afr_transaction_type type, afr_internal_lock_t *int_lock)
+static void
+afr_changelog_post_op_fail(call_frame_t *frame, xlator_t *this, int op_errno)
 {
-        unsigned char *locked_nodes = NULL;
-        switch (type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                locked_nodes = int_lock->locked_nodes;
-        break;
+    afr_local_t *local = frame->local;
+    local->op_ret = -1;
+    local->op_errno = op_errno;
 
-        case AFR_ENTRY_TRANSACTION:
-        case AFR_ENTRY_RENAME_TRANSACTION:
-                /*Because same set of subvols participate in all lockee
-                 * entities*/
-                locked_nodes = int_lock->lockee[0].locked_nodes;
-        break;
-        }
-        return locked_nodes;
+    gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_THIN_ARB,
+           "Failing %s for gfid %s. Fop state is:%d", gf_fop_list[local->op],
+           uuid_utoa(local->inode->gfid), local->fop_state);
+
+    afr_changelog_post_op_done(frame, this);
 }
 
+unsigned char *
+afr_locked_nodes_get(afr_transaction_type type, afr_internal_lock_t *int_lock)
+{
+    /*Because same set of subvols participate in all lockee
+     * entities*/
+    return int_lock->lockee[0].locked_nodes;
+}
 
 int
-afr_changelog_call_count (afr_transaction_type type,
-			  unsigned char *pre_op_subvols,
-                          unsigned char *failed_subvols,
-			  unsigned int child_count)
+afr_changelog_call_count(afr_transaction_type type,
+                         unsigned char *pre_op_subvols,
+                         unsigned char *failed_subvols,
+                         unsigned int child_count)
 {
-        int i = 0;
-        int call_count = 0;
+    int i = 0;
+    int call_count = 0;
 
-        for (i = 0; i < child_count; i++) {
-                if (pre_op_subvols[i] && !failed_subvols[i]) {
-                        call_count++;
-                }
+    for (i = 0; i < child_count; i++) {
+        if (pre_op_subvols[i] && !failed_subvols[i]) {
+            call_count++;
         }
+    }
 
-        if (type == AFR_ENTRY_RENAME_TRANSACTION)
-                call_count *= 2;
+    if (type == AFR_ENTRY_RENAME_TRANSACTION)
+        call_count *= 2;
 
-        return call_count;
+    return call_count;
 }
 
-
 gf_boolean_t
-afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this)
+afr_txn_nothing_failed(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-        local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        if (priv->thin_arbiter_count) {
-                /* We need to perform post-op even if 1 data brick was down
-                 * before the txn started.*/
-                if (AFR_COUNT (local->transaction.failed_subvols,
-                               priv->child_count))
-                        return _gf_false;
-        }
+    if (priv->thin_arbiter_count) {
+        /* We need to perform post-op even if 1 data brick was down
+         * before the txn started.*/
+        if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count))
+            return _gf_false;
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i] &&
-                    local->transaction.failed_subvols[i])
-                        return _gf_false;
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i] &&
+            local->transaction.failed_subvols[i])
+            return _gf_false;
+    }
 
-        return _gf_true;
+    return _gf_true;
 }
 
 void
-afr_handle_symmetric_errors (call_frame_t *frame, xlator_t *this)
+afr_handle_symmetric_errors(call_frame_t *frame, xlator_t *this)
 {
-	if (afr_is_symmetric_error (frame, this))
-		__mark_all_success (frame, this);
+    if (afr_is_symmetric_error(frame, this))
+        __mark_all_success(frame, this);
 }
 
 gf_boolean_t
-afr_has_quorum (unsigned char *subvols, xlator_t *this)
-{
-        unsigned int  quorum_count = 0;
-        afr_private_t *priv  = NULL;
-        unsigned int  up_children_count = 0;
-
-        priv = this->private;
-        up_children_count = AFR_COUNT (subvols, priv->child_count);
-
-        if (priv->quorum_count == AFR_QUORUM_AUTO) {
-                /*
-                 * Special case for auto-quorum with an even number of nodes.
-                 *
-                 * A replica set with even count N can only handle the same
-                 * number of failures as odd N-1 before losing "vanilla"
-                 * quorum, and the probability of more simultaneous failures is
-                 * actually higher.  For example, with a 1% chance of failure
-                 * we'd have a 0.03% chance of two simultaneous failures with
-                 * N=3 but a 0.06% chance with N=4.  However, the special case
-                 * is necessary for N=2 because there's no real quorum in that
-                 * case (i.e. can't normally survive *any* failures).  In that
-                 * case, we treat the first node as a tie-breaker, allowing
-                 * quorum to be retained in some cases while still honoring the
-                 * all-important constraint that there can not simultaneously
-                 * be two partitioned sets of nodes each believing they have
-                 * quorum.  Of two equally sized sets, the one without that
-                 * first node will lose.
-                 *
-                 * It turns out that the special case is beneficial for higher
-                 * values of N as well.  Continuing the example above, the
-                 * probability of losing quorum with N=4 and this type of
-                 * quorum is (very) slightly lower than with N=3 and vanilla
-                 * quorum.  The difference becomes even more pronounced with
-                 * higher N.  Therefore, even though such replica counts are
-                 * unlikely to be seen in practice, we might as well use the
-                 * "special" quorum then as well.
-                 */
-                if ((up_children_count * 2) == priv->child_count) {
-                        return subvols[0];
-                }
-        }
+afr_has_quorum(unsigned char *subvols, xlator_t *this, call_frame_t *frame)
+{
+    unsigned int quorum_count = 0;
+    afr_private_t *priv = NULL;
+    unsigned int up_children_count = 0;
 
-        if (priv->quorum_count == AFR_QUORUM_AUTO) {
-                quorum_count = priv->child_count/2 + 1;
-        } else {
-                quorum_count = priv->quorum_count;
+    priv = this->private;
+    up_children_count = AFR_COUNT(subvols, priv->child_count);
+
+    if (afr_lookup_has_quorum(frame, up_children_count))
+        return _gf_true;
+
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        /*
+         * Special case for auto-quorum with an even number of nodes.
+         *
+         * A replica set with even count N can only handle the same
+         * number of failures as odd N-1 before losing "vanilla"
+         * quorum, and the probability of more simultaneous failures is
+         * actually higher.  For example, with a 1% chance of failure
+         * we'd have a 0.03% chance of two simultaneous failures with
+         * N=3 but a 0.06% chance with N=4.  However, the special case
+         * is necessary for N=2 because there's no real quorum in that
+         * case (i.e. can't normally survive *any* failures).  In that
+         * case, we treat the first node as a tie-breaker, allowing
+         * quorum to be retained in some cases while still honoring the
+         * all-important constraint that there can not simultaneously
+         * be two partitioned sets of nodes each believing they have
+         * quorum.  Of two equally sized sets, the one without that
+         * first node will lose.
+         *
+         * It turns out that the special case is beneficial for higher
+         * values of N as well.  Continuing the example above, the
+         * probability of losing quorum with N=4 and this type of
+         * quorum is (very) slightly lower than with N=3 and vanilla
+         * quorum.  The difference becomes even more pronounced with
+         * higher N.  Therefore, even though such replica counts are
+         * unlikely to be seen in practice, we might as well use the
+         * "special" quorum then as well.
+         */
+        if ((up_children_count * 2) == priv->child_count) {
+            return subvols[0];
         }
+    }
 
-        if (up_children_count >= quorum_count)
-                return _gf_true;
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        quorum_count = priv->child_count / 2 + 1;
+    } else {
+        quorum_count = priv->quorum_count;
+    }
 
-        return _gf_false;
+    if (up_children_count >= quorum_count)
+        return _gf_true;
+
+    return _gf_false;
 }
 
 static gf_boolean_t
-afr_has_fop_quorum (call_frame_t *frame)
+afr_has_fop_quorum(call_frame_t *frame)
 {
-        xlator_t        *this = frame->this;
-        afr_local_t     *local = frame->local;
-        unsigned char *locked_nodes = NULL;
+    xlator_t *this = frame->this;
+    afr_local_t *local = frame->local;
+    unsigned char *locked_nodes = NULL;
 
-        locked_nodes = afr_locked_nodes_get (local->transaction.type,
-                                             &local->internal_lock);
-        return afr_has_quorum (locked_nodes, this);
+    locked_nodes = afr_locked_nodes_get(local->transaction.type,
+                                        &local->internal_lock);
+    return afr_has_quorum(locked_nodes, this, NULL);
 }
 
 static gf_boolean_t
-afr_has_fop_cbk_quorum (call_frame_t *frame)
+afr_has_fop_cbk_quorum(call_frame_t *frame)
 {
-        afr_local_t   *local   = frame->local;
-        xlator_t      *this    = frame->this;
-        afr_private_t *priv    = this->private;
-        unsigned char *success = alloca0(priv->child_count);
-        int           i        = 0;
+    afr_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+    unsigned char *success = alloca0(priv->child_count);
+    int i = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i])
-                        if (!local->transaction.failed_subvols[i])
-                                success[i] = 1;
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i])
+            if (!local->transaction.failed_subvols[i])
+                success[i] = 1;
+    }
 
-        return afr_has_quorum (success, this);
+    return afr_has_quorum(success, this, NULL);
 }
 
-void
-afr_handle_quorum (call_frame_t *frame)
+gf_boolean_t
+afr_need_dirty_marking(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int           i      = 0;
-        const char    *file  = NULL;
-        uuid_t        gfid   = {0};
+    afr_private_t *priv = this->private;
+    afr_local_t *local = NULL;
+    gf_boolean_t need_dirty = _gf_false;
 
-        local = frame->local;
-        priv  = frame->this->private;
+    local = frame->local;
 
-        if (priv->quorum_count == 0)
-                return;
+    if (!priv->quorum_count || !local->optimistic_change_log)
+        return _gf_false;
 
-        /* If the fop already failed return right away to preserve errno */
-        if (local->op_ret == -1)
-                return;
+    if (local->transaction.type == AFR_DATA_TRANSACTION ||
+        local->transaction.type == AFR_METADATA_TRANSACTION)
+        return _gf_false;
 
-        /*
-         * Network split may happen just after the fops are unwound, so check
-         * if the fop succeeded in a way it still follows quorum. If it doesn't,
-         * mark the fop as failure, mark the changelogs so it reflects that
-         * failure.
-         *
-         * Scenario:
-         * There are 3 mounts on 3 machines(node1, node2, node3) all writing to
-         * single file. Network split happened in a way that node1 can't see
-         * node2, node3. Node2, node3 both of them can't see node1. Now at the
-         * time of sending write all the bricks are up. Just after write fop is
-         * wound on node1, network split happens. Node1 thinks write fop failed
-         * on node2, node3 so marks pending changelog for those 2 extended
-         * attributes on node1. Node2, node3 thinks writes failed on node1 so
-         * they mark pending changelog for node1. When the network is stable
-         * again the file already is in split-brain. These checks prevent
-         * marking pending changelog on other subvolumes if the fop doesn't
-         * succeed in a way it is still following quorum. So with this fix what
-         * is happening is, node1 will have all pending changelog(FOOL) because
-         * the write succeeded only on node1 but failed on node2, node3 so
-         * instead of marking pending changelogs on node2, node3 it just treats
-         * the fop as failure and goes into DIRTY state. Where as node2, node3
-         * say they are sources and have pending changelog to node1 so there is
-         * no split-brain with the fix. The problem is eliminated completely.
-         */
+    if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count) ==
+        priv->child_count)
+        return _gf_false;
 
-        if (priv->arbiter_count) {
-                if (afr_has_arbiter_fop_cbk_quorum (frame))
-                        return;
-        } else if (afr_has_fop_cbk_quorum (frame)) {
-                return;
-        }
+    if (!afr_has_fop_cbk_quorum(frame))
+        need_dirty = _gf_true;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i])
-                        afr_transaction_fop_failed (frame, frame->this, i);
-        }
+    return need_dirty;
+}
 
-        local->op_ret = -1;
-        local->op_errno = afr_final_errno (local, priv);
-        if (local->op_errno == 0)
-                local->op_errno = afr_quorum_errno (priv);
+void
+afr_handle_quorum(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    const char *file = NULL;
+    uuid_t gfid = {0};
 
-        if (local->fd) {
-                gf_uuid_copy (gfid, local->fd->inode->gfid);
-                file = uuid_utoa (gfid);
-        } else {
-                loc_path (&local->loc, local->loc.name);
-                file = local->loc.path;
-        }
+    local = frame->local;
+    priv = frame->this->private;
 
-        gf_msg (frame->this->name, GF_LOG_WARNING, local->op_errno,
-                AFR_MSG_QUORUM_FAIL, "%s: Failing %s as quorum is not met",
-                file, gf_fop_list[local->op]);
+    if (priv->quorum_count == 0)
+        return;
 
-        switch (local->transaction.type) {
+    /* If the fop already failed return right away to preserve errno */
+    if (local->op_ret == -1)
+        return;
+
+    /*
+     * Network split may happen just after the fops are unwound, so check
+     * if the fop succeeded in a way it still follows quorum. If it doesn't,
+     * mark the fop as failure, mark the changelogs so it reflects that
+     * failure.
+     *
+     * Scenario:
+     * There are 3 mounts on 3 machines(node1, node2, node3) all writing to
+     * single file. Network split happened in a way that node1 can't see
+     * node2, node3. Node2, node3 both of them can't see node1. Now at the
+     * time of sending write all the bricks are up. Just after write fop is
+     * wound on node1, network split happens. Node1 thinks write fop failed
+     * on node2, node3 so marks pending changelog for those 2 extended
+     * attributes on node1. Node2, node3 thinks writes failed on node1 so
+     * they mark pending changelog for node1. When the network is stable
+     * again the file already is in split-brain. These checks prevent
+     * marking pending changelog on other subvolumes if the fop doesn't
+     * succeed in a way it is still following quorum. So with this fix what
+     * is happening is, node1 will have all pending changelog(FOOL) because
+     * the write succeeded only on node1 but failed on node2, node3 so
+     * instead of marking pending changelogs on node2, node3 it just treats
+     * the fop as failure and goes into DIRTY state. Where as node2, node3
+     * say they are sources and have pending changelog to node1 so there is
+     * no split-brain with the fix. The problem is eliminated completely.
+     */
+
+    if (afr_has_fop_cbk_quorum(frame))
+        return;
+
+    if (afr_need_dirty_marking(frame, this))
+        goto set_response;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i])
+            afr_transaction_fop_failed(frame, frame->this, i);
+    }
+
+set_response:
+    local->op_ret = -1;
+    local->op_errno = afr_final_errno(local, priv);
+    if (local->op_errno == 0)
+        local->op_errno = afr_quorum_errno(priv);
+
+    if (local->fd) {
+        gf_uuid_copy(gfid, local->fd->inode->gfid);
+        file = uuid_utoa(gfid);
+    } else {
+        loc_path(&local->loc, local->loc.name);
+        file = local->loc.path;
+    }
+
+    gf_msg(frame->this->name, GF_LOG_WARNING, local->op_errno,
+           AFR_MSG_QUORUM_FAIL, "%s: Failing %s as quorum is not met", file,
+           gf_fop_list[local->op]);
+
+    switch (local->transaction.type) {
         case AFR_ENTRY_TRANSACTION:
         case AFR_ENTRY_RENAME_TRANSACTION:
-                afr_pick_error_xdata (local, priv, local->parent,
-                                      local->readable, local->parent2,
-                                      local->readable2);
-                break;
+            afr_pick_error_xdata(local, priv, local->parent, local->readable,
+                                 local->parent2, local->readable2);
+            break;
         default:
-                afr_pick_error_xdata (local, priv, local->inode,
-                                      local->readable, NULL, NULL);
-                break;
-        }
+            afr_pick_error_xdata(local, priv, local->inode, local->readable,
+                                 NULL, NULL);
+            break;
+    }
 }
 
 int
-afr_fill_ta_loc (xlator_t *this, loc_t *loc)
-{
-        afr_private_t *priv = NULL;
-
-        priv = this->private;
-        loc->parent = inode_ref (priv->root_inode);
-        gf_uuid_copy (loc->pargfid, loc->parent->gfid);
-        loc->name = priv->pending_key[THIN_ARBITER_BRICK_INDEX];
-        gf_uuid_copy (loc->gfid, priv->ta_gfid);
-        loc->inode = inode_new (loc->parent->table);
-        if (!loc->inode)
-                return -ENOMEM;
-        return 0;
+afr_fill_ta_loc(xlator_t *this, loc_t *loc, gf_boolean_t is_gfid_based_fop)
+{
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    loc->parent = inode_ref(priv->root_inode);
+    gf_uuid_copy(loc->pargfid, loc->parent->gfid);
+    loc->name = priv->pending_key[THIN_ARBITER_BRICK_INDEX];
+    if (is_gfid_based_fop && gf_uuid_is_null(priv->ta_gfid)) {
+        /* Except afr_ta_id_file_check() which is path based, all other gluster
+         * FOPS need gfid.*/
+        return -EINVAL;
+    }
+    gf_uuid_copy(loc->gfid, priv->ta_gfid);
+    loc->inode = inode_new(loc->parent->table);
+    if (!loc->inode) {
+        loc_wipe(loc);
+        return -ENOMEM;
+    }
+    return 0;
+}
+
+static int
+afr_ta_post_op_done(int ret, call_frame_t *frame, void *opaque)
+{
+    xlator_t *this = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *txn_frame = NULL;
+    afr_ta_fop_state_t fop_state;
+
+    local = (afr_local_t *)opaque;
+    fop_state = local->fop_state;
+    txn_frame = local->transaction.frame;
+    this = frame->this;
+
+    if (ret == 0) {
+        /*Mark pending xattrs on the up data brick.*/
+        afr_post_op_handle_success(txn_frame, this);
+    } else {
+        afr_post_op_handle_failure(txn_frame, this, -ret);
+    }
+
+    STACK_DESTROY(frame->root);
+    afr_ta_process_onwireq(fop_state, this);
+
+    return 0;
+}
+
+int **
+afr_set_changelog_xattr(afr_private_t *priv, unsigned char *pending,
+                        dict_t *xattr, afr_local_t *local)
+{
+    int **changelog = NULL;
+    int idx = 0;
+    int ret = 0;
+    int i;
+
+    if (local->is_new_entry == _gf_true) {
+        changelog = afr_mark_pending_changelog(priv, pending, xattr,
+                                               local->cont.dir_fop.buf.ia_type);
+    } else {
+        idx = afr_index_for_transaction_type(local->transaction.type);
+        changelog = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+        if (!changelog) {
+            goto out;
+        }
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->transaction.failed_subvols[i])
+                changelog[i][idx] = hton32(1);
+        }
+        ret = afr_set_pending_dict(priv, xattr, changelog);
+        if (ret < 0) {
+            afr_matrix_cleanup(changelog, priv->child_count);
+            return NULL;
+        }
+    }
+
+out:
+    return changelog;
 }
 
-int
-afr_changelog_thin_arbiter_post_op (xlator_t *this, afr_local_t *local)
+static void
+afr_ta_locked_xattrop_validate(afr_private_t *priv, afr_local_t *local,
+                               gf_boolean_t *valid)
 {
-        int ret = 0;
-        afr_private_t *priv = NULL;
-        dict_t *xattr = NULL;
-        int failed_count = 0;
-        struct gf_flock flock = {0, };
-        loc_t loc = {0,};
-        int i = 0;
-
-        priv = this->private;
-        if (!priv->thin_arbiter_count)
-                return 0;
+    if (priv->ta_event_gen > local->ta_event_gen) {
+        /* We can't trust the ta's response anymore.*/
+        afr_ta_locked_priv_invalidate(priv);
+        *valid = _gf_false;
+        return;
+    }
+    return;
+}
+
+static int
+afr_ta_post_op_do(void *opaque)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    dict_t *xattr = NULL;
+    unsigned char *pending = NULL;
+    int **changelog = NULL;
+    int failed_subvol = -1;
+    int success_subvol = -1;
+    loc_t loc = {
+        0,
+    };
+    int i = 0;
+    int ret = 0;
+    gf_boolean_t valid = _gf_true;
+
+    local = (afr_local_t *)opaque;
+    this = local->transaction.frame->this;
+    priv = this->private;
+
+    ret = afr_fill_ta_loc(this, &loc, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate loc for thin-arbiter.");
+        goto out;
+    }
+
+    xattr = dict_new();
+    if (!xattr) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    pending = alloca0(priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i]) {
+            pending[i] = 1;
+            failed_subvol = i;
+        } else {
+            success_subvol = i;
+        }
+    }
+
+    changelog = afr_set_changelog_xattr(priv, pending, xattr, local);
+
+    if (!changelog) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = afr_ta_post_op_lock(this, &loc);
+    if (ret)
+        goto out;
+
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
+                         GF_XATTROP_ADD_ARRAY, xattr, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Post-op on thin-arbiter id file %s failed for gfid %s.",
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX],
+               uuid_utoa(local->inode->gfid));
+    }
+    LOCK(&priv->lock);
+    {
+        if (ret == 0) {
+            priv->ta_bad_child_index = failed_subvol;
+        } else if (ret == -EINVAL) {
+            priv->ta_bad_child_index = success_subvol;
+            ret = -EIO; /* TA failed the fop. Return EIO to application. */
+        }
+
+        afr_ta_locked_xattrop_validate(priv, local, &valid);
+    }
+    UNLOCK(&priv->lock);
+    if (valid == _gf_false) {
+        gf_msg(this->name, GF_LOG_ERROR, EIO, AFR_MSG_THIN_ARB,
+               "Post-op on thin-arbiter id file %s for gfid %s invalidated due "
+               "to event-gen mismatch.",
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX],
+               uuid_utoa(local->inode->gfid));
+        ret = -EIO;
+    }
+
+    afr_ta_post_op_unlock(this, &loc);
+out:
+    if (xattr)
+        dict_unref(xattr);
 
+    if (changelog)
+        afr_matrix_cleanup(changelog, priv->child_count);
 
-        failed_count = AFR_COUNT (local->transaction.failed_subvols,
-                                  priv->child_count);
-        if (!failed_count)
-                return 0;
+    loc_wipe(&loc);
 
-        GF_ASSERT (failed_count == 1);
-        ret = afr_fill_ta_loc (this, &loc);
-        if (ret)
-                goto out;
+    return ret;
+}
 
-        xattr = dict_new ();
-        if (!xattr) {
-                ret = -ENOMEM;
-                goto out;
-        }
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_set_static_bin (xattr, priv->pending_key[i],
-                                           local->pending[i],
-                                           AFR_NUM_CHANGE_LOGS * sizeof (int));
-                if (ret)
-                        goto out;
-        }
+static int
+afr_ta_post_op_synctask(xlator_t *this, afr_local_t *local)
+{
+    call_frame_t *ta_frame = NULL;
+    int ret = 0;
 
-        flock.l_type = F_WRLCK;
-        flock.l_start = 0;
-        flock.l_len = 0;
+    ta_frame = afr_ta_frame_create(this);
+    if (!ta_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to create ta_frame");
+        goto err;
+    }
+    ret = synctask_new(this->ctx->env, afr_ta_post_op_do, afr_ta_post_op_done,
+                       ta_frame, local);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to launch post-op on thin arbiter for gfid %s",
+               uuid_utoa(local->inode->gfid));
+        STACK_DESTROY(ta_frame->root);
+        goto err;
+    }
 
-        /*TODO: Convert to two domain locking. */
-        ret = syncop_inodelk (priv->children[THIN_ARBITER_BRICK_INDEX],
-                              THIN_ARBITER_DOM1, &loc, F_SETLKW, &flock,
-                              NULL, NULL);
-        if (ret)
-                goto out;
+    return ret;
+err:
+    afr_changelog_post_op_fail(local->transaction.frame, this, ENOMEM);
+    return ret;
+}
 
-        ret = syncop_xattrop (priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
-                              GF_XATTROP_ADD_ARRAY, xattr, NULL, NULL, NULL);
+static void
+afr_ta_set_fop_state(afr_private_t *priv, afr_local_t *local,
+                     int *on_wire_count)
+{
+    LOCK(&priv->lock);
+    {
+        if (priv->release_ta_notify_dom_lock == _gf_true) {
+            /* Put the fop in waitq until notify dom lock is released.*/
+            local->fop_state = TA_WAIT_FOR_NOTIFY_LOCK_REL;
+            list_add_tail(&local->ta_waitq, &priv->ta_waitq);
+        } else if (priv->ta_bad_child_index == AFR_CHILD_UNKNOWN) {
+            /* Post-op on thin-arbiter to decide success/failure. */
+            local->fop_state = TA_GET_INFO_FROM_TA_FILE;
+            *on_wire_count = ++priv->ta_on_wire_txn_count;
+            if (*on_wire_count > 1) {
+                /*Avoid sending multiple on-wire post-ops on TA*/
+                list_add_tail(&local->ta_onwireq, &priv->ta_onwireq);
+            }
+        } else if (local->ta_failed_subvol == priv->ta_bad_child_index) {
+            /* Post-op on TA not needed as the fop failed on the in-memory bad
+             * brick. Just mark pending xattrs on the good data brick.*/
+            local->fop_state = TA_INFO_IN_MEMORY_SUCCESS;
+            priv->ta_in_mem_txn_count++;
+        } else {
+            /* Post-op on TA not needed as the fop succeeded only on the
+             * in-memory bad data brick and not the good one. Fail the fop.*/
+            local->fop_state = TA_INFO_IN_MEMORY_FAILED;
+            priv->ta_in_mem_txn_count++;
+        }
+    }
+    UNLOCK(&priv->lock);
+}
 
-        if (ret == -EINVAL) {
-                gf_msg (this->name, GF_LOG_INFO, -ret, AFR_MSG_THIN_ARB,
-                        "Thin-arbiter has denied post-op on %s for gfid %s.",
-                        priv->pending_key[THIN_ARBITER_BRICK_INDEX],
-                        uuid_utoa (local->inode->gfid));
+static void
+afr_ta_fill_failed_subvol(afr_private_t *priv, afr_local_t *local)
+{
+    int i = 0;
 
-        } else if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
-                        "Post-op on thin-arbiter id file %s failed for gfid %s.",
-                        priv->pending_key[THIN_ARBITER_BRICK_INDEX],
-                        uuid_utoa (local->inode->gfid));
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i]) {
+            local->ta_failed_subvol = i;
+            break;
         }
-        flock.l_type = F_UNLCK;
-        syncop_inodelk (priv->children[THIN_ARBITER_BRICK_INDEX],
-                        THIN_ARBITER_DOM1, &loc, F_SETLKW, &flock, NULL, NULL);
-out:
-        if (xattr)
-                dict_unref (xattr);
+    }
+}
 
-        return ret;
+static void
+afr_post_op_handle_success(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+
+    local = frame->local;
+    if (local->is_new_entry == _gf_true) {
+        afr_mark_new_entry_changelog(frame, this);
+    }
+    afr_changelog_post_op_do(frame, this);
+
+    return;
 }
 
-int
-afr_changelog_post_op_now (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t           *priv           = this->private;
-        afr_local_t             *local          = NULL;
-        dict_t                  *xattr          = NULL;
-        int                     i               = 0;
-        int                     ret             = 0;
-        int                     idx             = 0;
-        int                     nothing_failed  = 1;
-        gf_boolean_t            need_undirty    = _gf_false;
-
-        afr_handle_quorum (frame);
-        local = frame->local;
-	idx = afr_index_for_transaction_type (local->transaction.type);
-
-        nothing_failed = afr_txn_nothing_failed (frame, this);
-
-	if (afr_changelog_pre_op_uninherit (frame, this))
-		need_undirty = _gf_false;
-	else
-		need_undirty = _gf_true;
-
-        if (local->op_ret < 0 && !nothing_failed) {
-                afr_changelog_post_op_done (frame, this);
-                goto out;
-        }
+static void
+afr_post_op_handle_failure(call_frame_t *frame, xlator_t *this, int op_errno)
+{
+    afr_changelog_post_op_fail(frame, this, op_errno);
 
-	if (nothing_failed && !need_undirty) {
-		afr_changelog_post_op_done (frame, this);
-                goto out;
-	}
+    return;
+}
 
-        if (local->transaction.in_flight_sb) {
-                local->op_ret = -1;
-                local->op_errno = local->transaction.in_flight_sb_errno;
-                afr_changelog_post_op_done (frame, this);
-                goto out;
-        }
+static void
+afr_ta_decide_post_op_state(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int on_wire_count = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    afr_ta_set_fop_state(priv, local, &on_wire_count);
+
+    switch (local->fop_state) {
+        case TA_GET_INFO_FROM_TA_FILE:
+            if (on_wire_count == 1)
+                afr_ta_post_op_synctask(this, local);
+            /*else, fop is queued in ta_onwireq.*/
+            break;
+        case TA_WAIT_FOR_NOTIFY_LOCK_REL:
+            /*Post releasing the notify lock, we will act on this queue*/
+            break;
+        case TA_INFO_IN_MEMORY_SUCCESS:
+            afr_post_op_handle_success(frame, this);
+            break;
+        case TA_INFO_IN_MEMORY_FAILED:
+            afr_post_op_handle_failure(frame, this, EIO);
+            break;
+        default:
+            break;
+    }
+    return;
+}
 
-	xattr = dict_new ();
-	if (!xattr) {
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-		afr_changelog_post_op_done (frame, this);
-		goto out;
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->transaction.failed_subvols[i])
-			local->pending[i][idx] = hton32(1);
-	}
-
-	ret = afr_set_pending_dict (priv, xattr, local->pending);
-	if (ret < 0) {
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-		afr_changelog_post_op_done (frame, this);
-		goto out;
-	}
-
-        ret = afr_changelog_thin_arbiter_post_op (this, local);
-        if (ret < 0) {
-                local->op_ret = -1;
-                local->op_errno = -ret;
-		afr_changelog_post_op_done (frame, this);
-                goto out;
-        }
+static void
+afr_handle_failure_using_thin_arbiter(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+
+    afr_ta_fill_failed_subvol(priv, local);
+    gf_msg_debug(this->name, 0,
+                 "Fop failed on data brick (%s) for gfid=%s. "
+                 "ta info needed to decide fop result.",
+                 priv->children[local->ta_failed_subvol]->name,
+                 uuid_utoa(local->inode->gfid));
+    afr_ta_decide_post_op_state(frame, this);
+}
 
-        if (need_undirty)
-		local->dirty[idx] = hton32(-1);
-	else
-		local->dirty[idx] = hton32(0);
-
-	ret = dict_set_static_bin (xattr, AFR_DIRTY, local->dirty,
-				   sizeof(int) * AFR_NUM_CHANGE_LOGS);
-	if (ret) {
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-		afr_changelog_post_op_done (frame, this);
-		goto out;
-	}
-
-        afr_changelog_do (frame, this, xattr, afr_changelog_post_op_done,
-                          AFR_TRANSACTION_POST_OP);
+void
+afr_changelog_post_op_do(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = NULL;
+    dict_t *xattr = NULL;
+    int i = 0;
+    int ret = 0;
+    int idx = 0;
+    int nothing_failed = 1;
+    gf_boolean_t need_undirty = _gf_false;
+
+    afr_handle_quorum(frame, this);
+    local = frame->local;
+    idx = afr_index_for_transaction_type(local->transaction.type);
+
+    xattr = dict_new();
+    if (!xattr) {
+        afr_changelog_post_op_fail(frame, this, ENOMEM);
+        goto out;
+    }
+
+    nothing_failed = afr_txn_nothing_failed(frame, this);
+
+    if (afr_changelog_pre_op_uninherit(frame, this))
+        need_undirty = _gf_false;
+    else
+        need_undirty = _gf_true;
+
+    if (local->op_ret < 0 && !nothing_failed) {
+        if (afr_need_dirty_marking(frame, this)) {
+            local->dirty[idx] = hton32(1);
+            goto set_dirty;
+        }
+
+        afr_changelog_post_op_done(frame, this);
+        goto out;
+    }
+
+    if (nothing_failed && !need_undirty) {
+        afr_changelog_post_op_done(frame, this);
+        goto out;
+    }
+
+    if (local->transaction.in_flight_sb) {
+        afr_changelog_post_op_fail(frame, this,
+                                   local->transaction.in_flight_sb_errno);
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i])
+            local->pending[i][idx] = hton32(1);
+    }
+
+    ret = afr_set_pending_dict(priv, xattr, local->pending);
+    if (ret < 0) {
+        afr_changelog_post_op_fail(frame, this, ENOMEM);
+        goto out;
+    }
+
+    if (need_undirty)
+        local->dirty[idx] = hton32(-1);
+    else
+        local->dirty[idx] = hton32(0);
+
+set_dirty:
+    ret = dict_set_static_bin(xattr, AFR_DIRTY, local->dirty,
+                              sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    if (ret) {
+        afr_changelog_post_op_fail(frame, this, ENOMEM);
+        goto out;
+    }
+
+    afr_changelog_do(frame, this, xattr, afr_changelog_post_op_done,
+                     AFR_TRANSACTION_POST_OP);
 out:
-	if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-        return 0;
+    return;
 }
 
+static int
+afr_changelog_post_op_now(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int failed_count = 0;
 
-gf_boolean_t
-afr_changelog_pre_op_uninherit (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-        afr_inode_ctx_t *ctx = NULL;
-	int i = 0;
-	gf_boolean_t ret = _gf_false;
-	int type = 0;
-
-	local = frame->local;
-	priv = this->private;
-        ctx = local->inode_ctx;
-
-	type = afr_index_for_transaction_type (local->transaction.type);
-	if (type != AFR_DATA_TRANSACTION)
-		return !local->transaction.dirtied;
-
-	if (local->transaction.no_uninherit)
-		return _gf_false;
-
-	/* This function must be idempotent. So check if we
-	   were called before and return the same answer again.
-
-	   It is important to keep this function idempotent for
-	   the call in afr_changelog_post_op_safe() to not have
-	   side effects on the call from afr_changelog_post_op_now()
-	*/
-	if (local->transaction.uninherit_done)
-		return local->transaction.uninherit_value;
-
-	LOCK(&local->inode->lock);
-	{
-		for (i = 0; i < priv->child_count; i++) {
-			if (local->transaction.pre_op[i] !=
-			    ctx->pre_op_done[type][i]) {
-				ret = !local->transaction.dirtied;
-				goto unlock;
-			}
-		}
-
-		if (ctx->inherited[type]) {
-			ret = _gf_true;
-			ctx->inherited[type]--;
-		} else if (ctx->on_disk[type]) {
-			ret = _gf_false;
-			ctx->on_disk[type]--;
-		} else {
-			/* ASSERT */
-			ret = _gf_false;
-		}
-
-		if (!ctx->inherited[type] && !ctx->on_disk[type]) {
-			for (i = 0; i < priv->child_count; i++)
-				ctx->pre_op_done[type][i] = 0;
-		}
-	}
-unlock:
-	UNLOCK(&local->inode->lock);
+    priv = this->private;
+    local = frame->local;
 
-	local->transaction.uninherit_done = _gf_true;
-	local->transaction.uninherit_value = ret;
+    if (priv->thin_arbiter_count) {
+        failed_count = AFR_COUNT(local->transaction.failed_subvols,
+                                 priv->child_count);
+        if (failed_count == 1) {
+            afr_handle_failure_using_thin_arbiter(frame, this);
+            return 0;
+        } else {
+            /* Txn either succeeded or failed on both data bricks. Let
+             * post_op_do handle it as the case might be. */
+        }
+    }
 
-	return ret;
+    afr_changelog_post_op_do(frame, this);
+    return 0;
 }
 
-
 gf_boolean_t
-afr_changelog_pre_op_inherit (call_frame_t *frame, xlator_t *this)
+afr_changelog_pre_op_uninherit(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-	gf_boolean_t ret = _gf_false;
-	int type = 0;
-
-	local = frame->local;
-	priv = this->private;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_inode_ctx_t *ctx = NULL;
+    int i = 0;
+    gf_boolean_t ret = _gf_false;
+    int type = 0;
 
-	if (local->transaction.type != AFR_DATA_TRANSACTION)
-		return _gf_false;
+    local = frame->local;
+    priv = this->private;
+    ctx = local->inode_ctx;
 
-	type = afr_index_for_transaction_type (local->transaction.type);
+    type = afr_index_for_transaction_type(local->transaction.type);
+    if (type != AFR_DATA_TRANSACTION)
+        return !local->transaction.dirtied;
 
-	LOCK(&local->inode->lock);
-	{
-		if (!local->inode_ctx->on_disk[type]) {
-			/* nothing to inherit yet */
-			ret = _gf_false;
-			goto unlock;
-		}
+    if (local->transaction.no_uninherit)
+        return _gf_false;
 
-		for (i = 0; i < priv->child_count; i++) {
-			if (local->transaction.pre_op[i] !=
-			    local->inode_ctx->pre_op_done[type][i]) {
-				/* either inherit exactly, or don't */
-				ret = _gf_false;
-				goto unlock;
-			}
-		}
+    /* This function must be idempotent. So check if we
+       were called before and return the same answer again.
 
-		local->inode_ctx->inherited[type]++;
+       It is important to keep this function idempotent for
+       the call in afr_changelog_post_op_safe() to not have
+       side effects on the call from afr_changelog_post_op_now()
+    */
+    if (local->transaction.uninherit_done)
+        return local->transaction.uninherit_value;
 
-		ret = _gf_true;
+    LOCK(&local->inode->lock);
+    {
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->transaction.pre_op[i] != ctx->pre_op_done[type][i]) {
+                ret = !local->transaction.dirtied;
+                goto unlock;
+            }
+        }
+
+        if (ctx->inherited[type]) {
+            ret = _gf_true;
+            ctx->inherited[type]--;
+        } else if (ctx->on_disk[type]) {
+            ret = _gf_false;
+            ctx->on_disk[type]--;
+        } else {
+            /* ASSERT */
+            ret = _gf_false;
+        }
 
-		local->transaction.inherited = _gf_true;
-	}
+        if (!ctx->inherited[type] && !ctx->on_disk[type]) {
+            for (i = 0; i < priv->child_count; i++)
+                ctx->pre_op_done[type][i] = 0;
+        }
+    }
 unlock:
-	UNLOCK(&local->inode->lock);
+    UNLOCK(&local->inode->lock);
 
-	return ret;
-}
+    local->transaction.uninherit_done = _gf_true;
+    local->transaction.uninherit_value = ret;
 
+    return ret;
+}
 
 gf_boolean_t
-afr_changelog_pre_op_update (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-	gf_boolean_t ret = _gf_false;
-	int type = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-        if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
-            local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION)
-                return _gf_false;
-
-	if (local->transaction.inherited)
-		/* was already inherited in afr_changelog_pre_op */
-		return _gf_false;
-
-	if (!local->transaction.dirtied)
-		return _gf_false;
-
-        if (!afr_txn_nothing_failed (frame, this))
-		return _gf_false;
-
-	type = afr_index_for_transaction_type (local->transaction.type);
-
-	ret = _gf_false;
-
-	LOCK(&local->inode->lock);
-	{
-		if (!local->inode_ctx->on_disk[type]) {
-			for (i = 0; i < priv->child_count; i++)
-				local->inode_ctx->pre_op_done[type][i] =
-                                        (!local->transaction.failed_subvols[i]);
-		} else {
-			for (i = 0; i < priv->child_count; i++)
-				if (local->inode_ctx->pre_op_done[type][i] !=
-				    (!local->transaction.failed_subvols[i])) {
-					local->transaction.no_uninherit = 1;
-					goto unlock;
-				}
-		}
-		local->inode_ctx->on_disk[type]++;
-
-		ret = _gf_true;
-	}
-unlock:
-	UNLOCK(&local->inode->lock);
+afr_changelog_pre_op_inherit(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    gf_boolean_t ret = _gf_false;
+    int type = 0;
 
-	return ret;
-}
+    local = frame->local;
+    priv = this->private;
 
+    if (local->transaction.type != AFR_DATA_TRANSACTION)
+        return _gf_false;
 
-int
-afr_changelog_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
-{
-        afr_local_t *local = NULL;
-        int call_count = -1;
-        int child_index = -1;
+    type = afr_index_for_transaction_type(local->transaction.type);
 
-        local = frame->local;
-        child_index = (long) cookie;
+    LOCK(&local->inode->lock);
+    {
+        if (!local->inode_ctx->on_disk[type]) {
+            /* nothing to inherit yet */
+            ret = _gf_false;
+            goto unlock;
+        }
 
-        if (op_ret == -1) {
-                local->op_errno = op_errno;
-                afr_transaction_fop_failed (frame, this, child_index);
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->transaction.pre_op[i] !=
+                local->inode_ctx->pre_op_done[type][i]) {
+                /* either inherit exactly, or don't */
+                ret = _gf_false;
+                goto unlock;
+            }
         }
 
-        if (xattr)
-                local->transaction.changelog_xdata[child_index] = dict_ref (xattr);
+        local->inode_ctx->inherited[type]++;
 
-        call_count = afr_frame_return (frame);
+        ret = _gf_true;
 
-        if (call_count == 0) {
-                local->transaction.changelog_resume (frame, this);
-        }
+        local->transaction.inherited = _gf_true;
+    }
+unlock:
+    UNLOCK(&local->inode->lock);
 
-        return 0;
+    return ret;
 }
 
-void
-afr_changelog_populate_xdata (call_frame_t *frame, afr_xattrop_type_t op,
-                              dict_t **xdata, dict_t **newloc_xdata)
-{
-        int              i                    = 0;
-        int              ret                  = 0;
-        char            *key                  = NULL;
-        const char      *name                 = NULL;
-        dict_t          *xdata1               = NULL;
-        dict_t          *xdata2               = NULL;
-        xlator_t        *this                 = NULL;
-        afr_local_t     *local                = NULL;
-        afr_private_t   *priv                 = NULL;
-        gf_boolean_t     need_entry_key_set   = _gf_true;
-
-        local = frame->local;
-        this = THIS;
-        priv = this->private;
-
-        if (local->transaction.type == AFR_DATA_TRANSACTION ||
-            local->transaction.type == AFR_METADATA_TRANSACTION)
-                goto out;
-
-        if (!priv->esh_granular)
-                goto out;
-
-        xdata1 = dict_new();
-        if (!xdata1)
-                goto out;
+gf_boolean_t
+afr_changelog_pre_op_update(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    gf_boolean_t ret = _gf_false;
+    int type = 0;
 
-        name = local->loc.name;
-        if (local->op == GF_FOP_LINK)
-                name = local->newloc.name;
+    local = frame->local;
+    priv = this->private;
 
-        switch (op) {
-        case AFR_TRANSACTION_PRE_OP:
-                key = GF_XATTROP_ENTRY_IN_KEY;
-                break;
-        case AFR_TRANSACTION_POST_OP:
-                if (afr_txn_nothing_failed (frame, this)) {
-                        key = GF_XATTROP_ENTRY_OUT_KEY;
-                        for (i = 0; i < priv->child_count; i++) {
-                                if (!local->transaction.failed_subvols[i])
-                                        continue;
-                                need_entry_key_set = _gf_false;
-                                break;
-                        }
-                        /* If the transaction itself did not fail and there
-                         * are no failed subvolumes, check whether the fop
-                         * failed due to a symmetric error. If it did, do
-                         * not set the ENTRY_OUT xattr which would end up
-                         * deleting a name index which was created possibly by
-                         * an earlier entry txn that may have failed on some
-                         * of the sub-volumes.
-                         */
-                        if (local->op_ret)
-                                need_entry_key_set = _gf_false;
-                } else {
-                        key = GF_XATTROP_ENTRY_IN_KEY;
-                }
-                break;
-        }
+    if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
+        local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION)
+        return _gf_false;
 
-        if (need_entry_key_set) {
-                ret = dict_set_str (xdata1, key, (char *)name);
-                if (ret)
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_DICT_SET_FAILED,
-                                "%s/%s: Could not set %s key during xattrop",
-                                uuid_utoa (local->loc.pargfid), local->loc.name,
-                                key);
-                if (local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
-                        xdata2 = dict_new ();
-                        if (!xdata2)
-                                goto out;
-
-                        ret = dict_set_str (xdata2, key,
-                                            (char *)local->newloc.name);
-                        if (ret)
-                                gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                        AFR_MSG_DICT_SET_FAILED,
-                                        "%s/%s: Could not set %s key during "
-                                        "xattrop",
-                                        uuid_utoa (local->newloc.pargfid),
-                                        local->newloc.name, key);
-                }
-        }
+    if (local->transaction.inherited)
+        /* was already inherited in afr_changelog_pre_op */
+        return _gf_false;
 
-        *xdata = xdata1;
-        *newloc_xdata = xdata2;
-        xdata1 = xdata2 = NULL;
-out:
-        if (xdata1)
-                dict_unref (xdata1);
-        return;
-}
+    if (!local->transaction.dirtied)
+        return _gf_false;
 
-int
-afr_changelog_prepare (xlator_t *this, call_frame_t *frame, int *call_count,
-                       afr_changelog_resume_t changelog_resume,
-                       afr_xattrop_type_t op, dict_t **xdata,
-                       dict_t **newloc_xdata)
-{
-        afr_private_t *priv  = NULL;
-        afr_local_t   *local = NULL;
+    if (!afr_txn_nothing_failed(frame, this))
+        return _gf_false;
 
-        local = frame->local;
-        priv = this->private;
+    type = afr_index_for_transaction_type(local->transaction.type);
 
-        *call_count = afr_changelog_call_count (local->transaction.type,
-                                               local->transaction.pre_op,
-                                              local->transaction.failed_subvols,
-                                               priv->child_count);
+    ret = _gf_false;
 
-        if (*call_count == 0) {
-                changelog_resume (frame, this);
-                return -1;
+    LOCK(&local->inode->lock);
+    {
+        if (!local->inode_ctx->on_disk[type]) {
+            for (i = 0; i < priv->child_count; i++)
+                local->inode_ctx->pre_op_done[type][i] =
+                    (!local->transaction.failed_subvols[i]);
+        } else {
+            for (i = 0; i < priv->child_count; i++)
+                if (local->inode_ctx->pre_op_done[type][i] !=
+                    (!local->transaction.failed_subvols[i])) {
+                    local->transaction.no_uninherit = 1;
+                    goto unlock;
+                }
         }
+        local->inode_ctx->on_disk[type]++;
 
-        afr_changelog_populate_xdata (frame, op, xdata, newloc_xdata);
-        local->call_count = *call_count;
+        ret = _gf_true;
+    }
+unlock:
+    UNLOCK(&local->inode->lock);
 
-        local->transaction.changelog_resume = changelog_resume;
-        return 0;
+    return ret;
 }
 
 int
-afr_changelog_do (call_frame_t *frame, xlator_t *this, dict_t *xattr,
-		  afr_changelog_resume_t changelog_resume,
-                  afr_xattrop_type_t op)
+afr_changelog_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, dict_t *xattr, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-        dict_t *xdata = NULL;
-        dict_t *newloc_xdata = NULL;
-	int i = 0;
-	int call_count = 0;
-        int ret = 0;
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = -1;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    child_index = (long)cookie;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.changelog_xdata[i]) {
-                        dict_unref (local->transaction.changelog_xdata[i]);
-                        local->transaction.changelog_xdata[i] = NULL;
-                }
-        }
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        afr_transaction_fop_failed(frame, this, child_index);
+    }
 
-        ret = afr_changelog_prepare (this, frame, &call_count, changelog_resume,
-                                     op, &xdata, &newloc_xdata);
+    if (xattr)
+        local->transaction.changelog_xdata[child_index] = dict_ref(xattr);
 
-        if (ret)
-                return 0;
+    call_count = afr_frame_return(frame);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->transaction.pre_op[i] ||
-                    local->transaction.failed_subvols[i])
-                        continue;
+    if (call_count == 0) {
+        local->transaction.changelog_resume(frame, this);
+    }
 
-                switch (local->transaction.type) {
-                case AFR_DATA_TRANSACTION:
-                case AFR_METADATA_TRANSACTION:
-                        if (!local->fd) {
-                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-						   (void *) (long) i,
-						   priv->children[i],
-						   priv->children[i]->fops->xattrop,
-						   &local->loc,
-						   GF_XATTROP_ADD_ARRAY, xattr,
-						   xdata);
-                        } else {
-                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-						   (void *) (long) i,
-						   priv->children[i],
-						   priv->children[i]->fops->fxattrop,
-						   local->fd,
-						   GF_XATTROP_ADD_ARRAY, xattr,
-						   xdata);
-                        }
-			break;
-                case AFR_ENTRY_RENAME_TRANSACTION:
-
-			STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->xattrop,
-					   &local->transaction.new_parent_loc,
-					   GF_XATTROP_ADD_ARRAY, xattr,
-					   newloc_xdata);
-                        call_count--;
-
-                /* fall through */
-
-                case AFR_ENTRY_TRANSACTION:
-                        if (local->fd)
-                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-						   (void *) (long) i,
-						   priv->children[i],
-						   priv->children[i]->fops->fxattrop,
-						   local->fd,
-						   GF_XATTROP_ADD_ARRAY, xattr,
-						   xdata);
-                        else
-                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-						   (void *) (long) i,
-						   priv->children[i],
-						   priv->children[i]->fops->xattrop,
-						   &local->transaction.parent_loc,
-						   GF_XATTROP_ADD_ARRAY, xattr,
-						   xdata);
-			break;
-		}
-
-                if (!--call_count)
-                        break;
-        }
-
-        if (xdata)
-                dict_unref (xdata);
-        if (newloc_xdata)
-                dict_unref (newloc_xdata);
-	return 0;
+    return 0;
 }
 
-static void
-afr_init_optimistic_changelog_for_txn (xlator_t *this, afr_local_t *local)
-{
-        int                locked_count   = 0;
-        afr_private_t     *priv           = NULL;
+void
+afr_changelog_populate_xdata(call_frame_t *frame, afr_xattrop_type_t op,
+                             dict_t **xdata, dict_t **newloc_xdata)
+{
+    int i = 0;
+    int ret = 0;
+    char *key = NULL;
+    int keylen = 0;
+    const char *name = NULL;
+    dict_t *xdata1 = NULL;
+    dict_t *xdata2 = NULL;
+    xlator_t *this = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t need_entry_key_set = _gf_true;
+
+    local = frame->local;
+    this = THIS;
+    priv = this->private;
+
+    if (local->transaction.type == AFR_DATA_TRANSACTION ||
+        local->transaction.type == AFR_METADATA_TRANSACTION)
+        goto out;
+
+    if (!priv->esh_granular)
+        goto out;
+
+    xdata1 = dict_new();
+    if (!xdata1)
+        goto out;
+
+    name = local->loc.name;
+    if (local->op == GF_FOP_LINK)
+        name = local->newloc.name;
+
+    switch (op) {
+        case AFR_TRANSACTION_PRE_OP:
+            key = GF_XATTROP_ENTRY_IN_KEY;
+            break;
+        case AFR_TRANSACTION_POST_OP:
+            if (afr_txn_nothing_failed(frame, this)) {
+                key = GF_XATTROP_ENTRY_OUT_KEY;
+                for (i = 0; i < priv->child_count; i++) {
+                    if (!local->transaction.failed_subvols[i])
+                        continue;
+                    need_entry_key_set = _gf_false;
+                    break;
+                }
+                /* If the transaction itself did not fail and there
+                 * are no failed subvolumes, check whether the fop
+                 * failed due to a symmetric error. If it did, do
+                 * not set the ENTRY_OUT xattr which would end up
+                 * deleting a name index which was created possibly by
+                 * an earlier entry txn that may have failed on some
+                 * of the sub-volumes.
+                 */
+                if (local->op_ret)
+                    need_entry_key_set = _gf_false;
+            } else {
+                key = GF_XATTROP_ENTRY_IN_KEY;
+            }
+            break;
+    }
 
-        priv = this->private;
+    if (need_entry_key_set) {
+        keylen = strlen(key);
+        ret = dict_set_strn(xdata1, key, keylen, (char *)name);
+        if (ret)
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                   "%s/%s: Could not set %s key during xattrop",
+                   uuid_utoa(local->loc.pargfid), local->loc.name, key);
+        if (local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
+            xdata2 = dict_new();
+            if (!xdata2)
+                goto out;
 
-        locked_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
-        if (priv->optimistic_change_log && locked_count == priv->child_count)
-                local->optimistic_change_log = 1;
+            ret = dict_set_strn(xdata2, key, keylen,
+                                (char *)local->newloc.name);
+            if (ret)
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                       "%s/%s: Could not set %s key during "
+                       "xattrop",
+                       uuid_utoa(local->newloc.pargfid), local->newloc.name,
+                       key);
+        }
+    }
 
-        return;
+    *xdata = xdata1;
+    *newloc_xdata = xdata2;
+    xdata1 = xdata2 = NULL;
+out:
+    if (xdata1)
+        dict_unref(xdata1);
+    return;
 }
 
 int
-afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t * priv = this->private;
-        int i = 0;
-        int ret = 0;
-        int call_count = 0;
-	int op_errno = 0;
-        afr_local_t *local = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        unsigned char       *locked_nodes = NULL;
-	int idx = -1;
-	gf_boolean_t pre_nop = _gf_true;
-	dict_t *xdata_req = NULL;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-	idx = afr_index_for_transaction_type (local->transaction.type);
-
-        locked_nodes = afr_locked_nodes_get (local->transaction.type, int_lock);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (locked_nodes[i]) {
-			local->transaction.pre_op[i] = 1;
-			call_count++;
-		} else {
-                        local->transaction.failed_subvols[i] = 1;
-                }
-	}
+afr_changelog_prepare(xlator_t *this, call_frame_t *frame, int *call_count,
+                      afr_changelog_resume_t changelog_resume,
+                      afr_xattrop_type_t op, dict_t **xdata,
+                      dict_t **newloc_xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        afr_init_optimistic_changelog_for_txn (this, local);
+    local = frame->local;
+    priv = this->private;
 
-        if (afr_changelog_pre_op_inherit (frame, this))
-                goto next;
+    *call_count = afr_changelog_call_count(
+        local->transaction.type, local->transaction.pre_op,
+        local->transaction.failed_subvols, priv->child_count);
 
-        /* This condition should not be met with present code, as
-         * transaction.done will be called if locks are not acquired on even a
-         * single node.
-         */
-        if (call_count == 0) {
-		op_errno = ENOTCONN;
-		goto err;
-	}
+    if (*call_count == 0) {
+        changelog_resume(frame, this);
+        return -1;
+    }
 
-        /* Check if the fop can be performed on at least
-         * quorum number of nodes.
-         */
-        if (priv->quorum_count && !afr_has_fop_quorum (frame)) {
-                op_errno = int_lock->lock_op_errno;
-                if (op_errno == 0)
-                        op_errno = afr_quorum_errno (priv);
-                goto err;
-        }
+    afr_changelog_populate_xdata(frame, op, xdata, newloc_xdata);
+    local->call_count = *call_count;
 
-	xdata_req = dict_new();
-	if (!xdata_req) {
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-        if (call_count < priv->child_count)
-                pre_nop = _gf_false;
+    local->transaction.changelog_resume = changelog_resume;
+    return 0;
+}
 
-        /* Set an all-zero pending changelog so that in the cbk, we can get the
-         * current on-disk values. In a replica 3 volume with arbiter enabled,
-         * these values are needed to arrive at a go/ no-go of the fop phase to
-         * avoid ending up in split-brain.*/
+int
+afr_changelog_do(call_frame_t *frame, xlator_t *this, dict_t *xattr,
+                 afr_changelog_resume_t changelog_resume, afr_xattrop_type_t op)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    dict_t *newloc_xdata = NULL;
+    int i = 0;
+    int call_count = 0;
+    int ret = 0;
 
-        ret = afr_set_pending_dict (priv, xdata_req, local->pending);
-	if (ret < 0) {
-		op_errno = ENOMEM;
-		goto err;
-	}
+    local = frame->local;
+    priv = this->private;
 
-	if (afr_needs_changelog_update (local)) {
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.changelog_xdata[i]) {
+            dict_unref(local->transaction.changelog_xdata[i]);
+            local->transaction.changelog_xdata[i] = NULL;
+        }
+    }
 
-		local->dirty[idx] = hton32(1);
+    ret = afr_changelog_prepare(this, frame, &call_count, changelog_resume, op,
+                                &xdata, &newloc_xdata);
 
-		ret = dict_set_static_bin (xdata_req, AFR_DIRTY, local->dirty,
-					   sizeof(int) * AFR_NUM_CHANGE_LOGS);
-		if (ret) {
-			op_errno = ENOMEM;
-			goto err;
-		}
+    if (ret)
+        return 0;
 
-		pre_nop = _gf_false;
-		local->transaction.dirtied = 1;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.pre_op[i] ||
+            local->transaction.failed_subvols[i])
+            continue;
 
-	if (pre_nop)
-		goto next;
+        switch (local->transaction.type) {
+            case AFR_DATA_TRANSACTION:
+            case AFR_METADATA_TRANSACTION:
+                if (!local->fd) {
+                    STACK_WIND_COOKIE(
+                        frame, afr_changelog_cbk, (void *)(long)i,
+                        priv->children[i], priv->children[i]->fops->xattrop,
+                        &local->loc, GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                } else {
+                    STACK_WIND_COOKIE(
+                        frame, afr_changelog_cbk, (void *)(long)i,
+                        priv->children[i], priv->children[i]->fops->fxattrop,
+                        local->fd, GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                }
+                break;
+            case AFR_ENTRY_RENAME_TRANSACTION:
 
-	if (!local->pre_op_compat) {
-		dict_copy (xdata_req, local->xdata_req);
-		goto next;
-	}
+                STACK_WIND_COOKIE(frame, afr_changelog_cbk, (void *)(long)i,
+                                  priv->children[i],
+                                  priv->children[i]->fops->xattrop,
+                                  &local->transaction.new_parent_loc,
+                                  GF_XATTROP_ADD_ARRAY, xattr, newloc_xdata);
+                call_count--;
 
-        afr_changelog_do (frame, this, xdata_req, afr_transaction_perform_fop,
-                          AFR_TRANSACTION_PRE_OP);
+                /* fall through */
 
-	if (xdata_req)
-		dict_unref (xdata_req);
+            case AFR_ENTRY_TRANSACTION:
+                if (local->fd)
+                    STACK_WIND_COOKIE(
+                        frame, afr_changelog_cbk, (void *)(long)i,
+                        priv->children[i], priv->children[i]->fops->fxattrop,
+                        local->fd, GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                else
+                    STACK_WIND_COOKIE(frame, afr_changelog_cbk, (void *)(long)i,
+                                      priv->children[i],
+                                      priv->children[i]->fops->xattrop,
+                                      &local->transaction.parent_loc,
+                                      GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                break;
+        }
 
-	return 0;
-next:
-	afr_transaction_perform_fop (frame, this);
+        if (!--call_count)
+            break;
+    }
 
-	if (xdata_req)
-		dict_unref (xdata_req);
+    if (xdata)
+        dict_unref(xdata);
+    if (newloc_xdata)
+        dict_unref(newloc_xdata);
+    return 0;
+}
 
-        return 0;
-err:
-	local->internal_lock.lock_cbk = afr_transaction_done;
-	local->op_ret = -1;
-	local->op_errno = op_errno;
+static void
+afr_init_optimistic_changelog_for_txn(xlator_t *this, afr_local_t *local)
+{
+    int locked_count = 0;
+    afr_private_t *priv = NULL;
 
-        afr_handle_lock_acquire_failure (local, _gf_true);
+    priv = this->private;
 
-	if (xdata_req)
-		dict_unref (xdata_req);
+    locked_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
+    if (priv->optimistic_change_log && locked_count == priv->child_count)
+        local->optimistic_change_log = 1;
 
-	return 0;
+    return;
 }
 
-
 int
-afr_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+afr_changelog_pre_op(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    int i = 0;
+    int ret = 0;
+    int call_count = 0;
+    int op_errno = 0;
+    afr_local_t *local = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    unsigned char *locked_nodes = NULL;
+    int idx = -1;
+    gf_boolean_t pre_nop = _gf_true;
+    dict_t *xdata_req = NULL;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    idx = afr_index_for_transaction_type(local->transaction.type);
+
+    locked_nodes = afr_locked_nodes_get(local->transaction.type, int_lock);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (locked_nodes[i]) {
+            local->transaction.pre_op[i] = 1;
+            call_count++;
+        } else {
+            local->transaction.failed_subvols[i] = 1;
+        }
+    }
+
+    afr_init_optimistic_changelog_for_txn(this, local);
+
+    if (afr_changelog_pre_op_inherit(frame, this))
+        goto next;
+
+    /* This condition should not be met with present code, as
+     * transaction.done will be called if locks are not acquired on even a
+     * single node.
+     */
+    if (call_count == 0) {
+        op_errno = ENOTCONN;
+        goto err;
+    }
+
+    /* Check if the fop can be performed on at least
+     * quorum number of nodes.
+     */
+    if (priv->quorum_count && !afr_has_fop_quorum(frame)) {
+        op_errno = int_lock->lock_op_errno;
+        if (op_errno == 0)
+            op_errno = afr_quorum_errno(priv);
+        goto err;
+    }
+
+    xdata_req = dict_new();
+    if (!xdata_req) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (call_count < priv->child_count)
+        pre_nop = _gf_false;
+
+    /* Set an all-zero pending changelog so that in the cbk, we can get the
+     * current on-disk values. In a replica 3 volume with arbiter enabled,
+     * these values are needed to arrive at a go/ no-go of the fop phase to
+     * avoid ending up in split-brain.*/
+
+    ret = afr_set_pending_dict(priv, xdata_req, local->pending);
+    if (ret < 0) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (afr_needs_changelog_update(local)) {
+        local->dirty[idx] = hton32(1);
+
+        ret = dict_set_static_bin(xdata_req, AFR_DIRTY, local->dirty,
+                                  sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+
+        pre_nop = _gf_false;
+        local->transaction.dirtied = 1;
+    }
+
+    if (pre_nop)
+        goto next;
+
+    if (!local->pre_op_compat) {
+        dict_copy(xdata_req, local->xdata_req);
+        goto next;
+    }
+
+    afr_changelog_do(frame, this, xdata_req, afr_transaction_perform_fop,
+                     AFR_TRANSACTION_PRE_OP);
+
+    if (xdata_req)
+        dict_unref(xdata_req);
+
+    return 0;
+next:
+    afr_transaction_perform_fop(frame, this);
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    if (xdata_req)
+        dict_unref(xdata_req);
 
-        /* Initiate blocking locks if non-blocking has failed */
-        if (int_lock->lock_op_ret < 0) {
-                gf_msg_debug (this->name, 0,
-                              "Non blocking inodelks failed. Proceeding to blocking");
-                int_lock->lock_cbk = afr_internal_lock_finish;
-                afr_blocking_lock (frame, this);
-        } else {
+    return 0;
+err:
+    local->internal_lock.lock_cbk = afr_transaction_done;
+    local->op_ret = -1;
+    local->op_errno = op_errno;
 
-                gf_msg_debug (this->name, 0,
-                              "Non blocking inodelks done. Proceeding to FOP");
-                afr_internal_lock_finish (frame, this);
-        }
+    afr_handle_lock_acquire_failure(local);
 
-        return 0;
-}
+    if (xdata_req)
+        dict_unref(xdata_req);
 
+    return 0;
+}
 
 int
-afr_post_nonblocking_entrylk_cbk (call_frame_t *frame, xlator_t *this)
+afr_post_nonblocking_lock_cbk(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        /* Initiate blocking locks if non-blocking has failed */
-        if (int_lock->lock_op_ret < 0) {
-                gf_msg_debug (this->name, 0,
-                              "Non blocking entrylks failed. Proceeding to blocking");
-                int_lock->lock_cbk = afr_internal_lock_finish;
-                afr_blocking_lock (frame, this);
-        } else {
-
-                gf_msg_debug (this->name, 0,
-                              "Non blocking entrylks done. Proceeding to FOP");
+    /* Initiate blocking locks if non-blocking has failed */
+    if (int_lock->lock_op_ret < 0) {
+        gf_msg_debug(this->name, 0,
+                     "Non blocking locks failed. Proceeding to blocking");
+        int_lock->lock_cbk = afr_internal_lock_finish;
+        afr_blocking_lock(frame, this);
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Non blocking locks done. Proceeding to FOP");
 
-                afr_internal_lock_finish (frame, this);
-        }
+        afr_internal_lock_finish(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_post_blocking_rename_cbk (call_frame_t *frame, xlator_t *this)
+afr_post_blocking_rename_cbk(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        AFR_MSG_BLOCKING_LKS_FAILED,
-                        "Blocking entrylks failed.");
+    if (int_lock->lock_op_ret < 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_INTERNAL_LKS_FAILED,
+               "Blocking entrylks failed.");
 
-                afr_transaction_done (frame, this);
-        } else {
-
-                gf_msg_debug (this->name, 0,
-                              "Blocking entrylks done. Proceeding to FOP");
+        afr_transaction_done(frame, this);
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Blocking entrylks done. Proceeding to FOP");
 
-                afr_internal_lock_finish (frame, this);
-        }
-        return 0;
+        afr_internal_lock_finish(frame, this);
+    }
+    return 0;
 }
 
 int
-afr_post_lower_unlock_cbk (call_frame_t *frame, xlator_t *this)
+afr_post_lower_unlock_cbk(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        GF_ASSERT (!int_lock->higher_locked);
+    GF_ASSERT(!int_lock->higher_locked);
 
-        int_lock->lock_cbk = afr_post_blocking_rename_cbk;
-        afr_blocking_lock (frame, this);
+    int_lock->lock_cbk = afr_post_blocking_rename_cbk;
+    afr_blocking_lock(frame, this);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_set_transaction_flock (xlator_t *this, afr_local_t *local)
+afr_set_transaction_flock(xlator_t *this, afr_local_t *local,
+                          afr_lockee_t *lockee)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
+    afr_private_t *priv = NULL;
+    struct gf_flock *flock = NULL;
 
-        int_lock = &local->internal_lock;
-        priv = this->private;
+    priv = this->private;
+    flock = &lockee->flock;
 
-        if ((priv->arbiter_count || local->transaction.eager_lock_on ||
-             priv->full_lock) &&
-            local->transaction.type == AFR_DATA_TRANSACTION) {
-                /*Lock entire file to avoid network split brains.*/
-                int_lock->flock.l_len   = 0;
-                int_lock->flock.l_start = 0;
-        } else {
-                int_lock->flock.l_len   = local->transaction.len;
-                int_lock->flock.l_start = local->transaction.start;
-        }
-        int_lock->flock.l_type  = F_WRLCK;
+    if ((priv->arbiter_count || local->transaction.eager_lock_on ||
+         priv->full_lock) &&
+        local->transaction.type == AFR_DATA_TRANSACTION) {
+        /*Lock entire file to avoid network split brains.*/
+        flock->l_len = 0;
+        flock->l_start = 0;
+    } else {
+        flock->l_len = local->transaction.len;
+        flock->l_start = local->transaction.start;
+    }
+    flock->l_type = F_WRLCK;
 
-        return 0;
+    return 0;
 }
 
 int
-afr_lock (call_frame_t *frame, xlator_t *this)
+afr_lock(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        int_lock->domain = this->name;
+    int_lock->lock_cbk = afr_post_nonblocking_lock_cbk;
+    int_lock->domain = this->name;
 
-        switch (local->transaction.type) {
+    switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
-                afr_set_transaction_flock (this, local);
-
-                int_lock->lock_cbk = afr_post_nonblocking_inodelk_cbk;
+            for (i = 0; i < int_lock->lockee_count; i++) {
+                afr_set_transaction_flock(this, local, &int_lock->lockee[i]);
+            }
 
-                afr_nonblocking_inodelk (frame, this);
-                break;
-
-        case AFR_ENTRY_RENAME_TRANSACTION:
-
-                int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk;
-                afr_nonblocking_entrylk (frame, this);
-                break;
+            break;
 
         case AFR_ENTRY_TRANSACTION:
-                int_lock->lk_basename = local->transaction.basename;
-                if (local->transaction.parent_loc.path)
-                        int_lock->lk_loc = &local->transaction.parent_loc;
-                else
-                        GF_ASSERT (local->fd);
-
-                int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk;
-                afr_nonblocking_entrylk (frame, this);
-                break;
-        }
+            int_lock->lk_basename = local->transaction.basename;
+            if (local->transaction.parent_loc.path)
+                int_lock->lk_loc = &local->transaction.parent_loc;
+            else
+                GF_ASSERT(local->fd);
+            break;
+        case AFR_ENTRY_RENAME_TRANSACTION:
+            break;
+    }
+    afr_lock_nonblocking(frame, this);
 
-        return 0;
+    return 0;
 }
 
 static gf_boolean_t
-afr_locals_overlap (afr_local_t *local1, afr_local_t *local2)
+afr_locals_overlap(afr_local_t *local1, afr_local_t *local2)
 {
-        uint64_t start1 = local1->transaction.start;
-        uint64_t start2 = local2->transaction.start;
-        uint64_t end1 = 0;
-        uint64_t end2 = 0;
+    uint64_t start1 = local1->transaction.start;
+    uint64_t start2 = local2->transaction.start;
+    uint64_t end1 = 0;
+    uint64_t end2 = 0;
 
-        if (local1->transaction.len)
-                end1 = start1 + local1->transaction.len - 1;
-        else
-                end1 = ULLONG_MAX;
+    if (local1->transaction.len)
+        end1 = start1 + local1->transaction.len - 1;
+    else
+        end1 = ULLONG_MAX;
 
-        if (local2->transaction.len)
-                end2 = start2 + local2->transaction.len - 1;
-        else
-                end2 = ULLONG_MAX;
+    if (local2->transaction.len)
+        end2 = start2 + local2->transaction.len - 1;
+    else
+        end2 = ULLONG_MAX;
 
-        return ((end1 >= start2) && (end2 >= start1));
+    return ((end1 >= start2) && (end2 >= start1));
 }
 
 gf_boolean_t
-afr_has_lock_conflict (afr_local_t *local, gf_boolean_t waitlist_check)
-{
-        afr_local_t     *each = NULL;
-        afr_lock_t      *lock = NULL;
-
-        lock = &local->inode_ctx->lock[local->transaction.type];
-        /*
-         * Once full file lock is acquired in eager-lock phase, overlapping
-         * writes do not compete for inode-locks, instead are transferred to the
-         * next writes. Because of this overlapping writes are not ordered.
-         * This can cause inconsistencies in replication.
-         * Example:
-         * Two overlapping writes w1, w2 are sent in parallel on same fd
-         * in two threads t1, t2.
-         * Both threads can execute afr_writev_wind in the following manner.
-         * t1 winds w1 on brick-0
-         * t2 winds w2 on brick-0
-         * t2 winds w2 on brick-1
-         * t1 winds w1 on brick-1
-         *
-         * This check makes sure the locks are not transferred for
-         * overlapping writes.
-         */
-        list_for_each_entry (each, &lock->owners, transaction.owner_list) {
-                if (afr_locals_overlap (each, local)) {
-                        return _gf_true;
-                }
-        }
-
-        if (!waitlist_check)
-                return _gf_false;
-        list_for_each_entry (each, &lock->waiting, transaction.wait_list) {
-                if (afr_locals_overlap (each, local)) {
-                        return _gf_true;
-                }
-        }
+afr_has_lock_conflict(afr_local_t *local, gf_boolean_t waitlist_check)
+{
+    afr_local_t *each = NULL;
+    afr_lock_t *lock = NULL;
+
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    /*
+     * Once full file lock is acquired in eager-lock phase, overlapping
+     * writes do not compete for inode-locks, instead are transferred to the
+     * next writes. Because of this overlapping writes are not ordered.
+     * This can cause inconsistencies in replication.
+     * Example:
+     * Two overlapping writes w1, w2 are sent in parallel on same fd
+     * in two threads t1, t2.
+     * Both threads can execute afr_writev_wind in the following manner.
+     * t1 winds w1 on brick-0
+     * t2 winds w2 on brick-0
+     * t2 winds w2 on brick-1
+     * t1 winds w1 on brick-1
+     *
+     * This check makes sure the locks are not transferred for
+     * overlapping writes.
+     */
+    list_for_each_entry(each, &lock->owners, transaction.owner_list)
+    {
+        if (afr_locals_overlap(each, local)) {
+            return _gf_true;
+        }
+    }
+
+    if (!waitlist_check)
         return _gf_false;
+    list_for_each_entry(each, &lock->waiting, transaction.wait_list)
+    {
+        if (afr_locals_overlap(each, local)) {
+            return _gf_true;
+        }
+    }
+    return _gf_false;
 }
 
-
 /* }}} */
 static void
-afr_copy_inodelk_vars (afr_internal_lock_t *dst, afr_internal_lock_t *src,
-                       xlator_t *this)
+afr_copy_inodelk_vars(afr_internal_lock_t *dst, afr_internal_lock_t *src,
+                      xlator_t *this, int lockee_num)
 {
-        afr_private_t *priv = this->private;
+    afr_private_t *priv = this->private;
+    afr_lockee_t *sl = &src->lockee[lockee_num];
+    afr_lockee_t *dl = &dst->lockee[lockee_num];
 
-        dst->domain = src->domain;
-        dst->flock.l_len  = src->flock.l_len;
-        dst->flock.l_start  = src->flock.l_start;
-        dst->flock.l_type  = src->flock.l_type;
-        dst->lock_count = src->lock_count;
-        memcpy (dst->locked_nodes, src->locked_nodes,
-                priv->child_count * sizeof (*dst->locked_nodes));
+    dst->domain = src->domain;
+    dl->flock.l_len = sl->flock.l_len;
+    dl->flock.l_start = sl->flock.l_start;
+    dl->flock.l_type = sl->flock.l_type;
+    dl->locked_count = sl->locked_count;
+    memcpy(dl->locked_nodes, sl->locked_nodes,
+           priv->child_count * sizeof(*dl->locked_nodes));
 }
 
 void
-__afr_transaction_wake_shared (afr_local_t *local, struct list_head *shared)
-{
-        gf_boolean_t conflict = _gf_false;
-        afr_local_t *each = NULL;
-        afr_lock_t *lock = &local->inode_ctx->lock[local->transaction.type];
-
-        while (!conflict) {
-                if (list_empty (&lock->waiting))
-                        return;
-                each = list_entry(lock->waiting.next, afr_local_t,
-                                  transaction.wait_list);
-                if (afr_has_lock_conflict (each, _gf_false)) {
-                        conflict = _gf_true;
-                }
-                if (conflict && !list_empty (&lock->owners))
-                        return;
-                afr_copy_inodelk_vars (&each->internal_lock,
-                                       &local->internal_lock,
-                                       each->transaction.frame->this);
-                list_move_tail (&each->transaction.wait_list, shared);
-                list_add_tail(&each->transaction.owner_list, &lock->owners);
+__afr_transaction_wake_shared(afr_local_t *local, struct list_head *shared)
+{
+    gf_boolean_t conflict = _gf_false;
+    afr_local_t *each = NULL;
+    afr_lock_t *lock = &local->inode_ctx->lock[local->transaction.type];
+
+    while (!conflict) {
+        if (list_empty(&lock->waiting))
+            return;
+        each = list_entry(lock->waiting.next, afr_local_t,
+                          transaction.wait_list);
+        if (afr_has_lock_conflict(each, _gf_false)) {
+            conflict = _gf_true;
         }
+        if (conflict && !list_empty(&lock->owners))
+            return;
+        afr_copy_inodelk_vars(&each->internal_lock, &local->internal_lock,
+                              each->transaction.frame->this, 0);
+        list_move_tail(&each->transaction.wait_list, shared);
+        list_add_tail(&each->transaction.owner_list, &lock->owners);
+    }
 }
 
 static void
-afr_lock_resume_shared (struct list_head *list)
+afr_lock_resume_shared(struct list_head *list)
 {
-        afr_local_t *each = NULL;
+    afr_local_t *each = NULL;
 
-        while (!list_empty(list)) {
-                each = list_entry(list->next, afr_local_t,
-                                  transaction.wait_list);
-                list_del_init(&each->transaction.wait_list);
-                afr_changelog_pre_op (each->transaction.frame,
-                                      each->transaction.frame->this);
-        }
+    while (!list_empty(list)) {
+        each = list_entry(list->next, afr_local_t, transaction.wait_list);
+        list_del_init(&each->transaction.wait_list);
+        afr_changelog_pre_op(each->transaction.frame,
+                             each->transaction.frame->this);
+    }
 }
 
 int
-afr_internal_lock_finish (call_frame_t *frame, xlator_t *this)
+afr_internal_lock_finish(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = frame->local;
-        afr_lock_t   *lock  = NULL;
+    afr_local_t *local = frame->local;
+    afr_lock_t *lock = NULL;
 
-
-        local->internal_lock.lock_cbk = NULL;
-        if (!local->transaction.eager_lock_on) {
-                if (local->internal_lock.lock_op_ret < 0) {
-                        afr_transaction_done (frame, this);
-                        return 0;
-                }
-                afr_changelog_pre_op (frame, this);
+    local->internal_lock.lock_cbk = NULL;
+    if (!local->transaction.eager_lock_on) {
+        if (local->internal_lock.lock_op_ret < 0) {
+            afr_transaction_done(frame, this);
+            return 0;
+        }
+        afr_changelog_pre_op(frame, this);
+    } else {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        if (local->internal_lock.lock_op_ret < 0) {
+            afr_handle_lock_acquire_failure(local);
         } else {
-                lock = &local->inode_ctx->lock[local->transaction.type];
-                if (local->internal_lock.lock_op_ret < 0) {
-                        afr_handle_lock_acquire_failure (local, _gf_false);
-                } else {
-                        lock->event_generation = local->event_generation;
-                        afr_changelog_pre_op (frame, this);
-                }
+            lock->event_generation = local->event_generation;
+            afr_changelog_pre_op(frame, this);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
 gf_boolean_t
-afr_are_multiple_fds_opened (afr_local_t *local, xlator_t *this)
+afr_are_conflicting_ops_waiting(afr_local_t *local, xlator_t *this)
 {
-        /* Lets say mount1 has eager-lock(full-lock) and after the eager-lock
-         * is taken mount2 opened the same file, it won't be able to
-         * perform any data operations until mount1 releases eager-lock.
-         * To avoid such scenario do not enable eager-lock for this transaction
-         * if open-fd-count is > 1
-         */
-
-        if (local->inode_ctx->open_fd_count > 1)
-                return _gf_true;
-
-        return _gf_false;
-}
+    afr_lock_t *lock = NULL;
+    lock = &local->inode_ctx->lock[local->transaction.type];
 
+    /* Lets say mount1 has eager-lock(full-lock) and after the eager-lock
+     * is taken mount2 opened the same file, it won't be able to
+     * perform any {meta,}data operations until mount1 releases eager-lock.
+     * To avoid such scenario do not enable eager-lock for this transaction
+     * if open-fd-count is > 1 for metadata transactions and if num-inodelks > 1
+     * for data transactions
+     */
 
-gf_boolean_t
-afr_is_delayed_changelog_post_op_needed (call_frame_t *frame, xlator_t *this,
-                                         int delay)
-{
-        afr_local_t  *local = NULL;
-        afr_lock_t   *lock  = NULL;
-        gf_boolean_t res    = _gf_false;
-
-        local = frame->local;
-        lock = &local->inode_ctx->lock[local->transaction.type];
-
-        if (!afr_txn_nothing_failed (frame, this)) {
-                lock->release = _gf_true;
-                goto out;
+    if (local->transaction.type == AFR_METADATA_TRANSACTION) {
+        if (local->inode_ctx->open_fd_count > 1) {
+            return _gf_true;
         }
-
-        if (afr_are_multiple_fds_opened (local, this)) {
-                lock->release = _gf_true;
-                goto out;
-        }
-
-        if (!list_empty (&lock->owners))
-                goto out;
-        else
-                GF_ASSERT (list_empty (&lock->waiting));
-
-        if (lock->release) {
-                goto out;
-        }
-
-        if (!delay) {
-                goto out;
+    } else if (local->transaction.type == AFR_DATA_TRANSACTION) {
+        if (lock->num_inodelks > 1) {
+            return _gf_true;
         }
+    }
 
-        if ((local->op != GF_FOP_WRITE) &&
-            (local->op != GF_FOP_FXATTROP)) {
-                /*Only allow writes but shard does [f]xattrops on writes, so
-                 * they are fine too*/
-                goto out;
-        }
+    return _gf_false;
+}
 
-        res = _gf_true;
+gf_boolean_t
+afr_is_delayed_changelog_post_op_needed(call_frame_t *frame, xlator_t *this,
+                                        int delay)
+{
+    afr_local_t *local = NULL;
+    afr_lock_t *lock = NULL;
+    gf_boolean_t res = _gf_false;
+
+    local = frame->local;
+    lock = &local->inode_ctx->lock[local->transaction.type];
+
+    if (!afr_txn_nothing_failed(frame, this)) {
+        lock->release = _gf_true;
+        goto out;
+    }
+
+    if (afr_are_conflicting_ops_waiting(local, this)) {
+        lock->release = _gf_true;
+        goto out;
+    }
+
+    if (!list_empty(&lock->owners))
+        goto out;
+    else
+        GF_ASSERT(list_empty(&lock->waiting));
+
+    if (lock->release) {
+        goto out;
+    }
+
+    if (!delay) {
+        goto out;
+    }
+
+    if (local->transaction.disable_delayed_post_op) {
+        goto out;
+    }
+
+    if ((local->op != GF_FOP_WRITE) && (local->op != GF_FOP_FXATTROP) &&
+        (local->op != GF_FOP_FSYNC)) {
+        /*Only allow writes/fsyncs but shard does [f]xattrops on writes, so
+         * they are fine too*/
+        goto out;
+    }
+
+    res = _gf_true;
 out:
-        return res;
+    return res;
 }
 
-
 void
-afr_delayed_changelog_wake_up_cbk (void *data)
-{
-        afr_lock_t  *lock  = NULL;
-        afr_local_t *local = data;
-        afr_local_t *timer_local = NULL;
-        struct list_head shared;
-
-        INIT_LIST_HEAD (&shared);
-        lock = &local->inode_ctx->lock[local->transaction.type];
-        LOCK (&local->inode->lock);
-        {
-                timer_local = list_entry(lock->post_op.next,
-                                         afr_local_t,
-                                        transaction.owner_list);
-                if (list_empty (&lock->owners) && (local == timer_local)) {
-                        GF_ASSERT (list_empty (&lock->waiting));
-                        /*Last owner*/
-                        lock->release = _gf_true;
-                        lock->delay_timer = NULL;
-                }
-        }
-        UNLOCK (&local->inode->lock);
-        afr_changelog_post_op_now (local->transaction.frame,
-                                   local->transaction.frame->this);
+afr_delayed_changelog_wake_up_cbk(void *data)
+{
+    afr_lock_t *lock = NULL;
+    afr_local_t *local = data;
+    afr_local_t *timer_local = NULL;
+    struct list_head shared;
+
+    INIT_LIST_HEAD(&shared);
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    LOCK(&local->inode->lock);
+    {
+        timer_local = list_entry(lock->post_op.next, afr_local_t,
+                                 transaction.owner_list);
+        if (list_empty(&lock->owners) && (local == timer_local)) {
+            GF_ASSERT(list_empty(&lock->waiting));
+            /*Last owner*/
+            lock->release = _gf_true;
+            lock->delay_timer = NULL;
+        }
+    }
+    UNLOCK(&local->inode->lock);
+    afr_changelog_post_op_now(local->transaction.frame,
+                              local->transaction.frame->this);
 }
 
-
 /* SET operation */
 int
-afr_fd_report_unstable_write (xlator_t *this, afr_local_t *local)
+afr_fd_report_unstable_write(xlator_t *this, afr_local_t *local)
 {
-        LOCK(&local->inode->lock);
-        {
-                local->inode_ctx->witnessed_unstable_write = _gf_true;
-        }
-        UNLOCK(&local->inode->lock);
+    LOCK(&local->inode->lock);
+    {
+        local->inode_ctx->witnessed_unstable_write = _gf_true;
+    }
+    UNLOCK(&local->inode->lock);
 
-        return 0;
+    return 0;
 }
 
 /* TEST and CLEAR operation */
 gf_boolean_t
-afr_fd_has_witnessed_unstable_write (xlator_t *this, inode_t *inode)
+afr_fd_has_witnessed_unstable_write(xlator_t *this, inode_t *inode)
 {
-        afr_inode_ctx_t *ctx = NULL;
-        gf_boolean_t witness = _gf_false;
+    afr_inode_ctx_t *ctx = NULL;
+    gf_boolean_t witness = _gf_false;
 
-        LOCK(&inode->lock);
-        {
-                (void)__afr_inode_ctx_get (this, inode, &ctx);
+    LOCK(&inode->lock);
+    {
+        (void)__afr_inode_ctx_get(this, inode, &ctx);
 
-                if (ctx->witnessed_unstable_write) {
-                        witness = _gf_true;
-                        ctx->witnessed_unstable_write = _gf_false;
-                }
+        if (ctx->witnessed_unstable_write) {
+            witness = _gf_true;
+            ctx->witnessed_unstable_write = _gf_false;
         }
-        UNLOCK (&inode->lock);
+    }
+    UNLOCK(&inode->lock);
 
-        return witness;
+    return witness;
 }
 
-
 int
-afr_changelog_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, struct iatt *pre,
-                         struct iatt *post, dict_t *xdata)
+afr_changelog_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, struct iatt *pre,
+                        struct iatt *post, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        int child_index = (long) cookie;
-        int call_count = -1;
-        afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+    afr_local_t *local = NULL;
 
-        priv = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        if (op_ret != 0) {
-                /* Failure of fsync() is as good as failure of previous
-                   write(). So treat it like one.
-		*/
-                gf_msg (this->name, GF_LOG_WARNING,
-                        op_errno, AFR_MSG_FSYNC_FAILED,
-                        "fsync(%s) failed on subvolume %s. Transaction was %s",
-                        uuid_utoa (local->fd->inode->gfid),
-                        priv->children[child_index]->name,
-                        gf_fop_list[local->op]);
+    if (op_ret != 0) {
+        /* Failure of fsync() is as good as failure of previous
+           write(). So treat it like one.
+        */
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, AFR_MSG_FSYNC_FAILED,
+               "fsync(%s) failed on subvolume %s. Transaction was %s",
+               uuid_utoa(local->fd->inode->gfid),
+               priv->children[child_index]->name, gf_fop_list[local->op]);
 
-                afr_transaction_fop_failed (frame, this, child_index);
-	}
+        afr_transaction_fop_failed(frame, this, child_index);
+    }
 
-	call_count = afr_frame_return (frame);
+    call_count = afr_frame_return(frame);
 
-	if (call_count == 0)
-		afr_changelog_post_op_now (frame, this);
+    if (call_count == 0)
+        afr_changelog_post_op_now(frame, this);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_changelog_fsync (call_frame_t *frame, xlator_t *this)
+afr_changelog_fsync(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        int i = 0;
-        int call_count = 0;
-        afr_private_t *priv = NULL;
-        dict_t *xdata = NULL;
-        GF_UNUSED int ret = -1;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = 0;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    GF_UNUSED int ret = -1;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        call_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
+    call_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
 
-        if (!call_count) {
-                /* will go straight to unlock */
-                afr_changelog_post_op_now (frame, this);
-                return 0;
-        }
+    if (!call_count) {
+        /* will go straight to unlock */
+        afr_changelog_post_op_now(frame, this);
+        return 0;
+    }
 
-        local->call_count = call_count;
+    local->call_count = call_count;
 
-        xdata = dict_new();
-        if (xdata)
-                ret = dict_set_int32 (xdata, "batch-fsync", 1);
+    xdata = dict_new();
+    if (xdata) {
+        ret = dict_set_int32_sizen(xdata, "batch-fsync", 1);
+        ret = dict_set_str(xdata, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->transaction.pre_op[i])
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.pre_op[i])
+            continue;
 
-                STACK_WIND_COOKIE (frame, afr_changelog_fsync_cbk,
-				   (void *) (long) i, priv->children[i],
-				   priv->children[i]->fops->fsync, local->fd,
-				   1, xdata);
-                if (!--call_count)
-                        break;
-        }
+        STACK_WIND_COOKIE(frame, afr_changelog_fsync_cbk, (void *)(long)i,
+                          priv->children[i], priv->children[i]->fops->fsync,
+                          local->fd, 1, xdata);
+        if (!--call_count)
+            break;
+    }
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_changelog_post_op_safe (call_frame_t *frame, xlator_t *this)
+afr_changelog_post_op_safe(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t    *local = NULL;
-        afr_private_t  *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        if (!local->fd || local->transaction.type != AFR_DATA_TRANSACTION) {
-                afr_changelog_post_op_now (frame, this);
-                return 0;
-        }
-
-        if (afr_changelog_pre_op_uninherit (frame, this) &&
-	    afr_txn_nothing_failed (frame, this)) {
-                /* just detected that this post-op is about to
-                   be optimized away as a new write() has
-                   already piggybacked on this frame's changelog.
-                   */
-                afr_changelog_post_op_now (frame, this);
-                return 0;
-        }
+    if (!local->fd || local->transaction.type != AFR_DATA_TRANSACTION) {
+        afr_changelog_post_op_now(frame, this);
+        return 0;
+    }
 
-        /* Calling afr_changelog_post_op_now() now will result in
-           issuing ->[f]xattrop().
-
-           Performing a hard POST-OP (->[f]xattrop() FOP) is a more
-           responsible operation that what it might appear on the surface.
-
-           The changelog of a file (in the xattr of the file on the server)
-           stores information (pending count) about the state of the file
-           on the OTHER server. This changelog is blindly trusted, and must
-           therefore be updated in such a way it remains trustworthy. This
-           implies that decrementing the pending count (essentially "clearing
-           the dirty flag") must be done STRICTLY after we are sure that the
-           operation on the other server has reached stable storage.
-
-           While the backend filesystem on that server will eventually flush
-           it to stable storage, we (being in userspace) have no mechanism
-           to get notified when the write became "stable".
-
-           This means we need take matter into our own hands and issue an
-           fsync() EVEN IF THE APPLICATION WAS PERFORMING UNSTABLE WRITES,
-           and get an acknowledgement for it. And we need to wait for the
-           fsync() acknowledgement before initiating the hard POST-OP.
-
-           However if the FD itself was opened in O_SYNC or O_DSYNC then
-           we are already guaranteed that the writes were made stable as
-           part of the FOP itself. The same holds true for NFS stable
-           writes which happen on an anonymous FD with O_DSYNC or O_SYNC
-           flag set in the writev() @flags param. For all other write types,
-           mark a flag in the fdctx whenever an unstable write is witnessed.
+    if (afr_changelog_pre_op_uninherit(frame, this) &&
+        afr_txn_nothing_failed(frame, this)) {
+        /* just detected that this post-op is about to
+           be optimized away as a new write() has
+           already piggybacked on this frame's changelog.
            */
+        afr_changelog_post_op_now(frame, this);
+        return 0;
+    }
+
+    /* Calling afr_changelog_post_op_now() now will result in
+       issuing ->[f]xattrop().
+
+       Performing a hard POST-OP (->[f]xattrop() FOP) is a more
+       responsible operation that what it might appear on the surface.
+
+       The changelog of a file (in the xattr of the file on the server)
+       stores information (pending count) about the state of the file
+       on the OTHER server. This changelog is blindly trusted, and must
+       therefore be updated in such a way it remains trustworthy. This
+       implies that decrementing the pending count (essentially "clearing
+       the dirty flag") must be done STRICTLY after we are sure that the
+       operation on the other server has reached stable storage.
+
+       While the backend filesystem on that server will eventually flush
+       it to stable storage, we (being in userspace) have no mechanism
+       to get notified when the write became "stable".
+
+       This means we need take matter into our own hands and issue an
+       fsync() EVEN IF THE APPLICATION WAS PERFORMING UNSTABLE WRITES,
+       and get an acknowledgement for it. And we need to wait for the
+       fsync() acknowledgement before initiating the hard POST-OP.
+
+       However if the FD itself was opened in O_SYNC or O_DSYNC then
+       we are already guaranteed that the writes were made stable as
+       part of the FOP itself. The same holds true for NFS stable
+       writes which happen on an anonymous FD with O_DSYNC or O_SYNC
+       flag set in the writev() @flags param. For all other write types,
+       mark a flag in the fdctx whenever an unstable write is witnessed.
+       */
+
+    if (!afr_fd_has_witnessed_unstable_write(this, local->inode)) {
+        afr_changelog_post_op_now(frame, this);
+        return 0;
+    }
 
-        if (!afr_fd_has_witnessed_unstable_write (this, local->inode)) {
-                afr_changelog_post_op_now (frame, this);
-                return 0;
-        }
-
-        /* Check whether users want durability and perform fsync/post-op
-         * accordingly.
-         */
-        if (priv->ensure_durability) {
-                /* Time to fsync() */
-                afr_changelog_fsync (frame, this);
-        } else {
-                afr_changelog_post_op_now (frame, this);
-        }
+    /* Check whether users want durability and perform fsync/post-op
+     * accordingly.
+     */
+    if (priv->ensure_durability) {
+        /* Time to fsync() */
+        afr_changelog_fsync(frame, this);
+    } else {
+        afr_changelog_post_op_now(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
 void
-afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
-{
-	struct timespec delta   = {0, };
-	afr_private_t   *priv   = NULL;
-	afr_local_t     *local  = frame->local;
-        afr_lock_t      *lock   = NULL;
-        gf_boolean_t    post_op = _gf_true;
-        struct list_head  shared;
-
-	priv = this->private;
-	delta.tv_sec = priv->post_op_delay_secs;
-	delta.tv_nsec = 0;
-
-        INIT_LIST_HEAD (&shared);
-        if (!local->transaction.eager_lock_on)
-                goto out;
-
-        lock = &local->inode_ctx->lock[local->transaction.type];
-        LOCK (&local->inode->lock);
-	{
-                list_del_init (&local->transaction.owner_list);
-                list_add (&local->transaction.owner_list, &lock->post_op);
-                __afr_transaction_wake_shared (local, &shared);
-
-                if (!afr_is_delayed_changelog_post_op_needed (frame, this,
-                                                              delta.tv_sec)) {
-                        if (list_empty (&lock->owners))
-                                lock->release = _gf_true;
-                        goto unlock;
-                }
-
-                GF_ASSERT (lock->delay_timer == NULL);
-		lock->delay_timer = gf_timer_call_after (this->ctx, delta,
-                                              afr_delayed_changelog_wake_up_cbk,
-                                              local);
-                if (!lock->delay_timer) {
-                        lock->release = _gf_true;
-                } else {
-                        post_op = _gf_false;
-                }
+afr_changelog_post_op(call_frame_t *frame, xlator_t *this)
+{
+    struct timespec delta = {
+        0,
+    };
+    afr_private_t *priv = NULL;
+    afr_local_t *local = frame->local;
+    afr_lock_t *lock = NULL;
+    gf_boolean_t post_op = _gf_true;
+    struct list_head shared;
+
+    priv = this->private;
+    delta.tv_sec = priv->post_op_delay_secs;
+    delta.tv_nsec = 0;
+
+    INIT_LIST_HEAD(&shared);
+    if (!local->transaction.eager_lock_on)
+        goto out;
+
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    LOCK(&local->inode->lock);
+    {
+        list_del_init(&local->transaction.owner_list);
+        list_add(&local->transaction.owner_list, &lock->post_op);
+        __afr_transaction_wake_shared(local, &shared);
+
+        if (!afr_is_delayed_changelog_post_op_needed(frame, this,
+                                                     delta.tv_sec)) {
+            if (list_empty(&lock->owners))
+                lock->release = _gf_true;
+            goto unlock;
+        }
 
-	}
+        GF_ASSERT(lock->delay_timer == NULL);
+        lock->delay_timer = gf_timer_call_after(
+            this->ctx, delta, afr_delayed_changelog_wake_up_cbk, local);
+        if (!lock->delay_timer) {
+            lock->release = _gf_true;
+        } else {
+            post_op = _gf_false;
+        }
+    }
 unlock:
-        UNLOCK (&local->inode->lock);
+    UNLOCK(&local->inode->lock);
 
-        if (!list_empty (&shared)) {
-                afr_lock_resume_shared (&shared);
-        }
+    if (!list_empty(&shared)) {
+        afr_lock_resume_shared(&shared);
+    }
 
 out:
-        if (post_op) {
-                if (!local->transaction.eager_lock_on || lock->release) {
-                        afr_changelog_post_op_safe (frame, this);
-                } else {
-                        afr_changelog_post_op_now (frame, this);
-                }
+    if (post_op) {
+        if (!local->transaction.eager_lock_on || lock->release) {
+            afr_changelog_post_op_safe(frame, this);
+        } else {
+            afr_changelog_post_op_now(frame, this);
         }
+    }
 }
 
 int
-afr_transaction_resume (call_frame_t *frame, xlator_t *this)
+afr_transaction_resume(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t         *local    = NULL;
+    afr_local_t *local = NULL;
 
-        local    = frame->local;
+    local = frame->local;
 
-        afr_restore_lk_owner (frame);
+    afr_restore_lk_owner(frame);
 
-        afr_handle_symmetric_errors (frame, this);
+    afr_handle_symmetric_errors(frame, this);
 
-	if (!local->pre_op_compat)
-		/* new mode, pre-op was done along
-		   with OP */
-		afr_changelog_pre_op_update (frame, this);
+    if (!local->pre_op_compat)
+        /* new mode, pre-op was done along
+           with OP */
+        afr_changelog_pre_op_update(frame, this);
 
-        afr_changelog_post_op (frame, this);
+    afr_changelog_post_op(frame, this);
 
-        return 0;
+    return 0;
 }
 
-
 /**
  * afr_transaction_fop_failed - inform that an fop failed
  */
 
 void
-afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
-                            int child_index)
+afr_transaction_fop_failed(call_frame_t *frame, xlator_t *this, int child_index)
 {
-        afr_local_t *   local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	local->transaction.failed_subvols[child_index] = 1;
+    local->transaction.failed_subvols[child_index] = 1;
 }
 
 static gf_boolean_t
-__need_previous_lock_unlocked (afr_local_t *local)
+__need_previous_lock_unlocked(afr_local_t *local)
 {
-        afr_lock_t      *lock = NULL;
-
-        if (!local->transaction.eager_lock_on)
-                return _gf_true;
+    afr_lock_t *lock = NULL;
 
-        lock = &local->inode_ctx->lock[local->transaction.type];
-        if (!lock->acquired)
-                return _gf_false;
-        if (lock->acquired && lock->event_generation != local->event_generation)
-                return _gf_true;
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    if (!lock->acquired)
         return _gf_false;
+    if (lock->acquired && lock->event_generation != local->event_generation)
+        return _gf_true;
+    return _gf_false;
 }
 
 void
-__afr_eager_lock_handle (afr_local_t *local, gf_boolean_t *take_lock,
-                         gf_boolean_t *do_pre_op, afr_local_t **timer_local)
-{
-        afr_lock_t      *lock = NULL;
-        afr_local_t     *owner_local = NULL;
-        xlator_t        *this = local->transaction.frame->this;
-
-        if (local->fd && !afr_are_multiple_fds_opened (local, this)) {
-                local->transaction.eager_lock_on = _gf_true;
-        }
-
-        lock = &local->inode_ctx->lock[local->transaction.type];
-        if (__need_previous_lock_unlocked (local)) {
-                if (!list_empty (&lock->owners)) {
-                        lock->release = _gf_true;
-                } else if (lock->delay_timer) {
-                        lock->release = _gf_true;
-                        if (gf_timer_call_cancel (this->ctx,
-                                                  lock->delay_timer)) {
-                                /* It will be put in frozen list
-                                 * in the code flow below*/
-                        } else {
-                                *timer_local = list_entry(lock->post_op.next,
-                                                          afr_local_t,
-                                                        transaction.owner_list);
-                                lock->delay_timer = NULL;
-                        }
-                }
-                if (!local->transaction.eager_lock_on)
-                        goto out;
-        }
-
-        if (lock->release) {
-                list_add_tail (&local->transaction.wait_list,
-                               &lock->frozen);
-                *take_lock = _gf_false;
-                goto out;
-        }
-
-        if (lock->delay_timer) {
-                *take_lock = _gf_false;
-                if (gf_timer_call_cancel (this->ctx,
-                                          lock->delay_timer)) {
-                        list_add_tail (&local->transaction.wait_list,
-                                       &lock->frozen);
-                } else {
-                        *timer_local = list_entry(lock->post_op.next,
-                                                  afr_local_t,
-                                                  transaction.owner_list);
-                        afr_copy_inodelk_vars (&local->internal_lock,
-                                               &(*timer_local)->internal_lock,
-                                               this);
-                        lock->delay_timer = NULL;
-                        *do_pre_op = _gf_true;
-                        list_add_tail (&local->transaction.owner_list,
-                                       &lock->owners);
-                }
-                goto out;
-        }
-
-        if (!list_empty (&lock->owners)) {
-                if (!lock->acquired ||
-                    afr_has_lock_conflict (local, _gf_true)) {
-                        list_add_tail (&local->transaction.wait_list,
-                                       &lock->waiting);
-                        *take_lock = _gf_false;
-                        goto out;
-                }
-                owner_local = list_entry (lock->owners.next,
-                                          afr_local_t,
+__afr_eager_lock_handle(afr_local_t *local, gf_boolean_t *take_lock,
+                        gf_boolean_t *do_pre_op, afr_local_t **timer_local)
+{
+    afr_lock_t *lock = NULL;
+    afr_local_t *owner_local = NULL;
+    xlator_t *this = local->transaction.frame->this;
+
+    local->transaction.eager_lock_on = _gf_true;
+    afr_set_lk_owner(local->transaction.frame, this, local->inode);
+
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    if (__need_previous_lock_unlocked(local)) {
+        if (!list_empty(&lock->owners)) {
+            lock->release = _gf_true;
+        } else if (lock->delay_timer) {
+            lock->release = _gf_true;
+            if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
+                /* It will be put in frozen list
+                 * in the code flow below*/
+            } else {
+                *timer_local = list_entry(lock->post_op.next, afr_local_t,
                                           transaction.owner_list);
-                afr_copy_inodelk_vars (&local->internal_lock,
-                                       &owner_local->internal_lock,
-                                       this);
-                *take_lock = _gf_false;
-                *do_pre_op = _gf_true;
+                lock->delay_timer = NULL;
+            }
         }
+    }
+
+    if (lock->release) {
+        list_add_tail(&local->transaction.wait_list, &lock->frozen);
+        *take_lock = _gf_false;
+        goto out;
+    }
 
-        if (lock->acquired)
-                GF_ASSERT (!(*take_lock));
-        list_add_tail (&local->transaction.owner_list, &lock->owners);
+    if (lock->delay_timer) {
+        *take_lock = _gf_false;
+        if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
+            list_add_tail(&local->transaction.wait_list, &lock->frozen);
+        } else {
+            *timer_local = list_entry(lock->post_op.next, afr_local_t,
+                                      transaction.owner_list);
+            afr_copy_inodelk_vars(&local->internal_lock,
+                                  &(*timer_local)->internal_lock, this, 0);
+            lock->delay_timer = NULL;
+            *do_pre_op = _gf_true;
+            list_add_tail(&local->transaction.owner_list, &lock->owners);
+        }
+        goto out;
+    }
+
+    if (!list_empty(&lock->owners)) {
+        if (!lock->acquired || afr_has_lock_conflict(local, _gf_true)) {
+            list_add_tail(&local->transaction.wait_list, &lock->waiting);
+            *take_lock = _gf_false;
+            goto out;
+        }
+        owner_local = list_entry(lock->owners.next, afr_local_t,
+                                 transaction.owner_list);
+        afr_copy_inodelk_vars(&local->internal_lock,
+                              &owner_local->internal_lock, this, 0);
+        *take_lock = _gf_false;
+        *do_pre_op = _gf_true;
+    }
+
+    if (lock->acquired)
+        GF_ASSERT(!(*take_lock));
+    list_add_tail(&local->transaction.owner_list, &lock->owners);
 out:
-        return;
+    return;
 }
 
 void
-afr_transaction_start (afr_local_t *local, xlator_t *this)
+afr_transaction_start(afr_local_t *local, xlator_t *this)
 {
-        afr_private_t   *priv = NULL;
-        gf_boolean_t    take_lock  = _gf_true;
-        gf_boolean_t    do_pre_op  = _gf_false;
-        afr_local_t     *timer_local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t take_lock = _gf_true;
+    gf_boolean_t do_pre_op = _gf_false;
+    afr_local_t *timer_local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (local->transaction.type != AFR_DATA_TRANSACTION &&
-            local->transaction.type != AFR_METADATA_TRANSACTION)
-                goto lock_phase;
+    if (local->transaction.type != AFR_DATA_TRANSACTION &&
+        local->transaction.type != AFR_METADATA_TRANSACTION)
+        goto lock_phase;
 
-        if (!priv->eager_lock)
-                goto lock_phase;
+    if (!priv->eager_lock)
+        goto lock_phase;
 
-        LOCK (&local->inode->lock);
-        {
-                __afr_eager_lock_handle (local, &take_lock, &do_pre_op,
-                                         &timer_local);
-        }
-        UNLOCK (&local->inode->lock);
+    LOCK(&local->inode->lock);
+    {
+        __afr_eager_lock_handle(local, &take_lock, &do_pre_op, &timer_local);
+    }
+    UNLOCK(&local->inode->lock);
 lock_phase:
-        if (!local->transaction.eager_lock_on) {
-                afr_set_lk_owner (local->transaction.frame, this,
-                                  local->transaction.frame->root);
-        } else {
-                afr_set_lk_owner (local->transaction.frame, this, local->inode);
-        }
-
-
-        if (take_lock) {
-                afr_lock (local->transaction.frame, this);
-        } else if (do_pre_op) {
-                afr_changelog_pre_op (local->transaction.frame, this);
-        }
-        /*Always call delayed_changelog_wake_up_cbk after calling pre-op above
-         * so that any inheriting can happen*/
-        if (timer_local)
-                afr_delayed_changelog_wake_up_cbk (timer_local);
+    if (!local->transaction.eager_lock_on) {
+        afr_set_lk_owner(local->transaction.frame, this,
+                         local->transaction.frame->root);
+    }
+
+    if (take_lock) {
+        afr_lock(local->transaction.frame, this);
+    } else if (do_pre_op) {
+        afr_changelog_pre_op(local->transaction.frame, this);
+    }
+    /*Always call delayed_changelog_wake_up_cbk after calling pre-op above
+     * so that any inheriting can happen*/
+    if (timer_local)
+        afr_delayed_changelog_wake_up_cbk(timer_local);
 }
 
 int
-afr_write_txn_refresh_done (call_frame_t *frame, xlator_t *this, int err)
+afr_write_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
 {
-        afr_local_t   *local           = frame->local;
+    afr_local_t *local = frame->local;
 
-        if (err) {
-                AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(-1, -err);
-                goto fail;
-        }
+    if (err) {
+        AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(-1, err);
+        goto fail;
+    }
 
-        afr_transaction_start (local, this);
-        return 0;
+    afr_transaction_start(local, this);
+    return 0;
 fail:
-        local->transaction.unwind (frame, this);
-        AFR_STACK_DESTROY (frame);
-        return 0;
+    local->transaction.unwind(frame, this);
+    AFR_STACK_DESTROY(frame);
+    return 0;
 }
 
 int
-afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type)
+afr_transaction_lockee_init(call_frame_t *frame)
 {
-        afr_local_t   *local           = NULL;
-        afr_private_t *priv            = NULL;
-        int           ret              = -1;
-        int           event_generation = 0;
-
-        local = frame->local;
-        priv  = this->private;
-        local->transaction.frame = frame;
-
-        local->transaction.type   = type;
+    afr_local_t *local = frame->local;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    afr_private_t *priv = frame->this->private;
+    int ret = 0;
 
-        if (!afr_is_consistent_io_possible (local, priv, &ret)) {
-                ret = -ret; /*op_errno to ret conversion*/
-                goto out;
-        }
+    switch (local->transaction.type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            ret = afr_add_inode_lockee(local, priv->child_count);
+            break;
 
-        ret = afr_transaction_local_init (local, this);
-        if (ret < 0)
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+            ret = afr_add_entry_lockee(local, &local->transaction.parent_loc,
+                                       local->transaction.basename,
+                                       priv->child_count);
+            if (ret) {
                 goto out;
+            }
+            if (local->op == GF_FOP_RENAME) {
+                ret = afr_add_entry_lockee(
+                    local, &local->transaction.new_parent_loc,
+                    local->transaction.new_basename, priv->child_count);
+                if (ret) {
+                    goto out;
+                }
 
+                if (local->newloc.inode &&
+                    IA_ISDIR(local->newloc.inode->ia_type)) {
+                    ret = afr_add_entry_lockee(local, &local->newloc, NULL,
+                                               priv->child_count);
+                    if (ret) {
+                        goto out;
+                    }
+                }
+            } else if (local->op == GF_FOP_RMDIR) {
+                ret = afr_add_entry_lockee(local, &local->loc, NULL,
+                                           priv->child_count);
+                if (ret) {
+                    goto out;
+                }
+            }
+
+            if (int_lock->lockee_count > 1) {
+                qsort(int_lock->lockee, int_lock->lockee_count,
+                      sizeof(*int_lock->lockee), afr_entry_lockee_cmp);
+            }
+            break;
+    }
+out:
+    return ret;
+}
 
-        if (type != AFR_METADATA_TRANSACTION) {
-                goto txn_start;
-        }
-
-        ret = afr_inode_get_readable (frame, local->inode, this,
-                                      local->readable, &event_generation, type);
-        if (ret < 0 || afr_is_inode_refresh_reqd (local->inode, this,
-                                                  priv->event_generation,
-                                                  event_generation)) {
-                afr_inode_refresh (frame, this, local->inode, local->loc.gfid,
-                                   afr_write_txn_refresh_done);
-                ret = 0;
-                goto out;
-        }
+int
+afr_transaction(call_frame_t *frame, xlator_t *this, afr_transaction_type type)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    int event_generation = 0;
+
+    local = frame->local;
+    priv = this->private;
+    local->transaction.frame = frame;
+
+    local->transaction.type = type;
+
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        ret = -afr_quorum_errno(priv);
+        goto out;
+    }
+
+    if (!afr_is_consistent_io_possible(local, priv, &ret)) {
+        ret = -ret; /*op_errno to ret conversion*/
+        goto out;
+    }
+
+    if (priv->thin_arbiter_count && !afr_ta_has_quorum(priv, local)) {
+        ret = -afr_quorum_errno(priv);
+        goto out;
+    }
+
+    ret = afr_transaction_local_init(local, this);
+    if (ret < 0)
+        goto out;
+
+    ret = afr_transaction_lockee_init(frame);
+    if (ret)
+        goto out;
+
+    if (type != AFR_METADATA_TRANSACTION) {
+        goto txn_start;
+    }
+
+    ret = afr_inode_get_readable(frame, local->inode, this, local->readable,
+                                 &event_generation, type);
+    if (ret < 0 ||
+        afr_is_inode_refresh_reqd(local->inode, this, priv->event_generation,
+                                  event_generation)) {
+        afr_inode_refresh(frame, this, local->inode, local->loc.gfid,
+                          afr_write_txn_refresh_done);
+        ret = 0;
+        goto out;
+    }
 
 txn_start:
-        ret = 0;
-        afr_transaction_start (local, this);
+    ret = 0;
+    afr_transaction_start(local, this);
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-transaction.h b/xlators/cluster/afr/src/afr-transaction.h
index 629f6dd557c..beefa26f4a6 100644
--- a/xlators/cluster/afr/src/afr-transaction.h
+++ b/xlators/cluster/afr/src/afr-transaction.h
@@ -14,53 +14,62 @@
 #include "afr.h"
 
 void
-afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
-			    int child_index);
+afr_transaction_fop_failed(call_frame_t *frame, xlator_t *this,
+                           int child_index);
 
 int32_t
-afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type);
+afr_transaction(call_frame_t *frame, xlator_t *this, afr_transaction_type type);
 
 int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending);
+afr_set_pending_dict(afr_private_t *priv, dict_t *xattr, int32_t **pending);
 
 void
-afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd);
+afr_delayed_changelog_wake_up(xlator_t *this, fd_t *fd);
 
 void
-__mark_all_success (call_frame_t *frame, xlator_t *this);
+__mark_all_success(call_frame_t *frame, xlator_t *this);
 
 gf_boolean_t
-afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this);
+afr_txn_nothing_failed(call_frame_t *frame, xlator_t *this);
 
-int afr_read_txn (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		  afr_read_txn_wind_t readfn, afr_transaction_type type);
+int
+afr_read_txn(call_frame_t *frame, xlator_t *this, inode_t *inode,
+             afr_read_txn_wind_t readfn, afr_transaction_type type);
 
-int afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol);
+int
+afr_read_txn_continue(call_frame_t *frame, xlator_t *this, int subvol);
 
 void
-afr_pending_read_increment (afr_private_t *priv, int child_index);
+afr_pending_read_increment(afr_private_t *priv, int child_index);
 
 void
-afr_pending_read_decrement (afr_private_t *priv, int child_index);
+afr_pending_read_decrement(afr_private_t *priv, int child_index);
 
-call_frame_t *afr_transaction_detach_fop_frame (call_frame_t *frame);
-gf_boolean_t afr_has_quorum (unsigned char *subvols, xlator_t *this);
-gf_boolean_t afr_needs_changelog_update (afr_local_t *local);
-void afr_zero_fill_stat (afr_local_t *local);
+call_frame_t *
+afr_transaction_detach_fop_frame(call_frame_t *frame);
+gf_boolean_t
+afr_has_quorum(unsigned char *subvols, xlator_t *this, call_frame_t *frame);
+gf_boolean_t
+afr_needs_changelog_update(afr_local_t *local);
+void
+afr_zero_fill_stat(afr_local_t *local);
 
 void
-afr_pick_error_xdata (afr_local_t *local, afr_private_t *priv,
-                      inode_t *inode1, unsigned char *readable1,
-                      inode_t *inode2, unsigned char *readable2);
+afr_pick_error_xdata(afr_local_t *local, afr_private_t *priv, inode_t *inode1,
+                     unsigned char *readable1, inode_t *inode2,
+                     unsigned char *readable2);
 int
-afr_transaction_resume (call_frame_t *frame, xlator_t *this);
+afr_transaction_resume(call_frame_t *frame, xlator_t *this);
 
 int
-afr_lock (call_frame_t *frame, xlator_t *this);
+afr_lock(call_frame_t *frame, xlator_t *this);
 
 void
-afr_delayed_changelog_wake_up_cbk (void *data);
+afr_delayed_changelog_wake_up_cbk(void *data);
+
+int
+afr_release_notify_lock_for_ta(void *opaque);
 
 int
-afr_fill_ta_loc (xlator_t *this, loc_t *loc);
+afr_ta_lock_release_done(int ret, call_frame_t *ta_frame, void *opaque);
 #endif /* __TRANSACTION_H__ */
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index 0d6773c96e8..df7366f0a65 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -21,1194 +21,1324 @@
 struct volume_options options[];
 
 static char *afr_favorite_child_policies[AFR_FAV_CHILD_POLICY_MAX + 1] = {
-        [AFR_FAV_CHILD_NONE] = "none",
-        [AFR_FAV_CHILD_BY_SIZE] = "size",
-        [AFR_FAV_CHILD_BY_CTIME] = "ctime",
-        [AFR_FAV_CHILD_BY_MTIME] = "mtime",
-        [AFR_FAV_CHILD_BY_MAJORITY] = "majority",
-        [AFR_FAV_CHILD_POLICY_MAX] = NULL,
+    [AFR_FAV_CHILD_NONE] = "none",
+    [AFR_FAV_CHILD_BY_SIZE] = "size",
+    [AFR_FAV_CHILD_BY_CTIME] = "ctime",
+    [AFR_FAV_CHILD_BY_MTIME] = "mtime",
+    [AFR_FAV_CHILD_BY_MAJORITY] = "majority",
+    [AFR_FAV_CHILD_POLICY_MAX] = NULL,
 };
 
 int32_t
-notify (xlator_t *this, int32_t event,
-        void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int ret = -1;
-        va_list         ap;
-        void *data2 = NULL;
+    int ret = -1;
+    va_list ap;
+    void *data2 = NULL;
 
-        va_start (ap, data);
-        data2 = va_arg (ap, dict_t*);
-        va_end (ap);
-        ret = afr_notify (this, event, data, data2);
+    va_start(ap, data);
+    data2 = va_arg(ap, dict_t *);
+    va_end(ap);
+    ret = afr_notify(this, event, data, data2);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_afr_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_afr_mt_end + 1);
 
+    if (ret != 0) {
         return ret;
-}
+    }
 
+    return ret;
+}
 
 int
-xlator_subvolume_index (xlator_t *this, xlator_t *subvol)
+xlator_subvolume_index(xlator_t *this, xlator_t *subvol)
 {
-        int index = -1;
-        int i = 0;
-        xlator_list_t *list = NULL;
-
-        list = this->children;
-
-        while (list) {
-                if (subvol == list->xlator ||
-                    strcmp (subvol->name, list->xlator->name) == 0) {
-                        index = i;
-                        break;
-                }
-                list = list->next;
-                i++;
+    int index = -1;
+    int i = 0;
+    xlator_list_t *list = NULL;
+
+    list = this->children;
+
+    while (list) {
+        if (subvol == list->xlator ||
+            strcmp(subvol->name, list->xlator->name) == 0) {
+            index = i;
+            break;
         }
+        list = list->next;
+        i++;
+    }
 
-        return index;
+    return index;
 }
 
 static void
-fix_quorum_options (xlator_t *this, afr_private_t *priv, char *qtype,
-                    dict_t *options)
+fix_quorum_options(xlator_t *this, afr_private_t *priv, char *qtype,
+                   dict_t *options)
 {
-
-        if (dict_get (options, "quorum-type") == NULL) {
-                /* If user doesn't configure anything enable auto-quorum if the
-                 * replica has more than two subvolumes */
-                if (priv->child_count > 2)
-                        qtype = "auto";
-        }
-
-        if (priv->quorum_count && strcmp (qtype, "fixed")) {
-                gf_msg (this->name,GF_LOG_WARNING, 0, AFR_MSG_QUORUM_OVERRIDE,
-                       "quorum-type %s overriding quorum-count %u",
-                       qtype, priv->quorum_count);
-        }
-
-        if (!strcmp (qtype, "none")) {
-                priv->quorum_count = 0;
-        } else if (!strcmp (qtype, "auto")) {
-                priv->quorum_count = AFR_QUORUM_AUTO;
-        }
-
+    if (dict_get_sizen(options, "quorum-type") == NULL) {
+        /* If user doesn't configure anything enable auto-quorum if the
+         * replica has more than two subvolumes */
+        if (priv->child_count > 2)
+            qtype = "auto";
+    }
+
+    if (priv->quorum_count && strcmp(qtype, "fixed")) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_OVERRIDE,
+               "quorum-type %s overriding quorum-count %u", qtype,
+               priv->quorum_count);
+    }
+
+    if (!strcmp(qtype, "none")) {
+        priv->quorum_count = 0;
+    } else if (!strcmp(qtype, "auto")) {
+        priv->quorum_count = AFR_QUORUM_AUTO;
+    }
 }
 
 int
-afr_set_favorite_child_policy (afr_private_t *priv, char *policy)
+afr_set_favorite_child_policy(afr_private_t *priv, char *policy)
 {
-        int index = -1;
+    int index = -1;
 
-        index = gf_get_index_by_elem (afr_favorite_child_policies, policy);
-        if (index  < 0 || index >= AFR_FAV_CHILD_POLICY_MAX)
-                return -1;
+    index = gf_get_index_by_elem(afr_favorite_child_policies, policy);
+    if (index < 0 || index >= AFR_FAV_CHILD_POLICY_MAX)
+        return -1;
 
-        priv->fav_child_policy = index;
+    priv->fav_child_policy = index;
 
-        return 0;
+    return 0;
 }
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        afr_private_t *priv        = NULL;
-        xlator_t      *read_subvol = NULL;
-        int            read_subvol_index = -1;
-        int            ret         = -1;
-        int            index       = -1;
-        char          *qtype       = NULL;
-        char          *fav_child_policy = NULL;
-        gf_boolean_t   consistent_io = _gf_false;
-        gf_boolean_t   choose_local_old = _gf_false;
 
-        priv = this->private;
+static void
+set_data_self_heal_algorithm(afr_private_t *priv, char *algo)
+{
+    if (!algo) {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_DYNAMIC;
+    } else if (strcmp(algo, "full") == 0) {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_FULL;
+    } else if (strcmp(algo, "diff") == 0) {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_DIFF;
+    } else {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_DYNAMIC;
+    }
+}
 
-	GF_OPTION_RECONF ("afr-dirty-xattr",
-			  priv->afr_dirty, options, str,
-			  out);
+void
+afr_handle_anon_inode_options(afr_private_t *priv, dict_t *options)
+{
+    char *volfile_id_str = NULL;
+    uuid_t anon_inode_gfid = {0};
+
+    /*If volume id is not present don't enable anything*/
+    if (dict_get_str(options, "volume-id", &volfile_id_str))
+        return;
+    GF_ASSERT(strlen(AFR_ANON_DIR_PREFIX) + strlen(volfile_id_str) <= NAME_MAX);
+    /*anon_inode_name is not supposed to change once assigned*/
+    if (!priv->anon_inode_name[0]) {
+        snprintf(priv->anon_inode_name, sizeof(priv->anon_inode_name), "%s-%s",
+                 AFR_ANON_DIR_PREFIX, volfile_id_str);
+        gf_uuid_parse(volfile_id_str, anon_inode_gfid);
+        /*Flip a bit to make sure volfile-id and anon-gfid are not same*/
+        anon_inode_gfid[0] ^= 1;
+        uuid_utoa_r(anon_inode_gfid, priv->anon_gfid_str);
+    }
+}
 
-	GF_OPTION_RECONF ("metadata-splitbrain-forced-heal",
-			  priv->metadata_splitbrain_forced_heal, options, bool,
-			  out);
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    afr_private_t *priv = NULL;
+    xlator_t *read_subvol = NULL;
+    int read_subvol_index = -1;
+    int timeout_old = 0;
+    int ret = -1;
+    int index = -1;
+    char *qtype = NULL;
+    char *fav_child_policy = NULL;
+    char *data_self_heal = NULL;
+    char *data_self_heal_algorithm = NULL;
+    char *locking_scheme = NULL;
+    gf_boolean_t consistent_io = _gf_false;
+    gf_boolean_t choose_local_old = _gf_false;
+    gf_boolean_t enabled_old = _gf_false;
 
-        GF_OPTION_RECONF ("background-self-heal-count",
-                          priv->background_self_heal_count, options, uint32,
-                          out);
+    priv = this->private;
 
-        GF_OPTION_RECONF ("heal-wait-queue-length",
-                          priv->heal_wait_qlen, options, uint32, out);
+    GF_OPTION_RECONF("metadata-splitbrain-forced-heal",
+                     priv->metadata_splitbrain_forced_heal, options, bool, out);
 
+    GF_OPTION_RECONF("background-self-heal-count",
+                     priv->background_self_heal_count, options, uint32, out);
 
-        GF_OPTION_RECONF ("metadata-self-heal",
-                          priv->metadata_self_heal, options, bool, out);
+    GF_OPTION_RECONF("heal-wait-queue-length", priv->heal_wait_qlen, options,
+                     uint32, out);
 
-        GF_OPTION_RECONF ("data-self-heal", priv->data_self_heal, options, str,
-                          out);
+    GF_OPTION_RECONF("metadata-self-heal", priv->metadata_self_heal, options,
+                     bool, out);
 
-        GF_OPTION_RECONF ("entry-self-heal", priv->entry_self_heal, options,
-                          bool, out);
+    GF_OPTION_RECONF("data-self-heal", data_self_heal, options, str, out);
+    if (gf_string2boolean(data_self_heal, &priv->data_self_heal) == -1)
+        goto out;
 
-        GF_OPTION_RECONF ("data-self-heal-window-size",
-                          priv->data_self_heal_window_size, options,
-                          uint32, out);
+    GF_OPTION_RECONF("entry-self-heal", priv->entry_self_heal, options, bool,
+                     out);
 
-        GF_OPTION_RECONF ("data-self-heal-algorithm",
-                          priv->data_self_heal_algorithm, options, str, out);
+    GF_OPTION_RECONF("data-self-heal-window-size",
+                     priv->data_self_heal_window_size, options, uint32, out);
 
-        GF_OPTION_RECONF ("halo-enabled",
-                          priv->halo_enabled, options, bool,
-                          out);
+    GF_OPTION_RECONF("data-self-heal-algorithm", data_self_heal_algorithm,
+                     options, str, out);
+    set_data_self_heal_algorithm(priv, data_self_heal_algorithm);
 
-        GF_OPTION_RECONF ("halo-shd-max-latency",
-                          priv->shd.halo_max_latency_msec, options, uint32,
-                          out);
+    GF_OPTION_RECONF("halo-enabled", priv->halo_enabled, options, bool, out);
 
-        GF_OPTION_RECONF ("halo-nfsd-max-latency",
-                          priv->nfsd.halo_max_latency_msec, options, uint32,
-                          out);
+    GF_OPTION_RECONF("halo-shd-max-latency", priv->shd.halo_max_latency_msec,
+                     options, uint32, out);
 
-        GF_OPTION_RECONF ("halo-max-latency", priv->halo_max_latency_msec,
-                          options, uint32, out);
+    GF_OPTION_RECONF("halo-nfsd-max-latency", priv->nfsd.halo_max_latency_msec,
+                     options, uint32, out);
 
-        GF_OPTION_RECONF ("halo-max-replicas", priv->halo_max_replicas, options,
-                              uint32, out);
+    GF_OPTION_RECONF("halo-max-latency", priv->halo_max_latency_msec, options,
+                     uint32, out);
 
-        GF_OPTION_RECONF ("halo-min-replicas", priv->halo_min_replicas, options,
-                              uint32, out);
+    GF_OPTION_RECONF("halo-max-replicas", priv->halo_max_replicas, options,
+                     uint32, out);
 
-        GF_OPTION_RECONF ("read-subvolume", read_subvol, options, xlator, out);
+    GF_OPTION_RECONF("halo-min-replicas", priv->halo_min_replicas, options,
+                     uint32, out);
 
-        choose_local_old = priv->choose_local;
-        GF_OPTION_RECONF ("choose-local", priv->choose_local, options, bool,
-                          out);
+    GF_OPTION_RECONF("read-subvolume", read_subvol, options, xlator, out);
 
-        if (choose_local_old != priv->choose_local) {
-                priv->read_child = -1;
-                if (choose_local_old == _gf_false)
-                        priv->did_discovery = _gf_false;
-        }
+    choose_local_old = priv->choose_local;
+    GF_OPTION_RECONF("choose-local", priv->choose_local, options, bool, out);
 
-        GF_OPTION_RECONF ("read-hash-mode", priv->hash_mode,
-                          options, uint32, out);
-
-        if (read_subvol) {
-                index = xlator_subvolume_index (this, read_subvol);
-                if (index == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_INVALID_SUBVOL, "%s not a subvolume",
-                                read_subvol->name);
-                        goto out;
-                }
-                priv->read_child = index;
+    if (choose_local_old != priv->choose_local) {
+        priv->read_child = -1;
+        if (choose_local_old == _gf_false)
+            priv->did_discovery = _gf_false;
+    }
+
+    GF_OPTION_RECONF("read-hash-mode", priv->hash_mode, options, uint32, out);
+
+    if (read_subvol) {
+        index = xlator_subvolume_index(this, read_subvol);
+        if (index == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%s not a subvolume", read_subvol->name);
+            goto out;
         }
-
-        GF_OPTION_RECONF ("read-subvolume-index",read_subvol_index, options,int32,out);
-
-        if (read_subvol_index >-1) {
-                index=read_subvol_index;
-                if (index >= priv->child_count) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_INVALID_SUBVOL,
-                                "%d not a subvolume-index", index);
-                        goto out;
-                }
-                priv->read_child = index;
+        priv->read_child = index;
+    }
+
+    GF_OPTION_RECONF("read-subvolume-index", read_subvol_index, options, int32,
+                     out);
+
+    if (read_subvol_index > -1) {
+        index = read_subvol_index;
+        if (index >= priv->child_count) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%d not a subvolume-index", index);
+            goto out;
         }
-
-        GF_OPTION_RECONF ("pre-op-compat", priv->pre_op_compat, options, bool,
-                          out);
-        GF_OPTION_RECONF ("locking-scheme", priv->locking_scheme, options, str,
-                          out);
-        GF_OPTION_RECONF ("full-lock", priv->full_lock, options, bool, out);
-        GF_OPTION_RECONF ("granular-entry-heal", priv->esh_granular, options,
-                          bool, out);
-
-        GF_OPTION_RECONF ("eager-lock", priv->eager_lock, options, bool, out);
-        GF_OPTION_RECONF ("quorum-type", qtype, options, str, out);
-        GF_OPTION_RECONF ("quorum-count", priv->quorum_count, options,
-                          uint32, out);
-        fix_quorum_options (this, priv, qtype, options);
-        if (priv->quorum_count && !afr_has_quorum (priv->child_up, this))
-                gf_msg (this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
-                        "Client-quorum is not met");
-
-
-	GF_OPTION_RECONF ("post-op-delay-secs", priv->post_op_delay_secs, options,
-			  uint32, out);
-
-        GF_OPTION_RECONF (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size,
-                          options, size_uint64, out);
-        /* Reset this so we re-discover in case the topology changed.  */
-        GF_OPTION_RECONF ("ensure-durability", priv->ensure_durability, options,
-                          bool, out);
-
-	GF_OPTION_RECONF ("self-heal-daemon", priv->shd.enabled, options,
-			  bool, out);
-
-	GF_OPTION_RECONF ("iam-self-heal-daemon", priv->shd.iamshd, options,
-			  bool, out);
-
-        GF_OPTION_RECONF ("heal-timeout", priv->shd.timeout, options,
-                          int32, out);
-
-        GF_OPTION_RECONF ("consistent-metadata", priv->consistent_metadata,
-                          options, bool, out);
-
-        GF_OPTION_RECONF ("shd-max-threads", priv->shd.max_threads,
-                          options, uint32, out);
-
-        GF_OPTION_RECONF ("shd-wait-qlength", priv->shd.wait_qlength,
-                          options, uint32, out);
-
-        GF_OPTION_RECONF ("favorite-child-policy", fav_child_policy, options,
-                          str, out);
-        if (afr_set_favorite_child_policy (priv, fav_child_policy) == -1)
-                goto out;
-
-        priv->did_discovery = _gf_false;
-
-        GF_OPTION_RECONF ("consistent-io", consistent_io, options, bool, out);
-        if (priv->quorum_count != 0)
-                consistent_io = _gf_false;
-        priv->consistent_io = consistent_io;
-
-        ret = 0;
+        priv->read_child = index;
+    }
+
+    GF_OPTION_RECONF("pre-op-compat", priv->pre_op_compat, options, bool, out);
+    GF_OPTION_RECONF("locking-scheme", locking_scheme, options, str, out);
+    priv->granular_locks = (strcmp(locking_scheme, "granular") == 0);
+    GF_OPTION_RECONF("full-lock", priv->full_lock, options, bool, out);
+    GF_OPTION_RECONF("granular-entry-heal", priv->esh_granular, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("eager-lock", priv->eager_lock, options, bool, out);
+    GF_OPTION_RECONF("optimistic-change-log", priv->optimistic_change_log,
+                     options, bool, out);
+    GF_OPTION_RECONF("quorum-type", qtype, options, str, out);
+    GF_OPTION_RECONF("quorum-count", priv->quorum_count, options, uint32, out);
+    fix_quorum_options(this, priv, qtype, options);
+    if (priv->quorum_count && !afr_has_quorum(priv->child_up, this, NULL))
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
+               "Client-quorum is not met");
+
+    GF_OPTION_RECONF("post-op-delay-secs", priv->post_op_delay_secs, options,
+                     uint32, out);
+
+    GF_OPTION_RECONF(AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, options,
+                     size_uint64, out);
+    /* Reset this so we re-discover in case the topology changed.  */
+    GF_OPTION_RECONF("ensure-durability", priv->ensure_durability, options,
+                     bool, out);
+
+    enabled_old = priv->shd.enabled;
+    GF_OPTION_RECONF("self-heal-daemon", priv->shd.enabled, options, bool, out);
+
+    GF_OPTION_RECONF("iam-self-heal-daemon", priv->shd.iamshd, options, bool,
+                     out);
+
+    timeout_old = priv->shd.timeout;
+    GF_OPTION_RECONF("heal-timeout", priv->shd.timeout, options, int32, out);
+
+    GF_OPTION_RECONF("consistent-metadata", priv->consistent_metadata, options,
+                     bool, out);
+
+    GF_OPTION_RECONF("shd-max-threads", priv->shd.max_threads, options, uint32,
+                     out);
+
+    GF_OPTION_RECONF("shd-wait-qlength", priv->shd.wait_qlength, options,
+                     uint32, out);
+
+    GF_OPTION_RECONF("favorite-child-policy", fav_child_policy, options, str,
+                     out);
+    if (afr_set_favorite_child_policy(priv, fav_child_policy) == -1)
+        goto out;
+
+    priv->did_discovery = _gf_false;
+
+    GF_OPTION_RECONF("consistent-io", consistent_io, options, bool, out);
+    if (priv->quorum_count != 0)
+        consistent_io = _gf_false;
+    priv->consistent_io = consistent_io;
+
+    afr_handle_anon_inode_options(priv, options);
+
+    GF_OPTION_RECONF("use-anonymous-inode", priv->use_anon_inode, options, bool,
+                     out);
+    if (priv->shd.enabled) {
+        if ((priv->shd.enabled != enabled_old) ||
+            (timeout_old != priv->shd.timeout))
+            afr_selfheal_childup(this, priv);
+    }
+
+    ret = 0;
 out:
-        return ret;
-
+    return ret;
 }
 
 static int
-afr_pending_xattrs_init (afr_private_t *priv, xlator_t *this)
+afr_pending_xattrs_init(afr_private_t *priv, xlator_t *this)
 {
-        int ret = -1;
-        int i = 0;
-        char *ptr = NULL;
-        char *ptr1 = NULL;
-        char *xattrs_list = NULL;
-        xlator_list_t *trav = NULL;
-        int child_count = -1;
-
-        trav = this->children;
-        child_count = priv->child_count;
-        if (priv->thin_arbiter_count) {
-                /* priv->pending_key[THIN_ARBITER_BRICK_INDEX] is used as the
-                 * name of the thin arbiter file for persistance across add/
-                 * removal of DHT subvols.*/
-                child_count++;
-        }
-
-        GF_OPTION_INIT ("afr-pending-xattr", xattrs_list, str, out);
-        priv->pending_key = GF_CALLOC (sizeof (*priv->pending_key),
-                                       child_count, gf_afr_mt_char);
-        if (!priv->pending_key) {
-                ret = -ENOMEM;
-                goto out;
-        }
-        if (!xattrs_list) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, AFR_MSG_NO_CHANGELOG,
-                        "Unable to fetch afr-pending-xattr option from volfile."
-                        " Falling back to using client translator names. ");
-
-                while (i < child_count) {
-                        ret = gf_asprintf (&priv->pending_key[i], "%s.%s",
-                                           AFR_XATTR_PREFIX,
-                                           trav->xlator->name);
-                        if (ret == -1) {
-                                ret = -ENOMEM;
-                                goto out;
-                        }
-                        trav = trav->next;
-                        i++;
-                }
-                ret = 0;
-                goto out;
-        }
+    int ret = -1;
+    int i = 0;
+    char *ptr = NULL;
+    char *ptr1 = NULL;
+    char *xattrs_list = NULL;
+    xlator_list_t *trav = NULL;
+    int child_count = -1;
+
+    trav = this->children;
+    child_count = priv->child_count;
+    if (priv->thin_arbiter_count) {
+        /* priv->pending_key[THIN_ARBITER_BRICK_INDEX] is used as the
+         * name of the thin arbiter file for persistence across add/
+         * removal of DHT subvols.*/
+        child_count++;
+    }
+
+    GF_OPTION_INIT("afr-pending-xattr", xattrs_list, str, out);
+    priv->pending_key = GF_CALLOC(sizeof(*priv->pending_key), child_count,
+                                  gf_afr_mt_char);
+    if (!priv->pending_key) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    if (!xattrs_list) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_NO_CHANGELOG,
+               "Unable to fetch afr-pending-xattr option from volfile."
+               " Falling back to using client translator names. ");
 
-        ptr = ptr1 = gf_strdup (xattrs_list);
-        if (!ptr) {
+        while (i < child_count) {
+            ret = gf_asprintf(&priv->pending_key[i], "%s.%s", AFR_XATTR_PREFIX,
+                              trav->xlator->name);
+            if (ret == -1) {
                 ret = -ENOMEM;
                 goto out;
-        }
-        for (i = 0, ptr = strtok (ptr, ","); ptr; ptr = strtok (NULL, ",")) {
-                ret = gf_asprintf (&priv->pending_key[i], "%s.%s",
-                                   AFR_XATTR_PREFIX, ptr);
-                if (ret == -1) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
-                i++;
+            }
+            trav = trav->next;
+            i++;
         }
         ret = 0;
+        goto out;
+    }
+
+    ptr = ptr1 = gf_strdup(xattrs_list);
+    if (!ptr) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    for (i = 0, ptr = strtok(ptr, ","); ptr; ptr = strtok(NULL, ",")) {
+        ret = gf_asprintf(&priv->pending_key[i], "%s.%s", AFR_XATTR_PREFIX,
+                          ptr);
+        if (ret == -1) {
+            ret = -ENOMEM;
+            goto out;
+        }
+        i++;
+    }
+    ret = 0;
 
 out:
-        GF_FREE (ptr1);
-        return ret;
+    GF_FREE(ptr1);
+    return ret;
+}
 
+void
+afr_ta_init(afr_private_t *priv)
+{
+    priv->thin_arbiter_count = 1;
+    priv->child_count--;
+    priv->ta_child_up = 0;
+    priv->ta_bad_child_index = AFR_CHILD_UNKNOWN;
+    priv->ta_notify_dom_lock_offset = 0;
+    priv->ta_in_mem_txn_count = 0;
+    priv->ta_on_wire_txn_count = 0;
+    priv->release_ta_notify_dom_lock = _gf_false;
+    INIT_LIST_HEAD(&priv->ta_waitq);
+    INIT_LIST_HEAD(&priv->ta_onwireq);
+    gf_uuid_clear(priv->ta_gfid);
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        afr_private_t *priv        = NULL;
-        int            child_count = 0;
-        xlator_list_t *trav        = NULL;
-        int            i           = 0;
-        int            ret         = -1;
-        GF_UNUSED int  op_errno    = 0;
-        xlator_t      *read_subvol = NULL;
-        int            read_subvol_index = -1;
-        char          *qtype       = NULL;
-        char          *fav_child_policy = NULL;
-        char          *thin_arbiter = NULL;
-
-        if (!this->children) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        AFR_MSG_CHILD_MISCONFIGURED,
-                        "replicate translator needs more than one "
-                        "subvolume defined.");
-                return -1;
+    afr_private_t *priv = NULL;
+    int child_count = 0;
+    xlator_list_t *trav = NULL;
+    int i = 0;
+    int ret = -1;
+    GF_UNUSED int op_errno = 0;
+    xlator_t *read_subvol = NULL;
+    int read_subvol_index = -1;
+    char *qtype = NULL;
+    char *fav_child_policy = NULL;
+    char *thin_arbiter = NULL;
+    char *data_self_heal = NULL;
+    char *locking_scheme = NULL;
+    char *data_self_heal_algorithm = NULL;
+
+    if (!this->children) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_CHILD_MISCONFIGURED,
+               "replicate translator needs more than one "
+               "subvolume defined.");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_VOL_MISCONFIGURED,
+               "Volume is dangling.");
+    }
+
+    this->private = GF_CALLOC(1, sizeof(afr_private_t),
+                              gf_afr_mt_afr_private_t);
+    if (!this->private)
+        goto out;
+
+    priv = this->private;
+    INIT_LIST_HEAD(&priv->saved_locks);
+    INIT_LIST_HEAD(&priv->lk_healq);
+    LOCK_INIT(&priv->lock);
+
+    child_count = xlator_subvolume_count(this);
+
+    priv->child_count = child_count;
+
+    priv->read_child = -1;
+
+    GF_OPTION_INIT("arbiter-count", priv->arbiter_count, uint32, out);
+    GF_OPTION_INIT("thin-arbiter", thin_arbiter, str, out);
+    if (thin_arbiter && strlen(thin_arbiter) > 0) {
+        afr_ta_init(priv);
+    }
+    INIT_LIST_HEAD(&priv->healing);
+    INIT_LIST_HEAD(&priv->heal_waiting);
+
+    priv->spb_choice_timeout = AFR_DEFAULT_SPB_CHOICE_TIMEOUT;
+
+    GF_OPTION_INIT("afr-dirty-xattr", priv->afr_dirty, str, out);
+
+    GF_OPTION_INIT("metadata-splitbrain-forced-heal",
+                   priv->metadata_splitbrain_forced_heal, bool, out);
+
+    GF_OPTION_INIT("read-subvolume", read_subvol, xlator, out);
+    if (read_subvol) {
+        priv->read_child = xlator_subvolume_index(this, read_subvol);
+        if (priv->read_child == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%s not a subvolume", read_subvol->name);
+            goto out;
         }
-
-        if (!this->parents) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        AFR_MSG_VOL_MISCONFIGURED, "Volume is dangling.");
+    }
+    GF_OPTION_INIT("read-subvolume-index", read_subvol_index, int32, out);
+    if (read_subvol_index > -1) {
+        if (read_subvol_index >= priv->child_count) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%d not a subvolume-index", read_subvol_index);
+            goto out;
         }
+        priv->read_child = read_subvol_index;
+    }
+    GF_OPTION_INIT("choose-local", priv->choose_local, bool, out);
 
-	this->private = GF_CALLOC (1, sizeof (afr_private_t),
-                                   gf_afr_mt_afr_private_t);
-        if (!this->private)
-                goto out;
-
-        priv = this->private;
-        LOCK_INIT (&priv->lock);
-
-        child_count = xlator_subvolume_count (this);
-
-        priv->child_count = child_count;
-
-        priv->read_child = -1;
-
-        GF_OPTION_INIT ("arbiter-count", priv->arbiter_count, uint32, out);
-        GF_OPTION_INIT ("thin-arbiter", thin_arbiter, str, out);
-        if (thin_arbiter && strlen(thin_arbiter) > 0) {
-                priv->thin_arbiter_count = 1;
-                priv->child_count--;
-        }
-        INIT_LIST_HEAD (&priv->healing);
-        INIT_LIST_HEAD (&priv->heal_waiting);
-
-        priv->spb_choice_timeout = AFR_DEFAULT_SPB_CHOICE_TIMEOUT;
-
-	GF_OPTION_INIT ("afr-dirty-xattr", priv->afr_dirty, str, out);
-
-	GF_OPTION_INIT ("metadata-splitbrain-forced-heal",
-			priv->metadata_splitbrain_forced_heal, bool, out);
-
-        GF_OPTION_INIT ("read-subvolume", read_subvol, xlator, out);
-        if (read_subvol) {
-                priv->read_child = xlator_subvolume_index (this, read_subvol);
-                if (priv->read_child == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_INVALID_SUBVOL, "%s not a subvolume",
-                                read_subvol->name);
-                        goto out;
-                }
-        }
-        GF_OPTION_INIT ("read-subvolume-index",read_subvol_index,int32,out);
-        if (read_subvol_index > -1) {
-                if (read_subvol_index >= priv->child_count) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_INVALID_SUBVOL,
-                                "%d not a subvolume-index", read_subvol_index);
-                        goto out;
-                }
-                priv->read_child = read_subvol_index;
-        }
-        GF_OPTION_INIT ("choose-local", priv->choose_local, bool, out);
+    priv->pending_reads = GF_CALLOC(sizeof(*priv->pending_reads),
+                                    priv->child_count, gf_afr_mt_atomic_t);
 
-        priv->pending_reads = GF_CALLOC (sizeof(*priv->pending_reads),
-                                         priv->child_count, gf_afr_mt_atomic_t);
+    GF_OPTION_INIT("read-hash-mode", priv->hash_mode, uint32, out);
 
-        GF_OPTION_INIT ("read-hash-mode", priv->hash_mode, uint32, out);
+    priv->favorite_child = -1;
 
-        priv->favorite_child = -1;
+    GF_OPTION_INIT("favorite-child-policy", fav_child_policy, str, out);
+    if (afr_set_favorite_child_policy(priv, fav_child_policy) == -1)
+        goto out;
 
-        GF_OPTION_INIT ("favorite-child-policy", fav_child_policy, str, out);
-        if (afr_set_favorite_child_policy(priv, fav_child_policy) == -1)
-                goto out;
+    GF_OPTION_INIT("shd-max-threads", priv->shd.max_threads, uint32, out);
 
-        GF_OPTION_INIT ("shd-max-threads", priv->shd.max_threads,
-                         uint32, out);
+    GF_OPTION_INIT("shd-wait-qlength", priv->shd.wait_qlength, uint32, out);
 
-        GF_OPTION_INIT ("shd-wait-qlength", priv->shd.wait_qlength,
-                         uint32, out);
+    GF_OPTION_INIT("background-self-heal-count",
+                   priv->background_self_heal_count, uint32, out);
 
-        GF_OPTION_INIT ("background-self-heal-count",
-                        priv->background_self_heal_count, uint32, out);
+    GF_OPTION_INIT("heal-wait-queue-length", priv->heal_wait_qlen, uint32, out);
 
-        GF_OPTION_INIT ("heal-wait-queue-length",
-                        priv->heal_wait_qlen, uint32, out);
+    GF_OPTION_INIT("data-self-heal", data_self_heal, str, out);
+    if (gf_string2boolean(data_self_heal, &priv->data_self_heal) == -1)
+        goto out;
 
-        GF_OPTION_INIT ("data-self-heal", priv->data_self_heal, str, out);
+    GF_OPTION_INIT("data-self-heal-algorithm", data_self_heal_algorithm, str,
+                   out);
+    set_data_self_heal_algorithm(priv, data_self_heal_algorithm);
 
-        GF_OPTION_INIT ("data-self-heal-algorithm",
-                        priv->data_self_heal_algorithm, str, out);
+    GF_OPTION_INIT("data-self-heal-window-size",
+                   priv->data_self_heal_window_size, uint32, out);
 
-        GF_OPTION_INIT ("data-self-heal-window-size",
-                        priv->data_self_heal_window_size, uint32, out);
+    GF_OPTION_INIT("metadata-self-heal", priv->metadata_self_heal, bool, out);
 
-        GF_OPTION_INIT ("metadata-self-heal", priv->metadata_self_heal, bool,
-                        out);
+    GF_OPTION_INIT("entry-self-heal", priv->entry_self_heal, bool, out);
 
-        GF_OPTION_INIT ("entry-self-heal", priv->entry_self_heal, bool, out);
+    GF_OPTION_INIT("halo-shd-max-latency", priv->shd.halo_max_latency_msec,
+                   uint32, out);
 
-        GF_OPTION_INIT ("halo-shd-max-latency", priv->shd.halo_max_latency_msec,
-                        uint32, out);
+    GF_OPTION_INIT("halo-max-latency", priv->halo_max_latency_msec, uint32,
+                   out);
+    GF_OPTION_INIT("halo-max-replicas", priv->halo_max_replicas, uint32, out);
+    GF_OPTION_INIT("halo-min-replicas", priv->halo_min_replicas, uint32, out);
 
-        GF_OPTION_INIT ("halo-max-latency", priv->halo_max_latency_msec,
-                        uint32, out);
-        GF_OPTION_INIT ("halo-max-replicas", priv->halo_max_replicas, uint32,
-                        out);
-        GF_OPTION_INIT ("halo-min-replicas", priv->halo_min_replicas, uint32,
-                        out);
+    GF_OPTION_INIT("halo-enabled", priv->halo_enabled, bool, out);
 
-        GF_OPTION_INIT ("halo-enabled",
-                        priv->halo_enabled, bool, out);
+    GF_OPTION_INIT("halo-nfsd-max-latency", priv->nfsd.halo_max_latency_msec,
+                   uint32, out);
 
-        GF_OPTION_INIT ("halo-nfsd-max-latency",
-                        priv->nfsd.halo_max_latency_msec, uint32, out);
+    GF_OPTION_INIT("iam-nfs-daemon", priv->nfsd.iamnfsd, bool, out);
 
-        GF_OPTION_INIT ("iam-nfs-daemon", priv->nfsd.iamnfsd, bool, out);
+    GF_OPTION_INIT("optimistic-change-log", priv->optimistic_change_log, bool,
+                   out);
 
-        GF_OPTION_INIT ("optimistic-change-log", priv->optimistic_change_log,
-                        bool, out);
+    GF_OPTION_INIT("pre-op-compat", priv->pre_op_compat, bool, out);
+    GF_OPTION_INIT("locking-scheme", locking_scheme, str, out);
+    priv->granular_locks = (strcmp(locking_scheme, "granular") == 0);
+    GF_OPTION_INIT("full-lock", priv->full_lock, bool, out);
+    GF_OPTION_INIT("granular-entry-heal", priv->esh_granular, bool, out);
 
-        GF_OPTION_INIT ("pre-op-compat", priv->pre_op_compat, bool, out);
-        GF_OPTION_INIT ("locking-scheme", priv->locking_scheme, str, out);
-        GF_OPTION_INIT ("full-lock", priv->full_lock, bool, out);
-        GF_OPTION_INIT ("granular-entry-heal", priv->esh_granular, bool, out);
+    GF_OPTION_INIT("eager-lock", priv->eager_lock, bool, out);
+    GF_OPTION_INIT("quorum-type", qtype, str, out);
+    GF_OPTION_INIT("quorum-count", priv->quorum_count, uint32, out);
+    GF_OPTION_INIT(AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, size_uint64,
+                   out);
+    fix_quorum_options(this, priv, qtype, this->options);
 
-        GF_OPTION_INIT ("eager-lock", priv->eager_lock, bool, out);
-        GF_OPTION_INIT ("quorum-type", qtype, str, out);
-        GF_OPTION_INIT ("quorum-count", priv->quorum_count, uint32, out);
-        GF_OPTION_INIT (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, size_uint64,
-                        out);
-        fix_quorum_options (this, priv, qtype, this->options);
+    GF_OPTION_INIT("post-op-delay-secs", priv->post_op_delay_secs, uint32, out);
+    GF_OPTION_INIT("ensure-durability", priv->ensure_durability, bool, out);
 
-	GF_OPTION_INIT ("post-op-delay-secs", priv->post_op_delay_secs, uint32, out);
-        GF_OPTION_INIT ("ensure-durability", priv->ensure_durability, bool,
-                        out);
+    GF_OPTION_INIT("self-heal-daemon", priv->shd.enabled, bool, out);
 
-	GF_OPTION_INIT ("self-heal-daemon", priv->shd.enabled, bool, out);
+    GF_OPTION_INIT("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
+    GF_OPTION_INIT("heal-timeout", priv->shd.timeout, int32, out);
 
-	GF_OPTION_INIT ("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
-        GF_OPTION_INIT ("heal-timeout", priv->shd.timeout, int32, out);
+    GF_OPTION_INIT("consistent-metadata", priv->consistent_metadata, bool, out);
+    GF_OPTION_INIT("consistent-io", priv->consistent_io, bool, out);
+    afr_handle_anon_inode_options(priv, this->options);
 
-        GF_OPTION_INIT ("consistent-metadata", priv->consistent_metadata, bool,
-                        out);
-        GF_OPTION_INIT ("consistent-io", priv->consistent_io, bool, out);
+    GF_OPTION_INIT("use-anonymous-inode", priv->use_anon_inode, bool, out);
+    if (priv->quorum_count != 0)
+        priv->consistent_io = _gf_false;
 
-        if (priv->quorum_count != 0)
-                priv->consistent_io = _gf_false;
+    priv->wait_count = 1;
 
-        priv->wait_count = 1;
+    priv->local = GF_CALLOC(sizeof(unsigned char), child_count, gf_afr_mt_char);
+    if (!priv->local) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        priv->local = GF_CALLOC (sizeof (unsigned char), child_count,
+    priv->anon_inode = GF_CALLOC(sizeof(unsigned char), child_count,
                                  gf_afr_mt_char);
-        if (!priv->local) {
-                ret = -ENOMEM;
-                goto out;
-        }
 
-        priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
-                                    gf_afr_mt_char);
+    priv->child_up = GF_CALLOC(sizeof(unsigned char), child_count,
+                               gf_afr_mt_char);
 
-        priv->child_latency = GF_CALLOC (sizeof (*priv->child_latency),
-                                         child_count,
-                                         gf_afr_mt_child_latency_t);
+    priv->child_latency = GF_MALLOC(sizeof(*priv->child_latency) * child_count,
+                                    gf_afr_mt_child_latency_t);
+    priv->halo_child_up = GF_CALLOC(sizeof(unsigned char), child_count,
+                                    gf_afr_mt_char);
 
-        if (!priv->child_up || !priv->child_latency) {
-                ret = -ENOMEM;
-                goto out;
-        }
-        /*Initialize to -ve ping timeout so that they are not considered
-         * in child-up events until ping-event comes*/
-        for (i = 0; i < child_count; i++)
-                priv->child_latency[i] = -1;
-
-        priv->children = GF_CALLOC (sizeof (xlator_t *), child_count,
-                                    gf_afr_mt_xlator_t);
-        if (!priv->children) {
-                ret = -ENOMEM;
-                goto out;
+    if (!priv->child_up || !priv->child_latency || !priv->halo_child_up ||
+        !priv->anon_inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    /*Initialize to -ve ping timeout so that they are not considered
+     * in child-up events until ping-event comes*/
+    for (i = 0; i < child_count; i++)
+        priv->child_latency[i] = -1;
+
+    priv->children = GF_CALLOC(sizeof(xlator_t *), child_count,
+                               gf_afr_mt_xlator_t);
+    if (!priv->children) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = afr_pending_xattrs_init(priv, this);
+    if (ret)
+        goto out;
+
+    trav = this->children;
+    i = 0;
+    while (i < child_count) {
+        priv->children[i] = trav->xlator;
+        trav = trav->next;
+        i++;
+    }
+
+    ret = gf_asprintf(&priv->sh_domain, AFR_SH_DATA_DOMAIN_FMT, this->name);
+    if (-1 == ret) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    priv->last_event = GF_CALLOC(child_count, sizeof(*priv->last_event),
+                                 gf_afr_mt_int32_t);
+    if (!priv->last_event) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    this->itable = inode_table_new(SHD_INODE_LRU_LIMIT, this);
+    if (!this->itable) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    if (priv->shd.iamshd) {
+        ret = afr_selfheal_daemon_init(this);
+        if (ret) {
+            ret = -ENOMEM;
+            goto out;
         }
+    }
 
-        ret = afr_pending_xattrs_init (priv, this);
-        if (ret)
-                goto out;
+    /* keep more local here as we may need them for self-heal etc */
+    this->local_pool = mem_pool_new(afr_local_t, 512);
+    if (!this->local_pool) {
+        ret = -1;
+        goto out;
+    }
 
-        trav = this->children;
-        i = 0;
-        while (i < child_count) {
-                priv->children[i] = trav->xlator;
-                trav = trav->next;
-                i++;
-        }
+    priv->root_inode = NULL;
 
-        ret = gf_asprintf (&priv->sh_domain, AFR_SH_DATA_DOMAIN_FMT,
-                           this->name);
-        if (-1 == ret) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    ret = 0;
+out:
+    return ret;
+}
+void
+afr_destroy_healer_object(xlator_t *this, struct subvol_healer *healer)
+{
+    int ret = -1;
 
-        priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
-                                      gf_afr_mt_int32_t);
-        if (!priv->last_event) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    if (!healer)
+        return;
 
-	ret = afr_selfheal_daemon_init (this);
-	if (ret) {
-		ret = -ENOMEM;
-		goto out;
-	}
+    if (healer->running) {
+        /*
+         * If there are any resources to cleanup, We need
+         * to do that gracefully using pthread_cleanup_push
+         */
+        ret = gf_thread_cleanup_xint(healer->thread);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SELF_HEAL_FAILED,
+                   "Failed to clean up healer threads.");
+        healer->thread = 0;
+    }
+    pthread_cond_destroy(&healer->cond);
+    pthread_mutex_destroy(&healer->mutex);
+}
 
-        /* keep more local here as we may need them for self-heal etc */
-        this->local_pool = mem_pool_new (afr_local_t, 512);
-        if (!this->local_pool) {
-                ret = -1;
-                goto out;
-        }
+void
+afr_selfheal_daemon_fini(xlator_t *this)
+{
+    struct subvol_healer *healer = NULL;
+    afr_self_heald_t *shd = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+    if (!priv)
+        return;
+
+    shd = &priv->shd;
+    if (!shd->iamshd)
+        return;
+
+    for (i = 0; i < priv->child_count; i++) {
+        healer = &shd->index_healers[i];
+        afr_destroy_healer_object(this, healer);
+
+        healer = &shd->full_healers[i];
+        afr_destroy_healer_object(this, healer);
+
+        if (shd->statistics[i])
+            eh_destroy(shd->statistics[i]);
+    }
+    GF_FREE(shd->index_healers);
+    GF_FREE(shd->full_healers);
+    GF_FREE(shd->statistics);
+    if (shd->split_brain)
+        eh_destroy(shd->split_brain);
+}
+void
+fini(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
 
-        priv->root_inode = NULL;
+    priv = this->private;
 
-        ret = 0;
-out:
-        return ret;
-}
+    afr_selfheal_daemon_fini(this);
+    GF_ASSERT(list_empty(&priv->saved_locks));
 
+    LOCK(&priv->lock);
+    if (priv->timer != NULL) {
+        gf_timer_call_cancel(this->ctx, priv->timer);
+        priv->timer = NULL;
+    }
+    UNLOCK(&priv->lock);
 
-int
-fini (xlator_t *this)
-{
-        afr_private_t *priv = NULL;
+    if (this->local_pool != NULL) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
 
-        priv = this->private;
-        LOCK (&priv->lock);
-        if (priv->timer != NULL) {
-                gf_timer_call_cancel(this->ctx, priv->timer);
-                priv->timer = NULL;
-        }
-        UNLOCK (&priv->lock);
-        this->private = NULL;
-        afr_priv_destroy (priv);
-        //if (this->itable);//I dont see any destroy func
+    this->private = NULL;
+    afr_priv_destroy(priv);
+    if (this->itable) {
+        inode_table_destroy(this->itable);
+        this->itable = NULL;
+    }
 
-        return 0;
+    return;
 }
 
-
 struct xlator_fops fops = {
-        .lookup      = afr_lookup,
-        .lk          = afr_lk,
-        .flush       = afr_flush,
-        .statfs      = afr_statfs,
-        .fsyncdir    = afr_fsyncdir,
-        .inodelk     = afr_inodelk,
-        .finodelk    = afr_finodelk,
-        .entrylk     = afr_entrylk,
-        .fentrylk    = afr_fentrylk,
-        .ipc         = afr_ipc,
-        .lease       = afr_lease,
-
-        /* inode read */
-        .access      = afr_access,
-        .stat        = afr_stat,
-        .fstat       = afr_fstat,
-        .readlink    = afr_readlink,
-        .getxattr    = afr_getxattr,
-        .fgetxattr   = afr_fgetxattr,
-        .readv       = afr_readv,
-
-        /* inode write */
-        .writev      = afr_writev,
-        .truncate    = afr_truncate,
-        .ftruncate   = afr_ftruncate,
-        .setxattr    = afr_setxattr,
-        .fsetxattr   = afr_fsetxattr,
-        .setattr     = afr_setattr,
-        .fsetattr    = afr_fsetattr,
-        .removexattr = afr_removexattr,
-        .fremovexattr = afr_fremovexattr,
-        .fallocate   = afr_fallocate,
-        .discard     = afr_discard,
-        .zerofill    = afr_zerofill,
-        .xattrop     = afr_xattrop,
-        .fxattrop    = afr_fxattrop,
-        .fsync       = afr_fsync,
-
-        /*inode open*/
-        .opendir     = afr_opendir,
-        .open        = afr_open,
-
-        /* dir read */
-        .readdir     = afr_readdir,
-        .readdirp    = afr_readdirp,
-
-        /* dir write */
-        .create      = afr_create,
-        .mknod       = afr_mknod,
-        .mkdir       = afr_mkdir,
-        .unlink      = afr_unlink,
-        .rmdir       = afr_rmdir,
-        .link        = afr_link,
-        .symlink     = afr_symlink,
-        .rename      = afr_rename,
+    .lookup = afr_lookup,
+    .lk = afr_lk,
+    .flush = afr_flush,
+    .statfs = afr_statfs,
+    .fsyncdir = afr_fsyncdir,
+    .inodelk = afr_inodelk,
+    .finodelk = afr_finodelk,
+    .entrylk = afr_entrylk,
+    .fentrylk = afr_fentrylk,
+    .ipc = afr_ipc,
+    .lease = afr_lease,
+
+    /* inode read */
+    .access = afr_access,
+    .stat = afr_stat,
+    .fstat = afr_fstat,
+    .readlink = afr_readlink,
+    .getxattr = afr_getxattr,
+    .fgetxattr = afr_fgetxattr,
+    .readv = afr_readv,
+    .seek = afr_seek,
+
+    /* inode write */
+    .writev = afr_writev,
+    .truncate = afr_truncate,
+    .ftruncate = afr_ftruncate,
+    .setxattr = afr_setxattr,
+    .fsetxattr = afr_fsetxattr,
+    .setattr = afr_setattr,
+    .fsetattr = afr_fsetattr,
+    .removexattr = afr_removexattr,
+    .fremovexattr = afr_fremovexattr,
+    .fallocate = afr_fallocate,
+    .discard = afr_discard,
+    .zerofill = afr_zerofill,
+    .xattrop = afr_xattrop,
+    .fxattrop = afr_fxattrop,
+    .fsync = afr_fsync,
+
+    /*inode open*/
+    .opendir = afr_opendir,
+    .open = afr_open,
+
+    /* dir read */
+    .readdir = afr_readdir,
+    .readdirp = afr_readdirp,
+
+    /* dir write */
+    .create = afr_create,
+    .mknod = afr_mknod,
+    .mkdir = afr_mkdir,
+    .unlink = afr_unlink,
+    .rmdir = afr_rmdir,
+    .link = afr_link,
+    .symlink = afr_symlink,
+    .rename = afr_rename,
 };
 
-
 struct xlator_dumpops dumpops = {
-        .priv       = afr_priv_dump,
+    .priv = afr_priv_dump,
 };
 
-
 struct xlator_cbks cbks = {
-        .release     = afr_release,
-        .releasedir  = afr_releasedir,
-        .forget      = afr_forget,
+    .release = afr_release,
+    .releasedir = afr_releasedir,
+    .forget = afr_forget,
 };
 
-
 struct volume_options options[] = {
-        { .key  = {"read-subvolume" },
-          .type = GF_OPTION_TYPE_XLATOR,
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "inode-read fops happen only on one of the bricks in "
-                         "replicate. Afr will prefer the one specified using "
-                         "this option if it is not stale. Option value must be "
-                         "one of the xlator names of the children. "
-                         "Ex: <volname>-client-0 till "
-                         "<volname>-client-<number-of-bricks - 1>"
-        },
-        { .key  = {"read-subvolume-index" },
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "-1",
-          .op_version = {2},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "inode-read fops happen only on one of the bricks in "
-                         "replicate. AFR will prefer the one specified using "
-                         "this option if it is not stale. allowed options"
-                         " include -1 till replica-count - 1"
-        },
-        { .key = {"read-hash-mode" },
-          .type = GF_OPTION_TYPE_INT,
-          .min = 0,
-          .max = 3,
-          .default_value = "1",
-          .op_version = {2},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "inode-read fops happen only on one of the bricks in "
-                         "replicate. AFR will prefer the one computed using "
-                         "the method specified using this option.\n"
-                         "0 = first readable child of AFR, starting from 1st child.\n"
-                         "1 = hash by GFID of file (all clients use "
-                                                    "same subvolume).\n"
-                         "2 = hash by GFID of file and client PID.\n"
-                         "3 = brick having the least outstanding read requests."
-        },
-        { .key  = {"choose-local" },
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "true",
-          .op_version = {2},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "Choose a local subvolume (i.e. Brick) to read from"
-	                 " if read-subvolume is not explicitly set.",
-        },
-        { .key  = {"background-self-heal-count"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 0,
-          .max  = 256,
-          .default_value = "8",
-          .validate = GF_OPT_VALIDATE_MIN,
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "This specifies the number of per client self-heal "
-                         "jobs that can perform parallel heals in the "
-                         "background."
-        },
-        { .key   = {"halo-shd-max-latency"},
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 1,
-          .max   = 99999,
-          .default_value = "99999",
-          .op_version = {GD_OP_VERSION_3_11_0},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate", "halo"},
-          .description = "Maximum latency for shd halo replication in msec."
-        },
-        { .key   = {"halo-enabled"},
-          .type  = GF_OPTION_TYPE_BOOL,
-          .default_value = "False",
-          .op_version = {GD_OP_VERSION_3_11_0},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate", "halo"},
-          .description = "Enable Halo (geo) replication mode."
-        },
-        { .key   = {"halo-nfsd-max-latency"},
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 1,
-          .max   = 99999,
-          .default_value = "5",
-          .op_version = {GD_OP_VERSION_3_11_0},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate", "halo"},
-          .description = "Maximum latency for nfsd halo replication in msec."
-        },
-        { .key   = {"halo-max-latency"},
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 1,
-          .max   = AFR_HALO_MAX_LATENCY,
-          .default_value = "5",
-          .op_version = {GD_OP_VERSION_3_11_0},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate", "halo"},
-          .description = "Maximum latency for halo replication in msec."
-        },
-        { .key   = {"halo-max-replicas"},
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 1,
-          .max   = 99999,
-          .default_value = "99999",
-          .op_version = {GD_OP_VERSION_3_11_0},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate", "halo"},
-          .description = "The maximum number of halo replicas; replicas"
-                          " beyond this value will be written asynchronously"
-                          "via the SHD."
-        },
-        { .key   = {"halo-min-replicas"},
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 1,
-          .max   = 99999,
-          .default_value = "2",
-          .op_version = {GD_OP_VERSION_3_11_0},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate", "halo"},
-          .description = "The minimmum number of halo replicas, before adding "
-                          "out of region replicas."
-         },
-         { .key  = {"heal-wait-queue-length"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 0,
-          .max  = 10000, /*Around 100MB with sizeof(afr_local_t)= 10496 bytes*/
-          .default_value = "128",
-          .validate = GF_OPT_VALIDATE_MIN,
-          .op_version = {GD_OP_VERSION_3_7_10},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "This specifies the number of heals that can be queued"
-                         " for the parallel background self heal jobs."
-        },
-        { .key  = {"data-self-heal"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"1", "on", "yes", "true", "enable",
-                    "0", "off", "no", "false", "disable",
-                    "open"},
-          .default_value = "on",
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description   = "Using this option we can enable/disable data "
-                           "self-heal on the file. \"open\" means data "
-                           "self-heal action will only be triggered by file "
-                           "open operations."
-        },
-        { .key  = {"data-self-heal-algorithm"},
-          .type = GF_OPTION_TYPE_STR,
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description   = "Select between \"full\", \"diff\". The "
-                           "\"full\" algorithm copies the entire file from "
-                           "source to sink. The \"diff\" algorithm copies to "
-                           "sink only those blocks whose checksums don't match "
-                           "with those of source. If no option is configured "
-                           "the option is chosen dynamically as follows: "
-                           "If the file does not exist on one of the sinks "
-                           "or empty file exists or if the source file size is "
-                           "about the same as page size the entire file will "
-                           "be read and written i.e \"full\" algo, "
-                           "otherwise \"diff\" algo is chosen.",
-          .value = { "diff", "full"}
-        },
-        { .key  = {"data-self-heal-window-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 1,
-          .max  = 1024,
-          .default_value = "1",
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "Maximum number blocks per file for which self-heal "
-                         "process would be applied simultaneously."
-        },
-        { .key  = {"metadata-self-heal"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          /*.validate_fn = validate_replica*/
-          .description = "Using this option we can enable/disable metadata "
-                         "i.e. Permissions, ownerships, xattrs self-heal on "
-                         "the file/directory."
-        },
-        { .key  = {"entry-self-heal"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          /*.validate_fn = validate_replica*/
-          .description = "Using this option we can enable/disable entry "
-                         "self-heal on the directory."
-        },
-        { .key  = {"data-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "This option exists only for backward compatibility "
-                         "and configuring it doesn't have any effect"
-        },
-        { .key  = {"metadata-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "This option exists only for backward compatibility "
-                         "and configuring it doesn't have any effect"
-        },
-        { .key  = {"entry-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "This option exists only for backward compatibility "
-                         "and configuring it doesn't have any effect"
-        },
-        { .key  = {"optimistic-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Entry/Metadata fops will not perform "
-                         "pre fop changelog operations in afr transaction "
-                         "if this option is enabled."
-        },
-        { .key = {"inodelk-trace"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Enabling this option logs inode lock/unlocks"
-        },
-        { .key = {"entrylk-trace"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Enabling this option logs entry lock/unlocks"
-        },
-	{ .key = {"pre-op-compat"},
-	  .type = GF_OPTION_TYPE_BOOL,
-	  .default_value = "on",
-	  .description = "Use separate pre-op xattrop() FOP rather than "
-	                 "overloading xdata of the OP"
-	},
-        { .key = {"eager-lock"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "Enable/Disable eager lock for replica volume. "
-                         "Lock phase of a transaction has two sub-phases. "
-                         "First is an attempt to acquire locks in parallel by "
-                         "broadcasting non-blocking lock requests. If lock "
-                         "acquisition fails on any server, then the held locks "
-                         "are unlocked and we revert to a blocking locks mode "
-                         "sequentially on one server after another.  If this "
-                         "option is enabled the initial broadcasting lock "
-                         "request attempts to acquire a full lock on the entire file. "
-                         "If this fails, we revert back to the sequential "
-                         "\"regional\" blocking locks as before. In the case "
-                         "where such an \"eager\" lock is granted in the "
-                         "non-blocking phase, it gives rise to an opportunity "
-                         "for optimization. i.e, if the next write transaction "
-                         "on the same FD arrives before the unlock phase of "
-                         "the first transaction, it \"takes over\" the full "
-                         "file lock. Similarly if yet another data transaction "
-                         "arrives before the unlock phase of the \"optimized\" "
-                         "transaction, that in turn \"takes over\" the lock as "
-                         "well. The actual unlock now happens at the end of "
-                         "the last \"optimized\" transaction."
-
-        },
-        { .key = {"self-heal-daemon"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
-          .tags = {"replicate"},
-          /*.validate_fn   = validate_replica_heal_enable_disable*/
-          .description = "This option applies to only self-heal-daemon. "
-                         "Index directory crawl and automatic healing of files "
-                         "will not be performed if this option is turned off."
-        },
-        { .key = {"iam-self-heal-daemon"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option differentiates if the replicate "
-                         "translator is running as part of self-heal-daemon "
-                         "or not."
-        },
-        { .key = {"iam-nfs-daemon"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option differentiates if the replicate "
-                         "translator is running as part of an NFS daemon "
-                         "or not."
-        },
-        { .key = {"quorum-type"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { "none", "auto", "fixed"},
-          .default_value = "none",
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          /*.option = quorum-type*/
-          .description = "If value is \"fixed\" only allow writes if "
-                         "quorum-count bricks are present.  If value is "
-                         "\"auto\" only allow writes if more than half of "
-                         "bricks, or exactly half including the first, are "
-                         "present.",
-        },
-        { .key = {"quorum-count"},
-          .type = GF_OPTION_TYPE_INT,
-          .min = 1,
-          .max = INT_MAX,
-          .default_value = 0,
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          /*.option = quorum-count*/
-          /*.validate_fn = validate_quorum_count*/
-          .description = "If quorum-type is \"fixed\" only allow writes if "
-                         "this many bricks are present.  Other quorum types "
-                         "will OVERWRITE this value.",
-        },
-        { .key = {"quorum-reads"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-          .op_version = {GD_OP_VERSION_3_7_0},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "This option has been removed. Reads are not allowed "
-                          "if quorum is not met.",
-        },
-        { .key  = {"node-uuid"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Local glusterd uuid string, used in starting "
-                         "self-heal-daemon so that it can crawl only on "
-                         "local index directories.",
-        },
-        { .key  = {"post-op-delay-secs"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 0,
-          .max  = INT_MAX,
-          .default_value = "1",
-          .op_version = {2},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "Time interval induced artificially before "
-	                 "post-operation phase of the transaction to "
-                         "enhance overlap of adjacent write operations.",
-        },
-        { .key = {AFR_SH_READDIR_SIZE_KEY},
-          .type = GF_OPTION_TYPE_SIZET,
-          .description = "readdirp size for performing entry self-heal",
-          .min = 1024,
-          .max = 131072,
-          .op_version = {2},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
-          .tags = {"replicate"},
-          .default_value = "1KB",
-        },
-        { .key = {"ensure-durability"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .op_version = {3},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "Afr performs fsyncs for transactions if this "
-                         "option is on to make sure the changelogs/data is "
-                         "written to the disk",
-          .default_value = "on",
-        },
-	{ .key = {"afr-dirty-xattr"},
-	  .type = GF_OPTION_TYPE_STR,
-	  .default_value = AFR_DIRTY_DEFAULT,
-	},
-	{ .key = {"afr-pending-xattr"},
-	  .type = GF_OPTION_TYPE_STR,
-          .description = "Comma separated list of xattrs that are used to  "
-                         "capture information on pending heals."
-	},
-	{ .key = {"metadata-splitbrain-forced-heal"},
-	  .type = GF_OPTION_TYPE_BOOL,
-	  .default_value = "off",
-	},
-        { .key  = {"heal-timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 5,
-          .max  = INT_MAX,
-          .default_value = "600",
-          .op_version = {2},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "time interval for checking the need to self-heal "
-                         "in self-heal-daemon"
-        },
-        { .key = {"consistent-metadata"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-          .op_version = {GD_OP_VERSION_3_7_0},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "If this option is enabled, readdirp will force "
-                         "lookups on those entries read whose read child is "
-                         "not the same as that of the parent. This will "
-                         "guarantee that all read operations on a file serve "
-                         "attributes from the same subvol as long as it holds "
-                         " a good copy of the file/dir.",
-        },
-        { .key = {"arbiter-count"},
-          .type = GF_OPTION_TYPE_INT,
-          .description = "subset of child_count. Has to be 0 or 1."
-        },
-        { .key = {"thin-arbiter"},
-          .type = GF_OPTION_TYPE_STR,
-          .op_version = {GD_OP_VERSION_4_1_0},
-          .flags = OPT_FLAG_SETTABLE,
-          .tags = {"replicate"},
-          .description = "contains host:path of thin abriter brick",
-        },
-        { .key   = {"shd-max-threads"},
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 1,
-          .max   = 64,
-          .default_value = "1",
-          .op_version = {GD_OP_VERSION_3_7_12},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "Maximum number of parallel heals SHD can do per "
-                          "local brick. This can substantially lower heal times"
-                          ", but can also crush your bricks if you don't have "
-                          "the storage hardware to support this."
-        },
-        { .key   = {"shd-wait-qlength"},
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 1,
-          .max   = 655536,
-          .default_value = "1024",
-          .op_version = {GD_OP_VERSION_3_7_12},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "This option can be used to control number of heals"
-                          " that can wait in SHD per subvolume",
-        },
-        { .key = {"locking-scheme"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { "full", "granular"},
-          .default_value = "full",
-          .op_version = {GD_OP_VERSION_3_7_12},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "If this option is set to granular, self-heal will "
-                         "stop being compatible with afr-v1, which helps afr "
-                         "be more granular while self-healing",
-        },
-        { .key = {"full-lock"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "yes",
-          .op_version = {GD_OP_VERSION_3_13_2},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
-          .tags = {"replicate"},
-          .description = "If this option is disabled, then the IOs will take "
-                         "range locks same as versions till 3.13.1."
-        },
-        { .key = {"granular-entry-heal"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-          .op_version = {GD_OP_VERSION_3_8_0},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "If this option is enabled, self-heal will resort to "
-                         "granular way of recording changelogs and doing entry "
-                         "self-heal.",
-        },
-        { .key   = {"favorite-child-policy"},
-          .type  = GF_OPTION_TYPE_STR,
-          .value = {"none", "size", "ctime", "mtime", "majority"},
-          .default_value = "none",
-          .op_version = {GD_OP_VERSION_3_7_12},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "This option can be used to automatically resolve "
-                         "split-brains using various policies without user "
-                         "intervention. \"size\" picks the file with the "
-                         "biggest size as the source. \"ctime\" and \"mtime\" "
-                         "pick the file with the latest ctime and mtime "
-                         "respectively as the source. \"majority\" picks a file"
-                         " with identical mtime and size in more than half the "
-                         "number of bricks in the replica.",
-        },
-        { .key = {"consistent-io"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-          .description = "If this option is enabled, i/o will fail even if "
-                         "one of the bricks is down in the replicas",
-        },
-        { .key   = {"use-compound-fops"},
-          .type  = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-          .op_version = {GD_OP_VERSION_3_8_4},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"replicate"},
-          .description = "This option exists only for backward compatibility "
-                         "and configuring it doesn't have any effect"
-        },
-        { .key  = {NULL} },
+    {.key = {"read-subvolume"},
+     .type = GF_OPTION_TYPE_XLATOR,
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "inode-read fops happen only on one of the bricks in "
+                    "replicate. Afr will prefer the one specified using "
+                    "this option if it is not stale. Option value must be "
+                    "one of the xlator names of the children. "
+                    "Ex: <volname>-client-0 till "
+                    "<volname>-client-<number-of-bricks - 1>"},
+    {.key = {"read-subvolume-index"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "-1",
+     .op_version = {2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "inode-read fops happen only on one of the bricks in "
+                    "replicate. AFR will prefer the one specified using "
+                    "this option if it is not stale. allowed options"
+                    " include -1 till replica-count - 1"},
+    {.key = {"read-hash-mode"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = 5,
+     .default_value = "1",
+     .op_version = {2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description =
+         "inode-read fops happen only on one of the bricks in "
+         "replicate. AFR will prefer the one computed using "
+         "the method specified using this option.\n"
+         "0 = first readable child of AFR, starting from 1st child.\n"
+         "1 = hash by GFID of file (all clients use "
+         "same subvolume).\n"
+         "2 = hash by GFID of file and client PID.\n"
+         "3 = brick having the least outstanding read requests.\n"
+         "4 = brick having the least network ping latency.\n"
+         "5 = Hybrid mode between 3 and 4, ie least value among "
+         "network-latency multiplied by outstanding-read-requests."},
+    {
+        .key = {"choose-local"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "true",
+        .op_version = {2},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "Choose a local subvolume (i.e. Brick) to read from"
+                       " if read-subvolume is not explicitly set.",
+    },
+    {.key = {"background-self-heal-count"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = 256,
+     .default_value = "8",
+     .validate = GF_OPT_VALIDATE_MIN,
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This specifies the number of per client self-heal "
+                    "jobs that can perform parallel heals in the "
+                    "background."},
+    {.key = {"halo-shd-max-latency"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "99999",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Maximum latency for shd halo replication in msec."},
+    {.key = {"halo-enabled"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "False",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Enable Halo (geo) replication mode."},
+    {.key = {"halo-nfsd-max-latency"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "5",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Maximum latency for nfsd halo replication in msec."},
+    {.key = {"halo-max-latency"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = AFR_HALO_MAX_LATENCY,
+     .default_value = "5",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Maximum latency for halo replication in msec."},
+    {.key = {"halo-max-replicas"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "99999",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "The maximum number of halo replicas; replicas"
+                    " beyond this value will be written asynchronously"
+                    "via the SHD."},
+    {.key = {"halo-min-replicas"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "2",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "The minimmum number of halo replicas, before adding "
+                    "out of region replicas."},
+    {.key = {"heal-wait-queue-length"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = 10000, /*Around 100MB with sizeof(afr_local_t)= 10496 bytes*/
+     .default_value = "128",
+     .validate = GF_OPT_VALIDATE_MIN,
+     .op_version = {GD_OP_VERSION_3_7_10},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This specifies the number of heals that can be queued"
+                    " for the parallel background self heal jobs."},
+    {.key = {"data-self-heal"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {"1", "on", "yes", "true", "enable", "0", "off", "no", "false",
+               "disable", "open"},
+     .default_value = "off",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Using this option we can enable/disable data "
+                    "self-heal on the file. \"open\" means data "
+                    "self-heal action will only be triggered by file "
+                    "open operations."},
+    {.key = {"data-self-heal-algorithm"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Select between \"full\", \"diff\". The "
+                    "\"full\" algorithm copies the entire file from "
+                    "source to sink. The \"diff\" algorithm copies to "
+                    "sink only those blocks whose checksums don't match "
+                    "with those of source. If no option is configured "
+                    "the option is chosen dynamically as follows: "
+                    "If the file does not exist on one of the sinks "
+                    "or empty file exists or if the source file size is "
+                    "about the same as page size the entire file will "
+                    "be read and written i.e \"full\" algo, "
+                    "otherwise \"diff\" algo is chosen.",
+     .value = {"diff", "full"}},
+    {.key = {"data-self-heal-window-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 1024,
+     .default_value = "1",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Maximum number blocks per file for which self-heal "
+                    "process would be applied simultaneously."},
+    {.key = {"metadata-self-heal"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     /*.validate_fn = validate_replica*/
+     .description = "Using this option we can enable/disable metadata "
+                    "i.e. Permissions, ownerships, xattrs self-heal on "
+                    "the file/directory."},
+    {.key = {"entry-self-heal"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     /*.validate_fn = validate_replica*/
+     .description = "Using this option we can enable/disable entry "
+                    "self-heal on the directory."},
+    {.key = {"data-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"metadata-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"entry-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"optimistic-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "Entry/Metadata fops will not perform "
+                    "pre fop changelog operations in afr transaction "
+                    "if this option is enabled."},
+    {.key = {"inodelk-trace"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "Enabling this option logs inode lock/unlocks"},
+    {.key = {"entrylk-trace"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "Enabling this option logs entry lock/unlocks"},
+    {.key = {"pre-op-compat"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "Use separate pre-op xattrop() FOP rather than "
+                    "overloading xdata of the OP"},
+    {.key = {"eager-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description =
+         "Enable/Disable eager lock for replica volume. "
+         "Lock phase of a transaction has two sub-phases. "
+         "First is an attempt to acquire locks in parallel by "
+         "broadcasting non-blocking lock requests. If lock "
+         "acquisition fails on any server, then the held locks "
+         "are unlocked and we revert to a blocking locks mode "
+         "sequentially on one server after another.  If this "
+         "option is enabled the initial broadcasting lock "
+         "request attempts to acquire a full lock on the entire file. "
+         "If this fails, we revert back to the sequential "
+         "\"regional\" blocking locks as before. In the case "
+         "where such an \"eager\" lock is granted in the "
+         "non-blocking phase, it gives rise to an opportunity "
+         "for optimization. i.e, if the next write transaction "
+         "on the same FD arrives before the unlock phase of "
+         "the first transaction, it \"takes over\" the full "
+         "file lock. Similarly if yet another data transaction "
+         "arrives before the unlock phase of the \"optimized\" "
+         "transaction, that in turn \"takes over\" the lock as "
+         "well. The actual unlock now happens at the end of "
+         "the last \"optimized\" transaction."
+
+    },
+    {.key = {"self-heal-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+     .tags = {"replicate"},
+     /*.validate_fn   = validate_replica_heal_enable_disable*/
+     .description = "This option applies to only self-heal-daemon. "
+                    "Index directory crawl and automatic healing of files "
+                    "will not be performed if this option is turned off."},
+    {.key = {"iam-self-heal-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option differentiates if the replicate "
+                    "translator is running as part of self-heal-daemon "
+                    "or not."},
+    {.key = {"iam-nfs-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option differentiates if the replicate "
+                    "translator is running as part of an NFS daemon "
+                    "or not."},
+    {
+        .key = {"quorum-type"},
+        .type = GF_OPTION_TYPE_STR,
+        .value = {"none", "auto", "fixed"},
+        .default_value = "none",
+        .op_version = {1},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        /*.option = quorum-type*/
+        .description = "If value is \"fixed\" only allow writes if "
+                       "quorum-count bricks are present.  If value is "
+                       "\"auto\" only allow writes if more than half of "
+                       "bricks, or exactly half including the first, are "
+                       "present.",
+    },
+    {
+        .key = {"quorum-count"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 1,
+        .max = INT_MAX,
+        .default_value = 0,
+        .op_version = {1},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        /*.option = quorum-count*/
+        /*.validate_fn = validate_quorum_count*/
+        .description = "If quorum-type is \"fixed\" only allow writes if "
+                       "this many bricks are present.  Other quorum types "
+                       "will OVERWRITE this value.",
+    },
+    {
+        .key = {"quorum-reads"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "This option has been removed. Reads are not allowed "
+                       "if quorum is not met.",
+    },
+    {
+        .key = {"node-uuid"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Local glusterd uuid string, used in starting "
+                       "self-heal-daemon so that it can crawl only on "
+                       "local index directories.",
+    },
+    {
+        .key = {"post-op-delay-secs"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 0,
+        .max = INT_MAX,
+        .default_value = "1",
+        .op_version = {2},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "Time interval induced artificially before "
+                       "post-operation phase of the transaction to "
+                       "enhance overlap of adjacent write operations.",
+    },
+    {
+        .key = {AFR_SH_READDIR_SIZE_KEY},
+        .type = GF_OPTION_TYPE_SIZET,
+        .description = "readdirp size for performing entry self-heal",
+        .min = 1024,
+        .max = 131072,
+        .op_version = {2},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+        .tags = {"replicate"},
+        .default_value = "1KB",
+    },
+    {
+        .key = {"ensure-durability"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .op_version = {3},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "Afr performs fsyncs for transactions if this "
+                       "option is on to make sure the changelogs/data is "
+                       "written to the disk",
+        .default_value = "on",
+    },
+    {
+        .key = {"afr-dirty-xattr"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = AFR_DIRTY_DEFAULT,
+    },
+    {.key = {"afr-pending-xattr"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Comma separated list of xattrs that are used to  "
+                    "capture information on pending heals."},
+    {
+        .key = {"metadata-splitbrain-forced-heal"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+    },
+    {.key = {"heal-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 5,
+     .max = INT_MAX,
+     .default_value = "600",
+     .op_version = {2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "time interval for checking the need to self-heal "
+                    "in self-heal-daemon"},
+    {
+        .key = {"consistent-metadata"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "If this option is enabled, readdirp will force "
+                       "lookups on those entries read whose read child is "
+                       "not the same as that of the parent. This will "
+                       "guarantee that all read operations on a file serve "
+                       "attributes from the same subvol as long as it holds "
+                       " a good copy of the file/dir.",
+    },
+    {.key = {"arbiter-count"},
+     .type = GF_OPTION_TYPE_INT,
+     .description = "subset of child_count. Has to be 0 or 1."},
+    {
+        .key = {"thin-arbiter"},
+        .type = GF_OPTION_TYPE_STR,
+        .op_version = {GD_OP_VERSION_4_1_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"replicate"},
+        .description = "contains host:path of thin abriter brick",
+    },
+    {.key = {"shd-max-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 64,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_3_7_12},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Maximum number of parallel heals SHD can do per "
+                    "local brick. This can substantially lower heal times"
+                    ", but can also crush your bricks if you don't have "
+                    "the storage hardware to support this."},
+    {
+        .key = {"shd-wait-qlength"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 1,
+        .max = 655536,
+        .default_value = "1024",
+        .op_version = {GD_OP_VERSION_3_7_12},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "This option can be used to control number of heals"
+                       " that can wait in SHD per subvolume",
+    },
+    {
+        .key = {"locking-scheme"},
+        .type = GF_OPTION_TYPE_STR,
+        .value = {"full", "granular"},
+        .default_value = "full",
+        .op_version = {GD_OP_VERSION_3_7_12},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "If this option is set to granular, self-heal will "
+                       "stop being compatible with afr-v1, which helps afr "
+                       "be more granular while self-healing",
+    },
+    {.key = {"full-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "yes",
+     .op_version = {GD_OP_VERSION_3_13_2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+     .tags = {"replicate"},
+     .description = "If this option is disabled, then the IOs will take "
+                    "range locks same as versions till 3.13.1."},
+    {
+        .key = {"granular-entry-heal"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .op_version = {GD_OP_VERSION_3_8_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "If this option is enabled, self-heal will resort to "
+                       "granular way of recording changelogs and doing entry "
+                       "self-heal.",
+    },
+    {
+        .key = {"favorite-child-policy"},
+        .type = GF_OPTION_TYPE_STR,
+        .value = {"none", "size", "ctime", "mtime", "majority"},
+        .default_value = "none",
+        .op_version = {GD_OP_VERSION_3_7_12},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "This option can be used to automatically resolve "
+                       "split-brains using various policies without user "
+                       "intervention. \"size\" picks the file with the "
+                       "biggest size as the source. \"ctime\" and \"mtime\" "
+                       "pick the file with the latest ctime and mtime "
+                       "respectively as the source. \"majority\" picks a file"
+                       " with identical mtime and size in more than half the "
+                       "number of bricks in the replica.",
+    },
+    {
+        .key = {"consistent-io"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .description = "If this option is enabled, i/o will fail even if "
+                       "one of the bricks is down in the replicas",
+    },
+    {.key = {"use-compound-fops"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "no",
+     .op_version = {GD_OP_VERSION_3_8_4},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"use-anonymous-inode"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "no",
+     .op_version = {GD_OP_VERSION_8_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+     .tags = {"replicate"},
+     .description = "Setting this option heals directory renames efficiently"},
+
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "replicate",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index c76b0c1c485..d62f9a9caf2 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -8,181 +8,291 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __AFR_H__
 #define __AFR_H__
 
-#include "call-stub.h"
-#include "compat-errno.h"
+#include <glusterfs/call-stub.h>
+#include <glusterfs/compat-errno.h>
 #include "afr-mem-types.h"
 
 #include "libxlator.h"
-#include "timer.h"
-#include "syncop.h"
+#include <glusterfs/timer.h>
+#include <glusterfs/syncop.h>
 
 #include "afr-self-heald.h"
 #include "afr-messages.h"
 
+#define SHD_INODE_LRU_LIMIT 1
 #define AFR_PATHINFO_HEADER "REPLICATE:"
 #define AFR_SH_READDIR_SIZE_KEY "self-heal-readdir-size"
 #define AFR_SH_DATA_DOMAIN_FMT "%s:self-heal"
 #define AFR_DIRTY_DEFAULT AFR_XATTR_PREFIX ".dirty"
-#define AFR_DIRTY (((afr_private_t *) (THIS->private))->afr_dirty)
+#define AFR_DIRTY (((afr_private_t *)(THIS->private))->afr_dirty)
 
-#define AFR_LOCKEE_COUNT_MAX    3
-#define AFR_DOM_COUNT_MAX    3
-#define AFR_NUM_CHANGE_LOGS            3 /*data + metadata + entry*/
+#define AFR_LOCKEE_COUNT_MAX 3
+#define AFR_DOM_COUNT_MAX 3
+#define AFR_NUM_CHANGE_LOGS 3              /*data + metadata + entry*/
 #define AFR_DEFAULT_SPB_CHOICE_TIMEOUT 300 /*in seconds*/
 
 #define ARBITER_BRICK_INDEX 2
 #define THIN_ARBITER_BRICK_INDEX 2
-#define THIN_ARBITER_DOM1 "afr.ta.domain-1"
+#define AFR_TA_DOM_NOTIFY "afr.ta.dom-notify"
+#define AFR_TA_DOM_MODIFY "afr.ta.dom-modify"
+
+#define AFR_LK_HEAL_DOM "afr.lock-heal.domain"
 
 #define AFR_HALO_MAX_LATENCY 99999
-typedef int (*afr_lock_cbk_t) (call_frame_t *frame, xlator_t *this);
+#define AFR_ANON_DIR_PREFIX ".glusterfs-anonymous-inode"
+
+#define PFLAG_PENDING (1 << 0)
+#define PFLAG_SBRAIN (1 << 1)
+
+typedef int (*afr_lock_cbk_t)(call_frame_t *frame, xlator_t *this);
+
+typedef int (*afr_read_txn_wind_t)(call_frame_t *frame, xlator_t *this,
+                                   int subvol);
+
+typedef int (*afr_inode_refresh_cbk_t)(call_frame_t *frame, xlator_t *this,
+                                       int err);
+
+typedef int (*afr_changelog_resume_t)(call_frame_t *frame, xlator_t *this);
+
+#define AFR_COUNT(array, max)                                                  \
+    ({                                                                         \
+        int __i;                                                               \
+        int __res = 0;                                                         \
+        for (__i = 0; __i < max; __i++)                                        \
+            if (array[__i])                                                    \
+                __res++;                                                       \
+        __res;                                                                 \
+    })
+#define AFR_INTERSECT(dst, src1, src2, max)                                    \
+    ({                                                                         \
+        int __i;                                                               \
+        for (__i = 0; __i < max; __i++)                                        \
+            dst[__i] = src1[__i] && src2[__i];                                 \
+    })
+#define AFR_CMP(a1, a2, len)                                                   \
+    ({                                                                         \
+        int __cmp = 0;                                                         \
+        int __i;                                                               \
+        for (__i = 0; __i < len; __i++)                                        \
+            if (a1[__i] != a2[__i]) {                                          \
+                __cmp = 1;                                                     \
+                break;                                                         \
+            }                                                                  \
+        __cmp;                                                                 \
+    })
+#define AFR_IS_ARBITER_BRICK(priv, index)                                      \
+    ((priv->arbiter_count == 1) && (index == ARBITER_BRICK_INDEX))
 
-typedef int (*afr_read_txn_wind_t) (call_frame_t *frame, xlator_t *this, int subvol);
+#define AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(ret, errnum)                       \
+    do {                                                                       \
+        local->op_ret = ret;                                                   \
+        local->op_errno = errnum;                                              \
+        if (local->op_errno == EIO)                                            \
+            gf_msg(this->name, GF_LOG_ERROR, local->op_errno,                  \
+                   AFR_MSG_SPLIT_BRAIN,                                        \
+                   "Failing %s on gfid %s: "                                   \
+                   "split-brain observed.",                                    \
+                   gf_fop_list[local->op], uuid_utoa(local->inode->gfid));     \
+    } while (0)
+
+#define AFR_ERROR_OUT_IF_FDCTX_INVALID(__fd, __this, __error, __label)         \
+    do {                                                                       \
+        afr_fd_ctx_t *__fd_ctx = NULL;                                         \
+        __fd_ctx = afr_fd_ctx_get(__fd, __this);                               \
+        if (__fd_ctx && __fd_ctx->is_fd_bad) {                                 \
+            __error = EBADF;                                                   \
+            goto __label;                                                      \
+        }                                                                      \
+    } while (0)
 
-typedef int (*afr_inode_refresh_cbk_t) (call_frame_t *frame, xlator_t *this, int err);
+typedef enum {
+    AFR_READ_POLICY_FIRST_UP,
+    AFR_READ_POLICY_GFID_HASH,
+    AFR_READ_POLICY_GFID_PID_HASH,
+    AFR_READ_POLICY_LESS_LOAD,
+    AFR_READ_POLICY_LEAST_LATENCY,
+    AFR_READ_POLICY_LOAD_LATENCY_HYBRID,
+} afr_read_hash_mode_t;
 
-typedef int (*afr_changelog_resume_t) (call_frame_t *frame, xlator_t *this);
+typedef enum {
+    AFR_FAV_CHILD_NONE,
+    AFR_FAV_CHILD_BY_SIZE,
+    AFR_FAV_CHILD_BY_CTIME,
+    AFR_FAV_CHILD_BY_MTIME,
+    AFR_FAV_CHILD_BY_MAJORITY,
+    AFR_FAV_CHILD_POLICY_MAX,
+} afr_favorite_child_policy;
 
-#define AFR_COUNT(array,max) ({int __i; int __res = 0; for (__i = 0; __i < max; __i++) if (array[__i]) __res++; __res;})
-#define AFR_INTERSECT(dst,src1,src2,max) ({int __i; for (__i = 0; __i < max; __i++) dst[__i] = src1[__i] && src2[__i];})
-#define AFR_CMP(a1,a2,len) ({int __cmp = 0; int __i; for (__i = 0; __i < len; __i++) if (a1[__i] != a2[__i]) { __cmp = 1; break;} __cmp;})
-#define AFR_IS_ARBITER_BRICK(priv, index) ((priv->arbiter_count == 1) && (index == ARBITER_BRICK_INDEX))
+typedef enum {
+    AFR_SELFHEAL_DATA_FULL = 0,
+    AFR_SELFHEAL_DATA_DIFF,
+    AFR_SELFHEAL_DATA_DYNAMIC,
+} afr_data_self_heal_type_t;
 
-#define AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(ret, errnum)                       \
-        do {                                                                   \
-                local->op_ret = ret;                                           \
-                local->op_errno = errnum;                                      \
-                if (local->op_errno == EIO)                                    \
-                        gf_msg (this->name, GF_LOG_ERROR, local->op_errno,     \
-                                AFR_MSG_SPLIT_BRAIN, "Failing %s on gfid %s: " \
-                                "split-brain observed.",                       \
-                                gf_fop_list[local->op],                        \
-                                uuid_utoa (local->inode->gfid));               \
-        } while (0)
+typedef enum {
+    AFR_CHILD_UNKNOWN = -1,
+    AFR_CHILD_ZERO,
+    AFR_CHILD_ONE,
+    AFR_CHILD_THIN_ARBITER,
+} afr_child_index;
 
 typedef enum {
-        AFR_FAV_CHILD_NONE,
-        AFR_FAV_CHILD_BY_SIZE,
-        AFR_FAV_CHILD_BY_CTIME,
-        AFR_FAV_CHILD_BY_MTIME,
-        AFR_FAV_CHILD_BY_MAJORITY,
-        AFR_FAV_CHILD_POLICY_MAX,
-} afr_favorite_child_policy;
+    TA_WAIT_FOR_NOTIFY_LOCK_REL, /*FOP came after notify domain lock upcall
+                                   notification and waiting for its release.*/
+    TA_GET_INFO_FROM_TA_FILE,    /*FOP needs post-op on ta file to get
+                                  *info about which brick is bad.*/
+    TA_INFO_IN_MEMORY_SUCCESS,   /*Bad brick info is in memory and fop failed
+                                  *on BAD brick - Success*/
+    TA_INFO_IN_MEMORY_FAILED,    /*Bad brick info is in memory and fop failed
+                                  *on GOOD brick - Failed*/
+    TA_SUCCESS,                  /*FOP succeeded on both data bricks.*/
+} afr_ta_fop_state_t;
 
 struct afr_nfsd {
-        gf_boolean_t     iamnfsd;
-        uint32_t         halo_max_latency_msec;
+    uint32_t halo_max_latency_msec;
+    gf_boolean_t iamnfsd;
 };
 
+typedef struct _afr_lk_heal_info {
+    fd_t *fd;
+    int32_t cmd;
+    struct gf_flock flock;
+    dict_t *xdata_req;
+    unsigned char *locked_nodes;
+    struct list_head pos;
+    gf_lkowner_t lk_owner;
+    pid_t pid;
+    int32_t *child_up_event_gen;
+    int32_t *child_down_event_gen;
+} afr_lk_heal_info_t;
+
 typedef struct _afr_private {
-        gf_lock_t lock;               /* to guard access to child_count, etc */
-        unsigned int child_count;     /* total number of children   */
-        unsigned int arbiter_count;   /*subset of child_count.
-                                        Has to be 0 or 1.*/
-        unsigned int thin_arbiter_count; /* 0 or 1 at the moment.*/
-
-        xlator_t **children;
-
-        inode_t *root_inode;
-        uuid_t ta_gfid; /*For thin arbiter.*/
-
-        unsigned char *child_up;
-        int64_t *child_latency;
-        unsigned char *local;
-
-        char **pending_key;
-
-        char         *data_self_heal;              /* on/off/open */
-        char *       data_self_heal_algorithm;    /* name of algorithm */
-        unsigned int data_self_heal_window_size;  /* max number of pipelined
-                                                     read/writes */
-
-        struct list_head heal_waiting; /*queue for files that need heal*/
-        uint32_t  heal_wait_qlen; /*configurable queue length for heal_waiting*/
-        int32_t  heal_waiters; /* No. of elements currently in wait queue.*/
-
-        struct list_head healing;/* queue for files that are undergoing
-                                    background heal*/
-        uint32_t  background_self_heal_count;/*configurable queue length for
-                                               healing queue*/
-        int32_t  healers;/* No. of elements currently undergoing background
-                          heal*/
-
-        gf_boolean_t metadata_self_heal;   /* on/off */
-        gf_boolean_t entry_self_heal;      /* on/off */
-
-	gf_boolean_t metadata_splitbrain_forced_heal; /* on/off */
-        int read_child;               /* read-subvolume */
-        unsigned int hash_mode;       /* for when read_child is not set */
-        gf_atomic_t *pending_reads; /*No. of pending read cbks per child.*/
-        int favorite_child;  /* subvolume to be preferred in resolving
-                                         split-brain cases */
-
-        afr_favorite_child_policy fav_child_policy;/*Policy to use for automatic
-                                                 resolution of split-brains.*/
-
-        unsigned int wait_count;      /* # of servers to wait for success */
-
-        gf_timer_t *timer;      /* launched when parent up is received */
-
-        gf_boolean_t      optimistic_change_log;
-        gf_boolean_t      eager_lock;
-        gf_boolean_t      pre_op_compat;      /* on/off */
-	uint32_t          post_op_delay_secs;
-        unsigned int      quorum_count;
-
-        char                   vol_uuid[UUID_SIZE + 1];
-        int32_t                *last_event;
-
-	/* @event_generation: Keeps count of number of events received which can
-	   potentially impact consistency decisions. The events are CHILD_UP
-	   and CHILD_DOWN, when we have to recalculate the freshness/staleness
-	   of copies to detect if changes had happened while the other server
-	   was down. CHILD_DOWN and CHILD_UP can also be received on network
-	   disconnect/reconnects and not necessarily server going down/up.
-	   Recalculating freshness/staleness on network events is equally
-	   important as we might have had a network split brain.
-	*/
-	uint32_t               event_generation;
-
-        gf_boolean_t           choose_local;
-        gf_boolean_t           did_discovery;
-        uint64_t               sh_readdir_size;
-        gf_boolean_t           ensure_durability;
-        char                   *sh_domain;
-	char                   *afr_dirty;
-        gf_boolean_t           halo_enabled;
-
-        uint32_t               halo_max_latency_msec;
-        uint32_t               halo_max_replicas;
-        uint32_t               halo_min_replicas;
-
-        afr_self_heald_t       shd;
-        struct afr_nfsd        nfsd;
-
-        gf_boolean_t           consistent_metadata;
-        uint64_t               spb_choice_timeout;
-        gf_boolean_t           need_heal;
-
-	/* pump dependencies */
-	void                   *pump_private;
-	gf_boolean_t           use_afr_in_pump;
-	char                   *locking_scheme;
-        gf_boolean_t           full_lock;
-        gf_boolean_t           esh_granular;
-        gf_boolean_t           consistent_io;
+    gf_lock_t lock;             /* to guard access to child_count, etc */
+    unsigned int child_count;   /* total number of children   */
+    unsigned int arbiter_count; /*subset of child_count.
+                                  Has to be 0 or 1.*/
+
+    xlator_t **children;
+
+    inode_t *root_inode;
+
+    int favorite_child; /* subvolume to be preferred in resolving
+                                    split-brain cases */
+    /* For thin-arbiter. */
+    uuid_t ta_gfid;
+    unsigned int thin_arbiter_count; /* 0 or 1 at the moment.*/
+    int ta_bad_child_index;
+    int ta_event_gen;
+    unsigned int ta_in_mem_txn_count;
+    unsigned int ta_on_wire_txn_count;
+    struct list_head ta_waitq;
+    struct list_head ta_onwireq;
+
+    unsigned char *anon_inode;
+    unsigned char *child_up;
+    unsigned char *halo_child_up;
+    int64_t *child_latency;
+    unsigned char *local;
+
+    char **pending_key;
+
+    afr_data_self_heal_type_t data_self_heal_algorithm;
+    unsigned int data_self_heal_window_size; /* max number of pipelined
+                                                read/writes */
+
+    struct list_head heal_waiting; /*queue for files that need heal*/
+    uint32_t heal_wait_qlen; /*configurable queue length for heal_waiting*/
+    int32_t heal_waiters;    /* No. of elements currently in wait queue.*/
+
+    struct list_head healing;            /* queue for files that are undergoing
+                                            background heal*/
+    uint32_t background_self_heal_count; /*configurable queue length for
+                                           healing queue*/
+    int32_t healers; /* No. of elements currently undergoing background
+                      heal*/
+
+    gf_boolean_t release_ta_notify_dom_lock;
+
+    gf_boolean_t metadata_self_heal; /* on/off */
+    gf_boolean_t entry_self_heal;    /* on/off */
+
+    gf_boolean_t metadata_splitbrain_forced_heal; /* on/off */
+    int read_child;                               /* read-subvolume */
+    gf_atomic_t *pending_reads; /*No. of pending read cbks per child.*/
+
+    gf_timer_t *timer; /* launched when parent up is received */
+
+    unsigned int wait_count; /* # of servers to wait for success */
+
+    unsigned char ta_child_up;
+    gf_boolean_t optimistic_change_log;
+    gf_boolean_t eager_lock;
+    gf_boolean_t pre_op_compat; /* on/off */
+    uint32_t post_op_delay_secs;
+    unsigned int quorum_count;
+
+    off_t ta_notify_dom_lock_offset;
+    afr_favorite_child_policy fav_child_policy; /*Policy to use for automatic
+                                              resolution of split-brains.*/
+    afr_read_hash_mode_t hash_mode; /* for when read_child is not set */
+
+    int32_t *last_event;
+
+    /* @event_generation: Keeps count of number of events received which can
+       potentially impact consistency decisions. The events are CHILD_UP
+       and CHILD_DOWN, when we have to recalculate the freshness/staleness
+       of copies to detect if changes had happened while the other server
+       was down. CHILD_DOWN and CHILD_UP can also be received on network
+       disconnect/reconnects and not necessarily server going down/up.
+       Recalculating freshness/staleness on network events is equally
+       important as we might have had a network split brain.
+    */
+    uint32_t event_generation;
+    char vol_uuid[UUID_SIZE + 1];
+
+    gf_boolean_t choose_local;
+    gf_boolean_t did_discovery;
+    gf_boolean_t ensure_durability;
+    gf_boolean_t halo_enabled;
+    gf_boolean_t consistent_metadata;
+    gf_boolean_t need_heal;
+    gf_boolean_t granular_locks;
+    uint64_t sh_readdir_size;
+    char *sh_domain;
+    char *afr_dirty;
+
+    uint64_t spb_choice_timeout;
+
+    afr_self_heald_t shd;
+    struct afr_nfsd nfsd;
+
+    uint32_t halo_max_latency_msec;
+    uint32_t halo_max_replicas;
+    uint32_t halo_min_replicas;
+
+    gf_boolean_t full_lock;
+    gf_boolean_t esh_granular;
+    gf_boolean_t consistent_io;
+    gf_boolean_t data_self_heal; /* on/off */
+    gf_boolean_t use_anon_inode;
+
+    /*For lock healing.*/
+    struct list_head saved_locks;
+    struct list_head lk_healq;
+
+    /*For anon-inode handling */
+    char anon_inode_name[NAME_MAX + 1];
+    char anon_gfid_str[UUID_SIZE + 1];
 } afr_private_t;
 
-
 typedef enum {
-        AFR_DATA_TRANSACTION,          /* truncate, write, ... */
-        AFR_METADATA_TRANSACTION,      /* chmod, chown, ... */
-        AFR_ENTRY_TRANSACTION,         /* create, rmdir, ... */
-        AFR_ENTRY_RENAME_TRANSACTION,  /* rename */
+    AFR_DATA_TRANSACTION,         /* truncate, write, ... */
+    AFR_METADATA_TRANSACTION,     /* chmod, chown, ... */
+    AFR_ENTRY_TRANSACTION,        /* create, rmdir, ... */
+    AFR_ENTRY_RENAME_TRANSACTION, /* rename */
 } afr_transaction_type;
 
 /*
@@ -193,914 +303,940 @@ typedef enum {
 */
 
 static inline int
-afr_index_for_transaction_type (afr_transaction_type type)
+afr_index_for_transaction_type(afr_transaction_type type)
 {
-        switch (type) {
-
+    switch (type) {
         case AFR_DATA_TRANSACTION:
-                return 0;
+            return 0;
 
         case AFR_METADATA_TRANSACTION:
-                return 1;
+            return 1;
 
         case AFR_ENTRY_TRANSACTION:
         case AFR_ENTRY_RENAME_TRANSACTION:
-                return 2;
-        }
+            return 2;
+    }
 
-        return -1;  /* make gcc happy */
+    return -1; /* make gcc happy */
 }
 
 static inline int
-afr_index_from_ia_type (ia_type_t type)
+afr_index_from_ia_type(ia_type_t type)
 {
-        switch (type) {
+    switch (type) {
         case IA_IFDIR:
-                return afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
+            return afr_index_for_transaction_type(AFR_ENTRY_TRANSACTION);
         case IA_IFREG:
-                return afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-        default: return -1;
-        }
+            return afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+        default:
+            return -1;
+    }
 }
 
 typedef struct {
-        loc_t                   loc;
-        char                    *basename;
-        unsigned char           *locked_nodes;
-        int                     locked_count;
+    struct gf_flock flock;
+    loc_t loc;
+    fd_t *fd;
+    char *basename;
+    unsigned char *locked_nodes;
+    int locked_count;
 
-} afr_entry_lockee_t;
+} afr_lockee_t;
 
 int
-afr_entry_lockee_cmp (const void *l1, const void *l2);
+afr_entry_lockee_cmp(const void *l1, const void *l2);
 
 typedef struct {
-        loc_t *lk_loc;
+    loc_t *lk_loc;
+
+    afr_lockee_t lockee[AFR_LOCKEE_COUNT_MAX];
 
-        int                     lockee_count;
-        afr_entry_lockee_t      lockee[AFR_LOCKEE_COUNT_MAX];
+    const char *lk_basename;
+    const char *lower_basename;
+    const char *higher_basename;
 
-        struct gf_flock flock;
-        const char *lk_basename;
-        const char *lower_basename;
-        const char *higher_basename;
-        char lower_locked;
-        char higher_locked;
+    unsigned char *lower_locked_nodes;
 
-        unsigned char *locked_nodes;
-        unsigned char *lower_locked_nodes;
+    afr_lock_cbk_t lock_cbk;
 
-        int32_t lock_count;
-        int32_t entrylk_lock_count;
+    int lockee_count;
 
-        int32_t lk_call_count;
-        int32_t lk_expected_count;
-        int32_t lk_attempted_count;
+    int32_t lk_call_count;
+    int32_t lk_expected_count;
+    int32_t lk_attempted_count;
 
-        int32_t lock_op_ret;
-        int32_t lock_op_errno;
-        afr_lock_cbk_t lock_cbk;
-        char *domain; /* Domain on which inode/entry lock/unlock in progress.*/
+    int32_t lock_op_ret;
+    int32_t lock_op_errno;
+    char *domain; /* Domain on which inode/entry lock/unlock in progress.*/
+    int32_t lock_count;
+    char lower_locked;
+    char higher_locked;
 } afr_internal_lock_t;
 
 struct afr_reply {
-	int	valid;
-	int32_t	op_ret;
-	int32_t	op_errno;
-	dict_t *xattr;/*For xattrop*/
-	dict_t *xdata;
-	struct iatt poststat;
-	struct iatt postparent;
-	struct iatt prestat;
-	struct iatt preparent;
-	struct iatt preparent2;
-	struct iatt postparent2;
-        /* For rchecksum */
-	uint8_t checksum[SHA256_DIGEST_LENGTH];
-        gf_boolean_t buf_has_zeroes;
-        gf_boolean_t fips_mode_rchecksum;
-        /* For lookup */
-        int8_t need_heal;
+    int valid;
+    int32_t op_ret;
+    dict_t *xattr; /*For xattrop*/
+    dict_t *xdata;
+    struct iatt poststat;
+    struct iatt postparent;
+    struct iatt prestat;
+    struct iatt preparent;
+    struct iatt preparent2;
+    struct iatt postparent2;
+    int32_t op_errno;
+    /* For rchecksum */
+    uint8_t checksum[SHA256_DIGEST_LENGTH];
+    gf_boolean_t buf_has_zeroes;
+    gf_boolean_t fips_mode_rchecksum;
+    /* For lookup */
+    int8_t need_heal;
 };
 
 typedef enum {
-        AFR_FD_NOT_OPENED,
-        AFR_FD_OPENED,
-        AFR_FD_OPENING
+    AFR_FD_NOT_OPENED,
+    AFR_FD_OPENED,
+    AFR_FD_OPENING
 } afr_fd_open_status_t;
 
 typedef struct {
-        afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
-        int flags;
-
-	/* the subvolume on which the latest sequence of readdirs (starting
-	   at offset 0) has begun. Till the next readdir request with 0 offset
-	   arrives, we continue to read off this subvol.
-	*/
-	int readdir_subvol;
+    afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
+    int flags;
+
+    /* the subvolume on which the latest sequence of readdirs (starting
+       at offset 0) has begun. Till the next readdir request with 0 offset
+       arrives, we continue to read off this subvol.
+    */
+    int readdir_subvol;
+    /* lock-healing related members. */
+    gf_boolean_t is_fd_bad;
+    afr_lk_heal_info_t *lk_heal_info;
+
 } afr_fd_ctx_t;
 
 typedef enum {
-        AFR_FOP_LOCK_PARALLEL,
-        AFR_FOP_LOCK_SERIAL,
-        AFR_FOP_LOCK_QUORUM_FAILED,
+    AFR_FOP_LOCK_PARALLEL,
+    AFR_FOP_LOCK_SERIAL,
+    AFR_FOP_LOCK_QUORUM_FAILED,
 } afr_fop_lock_state_t;
 
 typedef struct _afr_inode_lock_t {
-        unsigned int event_generation;
-        gf_boolean_t    release;
-        gf_boolean_t    acquired;
-        gf_timer_t        *delay_timer;
-        struct list_head  owners; /*Transactions that are performing fop*/
-        struct list_head  post_op;/*Transactions that are done with the fop
-                                   *So can not conflict with the fops*/
-        struct list_head waiting;/*Transaction that are waiting for
-                                   *conflicting transactions to complete*/
-        struct list_head frozen;/*Transactions that need to go as part of
-                                 * next batch of eager-lock*/
+    /* @num_inodelks:
+       Number of inodelks queried from the server, as queried through
+       xdata in FOPs. Currently, used to decide if eager-locking must be
+       temporarily disabled.
+    */
+    int32_t num_inodelks;
+    unsigned int event_generation;
+    gf_timer_t *delay_timer;
+    struct list_head owners;  /*Transactions that are performing fop*/
+    struct list_head post_op; /*Transactions that are done with the fop
+                               *So can not conflict with the fops*/
+    struct list_head waiting; /*Transaction that are waiting for
+                               *conflicting transactions to complete*/
+    struct list_head frozen;  /*Transactions that need to go as part of
+                               * next batch of eager-lock*/
+    gf_boolean_t release;
+    gf_boolean_t acquired;
 } afr_lock_t;
 
 typedef struct _afr_inode_ctx {
-        uint64_t        read_subvol;
-        uint64_t        write_subvol;
-        int             lock_count;
-        int             spb_choice;
-        gf_timer_t      *timer;
-        gf_boolean_t    need_refresh;
-        unsigned int *pre_op_done[AFR_NUM_CHANGE_LOGS];
-        int inherited[AFR_NUM_CHANGE_LOGS];
-        int on_disk[AFR_NUM_CHANGE_LOGS];
-
-        /* set if any write on this fd was a non stable write
-           (i.e, without O_SYNC or O_DSYNC)
-        */
-        gf_boolean_t      witnessed_unstable_write;
-
-        /* @open_fd_count:
-           Number of open FDs queried from the server, as queried through
-           xdata in FOPs. Currently, used to decide if eager-locking must be
-           temporarily disabled.
-        */
-        uint32_t        open_fd_count;
-        /*Only 2 types of transactions support eager-locks now. DATA/METADATA*/
-        afr_lock_t lock[2];
+    uint64_t read_subvol;
+    uint64_t write_subvol;
+    int lock_count;
+    int spb_choice;
+    gf_timer_t *timer;
+    unsigned int *pre_op_done[AFR_NUM_CHANGE_LOGS];
+    int inherited[AFR_NUM_CHANGE_LOGS];
+    int on_disk[AFR_NUM_CHANGE_LOGS];
+    /*Only 2 types of transactions support eager-locks now. DATA/METADATA*/
+    afr_lock_t lock[2];
+
+    /* @open_fd_count:
+       Number of open FDs queried from the server, as queried through
+       xdata in FOPs. Currently, used to decide if eager-locking must be
+       temporarily disabled.
+    */
+    uint32_t open_fd_count;
+    gf_boolean_t need_refresh;
+
+    /* set if any write on this fd was a non stable write
+       (i.e, without O_SYNC or O_DSYNC)
+    */
+    gf_boolean_t witnessed_unstable_write;
 } afr_inode_ctx_t;
 
-
 typedef struct _afr_local {
-	glusterfs_fop_t  op;
-        unsigned int call_count;
+    glusterfs_fop_t op;
+    unsigned int call_count;
 
-	/* @event_generation: copy of priv->event_generation taken at the
-	   time of starting the transaction. The copy is made so that we
-	   have a stable value through the various phases of the transaction.
-	*/
-	unsigned int event_generation;
+    /* @event_generation: copy of priv->event_generation taken at the
+       time of starting the transaction. The copy is made so that we
+       have a stable value through the various phases of the transaction.
+    */
+    unsigned int event_generation;
 
-        uint32_t     open_fd_count;
-        gf_boolean_t update_open_fd_count;
+    uint32_t open_fd_count;
+    int32_t num_inodelks;
 
-	gf_lkowner_t  saved_lk_owner;
+    int32_t op_ret;
+    int32_t op_errno;
 
-        int32_t op_ret;
-        int32_t op_errno;
+    int dirty[AFR_NUM_CHANGE_LOGS];
 
-        int32_t **pending;
+    int32_t **pending;
 
-	int dirty[AFR_NUM_CHANGE_LOGS];
+    loc_t loc;
+    loc_t newloc;
 
-        loc_t loc;
-        loc_t newloc;
+    fd_t *fd;
+    afr_fd_ctx_t *fd_ctx;
 
-        fd_t *fd;
-	afr_fd_ctx_t *fd_ctx;
+    /* @child_up: copy of priv->child_up taken at the time of transaction
+       start. The copy is taken so that we have a stable child_up array
+       through the phases of the transaction as priv->child_up[i] can keep
+       changing through time.
+    */
+    unsigned char *child_up;
 
-	/* @child_up: copy of priv->child_up taken at the time of transaction
-	   start. The copy is taken so that we have a stable child_up array
-	   through the phases of the transaction as priv->child_up[i] can keep
-	   changing through time.
-	*/
-        unsigned char *child_up;
+    /* @read_attempted:
+       array of flags representing subvolumes where read operations of
+       the read transaction have already been attempted. The array is
+       first pre-filled with down subvolumes, and as reads are performed
+       on other subvolumes, those are set as well. This way if the read
+       operation fails we do not retry on that subvolume again.
+    */
+    unsigned char *read_attempted;
 
-	/* @read_attempted:
-	   array of flags representing subvolumes where read operations of
-	   the read transaction have already been attempted. The array is
-	   first pre-filled with down subvolumes, and as reads are performed
-	   on other subvolumes, those are set as well. This way if the read
-	   operation fails we do not retry on that subvolume again.
-	*/
-	unsigned char *read_attempted;
+    /* @readfn:
 
-	/* @readfn:
+       pointer to function which will perform the read operation on a given
+       subvolume. Used in read transactions.
+    */
 
-	   pointer to function which will perform the read operation on a given
-	   subvolume. Used in read transactions.
-	*/
+    afr_read_txn_wind_t readfn;
 
-	afr_read_txn_wind_t readfn;
+    /* @inode:
 
-	/* @refreshed:
+       the inode on which the read txn is performed on. ref'ed and copied
+       from either fd->inode or loc.inode
+    */
 
-	   the inode was "refreshed" (i.e, pending xattrs from all subvols
-	   freshly inspected and inode ctx updated accordingly) as part of
-	   this transaction already.
-	*/
-	gf_boolean_t refreshed;
+    inode_t *inode;
 
-	/* @inode:
+    /* @parent[2]:
 
-	   the inode on which the read txn is performed on. ref'ed and copied
-	   from either fd->inode or loc.inode
-	*/
+       parent inode[s] on which directory transactions are performed.
+    */
 
-	inode_t *inode;
+    inode_t *parent;
+    inode_t *parent2;
 
-	/* @parent[2]:
+    /* @readable:
 
-	   parent inode[s] on which directory transactions are performed.
-	*/
+       array of flags representing servers from which a read can be
+       performed. This is the output of afr_inode_refresh()
+    */
+    unsigned char *readable;
+    unsigned char *readable2; /*For rename transaction*/
 
-	inode_t *parent;
-	inode_t *parent2;
+    afr_inode_refresh_cbk_t refreshfn;
 
-	/* @readable:
+    /* @refreshinode:
 
-	   array of flags representing servers from which a read can be
-	   performed. This is the output of afr_inode_refresh()
-	*/
-	unsigned char *readable;
-	unsigned char *readable2; /*For rename transaction*/
+       Inode currently getting refreshed.
+    */
+    inode_t *refreshinode;
 
-        int read_subvol; /* Current read subvolume */
+    dict_t *xattr_req;
 
-	afr_inode_refresh_cbk_t refreshfn;
+    dict_t *dict;
 
-	/* @refreshinode:
+    int read_subvol; /* Current read subvolume */
 
-	   Inode currently getting refreshed.
-	*/
-	inode_t *refreshinode;
+    int optimistic_change_log;
 
-        /*To handle setattr/setxattr on yet to be linked inode from dht*/
-        uuid_t  refreshgfid;
+    afr_internal_lock_t internal_lock;
 
-	/*
-	  @pre_op_compat:
+    /*To handle setattr/setxattr on yet to be linked inode from dht*/
+    uuid_t refreshgfid;
 
-	  compatibility mode of pre-op. send a separate pre-op and
-	  op operations as part of transaction, rather than combining
-	*/
+    /* @refreshed:
 
-	gf_boolean_t pre_op_compat;
+       the inode was "refreshed" (i.e, pending xattrs from all subvols
+       freshly inspected and inode ctx updated accordingly) as part of
+       this transaction already.
+    */
+    gf_boolean_t refreshed;
 
-        dict_t  *xattr_req;
+    gf_boolean_t update_num_inodelks;
+    gf_boolean_t update_open_fd_count;
 
-        afr_internal_lock_t internal_lock;
+    /*
+      @pre_op_compat:
 
-        dict_t  *dict;
+      compatibility mode of pre-op. send a separate pre-op and
+      op operations as part of transaction, rather than combining
+    */
 
-        int      optimistic_change_log;
+    gf_boolean_t pre_op_compat;
 
-	/* Is the current writev() going to perform a stable write?
-	   i.e, is fd->flags or @flags writev param have O_SYNC or
-	   O_DSYNC?
-	*/
-	gf_boolean_t      stable_write;
+    /* Is the current writev() going to perform a stable write?
+       i.e, is fd->flags or @flags writev param have O_SYNC or
+       O_DSYNC?
+    */
+    gf_boolean_t stable_write;
 
-	/* This write appended to the file. Nnot necessarily O_APPEND,
-	   just means the offset of write was at the end of file.
-	*/
-	gf_boolean_t      append_write;
+    /* This write appended to the file. Nnot necessarily O_APPEND,
+       just means the offset of write was at the end of file.
+    */
+    gf_boolean_t append_write;
 
-        /*
-          This struct contains the arguments for the "continuation"
-          (scheme-like) of fops
-        */
+    /*
+      This struct contains the arguments for the "continuation"
+      (scheme-like) of fops
+    */
 
+    struct {
         struct {
-                struct {
-                        gf_boolean_t needs_fresh_lookup;
-                        uuid_t gfid_req;
-                } lookup;
-
-                struct {
-                        unsigned char buf_set;
-                        struct statvfs buf;
-                } statfs;
-
-                struct {
-                        int32_t flags;
-                        fd_t *fd;
-                } open;
-
-                struct {
-                        int32_t cmd;
-                        struct gf_flock user_flock;
-                        struct gf_flock ret_flock;
-                        unsigned char *locked_nodes;
-                } lk;
-
-                /* inode read */
-
-                struct {
-                        int32_t mask;
-                        int last_index;  /* index of the child we tried previously */
-                } access;
-
-                struct {
-                        int last_index;
-                } stat;
-
-                struct {
-                        int last_index;
-                } fstat;
-
-                struct {
-                        size_t size;
-                        int last_index;
-                } readlink;
-
-                struct {
-                        char *name;
-                        int last_index;
-                        long xattr_len;
-                } getxattr;
-
-                struct {
-                        size_t size;
-                        off_t offset;
-                        int last_index;
-                        uint32_t flags;
-                } readv;
-
-                /* dir read */
-
-                struct {
-                        int success_count;
-                        int32_t op_ret;
-                        int32_t op_errno;
-
-                        uint32_t *checksum;
-                } opendir;
-
-                struct {
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        size_t size;
-                        off_t offset;
-                        dict_t *dict;
-                        gf_boolean_t failed;
-                        int last_index;
-                } readdir;
-                /* inode write */
-
-                struct {
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-                } inode_wfop; //common structure for all inode-write-fops
-
-                struct {
-                        int32_t op_ret;
-
-                        struct iovec *vector;
-                        struct iobref *iobref;
-                        int32_t count;
-                        off_t offset;
-                        uint32_t flags;
-                } writev;
-
-                struct {
-                        off_t offset;
-                } truncate;
-
-                struct {
-                        off_t offset;
-                } ftruncate;
-
-                struct {
-                        struct iatt in_buf;
-                        int32_t valid;
-                } setattr;
-
-                struct {
-                        struct iatt in_buf;
-                        int32_t valid;
-                } fsetattr;
-
-                struct {
-                        dict_t *dict;
-                        int32_t flags;
-                } setxattr;
-
-                struct {
-                        dict_t *dict;
-                        int32_t flags;
-                } fsetxattr;
-
-                struct {
-                        char *name;
-                } removexattr;
-
-                struct {
-                        dict_t *xattr;
-                        gf_xattrop_flags_t optype;
-                } xattrop;
-
-                /* dir write */
-
-                struct {
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                        struct iatt prenewparent;
-                        struct iatt postnewparent;
-                } dir_fop; //common structure for all dir fops
-
-                struct {
-                        fd_t *fd;
-                        dict_t *params;
-                        int32_t flags;
-                        mode_t mode;
-                } create;
-
-                struct {
-                        dev_t dev;
-                        mode_t mode;
-                        dict_t *params;
-                } mknod;
-
-                struct {
-                        int32_t mode;
-                        dict_t *params;
-                } mkdir;
-
-                struct {
-                        int flags;
-                } rmdir;
-
-                struct {
-                        dict_t *params;
-                        char *linkpath;
-                } symlink;
-
-		struct {
-			int32_t mode;
-			off_t offset;
-			size_t len;
-		} fallocate;
-
-		struct {
-			off_t offset;
-			size_t len;
-		} discard;
-
-                struct {
-                        off_t offset;
-                        off_t len;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-                } zerofill;
-
-                struct {
-                        char *volume;
-                        int32_t cmd;
-                        int32_t in_cmd;
-                        struct gf_flock in_flock;
-                        struct gf_flock flock;
-                        void *xdata;
-                } inodelk;
-
-                struct {
-                        char *volume;
-                        char *basename;
-                        entrylk_cmd in_cmd;
-                        entrylk_cmd cmd;
-                        entrylk_type type;
-                        void *xdata;
-                } entrylk;
-
-                struct {
-                        off_t offset;
-                        gf_seek_what_t what;
-                } seek;
-
-                struct {
-                        int32_t datasync;
-                } fsync;
-
-                struct {
-                        struct gf_lease user_lease;
-                        struct gf_lease ret_lease;
-                        unsigned char *locked_nodes;
-                } lease;
-
-
-        } cont;
+            struct statvfs buf;
+            unsigned char buf_set;
+        } statfs;
 
         struct {
-                off_t start, len;
+            fd_t *fd;
+            int32_t flags;
+        } open;
 
-                gf_boolean_t    eager_lock_on;
-                gf_boolean_t    do_eager_unlock;
+        struct {
+            struct gf_flock user_flock;
+            struct gf_flock ret_flock;
+            unsigned char *locked_nodes;
+            int32_t cmd;
+            /*For lock healing only.*/
+            unsigned char *dom_locked_nodes;
+            int32_t *dom_lock_op_ret;
+            int32_t *dom_lock_op_errno;
+            struct gf_flock *getlk_rsp;
+        } lk;
+
+        /* inode read */
 
-                char *basename;
-                char *new_basename;
+        struct {
+            int32_t mask;
+            int last_index; /* index of the child we tried previously */
+        } access;
 
-                loc_t parent_loc;
-                loc_t new_parent_loc;
+        struct {
+            int last_index;
+        } stat;
 
-                afr_transaction_type type;
+        struct {
+            int last_index;
+        } fstat;
 
-		/* stub to resume on destruction
-		   of the transaction frame */
-		call_stub_t      *resume_stub;
+        struct {
+            size_t size;
+            int last_index;
+        } readlink;
 
-		struct list_head  owner_list;
-                struct list_head  wait_list;
+        struct {
+            char *name;
+            long xattr_len;
+            int last_index;
+        } getxattr;
 
-                unsigned char   *pre_op;
+        struct {
+            size_t size;
+            off_t offset;
+            int last_index;
+            uint32_t flags;
+        } readv;
 
-                /* Changelog xattr dict for [f]xattrop*/
-                dict_t **changelog_xdata;
-                unsigned char *pre_op_sources;
+        /* dir read */
 
-		/* @failed_subvols: subvolumes on which a pre-op or a
-                    FOP failed. */
-                unsigned char   *failed_subvols;
+        struct {
+            uint32_t *checksum;
+            int success_count;
+            int32_t op_ret;
+            int32_t op_errno;
+        } opendir;
 
-		/* @dirtied: flag which indicates whether we set dirty flag
-		   in the OP. Typically true when we are performing operation
-		   on more than one subvol and optimistic changelog is disabled
+        struct {
+            int32_t op_ret;
+            int32_t op_errno;
+            size_t size;
+            off_t offset;
+            dict_t *dict;
+            int last_index;
+            gf_boolean_t failed;
+        } readdir;
+        /* inode write */
 
-		   A 'true' value set in @dirtied flag means an 'undirtying'
-		   has to be done in POST-OP phase.
-		*/
-		gf_boolean_t  dirtied;
+        struct {
+            struct iatt prebuf;
+            struct iatt postbuf;
+        } inode_wfop;  // common structure for all inode-write-fops
 
-		/* @inherited: flag which indicates that the dirty flags
-		   of the previous transaction were inherited
-		*/
-		gf_boolean_t  inherited;
+        struct {
+            struct iovec *vector;
+            struct iobref *iobref;
+            off_t offset;
+            int32_t op_ret;
+            int32_t count;
+            uint32_t flags;
+        } writev;
 
-		/*
-		  @no_uninherit: flag which indicates that a pre_op_uninherit()
-		  must _not_ be attempted (and returned as failure) always. This
-		  flag is set when a hard pre-op is performed, but not accounted
-		  for it in fd_ctx->on_disk[]. Such transactions are "isolated"
-		  from the pre-op piggybacking entirely and therefore uninherit
-		  must not be attempted.
-		*/
-		gf_boolean_t no_uninherit;
+        struct {
+            off_t offset;
+        } truncate;
 
-		/* @uninherit_done:
-		   @uninherit_value:
+        struct {
+            off_t offset;
+        } ftruncate;
 
-		   The above pair variables make pre_op_uninherit() idempotent.
-		   Both are FALSE initially. The first call to pre_op_uninherit
-		   sets @uninherit_done to TRUE and the return value to
-		   @uninherit_value. Further calls will check for @uninherit_done
-		   to be TRUE and if so will simply return @uninherit_value.
-		*/
-		gf_boolean_t uninherit_done;
-		gf_boolean_t uninherit_value;
+        struct {
+            struct iatt in_buf;
+            int32_t valid;
+        } setattr;
 
-                gf_boolean_t in_flight_sb; /* Indicator for occurrence of
-                                              split-brain while in the middle of
-                                              a txn. */
-                int32_t in_flight_sb_errno; /* This is where the cause of the
-                                               failure on the last good copy of
-                                               the file is stored.
-                                               */
+        struct {
+            struct iatt in_buf;
+            int32_t valid;
+        } fsetattr;
 
-		/* @changelog_resume: function to be called after changlogging
-		   (either pre-op or post-op) is done
-		*/
-		afr_changelog_resume_t changelog_resume;
+        struct {
+            dict_t *dict;
+            int32_t flags;
+        } setxattr;
 
-                call_frame_t *main_frame; /*Fop frame*/
-                call_frame_t *frame; /*Transaction frame*/
+        struct {
+            dict_t *dict;
+            int32_t flags;
+        } fsetxattr;
 
-                int (*wind) (call_frame_t *frame, xlator_t *this, int subvol);
+        struct {
+            char *name;
+        } removexattr;
 
-                int (*unwind) (call_frame_t *frame, xlator_t *this);
+        struct {
+            dict_t *xattr;
+            gf_xattrop_flags_t optype;
+        } xattrop;
 
-                /* post-op hook */
-        } transaction;
+        /* dir write */
 
-	syncbarrier_t barrier;
+        struct {
+            inode_t *inode;
+            struct iatt buf;
+            struct iatt preparent;
+            struct iatt postparent;
+            struct iatt prenewparent;
+            struct iatt postnewparent;
+        } dir_fop;  // common structure for all dir fops
 
-        /* extra data for fops */
-        dict_t         *xdata_req;
-        dict_t         *xdata_rsp;
+        struct {
+            fd_t *fd;
+            dict_t *params;
+            int32_t flags;
+            mode_t mode;
+        } create;
 
-        dict_t         *xattr_rsp; /*for [f]xattrop*/
+        struct {
+            dict_t *params;
+            dev_t dev;
+            mode_t mode;
+        } mknod;
 
-        mode_t          umask;
-        int             xflag;
-        gf_boolean_t    do_discovery;
-	struct afr_reply *replies;
+        struct {
+            dict_t *params;
+            int32_t mode;
+        } mkdir;
 
-        /* For  client side background heals. */
-        struct list_head healer;
-        call_frame_t *heal_frame;
+        struct {
+            dict_t *params;
+            char *linkpath;
+        } symlink;
 
-        gf_boolean_t need_full_crawl;
-        afr_fop_lock_state_t fop_lock_state;
+        struct {
+            off_t offset;
+            size_t len;
+            int32_t mode;
+        } fallocate;
 
-        gf_boolean_t is_read_txn;
-        afr_inode_ctx_t *inode_ctx;
-} afr_local_t;
+        struct {
+            off_t offset;
+            size_t len;
+        } discard;
+
+        struct {
+            off_t offset;
+            off_t len;
+            struct iatt prebuf;
+            struct iatt postbuf;
+        } zerofill;
+
+        struct {
+            char *volume;
+            int32_t cmd;
+            int32_t in_cmd;
+            struct gf_flock in_flock;
+            struct gf_flock flock;
+            void *xdata;
+        } inodelk;
+
+        struct {
+            char *volume;
+            char *basename;
+            void *xdata;
+            entrylk_cmd in_cmd;
+            entrylk_cmd cmd;
+            entrylk_type type;
+        } entrylk;
+
+        struct {
+            off_t offset;
+            gf_seek_what_t what;
+        } seek;
+
+        struct {
+            struct gf_lease user_lease;
+            struct gf_lease ret_lease;
+            unsigned char *locked_nodes;
+        } lease;
+
+        struct {
+            int flags;
+        } rmdir;
+
+        struct {
+            int32_t datasync;
+        } fsync;
+
+        struct {
+            uuid_t gfid_req;
+            gf_boolean_t needs_fresh_lookup;
+        } lookup;
+
+    } cont;
+
+    struct {
+        char *basename;
+        char *new_basename;
+
+        loc_t parent_loc;
+        loc_t new_parent_loc;
+
+        /* stub to resume on destruction
+           of the transaction frame */
+        call_stub_t *resume_stub;
+
+        struct list_head owner_list;
+        struct list_head wait_list;
 
+        unsigned char *pre_op;
+
+        /* Changelog xattr dict for [f]xattrop*/
+        dict_t **changelog_xdata;
+        unsigned char *pre_op_sources;
+
+        /* @failed_subvols: subvolumes on which a pre-op or a
+            FOP failed. */
+        unsigned char *failed_subvols;
+
+        call_frame_t *main_frame; /*Fop frame*/
+        call_frame_t *frame;      /*Transaction frame*/
+
+        int (*wind)(call_frame_t *frame, xlator_t *this, int subvol);
+
+        int (*unwind)(call_frame_t *frame, xlator_t *this);
+
+        off_t start, len;
+
+        afr_transaction_type type;
+
+        int32_t in_flight_sb_errno; /* This is where the cause of the
+                                       failure on the last good copy of
+                                       the file is stored.
+                                       */
+
+        /* @changelog_resume: function to be called after changlogging
+           (either pre-op or post-op) is done
+        */
+        afr_changelog_resume_t changelog_resume;
+
+        gf_boolean_t eager_lock_on;
+        gf_boolean_t do_eager_unlock;
+
+        /* @dirtied: flag which indicates whether we set dirty flag
+           in the OP. Typically true when we are performing operation
+           on more than one subvol and optimistic changelog is disabled
+
+           A 'true' value set in @dirtied flag means an 'undirtying'
+           has to be done in POST-OP phase.
+        */
+        gf_boolean_t dirtied;
+
+        /* @inherited: flag which indicates that the dirty flags
+           of the previous transaction were inherited
+        */
+        gf_boolean_t inherited;
+
+        /*
+          @no_uninherit: flag which indicates that a pre_op_uninherit()
+          must _not_ be attempted (and returned as failure) always. This
+          flag is set when a hard pre-op is performed, but not accounted
+          for it in fd_ctx->on_disk[]. Such transactions are "isolated"
+          from the pre-op piggybacking entirely and therefore uninherit
+          must not be attempted.
+        */
+        gf_boolean_t no_uninherit;
+
+        gf_boolean_t in_flight_sb; /* Indicator for occurrence of
+                                      split-brain while in the middle of
+                                      a txn. */
+
+        /* @uninherit_done:
+           @uninherit_value:
+
+           The above pair variables make pre_op_uninherit() idempotent.
+           Both are FALSE initially. The first call to pre_op_uninherit
+           sets @uninherit_done to TRUE and the return value to
+           @uninherit_value. Further calls will check for @uninherit_done
+           to be TRUE and if so will simply return @uninherit_value.
+        */
+        gf_boolean_t uninherit_done;
+        gf_boolean_t uninherit_value;
+
+        gf_boolean_t disable_delayed_post_op;
+    } transaction;
+
+    syncbarrier_t barrier;
+
+    /* extra data for fops */
+    dict_t *xdata_req;
+    dict_t *xdata_rsp;
+
+    dict_t *xattr_rsp; /*for [f]xattrop*/
+
+    mode_t umask;
+    int xflag;
+    struct afr_reply *replies;
+
+    /* For  client side background heals. */
+    struct list_head healer;
+    call_frame_t *heal_frame;
+
+    afr_inode_ctx_t *inode_ctx;
+
+    /*For thin-arbiter transactions.*/
+    int ta_failed_subvol;
+    int ta_event_gen;
+    struct list_head ta_waitq;
+    struct list_head ta_onwireq;
+    afr_ta_fop_state_t fop_state;
+    afr_fop_lock_state_t fop_lock_state;
+    gf_lkowner_t saved_lk_owner;
+    unsigned char read_txn_query_child;
+    unsigned char ta_child_up;
+    gf_boolean_t do_discovery;
+    gf_boolean_t need_full_crawl;
+    gf_boolean_t is_read_txn;
+    gf_boolean_t is_new_entry;
+} afr_local_t;
 
 typedef struct afr_spbc_timeout {
-        call_frame_t *frame;
-        gf_boolean_t d_spb;
-        gf_boolean_t m_spb;
-        loc_t        *loc;
-        int          spb_child_index;
+    call_frame_t *frame;
+    loc_t *loc;
+    int spb_child_index;
+    gf_boolean_t d_spb;
+    gf_boolean_t m_spb;
 } afr_spbc_timeout_t;
 
 typedef struct afr_spb_status {
-        call_frame_t *frame;
-        loc_t        *loc;
+    call_frame_t *frame;
+    loc_t *loc;
 } afr_spb_status_t;
 
 typedef struct afr_empty_brick_args {
-        call_frame_t *frame;
-        loc_t loc;
-        int empty_index;
-        char *op_type;
+    call_frame_t *frame;
+    char *op_type;
+    loc_t loc;
+    int empty_index;
 } afr_empty_brick_args_t;
 
 typedef struct afr_read_subvol_args {
-        ia_type_t ia_type;
-        uuid_t gfid;
+    ia_type_t ia_type;
+    uuid_t gfid;
 } afr_read_subvol_args_t;
 
 typedef struct afr_granular_esh_args {
-        fd_t *heal_fd;
-        xlator_t *xl;
-        call_frame_t *frame;
-        gf_boolean_t mismatch; /* flag to represent occurrence of type/gfid
-                                  mismatch */
+    fd_t *heal_fd;
+    xlator_t *xl;
+    call_frame_t *frame;
+    gf_boolean_t mismatch; /* flag to represent occurrence of type/gfid
+                              mismatch */
 } afr_granular_esh_args_t;
 
 int
-afr_inode_get_readable (call_frame_t *frame, inode_t *inode, xlator_t *this,
-                        unsigned char *readable, int *event_p, int type);
+afr_inode_get_readable(call_frame_t *frame, inode_t *inode, xlator_t *this,
+                       unsigned char *readable, int *event_p, int type);
 int
-afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
-			   unsigned char *data_subvols,
-			   unsigned char *metadata_subvols,
-			   int *event_generation);
+afr_inode_read_subvol_get(inode_t *inode, xlator_t *this,
+                          unsigned char *data_subvols,
+                          unsigned char *metadata_subvols,
+                          int *event_generation);
 int
-__afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
-			     unsigned char *data_subvols,
-			     unsigned char *metadata_subvols,
-			     int *event_generation);
+__afr_inode_read_subvol_get(inode_t *inode, xlator_t *this,
+                            unsigned char *data_subvols,
+                            unsigned char *metadata_subvols,
+                            int *event_generation);
 
 int
-__afr_inode_read_subvol_set (inode_t *inode, xlator_t *this,
-			     unsigned char *data_subvols,
-			     unsigned char *metadata_subvol,
-			     int event_generation);
+__afr_inode_read_subvol_set(inode_t *inode, xlator_t *this,
+                            unsigned char *data_subvols,
+                            unsigned char *metadata_subvol,
+                            int event_generation);
+int
+afr_inode_read_subvol_set(inode_t *inode, xlator_t *this,
+                          unsigned char *data_subvols,
+                          unsigned char *metadata_subvols,
+                          int event_generation);
+
 int
-afr_inode_read_subvol_set (inode_t *inode, xlator_t *this,
-			   unsigned char *data_subvols,
-			   unsigned char *metadata_subvols,
-			   int event_generation);
+__afr_inode_need_refresh_set(inode_t *inode, xlator_t *this);
 
 int
-afr_inode_event_gen_reset (inode_t *inode, xlator_t *this);
+afr_inode_need_refresh_set(inode_t *inode, xlator_t *this);
 
 int
-afr_read_subvol_select_by_policy (inode_t *inode, xlator_t *this,
-				  unsigned char *readable,
-                                  afr_read_subvol_args_t *args);
+afr_read_subvol_select_by_policy(inode_t *inode, xlator_t *this,
+                                 unsigned char *readable,
+                                 afr_read_subvol_args_t *args);
 
 int
-afr_inode_read_subvol_type_get (inode_t *inode, xlator_t *this,
-				unsigned char *readable, int *event_p,
-				int type);
+afr_inode_read_subvol_type_get(inode_t *inode, xlator_t *this,
+                               unsigned char *readable, int *event_p, int type);
 int
-afr_read_subvol_get (inode_t *inode, xlator_t *this, int *subvol_p,
-                     unsigned char *readables,
-		     int *event_p, afr_transaction_type type,
-                     afr_read_subvol_args_t *args);
+afr_read_subvol_get(inode_t *inode, xlator_t *this, int *subvol_p,
+                    unsigned char *readables, int *event_p,
+                    afr_transaction_type type, afr_read_subvol_args_t *args);
 
-#define afr_data_subvol_get(i, t, s, r, e, a) \
-	afr_read_subvol_get(i, t, s, r, e, AFR_DATA_TRANSACTION, a)
+#define afr_data_subvol_get(i, t, s, r, e, a)                                  \
+    afr_read_subvol_get(i, t, s, r, e, AFR_DATA_TRANSACTION, a)
 
-#define afr_metadata_subvol_get(i, t, s, r, e, a) \
-	afr_read_subvol_get(i, t, s, r, e, AFR_METADATA_TRANSACTION, a)
+#define afr_metadata_subvol_get(i, t, s, r, e, a)                              \
+    afr_read_subvol_get(i, t, s, r, e, AFR_METADATA_TRANSACTION, a)
 
 int
-afr_inode_refresh (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                   uuid_t gfid, afr_inode_refresh_cbk_t cbk);
+afr_inode_refresh(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                  uuid_t gfid, afr_inode_refresh_cbk_t cbk);
 
 int32_t
-afr_notify (xlator_t *this, int32_t event, void *data, void *data2);
+afr_notify(xlator_t *this, int32_t event, void *data, void *data2);
 
 int
-xattr_is_equal (dict_t *this, char *key1, data_t *value1, void *data);
+xattr_is_equal(dict_t *this, char *key1, data_t *value1, void *data);
 
 int
-afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
-                       loc_t *loc, char *basename, int child_count);
+afr_add_entry_lockee(afr_local_t *local, loc_t *loc, char *basename,
+                     int child_count);
+
+int
+afr_add_inode_lockee(afr_local_t *local, int child_count);
 
 void
-afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock);
+afr_lockees_cleanup(afr_internal_lock_t *int_lock);
 
 int
-afr_attempt_lock_recovery (xlator_t *this, int32_t child_index);
+afr_attempt_lock_recovery(xlator_t *this, int32_t child_index);
 
 int
-afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
-                       unsigned char *locked_nodes);
+afr_mark_locked_nodes(xlator_t *this, fd_t *fd, unsigned char *locked_nodes);
 
 void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner);
+afr_set_lk_owner(call_frame_t *frame, xlator_t *this, void *lk_owner);
 
 int
-afr_set_lock_number (call_frame_t *frame, xlator_t *this);
+afr_set_lock_number(call_frame_t *frame, xlator_t *this);
 
 int32_t
-afr_unlock (call_frame_t *frame, xlator_t *this);
-
-int
-afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this);
+afr_unlock(call_frame_t *frame, xlator_t *this);
 
 int
-afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this);
+afr_lock_nonblocking(call_frame_t *frame, xlator_t *this);
 
 int
-afr_blocking_lock (call_frame_t *frame, xlator_t *this);
+afr_blocking_lock(call_frame_t *frame, xlator_t *this);
 
 int
-afr_internal_lock_finish (call_frame_t *frame, xlator_t *this);
+afr_internal_lock_finish(call_frame_t *frame, xlator_t *this);
 
 int
-__afr_fd_ctx_set (xlator_t *this, fd_t *fd);
+__afr_fd_ctx_set(xlator_t *this, fd_t *fd);
 
 afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this);
+afr_fd_ctx_get(fd_t *fd, xlator_t *this);
 
 int
-afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno);
+afr_build_parent_loc(loc_t *parent, loc_t *child, int32_t *op_errno);
 
 int
-afr_locked_nodes_count (unsigned char *locked_nodes, int child_count);
+afr_locked_nodes_count(unsigned char *locked_nodes, int child_count);
 
 int
-afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                       gf_boolean_t *start_heal);
+afr_replies_interpret(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                      gf_boolean_t *start_heal);
 
 void
-afr_local_replies_wipe (afr_local_t *local, afr_private_t *priv);
+afr_local_replies_wipe(afr_local_t *local, afr_private_t *priv);
 
 void
-afr_local_cleanup (afr_local_t *local, xlator_t *this);
+afr_local_cleanup(afr_local_t *local, xlator_t *this);
 
 int
-afr_frame_return (call_frame_t *frame);
+afr_frame_return(call_frame_t *frame);
 
 int
-afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, dict_t *xdata);
+afr_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata);
 
 void
-afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this);
+afr_local_transaction_cleanup(afr_local_t *local, xlator_t *this);
 
 int
-afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd);
-
-#define AFR_STACK_UNWIND(fop, frame, op_ret, op_errno, params ...)\
-        do {                                                    \
-                afr_local_t *__local = NULL;                    \
-                xlator_t    *__this = NULL;                     \
-                int32_t     __op_ret   = 0;                     \
-                int32_t     __op_errno = 0;                     \
-                                                                \
-                __op_ret = op_ret;                              \
-                __op_errno = op_errno;                          \
-                if (frame) {                                    \
-                        __local = frame->local;                 \
-                        __this = frame->this;                   \
-                        afr_handle_inconsistent_fop (frame, &__op_ret,\
-                                                     &__op_errno);\
-                        if (__local && __local->is_read_txn) \
-                                afr_pending_read_decrement (__this->private, __local->read_subvol); \
-                        frame->local = NULL;                    \
-                }                                               \
-                                                                \
-                STACK_UNWIND_STRICT (fop, frame, __op_ret,      \
-                                     __op_errno, params);       \
-                if (__local) {                                  \
-                        afr_local_cleanup (__local, __this);    \
-                        mem_put (__local);                      \
-                }                                               \
-        } while (0)
-
-#define AFR_STACK_DESTROY(frame)                                \
-        do {                                                    \
-                afr_local_t *__local = NULL;                    \
-                xlator_t    *__this = NULL;                     \
-                __local = frame->local;                         \
-                __this = frame->this;                           \
-                frame->local = NULL;                            \
-                STACK_DESTROY (frame->root);                    \
-                if (__local) {                                  \
-                        afr_local_cleanup (__local, __this);    \
-                        mem_put (__local);                      \
-                }                                               \
-        } while (0);
-
-#define AFR_FRAME_INIT(frame, op_errno)				       \
-	({frame->local = mem_get0 (THIS->local_pool);		       \
-	if (afr_local_init (frame->local, THIS->private, &op_errno)) { \
-		afr_local_cleanup (frame->local, THIS);		       \
-		mem_put (frame->local);				       \
-		frame->local = NULL; };				       \
-	frame->local; })
-
-#define AFR_STACK_RESET(frame)                                         \
-        do {                                                           \
-                afr_local_t *__local = NULL;                           \
-                xlator_t    *__this = NULL;                            \
-                __local = frame->local;                                \
-                __this = frame->this;                                  \
-                frame->local = NULL;                                   \
-                int __opr;                                             \
-                STACK_RESET (frame->root);                             \
-                if (__local) {                                         \
-                        afr_local_cleanup (__local, __this);           \
-                        mem_put (__local);                             \
-                }                                                      \
-                AFR_FRAME_INIT (frame, __opr);                         \
-        } while (0)
+afr_cleanup_fd_ctx(xlator_t *this, fd_t *fd);
+
+#define AFR_STACK_UNWIND(fop, frame, op_ret, op_errno, params...)              \
+    do {                                                                       \
+        afr_local_t *__local = NULL;                                           \
+        xlator_t *__this = NULL;                                               \
+        int32_t __op_ret = 0;                                                  \
+        int32_t __op_errno = 0;                                                \
+                                                                               \
+        __op_ret = op_ret;                                                     \
+        __op_errno = op_errno;                                                 \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            __this = frame->this;                                              \
+            afr_handle_inconsistent_fop(frame, &__op_ret, &__op_errno);        \
+            if (__local && __local->is_read_txn)                               \
+                afr_pending_read_decrement(__this->private,                    \
+                                           __local->read_subvol);              \
+            if (__local && __local->xdata_req &&                               \
+                afr_is_lock_mode_mandatory(__local->xdata_req))                \
+                afr_dom_lock_release(frame);                                   \
+            frame->local = NULL;                                               \
+        }                                                                      \
+                                                                               \
+        STACK_UNWIND_STRICT(fop, frame, __op_ret, __op_errno, params);         \
+        if (__local) {                                                         \
+            afr_local_cleanup(__local, __this);                                \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0)
+
+#define AFR_STACK_DESTROY(frame)                                               \
+    do {                                                                       \
+        afr_local_t *__local = NULL;                                           \
+        xlator_t *__this = NULL;                                               \
+        __local = frame->local;                                                \
+        __this = frame->this;                                                  \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        if (__local) {                                                         \
+            afr_local_cleanup(__local, __this);                                \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0);
+
+#define AFR_FRAME_INIT(frame, op_errno)                                        \
+    ({                                                                         \
+        frame->local = mem_get0(THIS->local_pool);                             \
+        if (afr_local_init(frame->local, frame->this->private, &op_errno)) {   \
+            afr_local_cleanup(frame->local, frame->this);                      \
+            mem_put(frame->local);                                             \
+            frame->local = NULL;                                               \
+        };                                                                     \
+        frame->local;                                                          \
+    })
+
+#define AFR_STACK_RESET(frame)                                                 \
+    do {                                                                       \
+        afr_local_t *__local = NULL;                                           \
+        xlator_t *__this = NULL;                                               \
+        __local = frame->local;                                                \
+        __this = frame->this;                                                  \
+        frame->local = NULL;                                                   \
+        int __opr;                                                             \
+        STACK_RESET(frame->root);                                              \
+        if (__local) {                                                         \
+            afr_local_cleanup(__local, __this);                                \
+            mem_put(__local);                                                  \
+        }                                                                      \
+        AFR_FRAME_INIT(frame, __opr);                                          \
+    } while (0)
 
 /* allocate and return a string that is the basename of argument */
 static inline char *
-AFR_BASENAME (const char *str)
+AFR_BASENAME(const char *str)
 {
-        char *__tmp_str = NULL;
-        char *__basename_str = NULL;
-        __tmp_str = gf_strdup (str);
-        __basename_str = gf_strdup (basename (__tmp_str));
-        GF_FREE (__tmp_str);
-        return __basename_str;
+    char *__tmp_str = NULL;
+    char *__basename_str = NULL;
+    __tmp_str = gf_strdup(str);
+    __basename_str = gf_strdup(basename(__tmp_str));
+    GF_FREE(__tmp_str);
+    return __basename_str;
 }
 
 call_frame_t *
-afr_copy_frame (call_frame_t *base);
+afr_copy_frame(call_frame_t *base);
 
 int
-afr_transaction_local_init (afr_local_t *local, xlator_t *this);
+afr_transaction_local_init(afr_local_t *local, xlator_t *this);
 
 int32_t
-afr_marker_getxattr (call_frame_t *frame, xlator_t *this,
-                     loc_t *loc, const char *name,afr_local_t *local, afr_private_t *priv );
+afr_marker_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, afr_local_t *local, afr_private_t *priv);
 
 int
-afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno);
+afr_local_init(afr_local_t *local, afr_private_t *priv, int32_t *op_errno);
 
 int
-afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count);
+afr_internal_lock_init(afr_internal_lock_t *lk, size_t child_count);
 
 int
-afr_higher_errno (int32_t old_errno, int32_t new_errno);
+afr_higher_errno(int32_t old_errno, int32_t new_errno);
 
 int
-afr_final_errno (afr_local_t *local, afr_private_t *priv);
+afr_final_errno(afr_local_t *local, afr_private_t *priv);
 
 int
-afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req);
+afr_xattr_req_prepare(xlator_t *this, dict_t *xattr_req);
 
 void
-afr_fix_open (fd_t *fd, xlator_t *this);
+afr_fix_open(fd_t *fd, xlator_t *this);
 
 afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this);
+afr_fd_ctx_get(fd_t *fd, xlator_t *this);
 
 void
-afr_set_low_priority (call_frame_t *frame);
+afr_set_low_priority(call_frame_t *frame);
 int
-afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
-                      int flags);
+afr_child_fd_ctx_set(xlator_t *this, fd_t *fd, int32_t child, int flags);
 
 void
-afr_matrix_cleanup (int32_t **pending, unsigned int m);
+afr_matrix_cleanup(int32_t **pending, unsigned int m);
 
-int32_t**
-afr_matrix_create (unsigned int m, unsigned int n);
+int32_t **
+afr_matrix_create(unsigned int m, unsigned int n);
 
-int**
-afr_mark_pending_changelog (afr_private_t *priv, unsigned char *pending,
-                            dict_t *xattr, ia_type_t iat);
+int **
+afr_mark_pending_changelog(afr_private_t *priv, unsigned char *pending,
+                           dict_t *xattr, ia_type_t iat);
 
 void
-afr_filter_xattrs (dict_t *xattr);
+afr_filter_xattrs(dict_t *xattr);
 
 /*
  * Special value indicating we should use the "auto" quorum method instead of
@@ -1109,121 +1245,179 @@ afr_filter_xattrs (dict_t *xattr);
 #define AFR_QUORUM_AUTO INT_MAX
 
 int
-afr_fd_report_unstable_write (xlator_t *this, afr_local_t *local);
+afr_fd_report_unstable_write(xlator_t *this, afr_local_t *local);
 
 gf_boolean_t
-afr_fd_has_witnessed_unstable_write (xlator_t *this, inode_t *inode);
+afr_fd_has_witnessed_unstable_write(xlator_t *this, inode_t *inode);
 
 void
-afr_reply_wipe (struct afr_reply *reply);
+afr_reply_wipe(struct afr_reply *reply);
 
 void
-afr_replies_wipe (struct afr_reply *replies, int count);
+afr_replies_wipe(struct afr_reply *replies, int count);
 
 gf_boolean_t
-afr_xattrs_are_equal (dict_t *dict1, dict_t *dict2);
+afr_xattrs_are_equal(dict_t *dict1, dict_t *dict2);
 
 gf_boolean_t
-afr_is_xattr_ignorable (char *key);
+afr_is_xattr_ignorable(char *key);
 
 int
-afr_get_heal_info (call_frame_t *frame, xlator_t *this, loc_t *loc);
+afr_get_heal_info(call_frame_t *frame, xlator_t *this, loc_t *loc);
 
 int
 afr_heal_splitbrain_file(call_frame_t *frame, xlator_t *this, loc_t *loc);
 
 int
-afr_get_split_brain_status (void *opaque);
+afr_get_split_brain_status(void *opaque);
 
 int
-afr_get_split_brain_status_cbk (int ret, call_frame_t *frame, void *opaque);
+afr_get_split_brain_status_cbk(int ret, call_frame_t *frame, void *opaque);
 
 int
-afr_inode_split_brain_choice_set (inode_t *inode, xlator_t *this,
-                                  int spb_choice);
+afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this,
+                                 int spb_choice);
 int
-afr_inode_split_brain_choice_get (inode_t *inode, xlator_t *this,
-                                  int *spb_choice);
+afr_split_brain_read_subvol_get(inode_t *inode, xlator_t *this,
+                                call_frame_t *frame, int *spb_subvol);
 int
-afr_get_child_index_from_name (xlator_t *this, char *name);
+afr_get_child_index_from_name(xlator_t *this, char *name);
 
 int
-afr_is_split_brain (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                    uuid_t gfid, gf_boolean_t *d_spb, gf_boolean_t *m_spb);
+afr_is_split_brain(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                   uuid_t gfid, gf_boolean_t *d_spb, gf_boolean_t *m_spb);
 int
-afr_spb_choice_timeout_cancel (xlator_t *this, inode_t *inode);
+afr_spb_choice_timeout_cancel(xlator_t *this, inode_t *inode);
 
 int
-afr_set_split_brain_choice (int ret, call_frame_t *frame, void *opaque);
+afr_set_split_brain_choice(int ret, call_frame_t *frame, void *opaque);
 
 gf_boolean_t
-afr_get_need_heal (xlator_t *this);
+afr_get_need_heal(xlator_t *this);
 
 void
-afr_set_need_heal (xlator_t *this, afr_local_t *local);
+afr_set_need_heal(xlator_t *this, afr_local_t *local);
 
 int
-afr_selfheal_data_open (xlator_t *this, inode_t *inode, fd_t **fd);
+afr_selfheal_data_open(xlator_t *this, inode_t *inode, fd_t **fd);
 
 int
-afr_get_msg_id (char *op_type);
+afr_get_msg_id(char *op_type);
 
 int
-afr_set_in_flight_sb_status (xlator_t *this, call_frame_t *frame,
-                             inode_t *inode);
+afr_set_in_flight_sb_status(xlator_t *this, call_frame_t *frame,
+                            inode_t *inode);
 
 int32_t
-afr_quorum_errno (afr_private_t *priv);
+afr_quorum_errno(afr_private_t *priv);
 
 gf_boolean_t
-afr_is_consistent_io_possible (afr_local_t *local, afr_private_t *priv,
-                               int32_t *op_errno);
+afr_is_consistent_io_possible(afr_local_t *local, afr_private_t *priv,
+                              int32_t *op_errno);
 void
-afr_handle_inconsistent_fop (call_frame_t *frame, int32_t *op_ret,
-                             int32_t *op_errno);
+afr_handle_inconsistent_fop(call_frame_t *frame, int32_t *op_ret,
+                            int32_t *op_errno);
 
 void
-afr_inode_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
+afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                      struct iatt *postbuf, dict_t *xdata);
 void
-afr_process_post_writev (call_frame_t *frame, xlator_t *this);
+afr_process_post_writev(call_frame_t *frame, xlator_t *this);
 
 void
-afr_writev_unwind (call_frame_t *frame, xlator_t *this);
+afr_writev_unwind(call_frame_t *frame, xlator_t *this);
 
 void
-afr_writev_copy_outvars (call_frame_t *src_frame, call_frame_t *dst_frame);
+afr_writev_copy_outvars(call_frame_t *src_frame, call_frame_t *dst_frame);
 
 void
-afr_update_uninodelk (afr_local_t *local, afr_internal_lock_t *int_lock,
-                    int32_t child_index);
-int
-afr_is_inodelk_transaction(afr_transaction_type type);
-
+afr_update_uninodelk(afr_local_t *local, afr_internal_lock_t *int_lock,
+                     int32_t child_index);
 afr_fd_ctx_t *
-__afr_fd_ctx_get (fd_t *fd, xlator_t *this);
+__afr_fd_ctx_get(fd_t *fd, xlator_t *this);
 
 gf_boolean_t
-afr_is_inode_refresh_reqd (inode_t *inode, xlator_t *this,
-                           int event_gen1, int event_gen2);
+afr_is_inode_refresh_reqd(inode_t *inode, xlator_t *this, int event_gen1,
+                          int event_gen2);
 
 int
-afr_serialize_xattrs_with_delimiter (call_frame_t *frame, xlator_t *this,
-                                     char *buf, const char *default_str,
-                                     int32_t *serz_len, char delimiter);
+afr_serialize_xattrs_with_delimiter(call_frame_t *frame, xlator_t *this,
+                                    char *buf, const char *default_str,
+                                    int32_t *serz_len, char delimiter);
 gf_boolean_t
-afr_is_symmetric_error (call_frame_t *frame, xlator_t *this);
+afr_is_symmetric_error(call_frame_t *frame, xlator_t *this);
+
+int
+__afr_inode_ctx_get(xlator_t *this, inode_t *inode, afr_inode_ctx_t **ctx);
+
+uint64_t
+afr_write_subvol_get(call_frame_t *frame, xlator_t *this);
 
 int
-__afr_inode_ctx_get (xlator_t *this, inode_t *inode, afr_inode_ctx_t **ctx);
+afr_write_subvol_set(call_frame_t *frame, xlator_t *this);
 
 int
-afr_write_subvol_set (call_frame_t *frame, xlator_t *this);
+afr_write_subvol_reset(call_frame_t *frame, xlator_t *this);
 
 int
-afr_write_subvol_reset (call_frame_t *frame, xlator_t *this);
+afr_set_inode_local(xlator_t *this, afr_local_t *local, inode_t *inode);
 
 int
-afr_set_inode_local (xlator_t *this, afr_local_t *local, inode_t *inode);
+afr_fill_ta_loc(xlator_t *this, loc_t *loc, gf_boolean_t is_gfid_based_fop);
+
+int
+afr_ta_post_op_lock(xlator_t *this, loc_t *loc);
+
+int
+afr_ta_post_op_unlock(xlator_t *this, loc_t *loc);
+
+gf_boolean_t
+afr_is_pending_set(xlator_t *this, dict_t *xdata, int type);
+
+int
+__afr_get_up_children_count(afr_private_t *priv);
+
+call_frame_t *
+afr_ta_frame_create(xlator_t *this);
+
+gf_boolean_t
+afr_ta_has_quorum(afr_private_t *priv, afr_local_t *local);
+
+void
+afr_ta_lock_release_synctask(xlator_t *this);
+
+void
+afr_ta_locked_priv_invalidate(afr_private_t *priv);
+
+gf_boolean_t
+afr_lookup_has_quorum(call_frame_t *frame,
+                      const unsigned int up_children_count);
+
+void
+afr_mark_new_entry_changelog(call_frame_t *frame, xlator_t *this);
+
+void
+afr_handle_replies_quorum(call_frame_t *frame, xlator_t *this);
+
+gf_boolean_t
+afr_ta_dict_contains_pending_xattr(dict_t *dict, afr_private_t *priv,
+                                   int child);
+
+void
+afr_selfheal_childup(xlator_t *this, afr_private_t *priv);
+
+gf_boolean_t
+afr_is_lock_mode_mandatory(dict_t *xdata);
+
+void
+afr_dom_lock_release(call_frame_t *frame);
+
+void
+afr_fill_success_replies(afr_local_t *local, afr_private_t *priv,
+                         unsigned char *replies);
+
+gf_boolean_t
+afr_is_private_directory(afr_private_t *priv, uuid_t pargfid, const char *name,
+                         pid_t pid);
 #endif /* __AFR_H__ */
diff --git a/xlators/cluster/dht/src/Makefile.am b/xlators/cluster/dht/src/Makefile.am
index 525a214c24a..56f1f2ad7c8 100644
--- a/xlators/cluster/dht/src/Makefile.am
+++ b/xlators/cluster/dht/src/Makefile.am
@@ -1,7 +1,4 @@
 xlator_LTLIBRARIES = dht.la nufa.la switch.la
-if BUILD_GFDB
-   xlator_LTLIBRARIES += tier.la
-endif
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
@@ -16,41 +13,28 @@ dht_la_SOURCES = $(dht_common_source) dht.c
 
 nufa_la_SOURCES = $(dht_common_source) nufa.c
 switch_la_SOURCES = $(dht_common_source) switch.c
-tier_la_SOURCES = $(dht_common_source) tier.c tier-common.c
 
-dht_la_LDFLAGS = -module -avoid-version -export-symbols \
-	$(top_srcdir)/xlators/cluster/dht/src/dht.sym
+dht_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 dht_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-nufa_la_LDFLAGS = -module -avoid-version -export-symbols \
-	$(top_srcdir)/xlators/cluster/dht/src/nufa.sym
+nufa_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 nufa_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-switch_la_LDFLAGS = -module -avoid-version -export-symbols \
-	$(top_srcdir)/xlators/cluster/dht/src/switch.sym
+switch_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 switch_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-tier_la_LDFLAGS = -module -avoid-version -export-symbols \
-	$(top_srcdir)/xlators/cluster/dht/src/tier.sym
-tier_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
 noinst_HEADERS = dht-common.h dht-mem-types.h dht-messages.h \
-	dht-lock.h tier-common.h tier.h \
-	$(top_builddir)/xlators/lib/src/libxlator.h
+	dht-lock.h $(top_builddir)/xlators/lib/src/libxlator.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/libglusterfs/src/gfdb \
 	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
 	-I$(top_srcdir)/rpc/rpc-lib/src \
 	-I$(top_srcdir)/xlators/lib/src \
 	-DDATADIR=\"$(localstatedir)\" \
-	-DLIBDIR=\"$(libdir)\" \
-	-DLIBGFDB_VERSION=\"$(LIBGFDB_VERSION)\"
+	-DLIBDIR=\"$(libdir)\"
 
 CLEANFILES =
 
-EXTRA_DIST = dht.sym nufa.sym switch.sym tier.sym
-
 uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/distribute.so
 
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index c436ad48d9d..8ba0cc4c732 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -8,268 +8,209 @@
   cases as published by the Free Software Foundation.
 */
 
-
 /* TODO: add NS locking */
 
-#include "glusterfs.h"
-#include "xlator.h"
 #include "libxlator.h"
 #include "dht-common.h"
 #include "dht-lock.h"
-#include "defaults.h"
-#include "byte-order.h"
-#include "quota-common-utils.h"
-#include "upcall-utils.h"
+#include <glusterfs/byte-order.h>
+#include <glusterfs/quota-common-utils.h>
+#include <glusterfs/upcall-utils.h>
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
+#include <glusterfs/common-utils.h>
 
 #include <sys/time.h>
 #include <libgen.h>
 #include <signal.h>
 
-int run_defrag = 0;
-
-
-
-int dht_link2 (xlator_t *this, xlator_t *dst_node, call_frame_t *frame,
-               int ret);
-
-int
-dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame,
-                  int ret);
-
-int
-dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame,
-               int ret);
-
+static int
+dht_rmdir_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, gf_dirent_t *entries,
+                       dict_t *xdata);
 
-int
-dht_rmdir_readdirp_do (call_frame_t *readdirp_frame, xlator_t *this);
+static int
+dht_link2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
 
+static int
+dht_set_dir_xattr_req(xlator_t *this, loc_t *loc, dict_t *xattr_req);
 
-int
-dht_common_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *dict,
-                        dict_t *xdata);
+static int
+dht_lookup_everywhere_done(call_frame_t *frame, xlator_t *this);
 
+static int
+dht_common_mark_mdsxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, dict_t *xdata);
 
-/* Sets the blocks and size values to fixed values. This is to be called
- * only for dirs. The caller is responsible for checking the type
- */
-int32_t dht_set_fixed_dir_stat (struct iatt *stat)
-{
-        if (stat) {
-                stat->ia_blocks = DHT_DIR_STAT_BLOCKS;
-                stat->ia_size = DHT_DIR_STAT_SIZE;
-                return 0;
-        }
-        return -1;
-}
+static int
+dht_rmdir_unlock(call_frame_t *frame, xlator_t *this);
 
+static const char *dht_dbg_vxattrs[] = {DHT_DBG_HASHED_SUBVOL_PATTERN, NULL};
 
-/* Set both DHT_IATT_IN_XDATA_KEY and DHT_MODE_IN_XDATA_KEY
- * Use DHT_MODE_IN_XDATA_KEY if available. Else fall back to
- * DHT_IATT_IN_XDATA_KEY
- */
-int dht_request_iatt_in_xdata (xlator_t *this, dict_t *xattr_req)
+/* Check the xdata to make sure EBADF has been set by client xlator */
+int32_t
+dht_check_remote_fd_failed_error(dht_local_t *local, int op_ret, int op_errno)
 {
-        int ret = -1;
-
-        ret = dict_set_int8 (xattr_req, DHT_MODE_IN_XDATA_KEY, 1);
-        ret = dict_set_int8 (xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
-
-        /* At least one call succeeded */
-        return ret;
+    if (op_ret == -1 && (op_errno == EBADF || op_errno == EBADFD) &&
+        !(local->fd_checked)) {
+        return 1;
+    }
+    return 0;
 }
 
-
-/* Get both DHT_IATT_IN_XDATA_KEY and DHT_MODE_IN_XDATA_KEY
- * Use DHT_MODE_IN_XDATA_KEY if available, else fall back to
- * DHT_IATT_IN_XDATA_KEY
- * This will return a dummy iatt with only the mode and type set
+/* Sets the blocks and size values to fixed values. This is to be called
+ * only for dirs. The caller is responsible for checking the type
  */
-int dht_read_iatt_from_xdata (xlator_t *this, dict_t *xdata,
-                              struct iatt *stbuf)
+int32_t
+dht_set_fixed_dir_stat(struct iatt *stat)
 {
-        int ret = -1;
-        int32_t mode = 0;
-
-        ret = dict_get_int32 (xdata, DHT_MODE_IN_XDATA_KEY, &mode);
-
-        if (ret) {
-                ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY,
-                                    (void **)&stbuf);
-        } else {
-                stbuf->ia_prot = ia_prot_from_st_mode (mode);
-                stbuf->ia_type = ia_type_from_st_mode (mode);
-        }
-
-        return ret;
+    if (stat) {
+        stat->ia_blocks = DHT_DIR_STAT_BLOCKS;
+        stat->ia_size = DHT_DIR_STAT_SIZE;
+        return 0;
+    }
+    return -1;
 }
 
-
-
-int
-dht_rmdir_unlock (call_frame_t *frame, xlator_t *this);
-
-char *xattrs_to_heal[] = {
-        "user.",
-        POSIX_ACL_ACCESS_XATTR,
-        POSIX_ACL_DEFAULT_XATTR,
-        QUOTA_LIMIT_KEY,
-        QUOTA_LIMIT_OBJECTS_KEY,
-        GF_SELINUX_XATTR_KEY,
-        NULL
-};
-
-
-char *dht_dbg_vxattrs[] = {
-        DHT_DBG_HASHED_SUBVOL_KEY,
-        NULL
-};
-
 /* Return true if key exists in array
-*/
+ */
 static gf_boolean_t
-dht_match_xattr (const char *key)
+dht_match_xattr(const char *key)
 {
-        return gf_get_index_by_elem (xattrs_to_heal, (char *)key) >= 0;
-}
+    char **xattrs_to_heal = get_xattrs_to_heal();
 
-int
-dht_aggregate_quota_xattr (dict_t *dst, char *key, data_t *value)
-{
-        int              ret            = -1;
-        quota_meta_t    *meta_dst       = NULL;
-        quota_meta_t    *meta_src       = NULL;
-        int64_t         *size           = NULL;
-        int64_t          dst_dir_count  = 0;
-        int64_t          src_dir_count  = 0;
-
-        if (value == NULL) {
-                gf_msg ("dht", GF_LOG_WARNING, 0,
-                        DHT_MSG_DATA_NULL, "data value is NULL");
-                ret = -1;
-                goto out;
-        }
+    return gf_get_index_by_elem(xattrs_to_heal, (char *)key) >= 0;
+}
 
-        ret = dict_get_bin (dst, key, (void **)&meta_dst);
+static int
+dht_aggregate_quota_xattr(dict_t *dst, char *key, data_t *value)
+{
+    int ret = -1;
+    quota_meta_t *meta_dst = NULL;
+    quota_meta_t *meta_src = NULL;
+    int64_t *size = NULL;
+    int64_t dst_dir_count = 0;
+    int64_t src_dir_count = 0;
+
+    if (value == NULL) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DATA_NULL,
+               "data value is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_bin(dst, key, (void **)&meta_dst);
+    if (ret < 0) {
+        meta_dst = GF_CALLOC(1, sizeof(quota_meta_t), gf_common_quota_meta_t);
+        if (meta_dst == NULL) {
+            gf_msg("dht", GF_LOG_WARNING, ENOMEM, DHT_MSG_NO_MEMORY,
+                   "Memory allocation failed");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_bin(dst, key, meta_dst, sizeof(quota_meta_t));
         if (ret < 0) {
-                meta_dst = GF_CALLOC (1, sizeof (quota_meta_t),
-                                      gf_common_quota_meta_t);
-                if (meta_dst == NULL) {
-                        gf_msg ("dht", GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_NO_MEMORY,
-                                "Memory allocation failed");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_bin (dst, key, meta_dst,
-                                    sizeof (quota_meta_t));
-                if (ret < 0) {
-                        gf_msg ("dht", GF_LOG_WARNING, EINVAL,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "dht aggregate dict set failed");
-                        GF_FREE (meta_dst);
-                        ret = -1;
-                        goto out;
-                }
+            gf_msg("dht", GF_LOG_WARNING, EINVAL, DHT_MSG_DICT_SET_FAILED,
+                   "dht aggregate dict set failed");
+            GF_FREE(meta_dst);
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (value->len > sizeof (int64_t)) {
-                meta_src = data_to_bin (value);
+    if (value->len > sizeof(int64_t)) {
+        meta_src = data_to_bin(value);
 
-                meta_dst->size = hton64 (ntoh64 (meta_dst->size) +
-                                         ntoh64 (meta_src->size));
-                meta_dst->file_count = hton64 (ntoh64 (meta_dst->file_count) +
-                                               ntoh64 (meta_src->file_count));
+        meta_dst->size = hton64(ntoh64(meta_dst->size) +
+                                ntoh64(meta_src->size));
+        meta_dst->file_count = hton64(ntoh64(meta_dst->file_count) +
+                                      ntoh64(meta_src->file_count));
 
-                if (value->len > (2 * sizeof (int64_t))) {
-                        dst_dir_count = ntoh64 (meta_dst->dir_count);
-                        src_dir_count = ntoh64 (meta_src->dir_count);
+        if (value->len > (2 * sizeof(int64_t))) {
+            dst_dir_count = ntoh64(meta_dst->dir_count);
+            src_dir_count = ntoh64(meta_src->dir_count);
 
-                        if (src_dir_count > dst_dir_count)
-                                meta_dst->dir_count = meta_src->dir_count;
-                } else {
-                        meta_dst->dir_count = 0;
-                }
+            if (src_dir_count > dst_dir_count)
+                meta_dst->dir_count = meta_src->dir_count;
         } else {
-                size = data_to_bin (value);
-                meta_dst->size = hton64 (ntoh64 (meta_dst->size) +
-                                         ntoh64 (*size));
+            meta_dst->dir_count = 0;
         }
+    } else {
+        size = data_to_bin(value);
+        meta_dst->size = hton64(ntoh64(meta_dst->size) + ntoh64(*size));
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
-int add_opt(char **optsp, const char *opt)
-{
-        char *newopts = NULL;
-        unsigned oldsize = 0;
-        unsigned newsize = 0;
-
-        if (*optsp == NULL)
-                newopts = gf_strdup (opt);
-        else {
-                oldsize = strlen (*optsp);
-                newsize = oldsize + 1 + strlen (opt) + 1;
-                newopts = GF_REALLOC (*optsp, newsize);
-                if (newopts)
-                        sprintf (newopts + oldsize, ",%s", opt);
-        }
-        if (newopts == NULL) {
-                gf_msg ("dht", GF_LOG_WARNING, 0,
-                        DHT_MSG_NO_MEMORY,
-                        "Error to add choices in buffer in add_opt");
-                return -1;
-        }
-        *optsp = newopts;
-        return 0;
+static int
+add_opt(char **optsp, const char *opt)
+{
+    char *newopts = NULL;
+    unsigned oldsize = 0;
+    unsigned newsize = 0;
+
+    if (*optsp == NULL)
+        newopts = gf_strdup(opt);
+    else {
+        oldsize = strlen(*optsp);
+        newsize = oldsize + 1 + strlen(opt) + 1;
+        newopts = GF_REALLOC(*optsp, newsize);
+        if (newopts)
+            sprintf(newopts + oldsize, ",%s", opt);
+    }
+    if (newopts == NULL) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+               "Error to add choices in buffer in add_opt");
+        return -1;
+    }
+    *optsp = newopts;
+    return 0;
 }
 
 /* Return Choice list from Split brain status */
 static char *
-getChoices (const char *value)
+getChoices(const char *value)
 {
-        int i = 0;
-        char *ptr = NULL;
-        char *tok = NULL;
-        char *result = NULL;
-        char *newval = NULL;
+    int i = 0;
+    char *ptr = NULL;
+    char *tok = NULL;
+    char *result = NULL;
+    char *newval = NULL;
 
-        ptr = strstr (value, "Choices:");
-        if (!ptr) {
-                result = ptr;
-                goto out;
-        }
+    ptr = strstr(value, "Choices:");
+    if (!ptr) {
+        result = ptr;
+        goto out;
+    }
 
-        newval = gf_strdup (ptr);
-        if (!newval) {
-                result = newval;
-                goto out;
-        }
+    newval = gf_strdup(ptr);
+    if (!newval) {
+        result = newval;
+        goto out;
+    }
 
-        tok = strtok (newval, ":");
-        if (!tok) {
-                result = tok;
-                goto out;
-        }
+    tok = strtok(newval, ":");
+    if (!tok) {
+        result = tok;
+        goto out;
+    }
 
-        while (tok) {
-                i++;
-                if (i == 2)
-                        break;
-                tok = strtok (NULL, ":");
-        }
+    while (tok) {
+        i++;
+        if (i == 2)
+            break;
+        tok = strtok(NULL, ":");
+    }
 
-        result = gf_strdup (tok);
+    result = gf_strdup(tok);
 
 out:
-        if (newval)
-                GF_FREE (newval);
+    if (newval)
+        GF_FREE(newval);
 
-        return result;
+    return result;
 }
 
 /* This function prepare a list of choices for key
@@ -281,245 +222,227 @@ out:
 
 */
 
-int
-dht_aggregate_split_brain_xattr (dict_t *dst, char *key, data_t *value)
-{
+static int
+dht_aggregate_split_brain_xattr(dict_t *dst, char *key, data_t *value)
+{
+    int ret = 0;
+    char *oldvalue = NULL;
+    char *old_choice = NULL;
+    char *new_choice = NULL;
+    char *full_choice = NULL;
+    char *status = NULL;
+
+    if (value == NULL) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DATA_NULL,
+               "GF_AFR_SBRAIN_STATUS value is NULL");
+        ret = -1;
+        goto out;
+    }
 
-        int              ret            = 0;
-        char            *oldvalue       = NULL;
-        char            *old_choice     = NULL;
-        char            *new_choice     = NULL;
-        char            *full_choice    = NULL;
-        char            *status         = NULL;
+    ret = dict_get_str(dst, key, &oldvalue);
+    if (ret)
+        goto out;
+
+    /* skip code that is irrelevant if !oldvalue */
+    if (!oldvalue)
+        goto out;
 
-        if (value == NULL) {
-                gf_msg ("dht", GF_LOG_WARNING, 0,
-                        DHT_MSG_DATA_NULL,
-                        "GF_AFR_SBRAIN_STATUS value is NULL");
+    if (strstr(oldvalue, "not")) {
+        gf_msg_debug("dht", 0, "Need to update split-brain status in dict");
+        ret = -1;
+        goto out;
+    }
+    if (strstr(oldvalue, "metadata-split-brain:yes") &&
+        (strstr(oldvalue, "data-split-brain:no"))) {
+        if (strstr(value->data, "not")) {
+            gf_msg_debug("dht", 0, "No need to update split-brain status");
+            ret = 0;
+            goto out;
+        }
+        if (strstr(value->data, "yes") &&
+            (strncmp(oldvalue, value->data, strlen(oldvalue)))) {
+            old_choice = getChoices(oldvalue);
+            if (!old_choice) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to get choices");
                 ret = -1;
                 goto out;
-        }
+            }
 
-        ret = dict_get_str (dst, key, &oldvalue);
-        if (ret)
+            ret = add_opt(&full_choice, old_choice);
+            if (ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to add choices");
+                ret = -1;
                 goto out;
+            }
 
-        if (oldvalue && (strstr (oldvalue, "not"))) {
-                gf_msg_debug ("dht", 0,
-                              "Need to update split-brain status in dict");
+            new_choice = getChoices(value->data);
+            if (!new_choice) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to get choices");
                 ret = -1;
                 goto out;
-        }
-        if (oldvalue && (strstr (oldvalue, "metadata-split-brain:yes"))
-                     && (strstr (oldvalue, "data-split-brain:no"))) {
-                if (strstr (value->data, "not")) {
-                        gf_msg_debug ("dht", 0,
-                                      "No need to update split-brain status");
-                        ret = 0;
-                        goto out;
-                }
-                if (strstr (value->data, "yes") &&
-                        (strncmp (oldvalue, value->data, strlen(oldvalue)))) {
-                        old_choice = getChoices (oldvalue);
-                        if (!old_choice) {
-                                gf_msg ("dht", GF_LOG_WARNING, 0,
-                                        DHT_MSG_NO_MEMORY,
-                                        "Error to get choices");
-                                ret = -1;
-                                goto out;
-                        }
-
-                        ret = add_opt (&full_choice, old_choice);
-                        if (ret) {
-                                gf_msg ("dht", GF_LOG_WARNING, 0,
-                                         DHT_MSG_NO_MEMORY,
-                                         "Error to add choices");
-                                ret = -1;
-                                goto out;
-                        }
-
-                        new_choice = getChoices (value->data);
-                        if (!new_choice) {
-                                gf_msg ("dht", GF_LOG_WARNING, 0,
-                                        DHT_MSG_NO_MEMORY,
-                                        "Error to get choices");
-                                ret = -1;
-                                goto out;
-                        }
+            }
 
-                        ret = add_opt (&full_choice, new_choice);
-                        if (ret) {
-                                gf_msg ("dht", GF_LOG_WARNING, 0,
-                                       DHT_MSG_NO_MEMORY,
-                                       "Error to add choices ");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = gf_asprintf (&status,
-                                           "data-split-brain:%s    "
-                                           "metadata-split-brain:%s   Choices:%s",
-                                           "no", "yes", full_choice);
-
-                        if (-1 == ret) {
-                                gf_msg ("dht", GF_LOG_WARNING, 0,
-                                                 DHT_MSG_NO_MEMORY,
-                                                "Error to prepare status ");
-                                        goto out;
-                                }
-                        ret = dict_set_dynstr (dst, key, status);
-                        if (ret) {
-                                gf_msg ("dht", GF_LOG_WARNING, 0,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "Failed to set full choice");
-                        }
-                }
+            ret = add_opt(&full_choice, new_choice);
+            if (ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to add choices ");
+                ret = -1;
+                goto out;
+            }
+            ret = gf_asprintf(&status,
+                              "data-split-brain:%s    "
+                              "metadata-split-brain:%s   Choices:%s",
+                              "no", "yes", full_choice);
+
+            if (-1 == ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to prepare status ");
+                goto out;
+            }
+            ret = dict_set_dynstr(dst, key, status);
+            if (ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                       "Failed to set full choice");
+            }
         }
+    }
 
 out:
-        if (old_choice)
-                GF_FREE (old_choice);
-        if (new_choice)
-                GF_FREE (new_choice);
-        if (full_choice)
-                GF_FREE (full_choice);
+    if (old_choice)
+        GF_FREE(old_choice);
+    if (new_choice)
+        GF_FREE(new_choice);
+    if (full_choice)
+        GF_FREE(full_choice);
 
-        return ret;
+    return ret;
 }
 
-
-
-int
-dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
+static int
+dht_aggregate(dict_t *this, char *key, data_t *value, void *data)
 {
-        dict_t          *dst            = NULL;
-        int32_t          ret            = -1;
-        data_t          *dict_data      = NULL;
+    dict_t *dst = NULL;
+    int32_t ret = -1;
+    data_t *dict_data = NULL;
 
-        dst = data;
+    dst = data;
 
-        /* compare split brain xattr only */
-        if (strcmp (key, GF_AFR_SBRAIN_STATUS) == 0) {
-                ret = dht_aggregate_split_brain_xattr(dst, key, value);
+    /* compare split brain xattr only */
+    if (strcmp(key, GF_AFR_SBRAIN_STATUS) == 0) {
+        ret = dht_aggregate_split_brain_xattr(dst, key, value);
+        if (!ret)
+            goto out;
+    } else if (strcmp(key, QUOTA_SIZE_KEY) == 0) {
+        ret = dht_aggregate_quota_xattr(dst, key, value);
+        if (ret) {
+            gf_msg("dht", GF_LOG_WARNING, 0,
+                   DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED,
+                   "Failed to aggregate quota xattr");
+        }
+        goto out;
+    } else if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
+        ret = gf_get_min_stime(THIS, dst, key, value);
+        goto out;
+    } else {
+        /* compare user xattrs only */
+        if (!strncmp(key, "user.", SLEN("user."))) {
+            ret = dict_lookup(dst, key, &dict_data);
+            if (!ret && dict_data && value) {
+                ret = is_data_equal(dict_data, value);
                 if (!ret)
-                        goto out;
-        } else if (strcmp (key, QUOTA_SIZE_KEY) == 0) {
-                ret = dht_aggregate_quota_xattr (dst, key, value);
-                if (ret) {
-                        gf_msg ("dht", GF_LOG_WARNING, 0,
-                                DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED,
-                                "Failed to aggregate quota xattr");
-                }
-                goto out;
-        } else if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
-                ret = gf_get_min_stime (THIS, dst, key, value);
-                goto out;
-        } else {
-                /* compare user xattrs only */
-                if (!strncmp (key, "user.", strlen ("user."))) {
-                        ret = dict_lookup (dst, key, &dict_data);
-                        if (!ret && dict_data && value) {
-                                ret = is_data_equal (dict_data, value);
-                                if (!ret)
-                                        gf_msg_debug ("dht", 0,
-                                                      "xattr mismatch for %s",
-                                                      key);
-                        }
-                }
+                    gf_msg_debug("dht", 0, "xattr mismatch for %s", key);
+            }
         }
+    }
 
-        ret = dict_set (dst, key, value);
-        if (ret) {
-                gf_msg ("dht", GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value: key = %s",
-                        key);
-        }
+    ret = dict_set(dst, key, value);
+    if (ret) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s", key);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-
-void
-dht_aggregate_xattr (dict_t *dst, dict_t *src)
+static void
+dht_aggregate_xattr(dict_t *dst, dict_t *src)
 {
-        if ((dst == NULL) || (src == NULL)) {
-                goto out;
-        }
+    if ((dst == NULL) || (src == NULL)) {
+        goto out;
+    }
 
-        dict_foreach (src, dht_aggregate, dst);
+    dict_foreach(src, dht_aggregate, dst);
 out:
-        return;
+    return;
 }
 
 /* Code to save hashed subvol on inode ctx as a mds subvol
-*/
+ */
 int
-dht_inode_ctx_mdsvol_set (inode_t *inode, xlator_t *this, xlator_t *mds_subvol)
-{
-        dht_inode_ctx_t         *ctx            = NULL;
-        int                      ret            = -1;
-        uint64_t                 ctx_int        = 0;
-        gf_boolean_t             ctx_free       = _gf_false;
-
-
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this , &ctx_int);
-                if (ctx_int) {
-                        ctx = (dht_inode_ctx_t *)ctx_int;
-                        ctx->mds_subvol = mds_subvol;
-                } else {
-                        ctx = GF_CALLOC (1, sizeof(*ctx), gf_dht_mt_inode_ctx_t);
-                        if (!ctx)
-                                goto unlock;
-                        ctx->mds_subvol = mds_subvol;
-                        ctx_free        = _gf_true;
-                        ctx_int = (long) ctx;
-                        ret =  __inode_ctx_set (inode, this, &ctx_int);
-                }
-        }
+dht_inode_ctx_mdsvol_set(inode_t *inode, xlator_t *this, xlator_t *mds_subvol)
+{
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+    uint64_t ctx_int = 0;
+    gf_boolean_t ctx_free = _gf_false;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctx_int);
+        if (ctx_int) {
+            ctx = (dht_inode_ctx_t *)(uintptr_t)ctx_int;
+            ctx->mds_subvol = mds_subvol;
+        } else {
+            ctx = GF_CALLOC(1, sizeof(*ctx), gf_dht_mt_inode_ctx_t);
+            if (!ctx)
+                goto unlock;
+            ctx->mds_subvol = mds_subvol;
+            ctx_free = _gf_true;
+            ctx_int = (long)ctx;
+            ret = __inode_ctx_set(inode, this, &ctx_int);
+        }
+    }
 unlock:
-        UNLOCK (&inode->lock);
-        if (ret && ctx_free)
-                GF_FREE (ctx);
-        return ret;
+    UNLOCK(&inode->lock);
+    if (ret && ctx_free)
+        GF_FREE(ctx);
+    return ret;
 }
 
 /*Code to get mds subvol from inode ctx */
 
 int
-dht_inode_ctx_mdsvol_get (inode_t *inode, xlator_t *this, xlator_t **mdsvol)
+dht_inode_ctx_mdsvol_get(inode_t *inode, xlator_t *this, xlator_t **mdsvol)
 {
-        dht_inode_ctx_t         *ctx            = NULL;
-        int                      ret            = -1;
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        if (!mdsvol)
-                return ret;
+    if (!mdsvol)
+        return ret;
 
-        if (__is_root_gfid(inode->gfid)) {
-                (*mdsvol) = FIRST_CHILD (this);
-                return 0;
-        }
+    if (__is_root_gfid(inode->gfid)) {
+        (*mdsvol) = FIRST_CHILD(this);
+        return 0;
+    }
 
-        ret = dht_inode_ctx_get (inode, this, &ctx);
+    ret = dht_inode_ctx_get(inode, this, &ctx);
 
-        if (!ret && ctx) {
-                if (ctx->mds_subvol) {
-                        *mdsvol = ctx->mds_subvol;
-                        ret = 0;
-                } else {
-                        ret = -1;
-                }
+    if (!ret && ctx) {
+        if (ctx->mds_subvol) {
+            *mdsvol = ctx->mds_subvol;
+            ret = 0;
+        } else {
+            ret = -1;
         }
+    }
 
-        return ret;
+    return ret;
 }
 
-
-
-
-
 /* TODO:
    - use volumename in xattr instead of "dht"
    - use NS locks
@@ -527,293 +450,332 @@ dht_inode_ctx_mdsvol_get (inode_t *inode, xlator_t *this, xlator_t **mdsvol)
    - complete linkfile selfheal
 */
 
-
-int
-dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this,
-                         int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_lookup_selfheal_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        dht_conf_t   *conf   = NULL;
-        int           ret = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
 
-        local = frame->local;
-        conf  = this->private;
-        ret = op_ret;
+    local = frame->local;
+    conf = this->private;
+    ret = op_ret;
 
-        FRAME_SU_UNDO (frame, dht_local_t);
+    FRAME_SU_UNDO(frame, dht_local_t);
 
-        if (ret == 0) {
-                layout = local->selfheal.layout;
-                ret = dht_layout_set (this, local->inode, layout);
-        }
+    if (ret == 0) {
+        layout = local->selfheal.layout;
+        ret = dht_layout_set(this, local->inode, layout);
+    }
 
-        dht_inode_ctx_time_update (local->inode, this, &local->stbuf, 1);
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           &local->postparent, 1);
-        }
+    dht_inode_ctx_time_update(local->inode, this, &local->stbuf, 1);
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, &local->postparent,
+                                  1);
+    }
 
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        dht_set_fixed_dir_stat (&local->postparent);
-        /* Delete mds xattr at the time of STACK UNWIND */
-        GF_REMOVE_INTERNAL_XATTR (conf->mds_xattr_key, local->xattr);
+    DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+    dht_set_fixed_dir_stat(&local->postparent);
+    /* Delete mds xattr at the time of STACK UNWIND */
+    GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
 
-        DHT_STACK_UNWIND (lookup, frame, ret, local->op_errno, local->inode,
-                          &local->stbuf, local->xattr, &local->postparent);
+    DHT_STACK_UNWIND(lookup, frame, ret, local->op_errno, local->inode,
+                     &local->stbuf, local->xattr, &local->postparent);
 
 out:
-        return ret;
+    return ret;
 }
 
-int
-dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
-{
-        dht_local_t     *local           = NULL;
-        dht_local_t     *heal_local      = NULL;
-        call_frame_t    *main_frame      = NULL;
-        call_frame_t    *heal_frame      = NULL;
-        int              op_errno        = 0;
-        int              ret             = -1;
-        dht_layout_t    *layout          = NULL;
-        dht_conf_t      *conf            = NULL;
-        uint32_t         vol_commit_hash = 0;
-        xlator_t        *source          = NULL;
-        int              heal_path       = 0;
-        int              error_while_marking_mds   = 0;
-        int              i               = 0;
-        loc_t            loc             = {0 };
-        int8_t           is_read_only    = 0, layout_anomalies = 0;
-        char             gfid_local[GF_UUID_BUF_SIZE] = {0};
-
-        local = discover_frame->local;
-        layout = local->layout;
-        conf = this->private;
-        gf_uuid_unparse(local->gfid, gfid_local);
-
-        LOCK(&discover_frame->lock);
-        {
-                main_frame = local->main_frame;
-                local->main_frame = NULL;
-        }
-        UNLOCK(&discover_frame->lock);
-
-        if (!main_frame)
-                return 0;
-
-        /* Code to update all extended attributed from
-           subvol to local->xattr on that internal xattr has found
-        */
-        if (conf->subvolume_cnt == 1)
-                local->need_xattr_heal = 0;
-        if (local->need_xattr_heal && (local->mds_xattr)) {
-                dht_dir_set_heal_xattr (this, local, local->xattr,
-                                        local->mds_xattr, NULL, NULL);
-                dict_unref (local->mds_xattr);
-                local->mds_xattr = NULL;
-        }
+static int
+dht_discover_complete(xlator_t *this, call_frame_t *discover_frame)
+{
+    dht_local_t *local = NULL;
+    dht_local_t *heal_local = NULL;
+    call_frame_t *main_frame = NULL;
+    call_frame_t *heal_frame = NULL;
+    int op_errno = 0;
+    int ret = -1;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    uint32_t vol_commit_hash = 0;
+    xlator_t *source = NULL;
+    int heal_path = 0;
+    int error_while_marking_mds = 0;
+    int i = 0;
+    loc_t loc = {0};
+    int8_t is_read_only = 0, layout_anomalies = 0;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+
+    local = discover_frame->local;
+    layout = local->layout;
+    conf = this->private;
+    gf_uuid_unparse(local->gfid, gfid_local);
+
+    LOCK(&discover_frame->lock);
+    {
+        main_frame = local->main_frame;
+        local->main_frame = NULL;
+    }
+    UNLOCK(&discover_frame->lock);
+
+    if (!main_frame)
+        return 0;
+
+    /* Code to update all extended attributed from
+       subvol to local->xattr on that internal xattr has found
+    */
+    if (conf->subvolume_cnt == 1)
+        local->need_xattr_heal = 0;
+    if (local->need_xattr_heal && (local->mds_xattr)) {
+        dht_dir_set_heal_xattr(this, local, local->xattr, local->mds_xattr,
+                               NULL, NULL);
+        dict_unref(local->mds_xattr);
+        local->mds_xattr = NULL;
+    }
+
+    ret = dict_get_int8(local->xattr_req, QUOTA_READ_ONLY_KEY, &is_read_only);
+    if (ret < 0)
+        gf_msg_debug(this->name, 0, "key = %s not present in dict",
+                     QUOTA_READ_ONLY_KEY);
+
+    if (local->file_count && local->dir_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_FILE_TYPE_MISMATCH,
+               "path %s exists as a file on one subvolume "
+               "and directory on another. "
+               "Please fix it manually",
+               local->loc.path);
+        op_errno = EIO;
+        goto out;
+    }
 
-        ret = dict_get_int8 (local->xattr_req, QUOTA_READ_ONLY_KEY,
-                             &is_read_only);
-        if (ret < 0)
-                gf_msg_debug (this->name, 0, "key = %s not present in dict",
-                              QUOTA_READ_ONLY_KEY);
-
-        if (local->file_count && local->dir_count) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_FILE_TYPE_MISMATCH,
-                        "path %s exists as a file on one subvolume "
-                        "and directory on another. "
-                        "Please fix it manually",
-                        local->loc.path);
-                op_errno = EIO;
-                goto out;
+    if (local->cached_subvol) {
+        ret = dht_layout_preset(this, local->cached_subvol, local->inode);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_SET_FAILED,
+                   "failed to set layout for subvolume %s",
+                   local->cached_subvol ? local->cached_subvol->name : "<nil>");
+            op_errno = EINVAL;
+            goto out;
+        }
+    } else {
+        ret = dht_layout_normalize(this, &local->loc, layout);
+        if ((ret < 0) || ((ret > 0) && (local->op_ret != 0))) {
+            /* either the layout is incorrect or the directory is
+             * not found even in one subvolume.
+             */
+            gf_msg_debug(this->name, 0,
+                         "normalizing failed on %s "
+                         "(overlaps/holes present: %s, "
+                         "ENOENT errors: %d)",
+                         local->loc.path, (ret < 0) ? "yes" : "no",
+                         (ret > 0) ? ret : 0);
+            layout_anomalies = 1;
+        } else if (local->inode) {
+            dht_layout_set(this, local->inode, layout);
+        }
+    }
+
+    if (!conf->vch_forced) {
+        ret = dict_get_uint32(local->xattr, conf->commithash_xattr_name,
+                              &vol_commit_hash);
+        if (ret == 0) {
+            conf->vol_commit_hash = vol_commit_hash;
         }
+    }
 
-        if (local->cached_subvol) {
-                ret = dht_layout_preset (this, local->cached_subvol,
-                                         local->inode);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_LAYOUT_SET_FAILED,
-                                "failed to set layout for subvolume %s",
-                                local->cached_subvol ? local->cached_subvol->name : "<nil>");
-                        op_errno = EINVAL;
-                        goto out;
-                }
-        } else {
-                ret = dht_layout_normalize (this, &local->loc, layout);
-                if ((ret < 0) || ((ret > 0) && (local->op_ret != 0))) {
-                        /* either the layout is incorrect or the directory is
-                         * not found even in one subvolume.
-                         */
-                        gf_msg_debug (this->name, 0,
-                                      "normalizing failed on %s "
-                                      "(overlaps/holes present: %s, "
-                                      "ENOENT errors: %d)", local->loc.path,
-                                      (ret < 0) ? "yes" : "no", (ret > 0) ? ret : 0);
-                        layout_anomalies = 1;
-                } else if (local->inode) {
-                        dht_layout_set (this, local->inode, layout);
-                }
-        }
+    if (IA_ISDIR(local->stbuf.ia_type) && !is_read_only) {
+        for (i = 0; i < layout->cnt; i++) {
+            if (!source && !layout->list[i].err)
+                source = layout->list[i].xlator;
+            if (layout->list[i].err == ENOENT ||
+                layout->list[i].err == ESTALE) {
+                heal_path = 1;
+            }
 
-        if (!conf->vch_forced) {
-                ret = dict_get_uint32 (local->xattr,
-                                       conf->commithash_xattr_name,
-                                       &vol_commit_hash);
-                if (ret == 0) {
-                        conf->vol_commit_hash = vol_commit_hash;
-                }
+            if (source && heal_path)
+                break;
         }
+    }
 
-        if (IA_ISDIR (local->stbuf.ia_type) && !is_read_only) {
-                for (i = 0; i < layout->cnt; i++) {
-                       if (!source && !layout->list[i].err)
-                                source = layout->list[i].xlator;
-                        if (layout->list[i].err == ENOENT ||
-                            layout->list[i].err == ESTALE) {
-                                heal_path = 1;
-                        }
-
-                        if (source && heal_path)
-                                break;
-                }
+    if (IA_ISDIR(local->stbuf.ia_type)) {
+        /* Call function to save hashed subvol on inode ctx if
+           internal mds xattr is not present and all subvols are up
+        */
+        if (!local->op_ret && !__is_root_gfid(local->stbuf.ia_gfid))
+            (void)dht_common_mark_mdsxattr(discover_frame,
+                                           &error_while_marking_mds, 1);
+
+        if (local->need_xattr_heal && !heal_path) {
+            local->need_xattr_heal = 0;
+            ret = dht_dir_xattr_heal(this, local, &op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                       "xattr heal failed for "
+                       "directory  gfid is %s ",
+                       gfid_local);
+            }
         }
+    }
 
-        if (IA_ISDIR (local->stbuf.ia_type)) {
-                /* Call function to save hashed subvol on inode ctx if
-                   internal mds xattr is not present and all subvols are up
-                */
-                if (!local->op_ret && !__is_root_gfid (local->stbuf.ia_gfid))
-                        (void) dht_common_mark_mdsxattr (discover_frame,
-                                                         &error_while_marking_mds, 1);
-
-                if (local->need_xattr_heal && !heal_path) {
-                        local->need_xattr_heal = 0;
-                        ret =  dht_dir_xattr_heal (this, local);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_ERROR,
-                                        ret,
-                                        DHT_MSG_DIR_XATTR_HEAL_FAILED,
-                                        "xattr heal failed for "
-                                        "directory  gfid is %s ",
-                                        gfid_local);
-                }
+    if (source && (heal_path || layout_anomalies || error_while_marking_mds)) {
+        gf_uuid_copy(loc.gfid, local->gfid);
+        if (gf_uuid_is_null(loc.gfid)) {
+            goto done;
         }
 
-        if (source && (heal_path || layout_anomalies || error_while_marking_mds)) {
-                gf_uuid_copy (loc.gfid, local->gfid);
-                if (gf_uuid_is_null (loc.gfid)) {
-                        goto done;
-                }
-
-                if (local->inode)
-                        loc.inode = inode_ref (local->inode);
-                else
-                        goto done;
-
-               heal_frame = create_frame (this, this->ctx->pool);
-               if (heal_frame) {
-                        heal_local = dht_local_init (heal_frame, &loc,
-                                                     NULL, 0);
-                        if (!heal_local)
-                                goto cleanup;
-
-                        gf_uuid_copy (heal_local->gfid, local->gfid);
-                        heal_frame->cookie = source;
-                        heal_local->xattr = dict_ref (local->xattr);
-                        heal_local->stbuf = local->stbuf;
-                        heal_local->postparent = local->postparent;
-                        heal_local->inode = inode_ref (loc.inode);
-                        heal_local->main_frame = main_frame;
-                        FRAME_SU_DO (heal_frame, dht_local_t);
-                        ret = synctask_new (this->ctx->env,
-                                            dht_heal_full_path,
-                                            dht_heal_full_path_done,
-                                            heal_frame, heal_frame);
-                        if (!ret) {
-                                loc_wipe (&loc);
-                                return 0;
-                        }
-                        /*
-                         * Failed to spawn the synctask. Returning
-                         * with out doing heal.
-                         */
-cleanup:
-                        loc_wipe (&loc);
-                        DHT_STACK_DESTROY (heal_frame);
-                }
-
-        }
+        if (local->inode)
+            loc.inode = inode_ref(local->inode);
+        else
+            goto done;
+
+        heal_frame = create_frame(this, this->ctx->pool);
+        if (heal_frame) {
+            heal_local = dht_local_init(heal_frame, &loc, NULL, 0);
+            if (!heal_local)
+                goto cleanup;
+
+            gf_uuid_copy(heal_local->gfid, local->gfid);
+            heal_frame->cookie = source;
+            heal_local->xattr = dict_ref(local->xattr);
+            heal_local->stbuf = local->stbuf;
+            heal_local->postparent = local->postparent;
+            heal_local->inode = inode_ref(loc.inode);
+            heal_local->main_frame = main_frame;
+            FRAME_SU_DO(heal_frame, dht_local_t);
+            ret = synctask_new(this->ctx->env, dht_heal_full_path,
+                               dht_heal_full_path_done, heal_frame, heal_frame);
+            if (!ret) {
+                loc_wipe(&loc);
+                return 0;
+            }
+            /*
+             * Failed to spawn the synctask. Returning
+             * with out doing heal.
+             */
+        cleanup:
+            loc_wipe(&loc);
+            DHT_STACK_DESTROY(heal_frame);
+        }
+    }
 done:
-        dht_set_fixed_dir_stat (&local->postparent);
-        /* Delete mds xattr at the time of STACK UNWIND */
-        if (local->xattr)
-                GF_REMOVE_INTERNAL_XATTR (conf->mds_xattr_key, local->xattr);
+    dht_set_fixed_dir_stat(&local->postparent);
+    /* Delete mds xattr at the time of STACK UNWIND */
+    if (local->xattr)
+        GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
 
-        DHT_STACK_UNWIND (lookup, main_frame, local->op_ret, local->op_errno,
-                          local->inode, &local->stbuf, local->xattr,
-                          &local->postparent);
-        return 0;
+    DHT_STACK_UNWIND(lookup, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->stbuf, local->xattr,
+                     &local->postparent);
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (lookup, main_frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
+    DHT_STACK_UNWIND(lookup, main_frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-        return ret;
+    return ret;
 }
 
-int
-dht_common_mark_mdsxattr_cbk (call_frame_t *frame, void *cookie,
-                              xlator_t *this, int op_ret, int op_errno,
-                              dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        xlator_t     *prev  = cookie;
-        int           ret   = -1;
-        dht_conf_t   *conf  = 0;
-        dht_layout_t *layout = NULL;
+static int
+dht_common_mark_mdsxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = cookie;
+    int ret = -1;
+    dht_conf_t *conf = 0;
+    dht_layout_t *layout = NULL;
+    int32_t mds_heal_fresh_lookup = 0;
+
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+
+    local = frame->local;
+    conf = this->private;
+    layout = local->selfheal.layout;
+    mds_heal_fresh_lookup = local->mds_heal_fresh_lookup;
+
+    if (op_ret) {
+        gf_msg_debug(this->name, op_ret,
+                     "Failed to set %s on the MDS %s for path %s. ",
+                     conf->mds_xattr_key, prev->name, local->loc.path);
+    } else {
+        /* Save mds subvol on inode ctx */
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "Failed to set mds subvol on inode ctx"
+                   " %s for %s ",
+                   prev->name, local->loc.path);
+        }
+    }
+    if (!local->mds_heal_fresh_lookup && layout) {
+        dht_selfheal_dir_setattr(frame, &local->loc, &local->stbuf, 0xffffffff,
+                                 layout);
+    }
+out:
+    if (mds_heal_fresh_lookup)
+        DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
+static xlator_t *
+dht_inode_get_hashed_subvol(inode_t *inode, xlator_t *this, loc_t *loc)
+{
+    char *path = NULL;
+    loc_t populate_loc = {
+        0,
+    };
+    char *name = NULL;
+    xlator_t *hash_subvol = NULL;
 
-        local = frame->local;
-        conf = this->private;
-        layout = local->selfheal.layout;
+    if (!inode)
+        return hash_subvol;
 
-        if (op_ret) {
-                gf_msg_debug (this->name, op_ret,
-                              "Failed to set %s on the MDS %s for path %s. ",
-                               conf->mds_xattr_key, prev->name, local->loc.path);
-        } else {
-                /* Save mds subvol on inode ctx */
-                ret = dht_inode_ctx_mdsvol_set (local->inode, this, prev);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_SET_INODE_CTX_FAILED,
-                                "Failed to set mds subvol on inode ctx"
-                                " %s for %s ", prev->name,
-                                local->loc.path);
-                }
+    if (loc && loc->parent && loc->path) {
+        if (!loc->name) {
+            name = strrchr(loc->path, '/');
+            if (name) {
+                loc->name = name + 1;
+            } else {
+                goto out;
+            }
         }
-        if (!local->mds_heal_fresh_lookup && layout) {
-                dht_selfheal_dir_setattr (frame, &local->loc, &local->stbuf,
-                                          0xffffffff, layout);
+        hash_subvol = dht_subvol_get_hashed(this, loc);
+        goto out;
+    }
+
+    if (!gf_uuid_is_null(inode->gfid)) {
+        populate_loc.inode = inode_ref(inode);
+        populate_loc.parent = inode_parent(populate_loc.inode, NULL, NULL);
+        inode_path(populate_loc.inode, NULL, &path);
+
+        if (!path)
+            goto out;
+
+        populate_loc.path = path;
+        if (!populate_loc.name && populate_loc.path) {
+            name = strrchr(populate_loc.path, '/');
+            if (name) {
+                populate_loc.name = name + 1;
+
+            } else {
+                goto out;
+            }
         }
+        hash_subvol = dht_subvol_get_hashed(this, &populate_loc);
+    }
 out:
-        if (local && local->mds_heal_fresh_lookup)
-                DHT_STACK_DESTROY (frame);
-        return 0;
+    if (populate_loc.inode)
+        loc_wipe(&populate_loc);
+    return hash_subvol;
 }
 
-
-
 /* Common function call by revalidate/selfheal code path to populate
    internal xattr if it is not present, mark_during_fresh_lookup value
    determines either function is call by revalidate_cbk(discover_complete)
@@ -829,1446 +791,1508 @@ out:
    penalty.
 */
 int
-dht_common_mark_mdsxattr (call_frame_t *frame, int *errst, int mark_during_fresh_lookup)
-{
-        dht_local_t  *local          = NULL;
-        xlator_t     *this           = NULL;
-        xlator_t     *hashed_subvol  = NULL;
-        int           ret            = 0;
-        int           i              = 0;
-        dict_t       *xattrs         = NULL;
-        char          gfid_local[GF_UUID_BUF_SIZE] = {0,};
-        int32_t       zero[1]        = {0};
-        dht_conf_t   *conf           = 0;
-        dht_layout_t *layout         = NULL;
-        dht_local_t  *copy_local     = NULL;
-        call_frame_t *xattr_frame    = NULL;
-        gf_boolean_t  vol_down       = _gf_false;
-
-        this = frame->this;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        local = frame->local;
-        conf = this->private;
-        layout = local->selfheal.layout;
-        local->mds_heal_fresh_lookup = mark_during_fresh_lookup;
-        gf_uuid_unparse(local->gfid, gfid_local);
-
-        /* Code to update hashed subvol consider as a mds subvol
-           and wind a setxattr call on hashed subvol to update
-           internal xattr
+dht_common_mark_mdsxattr(call_frame_t *frame, int *errst,
+                         int mark_during_fresh_lookup)
+{
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    xlator_t *hashed_subvol = NULL;
+    int ret = 0;
+    int i = 0;
+    dict_t *xattrs = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+    int32_t zero[1] = {0};
+    dht_conf_t *conf = 0;
+    dht_layout_t *layout = NULL;
+    dht_local_t *copy_local = NULL;
+    call_frame_t *xattr_frame = NULL;
+    gf_boolean_t vol_down = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    local = frame->local;
+    conf = this->private;
+    layout = local->selfheal.layout;
+    local->mds_heal_fresh_lookup = mark_during_fresh_lookup;
+
+    gf_uuid_unparse(local->gfid, gfid_local);
+
+    /* Code to update hashed subvol consider as a mds subvol
+       and wind a setxattr call on hashed subvol to update
+       internal xattr
+    */
+    if (!local->xattr || !dict_get(local->xattr, conf->mds_xattr_key)) {
+        /* It means no internal MDS xattr has been set yet
+         */
+        /* Check the status of all subvol are up while call
+           this function call by lookup code path
         */
-        if (!local->xattr || !dict_get (local->xattr, conf->mds_xattr_key)) {
-                /* It means no internal MDS xattr has been set yet
-                */
-                /* Check the status of all subvol are up while call
-                   this function call by lookup code path
-                */
-                if (mark_during_fresh_lookup) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                if (!conf->subvolume_status[i]) {
-                                        vol_down = _gf_true;
-                                        break;
-                                }
-                        }
-                        if (vol_down) {
-                                gf_msg_debug (this->name, 0,
-                                              "subvol %s is down. Unable to "
-                                              " save mds subvol on inode for "
-                                              " path %s gfid is %s " ,
-                                              conf->subvolumes[i]->name,
-                                              local->loc.path, gfid_local);
-                                goto out;
-                        }
+        if (mark_during_fresh_lookup) {
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (!conf->subvolume_status[i]) {
+                    vol_down = _gf_true;
+                    break;
                 }
+            }
+            if (vol_down) {
+                gf_msg_debug(this->name, 0,
+                             "subvol %s is down. Unable to "
+                             " save mds subvol on inode for "
+                             " path %s gfid is %s ",
+                             conf->subvolumes[i]->name, local->loc.path,
+                             gfid_local);
+                goto out;
+            }
+        }
 
-                /* Calculate hashed subvol based on inode and parent node
-                */
-                hashed_subvol = dht_inode_get_hashed_subvol (local->inode, this,
-                                                             &local->loc);
-                if (!hashed_subvol) {
-                        gf_msg (this->name, GF_LOG_DEBUG, 0,
-                                DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                                "Failed to get hashed subvol for path %s"
-                                "gfid is %s ",
-                                local->loc.path, gfid_local);
-                        (*errst) = 1;
-                        ret = -1;
-                        goto out;
-                }
-                xattrs = dict_new ();
-                if (!xattrs) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                DHT_MSG_NO_MEMORY, "dict_new failed");
-                        ret = -1;
-                        goto out;
-                }
-                /* Add internal MDS xattr on disk for hashed subvol
-                */
-                ret = dht_dict_set_array (xattrs, conf->mds_xattr_key,
-                                          zero, 1);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary"
-                                "  value:key = %s for "
-                                "path %s", conf->mds_xattr_key,
-                                local->loc.path);
-                        ret = -1;
-                        goto out;
-                }
-                /* Create a new frame to wind a call only while
-                   this function call by revalidate_cbk code path
-                   To wind a call parallel need to create a new frame
-                */
-                if (mark_during_fresh_lookup) {
-                        xattr_frame = create_frame (this, this->ctx->pool);
-                        if (!xattr_frame) {
-                                ret = -1;
-                                goto out;
-                        }
-                        copy_local = dht_local_init (xattr_frame, &(local->loc),
-                                                     NULL, 0);
-                        if (!copy_local) {
-                                ret = -1;
-                                DHT_STACK_DESTROY (xattr_frame);
-                                goto out;
-                        }
-                        copy_local->stbuf = local->stbuf;
-                        copy_local->mds_heal_fresh_lookup = mark_during_fresh_lookup;
-                        if (!copy_local->inode)
-                                copy_local->inode = inode_ref (local->inode);
-                        gf_uuid_copy (copy_local->loc.gfid, local->gfid);
-                        FRAME_SU_DO (xattr_frame, dht_local_t);
-                        STACK_WIND_COOKIE (xattr_frame, dht_common_mark_mdsxattr_cbk,
-                                           hashed_subvol, hashed_subvol,
-                                           hashed_subvol->fops->setxattr,
-                                           &local->loc, xattrs, 0, NULL);
-                } else {
-                        STACK_WIND_COOKIE (frame,
-                                           dht_common_mark_mdsxattr_cbk,
-                                           (void *)hashed_subvol,
-                                           hashed_subvol,
-                                           hashed_subvol->fops->setxattr,
-                                           &local->loc, xattrs, 0,
-                                           NULL);
-                }
-        } else {
-                gf_msg_debug (this->name, 0,
-                              "internal xattr %s is present on subvol"
-                              "on path %s gfid is %s " , conf->mds_xattr_key,
-                               local->loc.path, gfid_local);
-                if (!mark_during_fresh_lookup)
-                        dht_selfheal_dir_setattr (frame, &local->loc,
-                                                  &local->stbuf, 0xffffffff,
-                                                  layout);
+        /* Calculate hashed subvol based on inode and parent node
+         */
+        hashed_subvol = dht_inode_get_hashed_subvol(local->inode, this,
+                                                    &local->loc);
+        if (!hashed_subvol) {
+            gf_msg(this->name, GF_LOG_DEBUG, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get hashed subvol for path %s"
+                   "gfid is %s ",
+                   local->loc.path, gfid_local);
+            if (errst)
+                (*errst) = 1;
+            ret = -1;
+            goto out;
+        }
+        xattrs = dict_new();
+        if (!xattrs) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                   "dict_new failed");
+            ret = -1;
+            goto out;
         }
+        /* Add internal MDS xattr on disk for hashed subvol
+         */
+        ret = dht_dict_set_array(xattrs, conf->mds_xattr_key, zero, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary"
+                   "  value:key = %s for "
+                   "path %s",
+                   conf->mds_xattr_key, local->loc.path);
+            ret = -1;
+            goto out;
+        }
+        /* Create a new frame to wind a call only while
+           this function call by revalidate_cbk code path
+           To wind a call parallel need to create a new frame
+        */
+        if (mark_during_fresh_lookup) {
+            xattr_frame = create_frame(this, this->ctx->pool);
+            if (!xattr_frame) {
+                ret = -1;
+                goto out;
+            }
+            copy_local = dht_local_init(xattr_frame, &(local->loc), NULL, 0);
+            if (!copy_local) {
+                ret = -1;
+                DHT_STACK_DESTROY(xattr_frame);
+                goto out;
+            }
+            copy_local->stbuf = local->stbuf;
+            copy_local->mds_heal_fresh_lookup = mark_during_fresh_lookup;
+            if (!copy_local->inode)
+                copy_local->inode = inode_ref(local->inode);
+            gf_uuid_copy(copy_local->loc.gfid, local->gfid);
+            FRAME_SU_DO(xattr_frame, dht_local_t);
+            STACK_WIND_COOKIE(xattr_frame, dht_common_mark_mdsxattr_cbk,
+                              hashed_subvol, hashed_subvol,
+                              hashed_subvol->fops->setxattr, &local->loc,
+                              xattrs, 0, NULL);
+        } else {
+            STACK_WIND_COOKIE(frame, dht_common_mark_mdsxattr_cbk,
+                              (void *)hashed_subvol, hashed_subvol,
+                              hashed_subvol->fops->setxattr, &local->loc,
+                              xattrs, 0, NULL);
+        }
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "internal xattr %s is present on subvol"
+                     "on path %s gfid is %s ",
+                     conf->mds_xattr_key, local->loc.path, gfid_local);
+        if (!mark_during_fresh_lookup)
+            dht_selfheal_dir_setattr(frame, &local->loc, &local->stbuf,
+                                     0xffffffff, layout);
+    }
 
 out:
-        if (xattrs)
-                dict_unref (xattrs);
-        return ret;
+    if (xattrs)
+        dict_unref(xattrs);
+    return ret;
 }
 
+/* Get the value of key from dict in the bytewise and save in array after
+   convert from network byte order to host byte order
+*/
+static int32_t
+dht_dict_get_array(dict_t *dict, char *key, int32_t value[], int32_t size,
+                   int *errst)
+{
+    void *ptr = NULL;
+    int32_t len = -1;
+    int32_t vindex = -1;
+    int32_t err = -1;
+    int ret = 0;
+
+    if (dict == NULL) {
+        (*errst) = -1;
+        return -EINVAL;
+    }
+    err = dict_get_ptr_and_len(dict, key, &ptr, &len);
+    if (err != 0) {
+        (*errst) = -1;
+        return err;
+    }
+
+    if (len != (size * sizeof(int32_t))) {
+        (*errst) = -1;
+        return -EINVAL;
+    }
+
+    for (vindex = 0; vindex < size; vindex++) {
+        value[vindex] = ntoh32(*((int32_t *)ptr + vindex));
+        if (value[vindex] < 0)
+            ret = -1;
+    }
+
+    return ret;
+}
 
-int
-dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno,
-                  inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                  struct iatt *postparent)
-{
-        dht_local_t  *local                   = NULL;
-        int           this_call_cnt           = 0;
-        xlator_t     *prev                    = NULL;
-        dht_layout_t *layout                  = NULL;
-        int           ret                     = -1;
-        int           is_dir                  = 0;
-        int32_t       check_mds               = 0;
-        int           is_linkfile             = 0;
-        int           attempt_unwind          = 0;
-        dht_conf_t   *conf                    = 0;
-        char          gfid_local[GF_UUID_BUF_SIZE] = {0};
-        char          gfid_node[GF_UUID_BUF_SIZE]  = {0};
-        int32_t       mds_xattr_val[1]             = {0};
-        int           errst                        = 0;
-
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+static int
+dht_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, inode_t *inode, struct iatt *stbuf,
+                 dict_t *xattr, struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    int is_dir = 0;
+    int32_t check_mds = 0;
+    int is_linkfile = 0;
+    int attempt_unwind = 0;
+    dht_conf_t *conf = 0;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char gfid_node[GF_UUID_BUF_SIZE] = {0};
+    int32_t mds_xattr_val[1] = {0};
+    int errst = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    layout = local->layout;
+
+    /* Check if the gfid is different for file from other node */
+    if (!op_ret && gf_uuid_compare(local->gfid, stbuf->ia_gfid)) {
+        gf_uuid_unparse(stbuf->ia_gfid, gfid_node);
+        gf_uuid_unparse(local->gfid, gfid_local);
 
-        local = frame->local;
-        prev  = cookie;
-        conf = this->private;
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "%s: gfid different on %s, gfid local = %s"
+               "gfid other = %s",
+               local->loc.path, prev->name, gfid_local, gfid_node);
+    }
 
-        layout = local->layout;
+    LOCK(&frame->lock);
+    {
+        /* TODO: assert equal mode on stbuf->st_mode and
+           local->stbuf->st_mode
 
+           else mkdir/chmod/chown and fix
+        */
 
-        /* Check if the gfid is different for file from other node */
-        if (!op_ret && gf_uuid_compare (local->gfid, stbuf->ia_gfid)) {
+        ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, xattr);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                   "%s: failed to merge layouts for subvol %s", local->loc.path,
+                   prev->name);
 
-                gf_uuid_unparse(stbuf->ia_gfid, gfid_node);
-                gf_uuid_unparse(local->gfid, gfid_local);
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            gf_msg_debug(this->name, op_errno,
+                         "lookup of %s on %s returned error", local->loc.path,
+                         prev->name);
 
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_GFID_MISMATCH,
-                        "%s: gfid different on %s, gfid local = %s"
-                        "gfid other = %s",
-                        local->loc.path, prev->name,
-                        gfid_local, gfid_node);
+            goto unlock;
         }
 
+        is_linkfile = check_is_linkfile(inode, stbuf, xattr,
+                                        conf->link_xattr_name);
+        is_dir = check_is_dir(inode, stbuf, xattr);
 
-        LOCK (&frame->lock);
-        {
-                /* TODO: assert equal mode on stbuf->st_mode and
-                   local->stbuf->st_mode
-
-                   else mkdir/chmod/chown and fix
-                */
-
-                ret = dht_layout_merge (this, layout, prev,
-                                        op_ret, op_errno, xattr);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_LAYOUT_MERGE_FAILED,
-                                "%s: failed to merge layouts for subvol %s",
-                                local->loc.path, prev->name);
-
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "lookup of %s on %s returned error",
-                                      local->loc.path, prev->name);
-
-                        goto unlock;
-                }
-
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                                 conf->link_xattr_name);
-                is_dir = check_is_dir (inode, stbuf, xattr);
-
-                if (is_dir) {
-                        local->dir_count ++;
-                } else {
-                        local->file_count ++;
-
-                        if (!is_linkfile && !local->cached_subvol) {
-                                /* real file */
-                                /* Ok, we somehow managed to find a file on
-                                 * more than one subvol. ignore this or we
-                                 * will end up overwriting information while a
-                                 * a thread is potentially unwinding from
-                                 * dht_discover_complete
-                                 */
-                                local->cached_subvol = prev;
-                                attempt_unwind = 1;
-                        } else {
-                                goto unlock;
-                        }
-                }
-
-                local->op_ret = 0;
+        if (is_dir) {
+            local->dir_count++;
+        } else {
+            local->file_count++;
+
+            if (!is_linkfile && !local->cached_subvol) {
+                /* real file */
+                /* Ok, we somehow managed to find a file on
+                 * more than one subvol. ignore this or we
+                 * will end up overwriting information while a
+                 * a thread is potentially unwinding from
+                 * dht_discover_complete
+                 */
+                local->cached_subvol = prev;
+                attempt_unwind = 1;
+            } else {
+                goto unlock;
+            }
+        }
 
-                if (local->xattr == NULL) {
-                        local->xattr = dict_ref (xattr);
-                } else {
-                        /* Don't aggregate for files. See BZ#1484113 */
-                        if (is_dir)
-                                dht_aggregate_xattr (local->xattr, xattr);
-                }
+        local->op_ret = 0;
 
-                if (local->inode == NULL)
-                        local->inode = inode_ref (inode);
+        if (local->xattr == NULL) {
+            local->xattr = dict_ref(xattr);
+        } else {
+            /* Don't aggregate for files. See BZ#1484709 */
+            if (is_dir)
+                dht_aggregate_xattr(local->xattr, xattr);
+        }
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev);
-                if (!dict_get (xattr, conf->mds_xattr_key)) {
-                        goto unlock;
-                } else {
-                        gf_msg_debug (this->name, 0,
-                                      "internal xattr %s is present on subvol"
-                                      "on path %s gfid is %s " ,
-                                      conf->mds_xattr_key,
-                                      local->loc.path, gfid_local);
-                }
-                check_mds = dht_dict_get_array (xattr, conf->mds_xattr_key,
-                                                mds_xattr_val, 1, &errst);
-                /* save mds subvol on inode ctx */
-                ret = dht_inode_ctx_mdsvol_set (local->inode, this,
-                                                prev);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_SET_INODE_CTX_FAILED,
-                                "Failed to set hashed subvol for %s vol is %s",
-                                local->loc.path, prev->name);
-                }
+        if (local->inode == NULL)
+            local->inode = inode_ref(inode);
 
-                if ((check_mds < 0) && !errst) {
-                        local->mds_xattr = dict_ref (xattr);
-                        gf_msg_debug (this->name, 0,
-                                      "Value of %s is not zero on mds subvol"
-                                      "so xattr needs to be healed on non mds"
-                                      " path is %s and vol name is %s "
-                                      " gfid is %s" ,
-                                      conf->mds_xattr_key,
-                                      local->loc.path,
-                                      prev->name, gfid_local);
-                        local->need_xattr_heal = 1;
-                        local->mds_subvol  = prev;
-                }
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->postparent, postparent);
 
-        }
+        if (!dict_get(xattr, conf->mds_xattr_key)) {
+            goto unlock;
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "internal xattr %s is present on subvol"
+                         "on path %s gfid is %s ",
+                         conf->mds_xattr_key, local->loc.path, gfid_local);
+        }
+        check_mds = dht_dict_get_array(xattr, conf->mds_xattr_key,
+                                       mds_xattr_val, 1, &errst);
+        /* save mds subvol on inode ctx */
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "Failed to set hashed subvol for %s vol is %s",
+                   local->loc.path, prev->name);
+        }
+
+        if ((check_mds < 0) && !errst) {
+            local->mds_xattr = dict_ref(xattr);
+            gf_msg_debug(this->name, 0,
+                         "Value of %s is not zero on mds subvol"
+                         "so xattr needs to be healed on non mds"
+                         " path is %s and vol name is %s "
+                         " gfid is %s",
+                         conf->mds_xattr_key, local->loc.path, prev->name,
+                         gfid_local);
+            local->need_xattr_heal = 1;
+            local->mds_subvol = prev;
+        }
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 out:
-        /* Make sure, the thread executing dht_discover_complete is the one
-         * which calls STACK_DESTROY (frame). In the case of "attempt_unwind",
-         * this makes sure that the thread don't call dht_frame_return, till
-         * call to dht_discover_complete is done.
-         */
-        if (attempt_unwind) {
-                dht_discover_complete (this, frame);
-        }
+    /* Make sure, the thread executing dht_discover_complete is the one
+     * which calls STACK_DESTROY (frame). In the case of "attempt_unwind",
+     * this makes sure that the thread don't call dht_frame_return, till
+     * call to dht_discover_complete is done.
+     */
+    if (attempt_unwind) {
+        dht_discover_complete(this, frame);
+    }
 
-        this_call_cnt = dht_frame_return (frame);
+    this_call_cnt = dht_frame_return(frame);
 
-        if (is_last_call (this_call_cnt) && !attempt_unwind) {
-                dht_discover_complete (this, frame);
-        }
+    if (is_last_call(this_call_cnt) && !attempt_unwind) {
+        dht_discover_complete(this, frame);
+    }
 
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_DESTROY (frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_DESTROY(frame);
 
-        return 0;
+    return 0;
 }
 
+static int
+dht_set_file_xattr_req(xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    int ret = -EINVAL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    if (!conf) {
+        goto err;
+    }
+
+    if (!xattr_req) {
+        goto err;
+    }
+
+    /* Used to check whether this is a linkto file.
+     */
+    ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->link_xattr_name, loc->path);
+        goto err;
+    }
+
+    /* This is used to make sure we don't unlink linkto files
+     * which are the target of an ongoing file migration.
+     */
+    ret = dict_set_uint32(xattr_req, GLUSTERFS_OPEN_FD_COUNT, 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               GLUSTERFS_OPEN_FD_COUNT, loc->path);
+        goto err;
+    }
+
+    ret = 0;
+err:
+    return ret;
+}
 
-int
-dht_discover (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        int          ret;
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf = NULL;
-        int          call_cnt = 0;
-        int          op_errno = EINVAL;
-        int          i = 0;
-        call_frame_t *discover_frame = NULL;
-
-        conf = this->private;
-        local = frame->local;
-
-        ret = dict_set_uint32 (local->xattr_req, conf->xattr_name, 4 * 4);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value:key = %s",
-                        loc->path, conf->xattr_name);
+/* This is a gfid based nameless lookup. Without a name, the hashed subvol
+ * cannot be calculated so a lookup is sent to all subvols.
+ */
+static int
+dht_do_discover(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    int ret;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int call_cnt = 0;
+    int op_errno = EINVAL;
+    int i = 0;
+    call_frame_t *discover_frame = NULL;
+
+    conf = this->private;
+    local = frame->local;
+
+    /* As we do not know if this is a file or directory, request
+     * both file and directory xattrs
+     */
+    ret = dht_set_file_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        goto err;
+    }
+
+    ret = dht_set_dir_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        goto err;
+    }
+
+    if (loc_is_root(loc)) {
+        /* Request the DHT commit hash xattr (trusted.glusterfs.dht.commithash)
+         * set on the brick root.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->commithash_xattr_name,
+                              sizeof(uint32_t));
+    }
 
-        ret = dict_set_uint32 (local->xattr_req, conf->link_xattr_name, 256);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value:key = %s",
-                        loc->path, conf->link_xattr_name);
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
 
-        if (__is_root_gfid(local->loc.gfid)) {
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->commithash_xattr_name,
-                                       sizeof(uint32_t));
-        }
+    local->layout = dht_layout_new(this, conf->subvolume_cnt);
 
-        call_cnt        = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
+    if (!local->layout) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        local->layout = dht_layout_new (this, conf->subvolume_cnt);
+    gf_uuid_copy(local->gfid, loc->gfid);
 
-        if (!local->layout) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    discover_frame = copy_frame(frame);
+    if (!discover_frame) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        gf_uuid_copy (local->gfid, loc->gfid);
+    discover_frame->local = local;
+    frame->local = NULL;
+    local->main_frame = frame;
 
-        discover_frame = copy_frame (frame);
-        if (!discover_frame) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(discover_frame, dht_discover_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->lookup, &local->loc,
+                          local->xattr_req);
+    }
 
-        discover_frame->local = local;
-        frame->local = NULL;
-        local->main_frame = frame;
-
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND_COOKIE (discover_frame, dht_discover_cbk,
-                                   conf->subvolumes[i], conf->subvolumes[i],
-                                   conf->subvolumes[i]->fops->lookup,
-                                   &local->loc, local->xattr_req);
-        }
-
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
-
-        return 0;
-}
-
-/* Get the value of key from dict in the bytewise and save in array after
-   convert from network byte order to host byte order
-*/
-int32_t
-dht_dict_get_array (dict_t *dict, char *key, int32_t value[], int32_t size, int *errst)
-{
-        void    *ptr          = NULL;
-        int32_t len           = -1;
-        int32_t vindex        = -1;
-        int32_t err           = -1;
-        int     ret          = 0;
-
-        if (dict == NULL) {
-                (*errst) = -1;
-                return -EINVAL;
-        }
-        err = dict_get_ptr_and_len(dict, key, &ptr, &len);
-        if (err != 0) {
-                (*errst) = -1;
-                return err;
-        }
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-        if (len != (size * sizeof (int32_t))) {
-                (*errst) = -1;
-                return -EINVAL;
-        }
-
-        memset (value, 0, size * sizeof(int32_t));
-        for (vindex = 0; vindex < size; vindex++) {
-                value[vindex] = ntoh32(*((int32_t *)ptr + vindex));
-                if (value[vindex] < 0)
-                        ret = -1;
-        }
-
-        return ret;
+    return 0;
 }
 
-
 /* Code to call syntask to heal custom xattr from hashed subvol
    to non hashed subvol
 */
 int
-dht_dir_xattr_heal (xlator_t *this, dht_local_t *local)
-{
-        dht_local_t  *copy_local                  = NULL;
-        call_frame_t *copy                        = NULL;
-        int          ret                          = -1;
-        char         gfid_local[GF_UUID_BUF_SIZE] = {0};
-
-        if (local->gfid) {
-                gf_uuid_unparse(local->gfid, gfid_local);
+dht_dir_xattr_heal(xlator_t *this, dht_local_t *local, int *op_errno)
+{
+    dht_local_t *copy_local = NULL;
+    call_frame_t *copy = NULL;
+    int ret = -1;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+
+    if (gf_uuid_is_null(local->gfid)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DIR_XATTR_HEAL_FAILED,
+               "No gfid exists for path %s "
+               "so healing xattr is not possible",
+               local->loc.path);
+        *op_errno = EIO;
+        goto out;
+    }
+
+    gf_uuid_unparse(local->gfid, gfid_local);
+    copy = create_frame(this, this->ctx->pool);
+    if (copy) {
+        copy_local = dht_local_init(copy, &(local->loc), NULL, 0);
+        if (!copy_local) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM,
+                   DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                   "Memory allocation failed "
+                   "for path %s gfid %s ",
+                   local->loc.path, gfid_local);
+            *op_errno = ENOMEM;
+            DHT_STACK_DESTROY(copy);
         } else {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DIR_XATTR_HEAL_FAILED,
-                        "No gfid exists for path %s "
-                        "so healing xattr is not possible",
-                        local->loc.path);
-                goto out;
-        }
-
-        copy = create_frame (this, this->ctx->pool);
-        if (copy) {
-                copy_local = dht_local_init (copy, &(local->loc), NULL, 0);
-                if (!copy_local) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                DHT_MSG_DIR_XATTR_HEAL_FAILED,
-                                "Memory allocation failed "
-                                "for path %s gfid %s ",
-                                local->loc.path, gfid_local);
-                        DHT_STACK_DESTROY (copy);
-                } else {
-                        copy_local->stbuf = local->stbuf;
-                        gf_uuid_copy (copy_local->loc.gfid, local->gfid);
-                        copy_local->mds_subvol = local->mds_subvol;
-                        FRAME_SU_DO (copy, dht_local_t);
-                        ret = synctask_new (this->ctx->env, dht_dir_heal_xattrs,
-                                            dht_dir_heal_xattrs_done,
-                                            copy, copy);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                        DHT_MSG_DIR_XATTR_HEAL_FAILED,
-                                        "Synctask creation failed to heal xattr "
-                                        "for path %s gfid %s ",
-                                        local->loc.path, gfid_local);
-                                DHT_STACK_DESTROY (copy);
-                        }
-                }
+            copy_local->stbuf = local->stbuf;
+            gf_uuid_copy(copy_local->loc.gfid, local->gfid);
+            copy_local->mds_subvol = local->mds_subvol;
+            FRAME_SU_DO(copy, dht_local_t);
+            ret = synctask_new(this->ctx->env, dht_dir_heal_xattrs,
+                               dht_dir_heal_xattrs_done, copy, copy);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM,
+                       DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                       "Synctask creation failed to heal xattr "
+                       "for path %s gfid %s ",
+                       local->loc.path, gfid_local);
+                *op_errno = ENOMEM;
+                DHT_STACK_DESTROY(copy);
+            }
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
+static int
+dht_needs_selfheal(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int needs_selfheal = 0;
+    int ret = 0;
 
+    local = frame->local;
+    layout = local->layout;
 
-int
-dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                    struct iatt *postparent)
-{
-        dht_local_t  *local                   = NULL;
-        dht_conf_t   *conf                    = NULL;
-        int           this_call_cnt           = 0;
-        xlator_t     *prev                    = NULL;
-        dht_layout_t *layout                  = NULL;
-        int           ret                     = -1;
-        int           is_dir                  = 0;
-        int32_t       check_mds               = 0;
-        int           errst                   = 0;
-        char          gfid_local[GF_UUID_BUF_SIZE] = {0};
-        char          gfid_node[GF_UUID_BUF_SIZE]  = {0};
-        int32_t       mds_xattr_val[1]                 = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+    if (local->need_attrheal || local->need_xattr_heal ||
+        local->need_selfheal) {
+        needs_selfheal = 1;
+    }
 
-        local = frame->local;
-        prev  = cookie;
-        conf  = this->private;
+    ret = dht_layout_normalize(this, &local->loc, layout);
 
-        layout = local->layout;
+    if (ret != 0) {
+        gf_msg_debug(this->name, 0, "fixing assignment on %s", local->loc.path);
+        needs_selfheal = 1;
+    }
+    return needs_selfheal;
+}
 
-        if (!op_ret && gf_uuid_is_null (local->gfid)) {
-                memcpy (local->gfid, stbuf->ia_gfid, 16);
-        }
-        if (local->gfid)
-                gf_uuid_unparse(local->gfid, gfid_local);
+static int
+is_permission_different(ia_prot_t *prot1, ia_prot_t *prot2)
+{
+    if ((prot1->owner.read != prot2->owner.read) ||
+        (prot1->owner.write != prot2->owner.write) ||
+        (prot1->owner.exec != prot2->owner.exec) ||
+        (prot1->group.read != prot2->group.read) ||
+        (prot1->group.write != prot2->group.write) ||
+        (prot1->group.exec != prot2->group.exec) ||
+        (prot1->other.read != prot2->other.read) ||
+        (prot1->other.write != prot2->other.write) ||
+        (prot1->other.exec != prot2->other.exec) ||
+        (prot1->suid != prot2->suid) || (prot1->sgid != prot2->sgid) ||
+        (prot1->sticky != prot2->sticky)) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
 
-        /* Check if the gfid is different for file from other node */
-        if (!op_ret && gf_uuid_compare (local->gfid, stbuf->ia_gfid)) {
+int
+dht_lookup_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    int is_dir = 0;
+    int32_t check_mds = 0;
+    int errst = 0;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char gfid_node[GF_UUID_BUF_SIZE] = {0};
+    int32_t mds_xattr_val[1] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    layout = local->layout;
+    gf_msg_debug(this->name, op_errno,
+                 "%s: lookup on %s returned with op_ret = %d, op_errno = %d",
+                 local->loc.path, prev->name, op_ret, op_errno);
+
+    /* The first successful lookup*/
+    if (!op_ret && gf_uuid_is_null(local->gfid)) {
+        memcpy(local->gfid, stbuf->ia_gfid, 16);
+    }
+    if (!gf_uuid_is_null(local->gfid)) {
+        gf_uuid_unparse(local->gfid, gfid_local);
+    }
+
+    /* Check if the gfid is different for file from other node */
+    if (!op_ret && gf_uuid_compare(local->gfid, stbuf->ia_gfid)) {
+        gf_uuid_unparse(stbuf->ia_gfid, gfid_node);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "%s: gfid different on %s."
+               " gfid local = %s, gfid subvol = %s",
+               local->loc.path, prev->name, gfid_local, gfid_node);
+    }
+
+    LOCK(&frame->lock);
+    {
+        /* TODO: assert equal mode on stbuf->st_mode and
+           local->stbuf->st_mode
+           else mkdir/chmod/chown and fix
+        */
+        ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, xattr);
 
-                gf_uuid_unparse(stbuf->ia_gfid, gfid_node);
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
 
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_GFID_MISMATCH,
-                        "%s: gfid different on %s."
-                        " gfid local = %s, gfid subvol = %s",
-                        local->loc.path, prev->name,
-                        gfid_local, gfid_node);
+            /* The GFID is missing on this subvol. Force a heal. */
+            if (op_errno == ENODATA) {
+                local->need_lookup_everywhere = 1;
+            }
+            goto unlock;
         }
 
-        LOCK (&frame->lock);
-        {
-                /* TODO: assert equal mode on stbuf->st_mode and
-                   local->stbuf->st_mode
+        is_dir = check_is_dir(inode, stbuf, xattr);
+        if (!is_dir) {
+            gf_msg_debug(this->name, 0,
+                         "%s: lookup on %s returned non dir 0%o"
+                         "calling lookup_everywhere",
+                         local->loc.path, prev->name, stbuf->ia_type);
 
-                   else mkdir/chmod/chown and fix
-                */
-                ret = dht_layout_merge (this, layout, prev, op_ret, op_errno,
-                                        xattr);
-
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "lookup of %s on %s returned error",
-                                      local->loc.path, prev->name);
-
-                        goto unlock;
-                }
-
-                is_dir = check_is_dir (inode, stbuf, xattr);
-                if (!is_dir) {
+            local->need_lookup_everywhere = 1;
+            goto unlock;
+        }
 
-                        gf_msg_debug (this->name, 0,
-                                      "lookup of %s on %s returned non"
-                                      "dir 0%o"
-                                      "calling lookup_everywhere",
-                                      local->loc.path, prev->name,
-                                      stbuf->ia_type);
+        local->op_ret = 0;
+        if (local->xattr == NULL) {
+            local->xattr = dict_ref(xattr);
+        } else {
+            dht_aggregate_xattr(local->xattr, xattr);
+        }
 
-                        local->need_selfheal = 1;
-                        goto unlock;
-                }
+        if (__is_root_gfid(stbuf->ia_gfid)) {
+            ret = dht_dir_has_layout(xattr, conf->xattr_name);
+            if (ret >= 0) {
+                if (is_greater_time(local->prebuf.ia_ctime,
+                                    local->prebuf.ia_ctime_nsec,
+                                    stbuf->ia_ctime, stbuf->ia_ctime_nsec)) {
+                    /* Choose source */
+                    local->prebuf.ia_gid = stbuf->ia_gid;
+                    local->prebuf.ia_uid = stbuf->ia_uid;
 
-                local->op_ret = 0;
-                if (local->xattr == NULL) {
-                        local->xattr = dict_ref (xattr);
-                } else {
-                        dht_aggregate_xattr (local->xattr, xattr);
+                    local->prebuf.ia_ctime = stbuf->ia_ctime;
+                    local->prebuf.ia_ctime_nsec = stbuf->ia_ctime_nsec;
+                    local->prebuf.ia_prot = stbuf->ia_prot;
                 }
+            }
+        }
 
-                if (local->inode == NULL)
-                        local->inode = inode_ref (inode);
+        if (local->stbuf.ia_type != IA_INVAL) {
+            /* This is not the first subvol to respond
+             * Compare values to see if attrs need to be healed
+             */
+            if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
+                (local->stbuf.ia_uid != stbuf->ia_uid) ||
+                (is_permission_different(&local->stbuf.ia_prot,
+                                         &stbuf->ia_prot))) {
+                local->need_attrheal = 1;
+            }
+        }
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev);
-                dht_iatt_merge (this, &local->postparent, postparent, prev);
+        if (local->inode == NULL)
+            local->inode = inode_ref(inode);
 
-                if (!dict_get (xattr, conf->mds_xattr_key)) {
-                        gf_msg_debug (this->name, 0,
-                                      "Internal xattr %s is not present "
-                                      " on path %s gfid is %s " ,
-                                      conf->mds_xattr_key,
-                                      local->loc.path, gfid_local);
-                        goto unlock;
-                } else {
-                        /* Save mds subvol on inode ctx */
-                        ret = dht_inode_ctx_mdsvol_set (local->inode, this,
-                                                        prev);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_SET_INODE_CTX_FAILED,
-                                        "Failed to set hashed subvol for %s vol is %s",
-                                        local->loc.path, prev->name);
-                         }
-                }
-                check_mds = dht_dict_get_array (xattr, conf->mds_xattr_key,
-                                                mds_xattr_val, 1, &errst);
-                if ((check_mds < 0) && !errst) {
-                        local->mds_xattr = dict_ref (xattr);
-                        gf_msg_debug (this->name, 0,
-                                      "Value of %s is not zero on hashed subvol "
-                                      "so xattr needs to be heal on non hashed"
-                                      " path is %s and vol name is %s "
-                                      " gfid is %s" ,
-                                      conf->mds_xattr_key,
-                                      local->loc.path,
-                                      prev->name, gfid_local);
-                        local->need_xattr_heal = 1;
-                        local->mds_subvol  = prev;
-                }
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->postparent, postparent);
 
+        if (!dict_get(xattr, conf->mds_xattr_key)) {
+            gf_msg_debug(this->name, 0,
+                         "%s: mds xattr %s is not present "
+                         "on %s(gfid = %s)",
+                         local->loc.path, conf->mds_xattr_key, prev->name,
+                         gfid_local);
+            goto unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
+        /* Save the mds subvol info and stbuf. This is the value that will
+         * be used for healing
+         */
+        local->mds_subvol = prev;
+        local->mds_stbuf = *stbuf;
 
-        this_call_cnt = dht_frame_return (frame);
+        /* Save mds subvol on inode ctx */
 
-        if (is_last_call (this_call_cnt)) {
-                /* No need to call xattr heal code if volume count is 1
-                */
-                if (conf->subvolume_cnt == 1)
-                        local->need_xattr_heal = 0;
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "%s: Failed to set mds (%s)", local->loc.path, prev->name);
+        }
+        check_mds = dht_dict_get_array(xattr, conf->mds_xattr_key,
+                                       mds_xattr_val, 1, &errst);
+        if ((check_mds < 0) && !errst) {
+            /* Check if xattrs need to be healed on the directories */
+            local->mds_xattr = dict_ref(xattr);
+            gf_msg_debug(this->name, 0,
+                         "%s: %s is not zero on %s. Xattrs need to be healed."
+                         "(gfid = %s)",
+                         local->loc.path, conf->mds_xattr_key, prev->name,
+                         gfid_local);
+            local->need_xattr_heal = 1;
+        }
+    }
 
-                /* Code to update all extended attributed from hashed subvol
-                   to local->xattr
-                */
-                if (local->need_xattr_heal && (local->mds_xattr)) {
-                        dht_dir_set_heal_xattr (this, local, local->xattr,
-                                                local->mds_xattr, NULL, NULL);
-                        dict_unref (local->mds_xattr);
-                        local->mds_xattr = NULL;
-                }
+unlock:
+    UNLOCK(&frame->lock);
 
-                if (local->need_selfheal) {
-                        local->need_selfheal = 0;
-                        dht_lookup_everywhere (frame, this, &local->loc);
-                        return 0;
-                }
+    this_call_cnt = dht_frame_return(frame);
 
-                if (local->op_ret == 0) {
-                        ret = dht_layout_normalize (this, &local->loc, layout);
+    if (is_last_call(this_call_cnt)) {
+        /* If the mds subvol is not set correctly*/
+        if (!__is_root_gfid(local->gfid) &&
+            (!dict_get(local->xattr, conf->mds_xattr_key))) {
+            local->need_selfheal = 1;
+        }
 
-                        if (ret != 0) {
-                                gf_msg_debug (this->name, 0,
-                                              "fixing assignment on %s",
-                                              local->loc.path);
-                                goto selfheal;
-                        }
+        /* No need to call xattr heal code if volume count is 1
+         */
+        if (conf->subvolume_cnt == 1) {
+            local->need_xattr_heal = 0;
+        }
+
+        if (local->need_selfheal || local->need_lookup_everywhere) {
+            /* Set the gfid-req so posix will set the GFID*/
+            if (!gf_uuid_is_null(local->gfid)) {
+                /* Ok, this should _never_ happen */
+                ret = dict_set_static_bin(local->xattr_req, "gfid-req",
+                                          local->gfid, 16);
+            } else {
+                if (!gf_uuid_is_null(local->gfid_req))
+                    ret = dict_set_static_bin(local->xattr_req, "gfid-req",
+                                              local->gfid_req, 16);
+            }
+        }
 
-                        dht_layout_set (this, local->inode, layout);
-                        if (!dict_get (local->xattr, conf->mds_xattr_key) ||
-                            local->need_xattr_heal)
-                                goto selfheal;
-                }
+        if (local->need_lookup_everywhere) {
+            local->need_lookup_everywhere = 0;
+            dht_lookup_everywhere(frame, this, &local->loc);
+            return 0;
+        }
 
-                if (local->inode) {
-                        dht_inode_ctx_time_update (local->inode, this,
-                                                   &local->stbuf, 1);
-                }
+        if (local->op_ret == 0) {
+            if (dht_needs_selfheal(frame, this)) {
+                goto selfheal;
+            }
 
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+            dht_layout_set(this, local->inode, layout);
+            if (local->inode) {
+                dht_inode_ctx_time_update(local->inode, this, &local->stbuf, 1);
+            }
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                dht_set_fixed_dir_stat (&local->postparent);
-                /* Delete mds xattr at the time of STACK UNWIND */
-                GF_REMOVE_INTERNAL_XATTR (conf->mds_xattr_key, local->xattr);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                                  local->inode, &local->stbuf, local->xattr,
-                                  &local->postparent);
+            if (local->loc.parent) {
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->postparent, 1);
+            }
         }
 
-        return 0;
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+        /* Delete mds xattr at the time of STACK UNWIND */
+        if (local->xattr)
+            GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
+
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+    }
+
+    return 0;
 
 selfheal:
-        FRAME_SU_DO (frame, dht_local_t);
-        ret = dht_selfheal_directory (frame, dht_lookup_selfheal_cbk,
-                                      &local->loc, layout);
+    FRAME_SU_DO(frame, dht_local_t);
+    ret = dht_selfheal_directory(frame, dht_lookup_selfheal_cbk, &local->loc,
+                                 layout);
 out:
-        return ret;
+    return ret;
 }
 
-int static
-is_permission_different (ia_prot_t *prot1, ia_prot_t *prot2)
-{
-        if ((prot1->owner.read != prot2->owner.read) ||
-            (prot1->owner.write != prot2->owner.write) ||
-            (prot1->owner.exec != prot2->owner.exec) ||
-            (prot1->group.read != prot2->group.read) ||
-            (prot1->group.write != prot2->group.write) ||
-            (prot1->group.exec != prot2->group.exec) ||
-            (prot1->other.read != prot2->other.read) ||
-            (prot1->other.write != prot2->other.write) ||
-            (prot1->other.exec != prot2->other.exec) ||
-            (prot1->suid != prot2->suid) ||
-            (prot1->sgid != prot2->sgid) ||
-            (prot1->sticky != prot2->sticky)) {
-                return 1;
-        } else {
-                return 0;
-        }
+static int
+dht_lookup_directory(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    int call_cnt = 0;
+    int i = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, unwind);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, unwind);
+    GF_VALIDATE_OR_GOTO("dht", this->private, unwind);
+    GF_VALIDATE_OR_GOTO("dht", loc, unwind);
+
+    conf = this->private;
+    local = frame->local;
+
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
+
+    local->layout = dht_layout_new(this, conf->subvolume_cnt);
+    if (!local->layout) {
+        goto unwind;
+    }
+
+    if (local->xattr != NULL) {
+        dict_unref(local->xattr);
+        local->xattr = NULL;
+    }
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        /* use this gfid in order to heal any missing ones */
+        ret = dict_set_gfuuid(local->xattr_req, "gfid-req", local->gfid, true);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                   "%s: Failed to set dictionary value:"
+                   " key = gfid-req",
+                   local->loc.path);
+    }
+
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(
+            frame, dht_lookup_dir_cbk, conf->subvolumes[i], conf->subvolumes[i],
+            conf->subvolumes[i]->fops->lookup, &local->loc, local->xattr_req);
+    }
+    return 0;
+unwind:
+    DHT_STACK_UNWIND(lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+out:
+    return 0;
 }
 
 int
-dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                    struct iatt *postparent)
-{
-        dht_local_t  *local         = NULL;
-        int           this_call_cnt = 0;
-        xlator_t     *prev          = NULL;
-        dht_layout_t *layout        = NULL;
-        dht_conf_t   *conf          = NULL;
-        int           ret  = -1;
-        int           is_dir = 0;
-        int           is_linkfile = 0;
-        int           follow_link = 0;
-        call_frame_t *copy          = NULL;
-        dht_local_t  *copy_local    = NULL;
-        char gfid[GF_UUID_BUF_SIZE] = {0};
-        uint32_t      vol_commit_hash = 0;
-        xlator_t      *subvol = NULL;
-        int32_t       check_mds       = 0;
-        int           errst           = 0;
-        int32_t       mds_xattr_val[1] = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, err);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, err);
+dht_revalidate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int is_dir = 0;
+    int is_linkfile = 0;
+    int follow_link = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    uint32_t vol_commit_hash = 0;
+    xlator_t *subvol = NULL;
+    int32_t check_mds = 0;
+    int errst = 0, i = 0;
+    int32_t mds_xattr_val[1] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, err);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, err);
+    GF_VALIDATE_OR_GOTO("dht", cookie, err);
+    GF_VALIDATE_OR_GOTO("dht", this->private, err);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    if (!conf->vch_forced) {
+        /* Update the commithash value if available
+         */
+        ret = dict_get_uint32(xattr, conf->commithash_xattr_name,
+                              &vol_commit_hash);
+        if (ret == 0) {
+            conf->vol_commit_hash = vol_commit_hash;
+        }
+    }
 
-        local = frame->local;
-        prev  = cookie;
-        conf = this->private;
-        if (!conf)
-                goto out;
+    gf_uuid_unparse(local->loc.gfid, gfid);
 
-        if (!conf->vch_forced) {
-                ret = dict_get_uint32 (xattr, conf->commithash_xattr_name,
-                                       &vol_commit_hash);
-                if (ret == 0) {
-                        conf->vol_commit_hash = vol_commit_hash;
-                }
-        }
+    gf_msg_debug(this->name, op_errno,
+                 "%s: revalidate lookup on %s returned op_ret %d",
+                 local->loc.path, prev->name, op_ret);
 
-        gf_uuid_unparse (local->loc.gfid, gfid);
+    LOCK(&frame->lock);
+    {
+        if (gf_uuid_is_null(local->gfid)) {
+            memcpy(local->gfid, local->loc.gfid, 16);
+        }
 
-        LOCK (&frame->lock);
-        {
-                if (gf_uuid_is_null (local->gfid)) {
-                        memcpy (local->gfid, local->loc.gfid, 16);
-                }
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+
+            if ((op_errno != ENOTCONN) && (op_errno != ENOENT) &&
+                (op_errno != ESTALE)) {
+                gf_msg(this->name, GF_LOG_INFO, op_errno,
+                       DHT_MSG_REVALIDATE_CBK_INFO,
+                       "Revalidate: subvolume %s for %s "
+                       "(gfid = %s) returned -1",
+                       prev->name, local->loc.path, gfid);
+            }
+            if (op_errno == ESTALE) {
+                /* propagate the ESTALE to parent.
+                 * setting local->return_estale would send
+                 * ESTALE to parent. */
+                local->return_estale = 1;
+            }
 
-                gf_msg_debug (this->name, op_errno,
-                              "revalidate lookup of %s "
-                              "returned with op_ret %d",
-                              local->loc.path, op_ret);
-
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-
-                        if ((op_errno != ENOTCONN)
-                            && (op_errno != ENOENT)
-                            && (op_errno != ESTALE)) {
-                                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                                        DHT_MSG_REVALIDATE_CBK_INFO,
-                                        "Revalidate: subvolume %s for %s "
-                                        "(gfid = %s) returned -1",
-                                        prev->name, local->loc.path,
-                                        gfid);
-                        }
-                        if (op_errno == ESTALE) {
-                                /* propagate the ESTALE to parent.
-                                 * setting local->return_estale would send
-                                 * ESTALE to parent. */
-                                local->return_estale = 1;
+            /* if it is ENOENT, we may have to do a
+             * 'lookup_everywhere()' to make sure
+             * the file is not migrated */
+            if (op_errno == ENOENT) {
+                if (IA_ISREG(local->loc.inode->ia_type)) {
+                    gf_msg_debug(this->name, 0,
+                                 "found ENOENT for %s. "
+                                 "Setting "
+                                 "need_lookup_everywhere"
+                                 " flag to 1",
+                                 local->loc.path);
+
+                    local->need_lookup_everywhere = 1;
+                } else if (IA_ISDIR(local->loc.inode->ia_type)) {
+                    layout = local->layout;
+                    for (i = 0; i < layout->cnt; i++) {
+                        if (layout->list[i].xlator == prev) {
+                            layout->list[i].err = op_errno;
+                            break;
                         }
+                    }
 
-                        /* if it is ENOENT, we may have to do a
-                         * 'lookup_everywhere()' to make sure
-                         * the file is not migrated */
-                        if (op_errno == ENOENT) {
-                                if (IA_ISREG (local->loc.inode->ia_type)) {
-
-                                        gf_msg_debug (this->name, 0,
-                                                      "found ENOENT for %s. "
-                                                      "Setting "
-                                                      "need_lookup_everywhere"
-                                                      " flag to 1",
-                                                      local->loc.path);
-
-                                        local->need_lookup_everywhere = 1;
-                                }
-                        }
-                        goto unlock;
+                    local->need_selfheal = 1;
                 }
+            }
 
-                if ((!IA_ISINVAL(local->inode->ia_type)) &&
-                    stbuf->ia_type != local->inode->ia_type) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_FILE_TYPE_MISMATCH,
-                                "mismatching filetypes 0%o v/s 0%o for %s,"
-                                " gfid = %s",
-                                (stbuf->ia_type), (local->inode->ia_type),
-                                local->loc.path, gfid);
+            /* The GFID is missing on this subvol. Lookup everywhere to force a
+             * gfid heal
+             */
+            if ((op_errno == ENODATA) &&
+                (IA_ISDIR(local->loc.inode->ia_type))) {
+                local->need_lookup_everywhere = 1;
+            }
 
-                        local->op_ret = -1;
-                        local->op_errno = EINVAL;
+            goto unlock;
+        }
 
-                        goto unlock;
+        if ((!IA_ISINVAL(local->inode->ia_type)) &&
+            stbuf->ia_type != local->inode->ia_type) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FILE_TYPE_MISMATCH,
+                   "mismatching filetypes 0%o v/s 0%o for %s,"
+                   " gfid = %s",
+                   (stbuf->ia_type), (local->inode->ia_type), local->loc.path,
+                   gfid);
 
-                }
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
 
-                layout = local->layout;
+            goto unlock;
+        }
 
-                is_dir = check_is_dir (inode, stbuf, xattr);
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                                 conf->link_xattr_name);
-                if (is_linkfile) {
-                        follow_link = 1;
-                        goto unlock;
-                }
-                if (is_dir) {
-                        ret = dht_dir_has_layout (xattr, conf->xattr_name);
-                        if (ret >= 0) {
-                                if (is_greater_time(local->stbuf.ia_ctime,
-                                                    local->stbuf.ia_ctime_nsec,
-                                                    stbuf->ia_ctime,
-                                                    stbuf->ia_ctime_nsec)) {
-                                        /* Choose source */
-                                        local->prebuf.ia_gid = stbuf->ia_gid;
-                                        local->prebuf.ia_uid = stbuf->ia_uid;
-
-                                        if (__is_root_gfid (stbuf->ia_gfid))
-                                                local->prebuf.ia_prot = stbuf->ia_prot;
-                                }
-                        }
+        layout = local->layout;
 
-                        if (local->stbuf.ia_type != IA_INVAL)
-                        {
-                                if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
-                                    (local->stbuf.ia_uid != stbuf->ia_uid) ||
-                                    (__is_root_gfid (stbuf->ia_gfid) &&
-                                     is_permission_different (&local->stbuf.ia_prot,
-                                                              &stbuf->ia_prot))) {
-                                        local->need_selfheal = 1;
-                                }
-                        }
-                        if (!dict_get (xattr, conf->mds_xattr_key)) {
-                                gf_msg_debug (this->name, 0,
-                                              "internal xattr %s is not present"
-                                              " on path %s gfid is %s " ,
-                                              conf->mds_xattr_key,
-                                              local->loc.path, gfid);
-                        } else {
-                                check_mds = dht_dict_get_array (xattr, conf->mds_xattr_key,
-                                                                mds_xattr_val, 1, &errst);
-                                local->mds_subvol  = prev;
-                                local->mds_stbuf.ia_gid = stbuf->ia_gid;
-                                local->mds_stbuf.ia_uid = stbuf->ia_uid;
-                                local->mds_stbuf.ia_prot = stbuf->ia_prot;
-
-                                /* save mds subvol on inode ctx */
-                                ret = dht_inode_ctx_mdsvol_set (local->inode, this,
-                                                                prev);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                DHT_MSG_SET_INODE_CTX_FAILED,
-                                                "Failed to set MDS subvol for %s vol is %s",
-                                                local->loc.path, prev->name);
-                                }
-                                if ((check_mds < 0) && !errst) {
-                                        local->mds_xattr = dict_ref (xattr);
-                                        gf_msg_debug (this->name, 0,
-                                                      "Value of %s is not zero on "
-                                                      "hashed subvol so xattr needs to"
-                                                      " be healed on non hashed"
-                                                      " path is %s and vol name is %s "
-                                                      " gfid is %s" ,
-                                                      conf->mds_xattr_key,
-                                                      local->loc.path,
-                                                      prev->name, gfid);
-                                        local->need_xattr_heal = 1;
-                                }
-                        }
-                        ret = dht_layout_dir_mismatch (this, layout,
-                                                       prev, &local->loc,
-                                                       xattr);
-                        if (ret != 0) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_LAYOUT_MISMATCH,
-                                        "Mismatching layouts for %s, gfid = %s",
-                                        local->loc.path, gfid);
-
-                                local->layout_mismatch = 1;
-
-                                goto unlock;
-                        }
-                }
+        is_dir = check_is_dir(inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile(inode, stbuf, xattr,
+                                        conf->link_xattr_name);
+        if (is_linkfile) {
+            follow_link = 1;
+            goto unlock;
+        }
+        if (is_dir) {
+            ret = dht_dir_has_layout(xattr, conf->xattr_name);
+            if (ret >= 0) {
+                if (is_greater_time(local->prebuf.ia_ctime,
+                                    local->prebuf.ia_ctime_nsec,
+                                    stbuf->ia_ctime, stbuf->ia_ctime_nsec)) {
+                    /* Choose source */
+                    local->prebuf.ia_gid = stbuf->ia_gid;
+                    local->prebuf.ia_uid = stbuf->ia_uid;
 
+                    local->prebuf.ia_ctime = stbuf->ia_ctime;
+                    local->prebuf.ia_ctime_nsec = stbuf->ia_ctime_nsec;
 
-                /* Update stbuf from the servers where layout is present. This
-                 * is an indication that the server is not a newly added brick.
-                 * Merging stbuf from newly added brick may result in the added
-                 * brick being the source of heal for uid/gid */
-                if (!is_dir || (is_dir &&
-                    dht_dir_has_layout (xattr, conf->xattr_name) >= 0)
-                    || conf->subvolume_cnt == 1) {
+                    if (__is_root_gfid(stbuf->ia_gfid))
+                        local->prebuf.ia_prot = stbuf->ia_prot;
+                }
+            }
 
-                        dht_iatt_merge (this, &local->stbuf, stbuf, prev);
-                        dht_iatt_merge (this, &local->postparent, postparent,
-                                        prev);
-                } else {
-                        /* copy the gfid anyway */
-                        gf_uuid_copy (local->stbuf.ia_gfid, stbuf->ia_gfid);
+            if (local->stbuf.ia_type != IA_INVAL) {
+                if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
+                    (local->stbuf.ia_uid != stbuf->ia_uid) ||
+                    is_permission_different(&local->stbuf.ia_prot,
+                                            &stbuf->ia_prot)) {
+                    local->need_attrheal = 1;
                 }
+            }
 
-                local->op_ret = 0;
+            if (!dict_get(xattr, conf->mds_xattr_key)) {
+                gf_msg_debug(this->name, 0,
+                             "%s: internal xattr %s is not present"
+                             " on subvol %s(gfid is %s)",
+                             local->loc.path, conf->mds_xattr_key, prev->name,
+                             gfid);
+            } else {
+                check_mds = dht_dict_get_array(xattr, conf->mds_xattr_key,
+                                               mds_xattr_val, 1, &errst);
+                local->mds_subvol = prev;
+                local->mds_stbuf.ia_gid = stbuf->ia_gid;
+                local->mds_stbuf.ia_uid = stbuf->ia_uid;
+                local->mds_stbuf.ia_prot = stbuf->ia_prot;
 
-                if (!local->xattr) {
-                        local->xattr = dict_ref (xattr);
-                } else if (is_dir) {
-                        dht_aggregate_xattr (local->xattr, xattr);
+                /* save mds subvol on inode ctx */
+                ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           DHT_MSG_SET_INODE_CTX_FAILED,
+                           "Failed to set MDS subvol for %s vol is %s",
+                           local->loc.path, prev->name);
+                }
+                if ((check_mds < 0) && !errst) {
+                    /* Check if xattrs need to be healed on the directory
+                     */
+                    local->mds_xattr = dict_ref(xattr);
+                    gf_msg_debug(this->name, 0,
+                                 "Value of %s is not zero on "
+                                 "hashed subvol so xattr needs to"
+                                 " be healed on non hashed"
+                                 " path is %s and vol name is %s "
+                                 " gfid is %s",
+                                 conf->mds_xattr_key, local->loc.path,
+                                 prev->name, gfid);
+                    local->need_xattr_heal = 1;
                 }
+            }
+            ret = dht_layout_dir_mismatch(this, layout, prev, &local->loc,
+                                          xattr);
+            if (ret != 0) {
+                /* In memory layout does not match on-disk layout.
+                 */
+                gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_MISMATCH,
+                       "Mismatching layouts for %s, gfid = %s", local->loc.path,
+                       gfid);
+
+                local->layout_mismatch = 1;
+
+                goto unlock;
+            }
         }
+
+        gf_uuid_copy(local->stbuf.ia_gfid, stbuf->ia_gfid);
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->postparent, postparent);
+
+        local->op_ret = 0;
+
+        if (!local->xattr) {
+            local->xattr = dict_ref(xattr);
+        } else if (is_dir) {
+            dht_aggregate_xattr(local->xattr, xattr);
+        }
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 
-        if (follow_link) {
-                gf_uuid_copy (local->gfid, stbuf->ia_gfid);
+    if (follow_link) {
+        /* Found a linkto file. Follow it to see if the target file exists
+         */
+        gf_uuid_copy(local->gfid, stbuf->ia_gfid);
 
-                subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
-                if (!subvol) {
-                        op_errno = ESTALE;
-                        local->op_ret = -1;
-                } else {
+        subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
+        if (!subvol) {
+            op_errno = ESTALE;
+            local->op_ret = -1;
+        } else {
+            STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                              subvol->fops->lookup, &local->loc,
+                              local->xattr_req);
+            return 0;
+        }
+    }
 
-                        STACK_WIND_COOKIE (frame, dht_lookup_linkfile_cbk,
-                                           subvol, subvol, subvol->fops->lookup,
-                                           &local->loc, local->xattr_req);
-                        return 0;
+    this_call_cnt = dht_frame_return(frame);
+
+    if (is_last_call(this_call_cnt)) {
+        if (!IA_ISDIR(local->stbuf.ia_type) &&
+            (local->hashed_subvol != local->cached_subvol) &&
+            (local->stbuf.ia_nlink == 1) &&
+            (conf && conf->unhashed_sticky_bit)) {
+            local->stbuf.ia_prot.sticky = 1;
+        }
+        /* No need to call heal code if volume count is 1
+         */
+        if (conf->subvolume_cnt == 1)
+            local->need_xattr_heal = 0;
+
+        if (IA_ISDIR(local->stbuf.ia_type)) {
+            /* No mds xattr found. Trigger a heal to set it */
+            if (!__is_root_gfid(local->loc.inode->gfid) &&
+                (!dict_get(local->xattr, conf->mds_xattr_key)))
+                local->need_selfheal = 1;
+
+            if (dht_needs_selfheal(frame, this)) {
+                if (!__is_root_gfid(local->loc.inode->gfid)) {
+                    if (local->mds_subvol) {
+                        local->stbuf.ia_gid = local->mds_stbuf.ia_gid;
+                        local->stbuf.ia_uid = local->mds_stbuf.ia_uid;
+                        local->stbuf.ia_prot = local->mds_stbuf.ia_prot;
+                    }
+                } else {
+                    local->stbuf.ia_gid = local->prebuf.ia_gid;
+                    local->stbuf.ia_uid = local->prebuf.ia_uid;
+                    local->stbuf.ia_prot = local->prebuf.ia_prot;
                 }
+
+                layout = local->layout;
+                dht_selfheal_directory(frame, dht_lookup_selfheal_cbk,
+                                       &local->loc, layout);
+                return 0;
+            }
         }
 
-out:
-        this_call_cnt = dht_frame_return (frame);
-
-        if (is_last_call (this_call_cnt)) {
-                if (!IA_ISDIR (local->stbuf.ia_type)
-                    && (local->hashed_subvol != local->cached_subvol)
-                    && (local->stbuf.ia_nlink == 1)
-                    && (conf && conf->unhashed_sticky_bit)) {
-                        local->stbuf.ia_prot.sticky = 1;
-                }
-                /* No need to call heal code if volume count is 1
-                */
-                if (conf->subvolume_cnt == 1)
-                        local->need_xattr_heal = 0;
-
-                if (IA_ISDIR (local->stbuf.ia_type)) {
-                        /* Code to update all extended attributed from hashed
-                           subvol to local->xattr and call heal code to heal
-                           custom xattr from hashed subvol to non-hashed subvol
-                        */
-                        if (local->need_xattr_heal && (local->mds_xattr)) {
-                                dht_dir_set_heal_xattr (this, local,
-                                                        local->xattr,
-                                                        local->mds_xattr, NULL,
-                                                        NULL);
-                                dict_unref (local->mds_xattr);
-                                local->mds_xattr = NULL;
-                                local->need_xattr_heal = 0;
-                                ret =  dht_dir_xattr_heal (this, local);
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_ERROR,
-                                                ret, DHT_MSG_DIR_XATTR_HEAL_FAILED,
-                                                "xattr heal failed for directory %s "
-                                                " gfid %s ", local->loc.path,
-                                                gfid);
-                        } else {
-                                /* Call function to save hashed subvol on inode
-                                   ctx if internal mds xattr is not present and
-                                   all subvols are up
-                                */
-                                if (inode && !__is_root_gfid (inode->gfid) &&
-                                    (!local->op_ret))
-                                        (void) dht_common_mark_mdsxattr (frame, NULL, 1);
-                        }
-                }
-                if (local->need_selfheal) {
-                        local->need_selfheal = 0;
-                        if (!__is_root_gfid (inode->gfid)) {
-                                gf_uuid_copy (local->gfid, local->mds_stbuf.ia_gfid);
-                                if (local->mds_stbuf.ia_gid || local->mds_stbuf.ia_uid) {
-                                        local->stbuf.ia_gid = local->mds_stbuf.ia_gid;
-                                        local->stbuf.ia_uid = local->mds_stbuf.ia_uid;
-                                }
-                        } else {
-                                gf_uuid_copy (local->gfid, local->stbuf.ia_gfid);
-                                local->stbuf.ia_gid = local->prebuf.ia_gid;
-                                local->stbuf.ia_uid = local->prebuf.ia_uid;
-                                local->stbuf.ia_prot = local->prebuf.ia_prot;
-                        }
+        if (local->layout_mismatch) {
+            /* Found layout mismatch in the directory, need to
+               fix this in the inode context */
+            dht_layout_unref(this, local->layout);
+            local->layout = NULL;
+            dht_lookup_directory(frame, this, &local->loc);
+            return 0;
+        }
 
-                        copy = create_frame (this, this->ctx->pool);
-                        if (copy) {
-                                copy_local = dht_local_init (copy, &local->loc,
-                                                             NULL, 0);
-                                if (!copy_local)
-                                        goto cont;
-                                copy_local->stbuf = local->stbuf;
-                                copy_local->mds_stbuf = local->mds_stbuf;
-                                copy_local->mds_subvol = local->mds_subvol;
-                                copy->local = copy_local;
-                                FRAME_SU_DO (copy, dht_local_t);
-                                ret = synctask_new (this->ctx->env,
-                                                    dht_dir_attr_heal,
-                                                    dht_dir_attr_heal_done,
-                                                    copy, copy);
-                        }
-                }
-cont:
-                if (local->layout_mismatch) {
-                        /* Found layout mismatch in the directory, need to
-                           fix this in the inode context */
-                        dht_layout_unref (this, local->layout);
-                        local->layout = NULL;
-                        dht_lookup_directory (frame, this, &local->loc);
-                        return 0;
-                }
+        if (local->need_lookup_everywhere) {
+            /* As the current layout gave ENOENT error, we would
+               need a new layout */
+            dht_layout_unref(this, local->layout);
+            local->layout = NULL;
 
-                if (local->need_lookup_everywhere) {
-                        /* As the current layout gave ENOENT error, we would
-                           need a new layout */
-                        dht_layout_unref (this, local->layout);
-                        local->layout = NULL;
-
-                        /* We know that current cached subvol is no more
-                           valid, get the new one */
-                        local->cached_subvol = NULL;
-                        dht_lookup_everywhere (frame, this, &local->loc);
-                        return 0;
-                }
-                if (local->return_estale) {
-                        local->op_ret = -1;
-                        local->op_errno = ESTALE;
+            /* We know that current cached subvol is no longer
+               valid, get the new one */
+            local->cached_subvol = NULL;
+            if (local->xattr_req) {
+                if (!gf_uuid_is_null(local->gfid)) {
+                    ret = dict_set_static_bin(local->xattr_req, "gfid-req",
+                                              local->gfid, 16);
                 }
+            }
 
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+            dht_lookup_everywhere(frame, this, &local->loc);
+            return 0;
+        }
+        if (local->return_estale) {
+            local->op_ret = -1;
+            local->op_errno = ESTALE;
+        }
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                dht_set_fixed_dir_stat (&local->postparent);
-
-                /* local->stbuf is udpated only from subvols which have a layout
-                 * The reason is to avoid choosing attr heal source from newly
-                 * added bricks. In case e.g we have only one subvol and for
-                 * some reason layout is not present on it, then local->stbuf
-                 * will be EINVAL. This is an indication that the subvols
-                 * active in the cluster do not have layouts on disk.
-                 * Unwind with ESTALE to trigger a fresh lookup */
-                if (is_dir && local->stbuf.ia_type == IA_INVAL) {
-                        local->op_ret = -1;
-                        local->op_errno = ESTALE;
-                }
-                /* Delete mds xattr at the time of STACK UNWIND */
-                GF_REMOVE_INTERNAL_XATTR (conf->mds_xattr_key, local->xattr);
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
+        }
+
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
 
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                                  local->inode, &local->stbuf, local->xattr,
-                                  &local->postparent);
+        /* local->stbuf is updated only from subvols which have a layout
+         * The reason is to avoid choosing attr heal source from newly
+         * added bricks. In case e.g we have only one subvol and for
+         * some reason layout is not present on it, then local->stbuf
+         * will be EINVAL. This is an indication that the subvols
+         * active in the cluster do not have layouts on disk.
+         * Unwind with ESTALE to trigger a fresh lookup */
+        if (is_dir && local->stbuf.ia_type == IA_INVAL) {
+            local->op_ret = -1;
+            local->op_errno = ESTALE;
         }
+        /* Delete mds xattr at the time of STACK UNWIND */
+        if (local->xattr)
+            GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
+
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+    }
 
 err:
-        return ret;
+    return ret;
 }
 
+static int
+dht_lookup_linkfile_create_cbk(call_frame_t *frame, void *cooie, xlator_t *this,
+                               int32_t op_ret, int32_t op_errno, inode_t *inode,
+                               struct iatt *stbuf, struct iatt *preparent,
+                               struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-int
-dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent,
-                                dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        xlator_t     *cached_subvol = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret = -1;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
 
-        local = frame->local;
-        cached_subvol = local->cached_subvol;
-        conf = this->private;
+    local = frame->local;
+    cached_subvol = local->cached_subvol;
+    conf = this->private;
 
-        gf_uuid_unparse(local->loc.gfid, gfid);
+    gf_uuid_unparse(local->loc.gfid, gfid);
 
-        if (local->locked)
-                dht_unlock_namespace (frame, &local->lock[0]);
+    if (local->locked)
+        dht_unlock_namespace(frame, &local->lock[0]);
 
-        ret = dht_layout_preset (this, local->cached_subvol, local->loc.inode);
-        if (ret < 0) {
-                gf_msg_debug (this->name, EINVAL,
-                              "Failed to set layout for subvolume %s, "
-                              "(gfid = %s)",
-                              cached_subvol ? cached_subvol->name : "<nil>",
-                              gfid);
-                local->op_ret = -1;
-                local->op_errno = EINVAL;
-                goto unwind;
-        }
+    ret = dht_layout_preset(this, local->cached_subvol, local->loc.inode);
+    if (ret < 0) {
+        gf_msg_debug(this->name, EINVAL,
+                     "Failed to set layout for subvolume %s, "
+                     "(gfid = %s)",
+                     cached_subvol ? cached_subvol->name : "<nil>", gfid);
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
 
-        local->op_ret = 0;
-        if ((local->stbuf.ia_nlink == 1)
-            && (conf && conf->unhashed_sticky_bit)) {
-                local->stbuf.ia_prot.sticky = 1;
-        }
-
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
+    local->op_ret = 0;
+    if ((local->stbuf.ia_nlink == 1) && (conf && conf->unhashed_sticky_bit)) {
+        local->stbuf.ia_prot.sticky = 1;
+    }
 
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
 
 unwind:
-        gf_msg_debug (this->name, 0,
-                      "creation of linkto on hashed subvol:%s, "
-                      "returned with op_ret %d and op_errno %d: %s",
-                      local->hashed_subvol->name,
-                      op_ret, op_errno, uuid_utoa (local->loc.gfid));
+    gf_msg_debug(this->name, 0,
+                 "creation of linkto on hashed subvol:%s, "
+                 "returned with op_ret %d and op_errno %d: %s",
+                 local->hashed_subvol->name, op_ret, op_errno,
+                 uuid_utoa(local->loc.gfid));
 
-        if (local->linked == _gf_true)
-                dht_linkfile_attr_heal (frame, this);
+    if (local->linked == _gf_true)
+        dht_linkfile_attr_heal(frame, this);
 
+    dht_set_fixed_dir_stat(&local->postparent);
 
-        dht_set_fixed_dir_stat (&local->postparent);
-
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                          local->inode, &local->stbuf, local->xattr,
-                          &local->postparent);
+    DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+    DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->stbuf, local->xattr,
+                     &local->postparent);
 out:
-        return ret;
+    return ret;
 }
 
-int
-dht_lookup_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       struct iatt *preparent, struct iatt *postparent,
-                       dict_t *xdata)
+static int
+dht_lookup_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        int             this_call_cnt = 0;
-        dht_local_t     *local = NULL;
-        const char      *path =  NULL;
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    const char *path = NULL;
 
-        local =  (dht_local_t*)frame->local;
-        path = local->loc.path;
-        FRAME_SU_UNDO (frame, dht_local_t);
+    local = (dht_local_t *)frame->local;
+    path = local->loc.path;
+    FRAME_SU_UNDO(frame, dht_local_t);
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                DHT_MSG_UNLINK_LOOKUP_INFO, "lookup_unlink returned with "
-                "op_ret -> %d and op-errno -> %d for %s", op_ret, op_errno,
-                ((path == NULL)? "null" : path ));
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_UNLINK_LOOKUP_INFO,
+           "lookup_unlink returned with "
+           "op_ret -> %d and op-errno -> %d for %s",
+           op_ret, op_errno, ((path == NULL) ? "null" : path));
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                dht_lookup_everywhere_done (frame, this);
-        }
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_lookup_everywhere_done(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_lookup_unlink_of_false_linkto_cbk (call_frame_t *frame, void *cookie,
-                                       xlator_t *this, int op_ret, int op_errno,
-                                       struct iatt *preparent,
-                                       struct iatt *postparent, dict_t *xdata)
+static int
+dht_lookup_unlink_of_false_linkto_cbk(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int op_ret, int op_errno,
+                                      struct iatt *preparent,
+                                      struct iatt *postparent, dict_t *xdata)
 {
-        int             this_call_cnt = 0;
-        dht_local_t     *local = NULL;
-        const char      *path =  NULL;
-
-        local =  (dht_local_t*)frame->local;
-        path = local->loc.path;
-
-        FRAME_SU_UNDO (frame, dht_local_t);
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    const char *path = NULL;
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                DHT_MSG_UNLINK_LOOKUP_INFO, "lookup_unlink returned with "
-                "op_ret -> %d and op-errno -> %d for %s", op_ret, op_errno,
-                ((path == NULL)? "null" : path ));
+    local = (dht_local_t *)frame->local;
+    path = local->loc.path;
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
+    FRAME_SU_UNDO(frame, dht_local_t);
 
-                if (op_ret == 0) {
-                        dht_lookup_everywhere_done (frame, this);
-                } else {
-                       /*When dht_lookup_everywhere is performed, one cached
-                         *and one hashed file was found and hashed file does
-                         *not point to the above mentioned cached node. So it
-                         *was considered as stale and an unlink was performed.
-                         *But unlink fails. So may be rebalance is in progress.
-                        *now ideally we have two data-files. One obtained during
-                         *lookup_everywhere and one where unlink-failed. So
-                         *at this point in time we cannot decide which one to
-                         *choose because there are chances of first cached
-                         *file is truncated after rebalance and if it is chosen
-                        *as cached node, application will fail. So return EIO.*/
-
-                        if (op_errno == EBUSY) {
-
-                                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                        DHT_MSG_UNLINK_FAILED,
-                                        "Could not unlink the linkto file as "
-                                        "either fd is open and/or linkto xattr "
-                                        "is set for %s",
-                                        ((path == NULL)? "null":path));
-
-                        }
-                        DHT_STACK_UNWIND (lookup, frame, -1, EIO, NULL, NULL,
-                                          NULL, NULL);
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_UNLINK_LOOKUP_INFO,
+           "lookup_unlink returned with "
+           "op_ret -> %d and op-errno -> %d for %s",
+           op_ret, op_errno, ((path == NULL) ? "null" : path));
 
-                }
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if ((op_ret == 0) || ((op_errno != EBUSY) && (op_errno != ENOTCONN))) {
+            dht_lookup_everywhere_done(frame, this);
+        } else {
+            /*When dht_lookup_everywhere is performed, one cached
+             *and one hashed file was found and hashed file does
+             *not point to the above mentioned cached node. So it
+             *was considered as stale and an unlink was performed.
+             *But unlink fails. So may be rebalance is in progress.
+             *now ideally we have two data-files. One obtained during
+             *lookup_everywhere and one where unlink-failed. So
+             *at this point in time we cannot decide which one to
+             *choose because there are chances of first cached
+             *file is truncated after rebalance and if it is chosen
+             *as cached node, application will fail. So return EIO.*/
+
+            if (op_errno == EBUSY) {
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_UNLINK_FAILED,
+                       "Could not unlink the linkto file as "
+                       "either fd is open and/or linkto xattr "
+                       "is set for %s",
+                       ((path == NULL) ? "null" : path));
+            }
+            DHT_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_lookup_unlink_stale_linkto_cbk (call_frame_t *frame, void *cookie,
-                                    xlator_t *this, int op_ret, int op_errno,
-                                    struct iatt *preparent,
-                                    struct iatt *postparent, dict_t *xdata)
+static int
+dht_lookup_unlink_stale_linkto_cbk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int op_ret, int op_errno,
+                                   struct iatt *preparent,
+                                   struct iatt *postparent, dict_t *xdata)
 {
+    dht_local_t *local = NULL;
+    const char *path = NULL;
 
-        dht_local_t     *local = NULL;
-        const char      *path  = NULL;
-
-        /* NOTE:
-         * If stale file unlink fails either there is an open-fd or is not an
-         * dht-linkto-file then posix_unlink returns EBUSY, which is overwritten
-         *  to ENOENT
-         */
+    /* NOTE:
+     * If stale file unlink fails either there is an open-fd or is not an
+     * dht-linkto-file then posix_unlink returns EBUSY, which is overwritten
+     *  to ENOENT
+     */
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local && local->loc.path)
-                path = local->loc.path;
+    if (local) {
+        FRAME_SU_UNDO(frame, dht_local_t);
+        if (local->loc.path)
+            path = local->loc.path;
+    }
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                DHT_MSG_UNLINK_LOOKUP_INFO,
-                "Returned with op_ret %d and "
-                "op_errno %d for %s", op_ret, op_errno,
-                ((path==NULL)?"null":path));
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_UNLINK_LOOKUP_INFO,
+           "Returned with op_ret %d and "
+           "op_errno %d for %s",
+           op_ret, op_errno, ((path == NULL) ? "null" : path));
 
-        FRAME_SU_UNDO (frame, dht_local_t);
-        DHT_STACK_UNWIND (lookup, frame, -1, ENOENT, NULL, NULL, NULL,
-                          NULL);
+    DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_fill_dict_to_avoid_unlink_of_migrating_file (dict_t *dict) {
-
-        int                      ret = 0;
-        xlator_t                *this           = NULL;
-        char                    *linktoskip_key = NULL;
-
-        this    = THIS;
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-
-        if (dht_is_tier_xlator (this))
-                linktoskip_key = TIER_SKIP_NON_LINKTO_UNLINK;
-        else
-                linktoskip_key = DHT_SKIP_NON_LINKTO_UNLINK;
-
-        ret = dict_set_int32 (dict, linktoskip_key, 1);
-
-        if (ret)
-                goto err;
+static int
+dht_fill_dict_to_avoid_unlink_of_migrating_file(dict_t *dict)
+{
+    int ret = 0;
 
-        ret =  dict_set_int32 (dict, DHT_SKIP_OPEN_FD_UNLINK, 1);
-
-        if (ret)
-                goto err;
+    ret = dict_set_int32_sizen(dict, DHT_SKIP_NON_LINKTO_UNLINK, 1);
 
+    if (ret)
+        return -1;
 
-        return 0;
+    ret = dict_set_int32_sizen(dict, DHT_SKIP_OPEN_FD_UNLINK, 1);
 
-err:
+    if (ret)
         return -1;
 
+    return 0;
 }
 
-int32_t
-dht_linkfile_create_lookup_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this, int32_t op_ret,
-                                int32_t op_errno, inode_t *inode,
-                                struct iatt *buf, dict_t *xdata,
-                                struct iatt *postparent)
+static int32_t
+dht_linkfile_create_lookup_cbk(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               inode_t *inode, struct iatt *buf, dict_t *xdata,
+                               struct iatt *postparent)
 {
-        dht_local_t *local                      = NULL;
-        int          call_cnt                   = 0, ret = 0;
-        xlator_t    *subvol                     = NULL;
-        uuid_t       gfid                       = {0, };
-        char         gfid_str[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *local = NULL;
+    int call_cnt = 0, ret = 0;
+    xlator_t *subvol = NULL;
+    uuid_t gfid = {
+        0,
+    };
+    char gfid_str[GF_UUID_BUF_SIZE] = {0};
 
-        subvol = cookie;
-        local = frame->local;
+    subvol = cookie;
+    local = frame->local;
 
-        if (subvol == local->hashed_subvol) {
-                if ((op_ret == 0) || (op_errno != ENOENT))
-                        local->dont_create_linkto = _gf_true;
-        } else {
-                if (gf_uuid_is_null (local->gfid))
-                        gf_uuid_copy (gfid, local->loc.gfid);
-                else
-                        gf_uuid_copy (gfid, local->gfid);
-
-                if ((op_ret == 0) && gf_uuid_compare (gfid, buf->ia_gfid)) {
-                        gf_uuid_unparse (gfid, gfid_str);
-                        gf_msg_debug (this->name, 0,
-                                      "gfid (%s) different on cached subvol "
-                                      "(%s) and looked up inode (%s), not "
-                                      "creating linkto",
-                                      uuid_utoa (buf->ia_gfid), subvol->name,
-                                      gfid_str);
-                        local->dont_create_linkto = _gf_true;
-                } else if (op_ret == -1) {
-                        local->dont_create_linkto = _gf_true;
-                }
-        }
+    if (subvol == local->hashed_subvol) {
+        if ((op_ret == 0) || (op_errno != ENOENT))
+            local->dont_create_linkto = _gf_true;
+    } else {
+        if (gf_uuid_is_null(local->gfid))
+            gf_uuid_copy(gfid, local->loc.gfid);
+        else
+            gf_uuid_copy(gfid, local->gfid);
+
+        if ((op_ret == 0) && gf_uuid_compare(gfid, buf->ia_gfid)) {
+            gf_uuid_unparse(gfid, gfid_str);
+            gf_msg_debug(this->name, 0,
+                         "gfid (%s) different on cached subvol "
+                         "(%s) and looked up inode (%s), not "
+                         "creating linkto",
+                         uuid_utoa(buf->ia_gfid), subvol->name, gfid_str);
+            local->dont_create_linkto = _gf_true;
+        } else if (op_ret == -1) {
+            local->dont_create_linkto = _gf_true;
+        }
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        if (local->dont_create_linkto)
+            goto no_linkto;
+        else {
+            gf_msg_debug(this->name, 0,
+                         "Creating linkto file on %s(hash) to "
+                         "%s on %s (gfid = %s)",
+                         local->hashed_subvol->name, local->loc.path,
+                         local->cached_subvol->name, gfid_str);
 
-        call_cnt = dht_frame_return (frame);
-        if (is_last_call (call_cnt)) {
-                if (local->dont_create_linkto)
-                        goto no_linkto;
-                else {
-                        gf_msg_debug (this->name, 0,
-                                      "Creating linkto file on %s(hash) to "
-                                      "%s on %s (gfid = %s)",
-                                      local->hashed_subvol->name,
-                                      local->loc.path,
-                                      local->cached_subvol->name, gfid);
-
-                        ret = dht_linkfile_create
-                                (frame, dht_lookup_linkfile_create_cbk,
-                                 this, local->cached_subvol,
-                                 local->hashed_subvol, &local->loc);
-
-                        if (ret < 0)
-                                goto no_linkto;
-                }
+            ret = dht_linkfile_create(frame, dht_lookup_linkfile_create_cbk,
+                                      this, local->cached_subvol,
+                                      local->hashed_subvol, &local->loc);
+
+            if (ret < 0)
+                goto no_linkto;
         }
+    }
 
-        return 0;
+    return 0;
 
 no_linkto:
-        gf_msg_debug (this->name, 0,
-                      "skipped linkto creation (path:%s) (gfid:%s) "
-                      "(hashed-subvol:%s) (cached-subvol:%s)",
-                      local->loc.path, gfid_str, local->hashed_subvol->name,
-                      local->cached_subvol->name);
-
-        dht_lookup_linkfile_create_cbk (frame, NULL, this, 0, 0,
-                                        local->loc.inode, &local->stbuf,
-                                        &local->preparent, &local->postparent,
-                                        local->xattr);
-        return 0;
-}
+    gf_msg_debug(this->name, 0,
+                 "skipped linkto creation (path:%s) (gfid:%s) "
+                 "(hashed-subvol:%s) (cached-subvol:%s)",
+                 local->loc.path, gfid_str, local->hashed_subvol->name,
+                 local->cached_subvol->name);
 
+    dht_lookup_linkfile_create_cbk(frame, NULL, this, 0, 0, local->loc.inode,
+                                   &local->stbuf, &local->preparent,
+                                   &local->postparent, local->xattr);
+    return 0;
+}
 
-int32_t
-dht_call_lookup_linkfile_create (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *xdata)
+static int32_t
+dht_call_lookup_linkfile_create(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local          = NULL;
-        char gfid[GF_UUID_BUF_SIZE] = {0};
-        int          i              = 0;
-        xlator_t    *subvol         = NULL;
-
-        local = frame->local;
-        if (gf_uuid_is_null (local->gfid))
-                gf_uuid_unparse (local->loc.gfid, gfid);
-        else
-                gf_uuid_unparse (local->gfid, gfid);
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int i = 0;
+    xlator_t *subvol = NULL;
 
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "protecting namespace failed, skipping linkto "
-                        "creation (path:%s)(gfid:%s)(hashed-subvol:%s)"
-                        "(cached-subvol:%s)", local->loc.path, gfid,
-                        local->hashed_subvol->name, local->cached_subvol->name);
-                goto err;
-        }
+    local = frame->local;
+    if (gf_uuid_is_null(local->gfid))
+        gf_uuid_unparse(local->loc.gfid, gfid);
+    else
+        gf_uuid_unparse(local->gfid, gfid);
 
-        local->locked = _gf_true;
+    if (op_ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "protecting namespace failed, skipping linkto "
+               "creation (path:%s)(gfid:%s)(hashed-subvol:%s)"
+               "(cached-subvol:%s)",
+               local->loc.path, gfid, local->hashed_subvol->name,
+               local->cached_subvol->name);
+        goto err;
+    }
 
+    local->locked = _gf_true;
 
-        local->call_cnt = 2;
+    local->call_cnt = 2;
 
-        for (i = 0; i < 2; i++) {
-                subvol = (subvol == NULL) ? local->hashed_subvol
-                        : local->cached_subvol;
+    for (i = 0; i < 2; i++) {
+        subvol = (subvol == NULL) ? local->hashed_subvol : local->cached_subvol;
 
-                STACK_WIND_COOKIE (frame, dht_linkfile_create_lookup_cbk,
-                                   subvol, subvol, subvol->fops->lookup,
-                                   &local->loc, NULL);
-        }
+        STACK_WIND_COOKIE(frame, dht_linkfile_create_lookup_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, NULL);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        dht_lookup_linkfile_create_cbk (frame, NULL, this, 0, 0,
-                                        local->loc.inode,
-                                        &local->stbuf, &local->preparent,
-                                        &local->postparent, local->xattr);
-        return 0;
+    dht_lookup_linkfile_create_cbk(frame, NULL, this, 0, 0, local->loc.inode,
+                                   &local->stbuf, &local->preparent,
+                                   &local->postparent, local->xattr);
+    return 0;
 }
 
 /* Rebalance is performed from cached_node to hashed_node. Initial cached_node
@@ -2311,763 +2335,654 @@ err:
  * dht_lookup_everywhere_done takes decision based on any of the above case
  */
 
-int
-dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
-{
-        int           ret                        = 0;
-        dht_local_t  *local                      = NULL;
-        xlator_t     *hashed_subvol              = NULL;
-        xlator_t     *cached_subvol              = NULL;
-        dht_layout_t *layout                     = NULL;
-        char gfid[GF_UUID_BUF_SIZE]              = {0};
-        gf_boolean_t  found_non_linkto_on_hashed = _gf_false;
+static int
+dht_lookup_everywhere_done(call_frame_t *frame, xlator_t *this)
+{
+    int ret = 0;
+    dht_local_t *local = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_layout_t *layout = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    gf_boolean_t found_non_linkto_on_hashed = _gf_false;
+
+    local = frame->local;
+    hashed_subvol = local->hashed_subvol;
+    cached_subvol = local->cached_subvol;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    if (local->file_count && local->dir_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_FILE_TYPE_MISMATCH,
+               "path %s (gfid = %s)exists as a file on one "
+               "subvolume and directory on another. "
+               "Please fix it manually",
+               local->loc.path, gfid);
+        DHT_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
+        return 0;
+    }
+    if (local->op_ret && local->gfid_missing) {
+        if (gf_uuid_is_null(local->gfid_req)) {
+            DHT_STACK_UNWIND(lookup, frame, -1, ENODATA, NULL, NULL, NULL,
+                             NULL);
+            return 0;
+        }
+        /* A hack */
+        dht_lookup_directory(frame, this, &local->loc);
+        return 0;
+    }
+
+    if (local->dir_count) {
+        dht_lookup_directory(frame, this, &local->loc);
+        return 0;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "STATUS: hashed_subvol %s "
+                 "cached_subvol %s",
+                 (hashed_subvol == NULL) ? "null" : hashed_subvol->name,
+                 (cached_subvol == NULL) ? "null" : cached_subvol->name);
+
+    if (!cached_subvol) {
+        if (local->skip_unlink.handle_valid_link && hashed_subvol) {
+            /*Purpose of "DHT_SKIP_NON_LINKTO_UNLINK":
+             * If this lookup is performed by rebalance and this
+             * rebalance process detected hashed file and by
+             * the time it sends the lookup request to cached node,
+             * file got migrated and now at initial hashed_node,
+             * final migrated file is present. With current logic,
+             * because this process fails to find the cached_node,
+             * it will unlink the file at initial hashed_node.
+             *
+             * So we avoid this by setting key, and checking at the
+             * posix_unlink that unlink the file only if file is a
+             * linkto file and not a migrated_file.
+             */
+
+            ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(
+                local->xattr_req);
+
+            if (ret) {
+                /* If for some reason, setting key in the dict
+                 * fails, return with ENOENT, as with respect to
+                 * this process, it detected only a stale link
+                 * file.
+                 *
+                 * Next lookup will delete it.
+                 *
+                 * Performing deletion of stale link file when
+                 * setting key in dict fails, may cause the data
+                 * loss because of the above mentioned race.
+                 */
 
-        local = frame->local;
-        hashed_subvol = local->hashed_subvol;
-        cached_subvol = local->cached_subvol;
+                DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL,
+                                 NULL);
+            } else {
+                local->skip_unlink.handle_valid_link = _gf_false;
+
+                gf_msg_debug(this->name, 0,
+                             "No Cached was found and "
+                             "unlink on hashed was skipped"
+                             " so performing now: %s",
+                             local->loc.path);
+                FRAME_SU_DO(frame, dht_local_t);
+                STACK_WIND(frame, dht_lookup_unlink_stale_linkto_cbk,
+                           hashed_subvol, hashed_subvol->fops->unlink,
+                           &local->loc, 0, local->xattr_req);
+            }
 
-        gf_uuid_unparse (local->loc.gfid, gfid);
-
-        if (local->file_count && local->dir_count) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_FILE_TYPE_MISMATCH,
-                        "path %s (gfid = %s)exists as a file on one "
-                        "subvolume and directory on another. "
-                        "Please fix it manually",
-                        local->loc.path, gfid);
-                DHT_STACK_UNWIND (lookup, frame, -1, EIO, NULL, NULL, NULL,
-                                  NULL);
-                return 0;
-        }
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "There was no cached file and  "
+                         "unlink on hashed is not skipped %s",
+                         local->loc.path);
 
-        if (local->dir_count) {
-                dht_lookup_directory (frame, this, &local->loc);
-                return 0;
+            DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL, NULL);
         }
+        return 0;
+    }
 
-        gf_msg_debug (this->name, 0, "STATUS: hashed_subvol %s "
-                      "cached_subvol %s",
-                      (hashed_subvol == NULL)?"null":hashed_subvol->name,
-                      (cached_subvol == NULL)?"null":cached_subvol->name);
-
-        if (!cached_subvol) {
-
-                if (local->skip_unlink.handle_valid_link && hashed_subvol) {
-
-                        /*Purpose of "DHT_SKIP_NON_LINKTO_UNLINK":
-                         * If this lookup is performed by rebalance and this
-                         * rebalance process detected hashed file and by
-                         * the time it sends the lookup request to cached node,
-                         * file got migrated and now at initial hashed_node,
-                         * final migrated file is present. With current logic,
-                         * because this process fails to find the cached_node,
-                         * it will unlink the file at initial hashed_node.
-                         *
-                         * So we avoid this by setting key, and checking at the
-                         * posix_unlink that unlink the file only if file is a
-                         * linkto file and not a migrated_file.
-                         */
-
-
-                        ret = dht_fill_dict_to_avoid_unlink_of_migrating_file
-                              (local->xattr_req);
-
-                        if (ret) {
-                                /* If for some reason, setting key in the dict
-                                 * fails, return with ENOENT, as with respect to
-                                 * this process, it detected only a stale link
-                                 * file.
-                                 *
-                                 * Next lookup will delete it.
-                                 *
-                                 * Performing deletion of stale link file when
-                                 * setting key in dict fails, may cause the data
-                                 * loss becase of the above mentioned race.
-                                 */
-
-
-                                DHT_STACK_UNWIND (lookup, frame, -1, ENOENT,
-                                                  NULL, NULL, NULL, NULL);
-                        } else {
-                                local->skip_unlink.handle_valid_link = _gf_false;
-
-                                gf_msg_debug (this->name, 0,
-                                              "No Cached was found and "
-                                              "unlink on hashed was skipped"
-                                              " so performing now: %s",
-                                              local->loc.path);
-                                FRAME_SU_DO (frame, dht_local_t);
-                                STACK_WIND (frame,
-                                            dht_lookup_unlink_stale_linkto_cbk,
-                                            hashed_subvol,
-                                            hashed_subvol->fops->unlink,
-                                            &local->loc, 0, local->xattr_req);
-                        }
+    /* At the time of dht_lookup, no file was found on hashed and that is
+     * why dht_lookup_everywhere is called, but by the time
+     * dht_lookup_everywhere
+     * reached to server, file might have already migrated. In that case we
+     * will find a migrated file at the hashed_node. In this case store the
+     * layout in context and return successfully.
+     */
 
-                } else  {
+    if (hashed_subvol || local->need_lookup_everywhere) {
+        if (local->need_lookup_everywhere) {
+            found_non_linkto_on_hashed = _gf_true;
 
-                        gf_msg_debug (this->name, 0,
-                                      "There was no cached file and  "
-                                      "unlink on hashed is not skipped %s",
-                                      local->loc.path);
+        } else if ((local->file_count == 1) &&
+                   (hashed_subvol == cached_subvol)) {
+            gf_msg_debug(this->name, 0,
+                         "found cached file on hashed subvolume "
+                         "so store in context and return for %s",
+                         local->loc.path);
 
-                        DHT_STACK_UNWIND (lookup, frame, -1, ENOENT,
-                                          NULL, NULL, NULL, NULL);
-                }
-                return 0;
+            found_non_linkto_on_hashed = _gf_true;
         }
 
-        /* At the time of dht_lookup, no file was found on hashed and that is
-         * why dht_lookup_everywhere is called, but by the time
-         * dht_lookup_everywhere
-         * reached to server, file might have already migrated. In that case we
-         * will find a migrated file at the hashed_node. In this case store the
-         * layout in context and return successfully.
-         */
-
-        if (hashed_subvol || local->need_lookup_everywhere) {
-
-                if (local->need_lookup_everywhere) {
-
-                        found_non_linkto_on_hashed = _gf_true;
+        if (found_non_linkto_on_hashed)
+            goto preset_layout;
+    }
 
-                } else if ((local->file_count == 1) &&
-                            (hashed_subvol == cached_subvol)) {
+    if (hashed_subvol) {
+        if (local->skip_unlink.handle_valid_link == _gf_true) {
+            if (cached_subvol == local->skip_unlink.hash_links_to) {
+                if (gf_uuid_compare(local->skip_unlink.cached_gfid,
+                                    local->skip_unlink.hashed_gfid)) {
+                    /*GFID different, return error*/
+                    DHT_STACK_UNWIND(lookup, frame, -1, ESTALE, NULL, NULL,
+                                     NULL, NULL);
 
-                        gf_msg_debug (this->name, 0,
-                                      "found cached file on hashed subvolume "
-                                      "so store in context and return for %s",
-                                      local->loc.path);
-
-                        found_non_linkto_on_hashed = _gf_true;
+                    return 0;
                 }
 
-                if (found_non_linkto_on_hashed)
-                        goto preset_layout;
-
-        }
-
-
-        if (hashed_subvol) {
-                if (local->skip_unlink.handle_valid_link == _gf_true) {
-                        if (cached_subvol == local->skip_unlink.hash_links_to) {
-
-                             if (gf_uuid_compare (local->skip_unlink.cached_gfid,
-                                               local->skip_unlink.hashed_gfid)){
-
-                                        /*GFID different, return error*/
-                                        DHT_STACK_UNWIND (lookup, frame, -1,
-                                                          ESTALE, NULL, NULL,
-                                                          NULL, NULL);
-
-                                        return 0;
-                                }
-
-                                ret = dht_layout_preset (this, cached_subvol,
-                                                         local->loc.inode);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                                DHT_MSG_LAYOUT_PRESET_FAILED,
-                                                "Could not set pre-set layout "
-                                                "for subvolume %s",
-                                                cached_subvol->name);
-                                }
-
-                                local->op_ret = (ret == 0) ? ret : -1;
-                                local->op_errno = (ret == 0) ? ret : EINVAL;
-
-                                /* Presence of local->cached_subvol validates
-                                 * that lookup from cached node is successful
-                                 */
-
-                                if (!local->op_ret && local->loc.parent) {
-                                        dht_inode_ctx_time_update
-                                                (local->loc.parent, this,
-                                                 &local->postparent, 1);
-                                }
-
-                                gf_msg_debug (this->name, 0,
-                                              "Skipped unlinking linkto file "
-                                              "on the hashed subvolume. "
-                                              "Returning success as it is a "
-                                              "valid linkto file. Path:%s"
-                                              ,local->loc.path);
-
-                                goto unwind_hashed_and_cached;
-                        } else {
-
-                               local->skip_unlink.handle_valid_link = _gf_false;
-
-                               gf_msg_debug (this->name, 0,
-                                             "Linkto file found on hashed "
-                                             "subvol "
-                                             "and data file found on cached "
-                                             "subvolume. But linkto points to "
-                                             "different cached subvolume (%s) "
-                                             "path %s",
-                                            (local->skip_unlink.hash_links_to ?
-                                        local->skip_unlink.hash_links_to->name :
-                                             " <nil>"), local->loc.path);
-
-                               if (local->skip_unlink.opend_fd_count == 0) {
-
-
-                          ret = dht_fill_dict_to_avoid_unlink_of_migrating_file
-                                  (local->xattr_req);
-
-
-                                        if (ret) {
-                                          DHT_STACK_UNWIND (lookup, frame, -1,
-                                                            EIO, NULL, NULL,
-                                                            NULL, NULL);
-                                        } else {
-                                                local->call_cnt = 1;
-                                                FRAME_SU_DO (frame, dht_local_t);
-                                                STACK_WIND (frame,
-                                          dht_lookup_unlink_of_false_linkto_cbk,
-                                                    hashed_subvol,
-                                                    hashed_subvol->fops->unlink,
-                                                    &local->loc, 0,
-                                                    local->xattr_req);
-                                        }
+                ret = dht_layout_preset(this, cached_subvol, local->loc.inode);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_INFO, 0,
+                           DHT_MSG_LAYOUT_PRESET_FAILED,
+                           "Could not set pre-set layout "
+                           "for subvolume %s",
+                           cached_subvol->name);
+                }
 
-                                        return 0;
+                local->op_ret = (ret == 0) ? ret : -1;
+                local->op_errno = (ret == 0) ? ret : EINVAL;
 
-                                }
-                        }
+                /* Presence of local->cached_subvol validates
+                 * that lookup from cached node is successful
+                 */
 
+                if (!local->op_ret && local->loc.parent) {
+                    dht_inode_ctx_time_update(local->loc.parent, this,
+                                              &local->postparent, 1);
+                }
+
+                gf_msg_debug(this->name, 0,
+                             "Skipped unlinking linkto file "
+                             "on the hashed subvolume. "
+                             "Returning success as it is a "
+                             "valid linkto file. Path:%s",
+                             local->loc.path);
+
+                goto unwind_hashed_and_cached;
+            } else {
+                local->skip_unlink.handle_valid_link = _gf_false;
+
+                gf_msg_debug(this->name, 0,
+                             "Linkto file found on hashed "
+                             "subvol "
+                             "and data file found on cached "
+                             "subvolume. But linkto points to "
+                             "different cached subvolume (%s) "
+                             "path %s",
+                             (local->skip_unlink.hash_links_to
+                                  ? local->skip_unlink.hash_links_to->name
+                                  : " <nil>"),
+                             local->loc.path);
+
+                if (local->skip_unlink.opend_fd_count == 0) {
+                    ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(
+                        local->xattr_req);
+
+                    if (ret) {
+                        DHT_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL,
+                                         NULL, NULL);
+                    } else {
+                        local->call_cnt = 1;
+                        FRAME_SU_DO(frame, dht_local_t);
+                        STACK_WIND(frame, dht_lookup_unlink_of_false_linkto_cbk,
+                                   hashed_subvol, hashed_subvol->fops->unlink,
+                                   &local->loc, 0, local->xattr_req);
+                    }
+
+                    return 0;
                 }
+            }
         }
-
+    }
 
 preset_layout:
 
-        if (found_non_linkto_on_hashed) {
-
-                if (local->need_lookup_everywhere) {
-                        if (gf_uuid_compare (local->gfid, local->inode->gfid)) {
-                                /* GFID different, return error */
-                                DHT_STACK_UNWIND (lookup, frame, -1, ENOENT,
-                                                  NULL, NULL, NULL, NULL);
-                                return 0;
-                        }
-                }
-
-                local->op_ret = 0;
-                local->op_errno = 0;
-                layout = dht_layout_for_subvol (this, cached_subvol);
-                if (!layout) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_SUBVOL_INFO,
-                                "%s: no pre-set layout for subvolume %s,"
-                                " gfid = %s",
-                                local->loc.path, (cached_subvol ?
-                                                  cached_subvol->name :
-                                                  "<nil>"), gfid);
-                }
-
-                ret = dht_layout_set (this, local->inode, layout);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_SUBVOL_INFO,
-                                "%s: failed to set layout for subvol %s, "
-                                "gfid = %s",
-                                local->loc.path, (cached_subvol ?
-                                                  cached_subvol->name :
-                                                  "<nil>"), gfid);
-                }
-
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
-
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                dht_set_fixed_dir_stat (&local->postparent);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret,
-                                  local->op_errno, local->inode,
-                                  &local->stbuf, local->xattr,
-                                  &local->postparent);
+    if (found_non_linkto_on_hashed) {
+        if (local->need_lookup_everywhere) {
+            if (gf_uuid_compare(local->gfid, local->inode->gfid)) {
+                /* GFID different, return error */
+                DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL,
+                                 NULL);
                 return 0;
+            }
         }
 
-        if (!hashed_subvol) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        layout = dht_layout_for_subvol(this, cached_subvol);
+        if (!layout) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                   "%s: no pre-set layout for subvolume %s,"
+                   " gfid = %s",
+                   local->loc.path,
+                   (cached_subvol ? cached_subvol->name : "<nil>"), gfid);
+        }
 
-                gf_msg_debug (this->name, 0,
-                              "Cannot create linkfile for %s on %s: "
-                              "hashed subvolume cannot be found, gfid = %s.",
-                              local->loc.path, cached_subvol->name, gfid);
+        ret = dht_layout_set(this, local->inode, layout);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                   "%s: failed to set layout for subvol %s, "
+                   "gfid = %s",
+                   local->loc.path,
+                   (cached_subvol ? cached_subvol->name : "<nil>"), gfid);
+        }
 
-                local->op_ret = 0;
-                local->op_errno = 0;
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
+        }
 
-                ret = dht_layout_preset (frame->this, cached_subvol,
-                                         local->inode);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_LAYOUT_PRESET_FAILED,
-                                "Failed to set layout for subvol %s"
-                                ", gfid = %s",
-                                cached_subvol ? cached_subvol->name :
-                                "<nil>", gfid);
-                        local->op_ret = -1;
-                        local->op_errno = EINVAL;
-                }
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+        return 0;
+    }
 
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+    if (!hashed_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "Cannot create linkfile for %s on %s: "
+                     "hashed subvolume cannot be found, gfid = %s.",
+                     local->loc.path, cached_subvol->name, gfid);
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                dht_set_fixed_dir_stat (&local->postparent);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret,
-                                  local->op_errno, local->inode,
-                                  &local->stbuf, local->xattr,
-                                  &local->postparent);
-                return 0;
-        }
+        local->op_ret = 0;
+        local->op_errno = 0;
 
-        if (frame->root->op != GF_FOP_RENAME) {
-                local->current = &local->lock[0];
-                ret = dht_protect_namespace (frame, &local->loc, hashed_subvol,
-                                             &local->current->ns,
-                                             dht_call_lookup_linkfile_create);
-        } else {
-                gf_msg_debug (this->name, 0,
-                              "Creating linkto file on %s(hash) to %s on %s "
-                              "(gfid = %s)",
-                              hashed_subvol->name, local->loc.path,
-                              cached_subvol->name, gfid);
+        ret = dht_layout_preset(frame->this, cached_subvol, local->inode);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_PRESET_FAILED,
+                   "Failed to set layout for subvol %s"
+                   ", gfid = %s",
+                   cached_subvol ? cached_subvol->name : "<nil>", gfid);
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
+        }
 
-                ret = dht_linkfile_create (frame,
-                                           dht_lookup_linkfile_create_cbk, this,
-                                           cached_subvol, hashed_subvol,
-                                           &local->loc);
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
         }
 
-        return ret;
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+        return 0;
+    }
+
+    if (frame->root->op != GF_FOP_RENAME) {
+        local->current = &local->lock[0];
+        ret = dht_protect_namespace(frame, &local->loc, hashed_subvol,
+                                    &local->current->ns,
+                                    dht_call_lookup_linkfile_create);
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Creating linkto file on %s(hash) to %s on %s "
+                     "(gfid = %s)",
+                     hashed_subvol->name, local->loc.path, cached_subvol->name,
+                     gfid);
+
+        ret = dht_linkfile_create(frame, dht_lookup_linkfile_create_cbk, this,
+                                  cached_subvol, hashed_subvol, &local->loc);
+    }
+
+    return ret;
 
 unwind_hashed_and_cached:
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        dht_set_fixed_dir_stat (&local->postparent);
-        DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                          local->inode, &local->stbuf, local->xattr,
-                          &local->postparent);
-        return 0;
+    DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+    dht_set_fixed_dir_stat(&local->postparent);
+    DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->stbuf, local->xattr,
+                     &local->postparent);
+    return 0;
 }
 
-int
-dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           inode_t *inode, struct iatt *buf, dict_t *xattr,
-                           struct iatt *postparent)
-{
-        dht_local_t  *local         = NULL;
-        int           this_call_cnt = 0;
-        xlator_t     *prev          = NULL;
-        int           is_linkfile   = 0;
-        int           is_dir        = 0;
-        xlator_t     *subvol        = NULL;
-        loc_t        *loc           = NULL;
-        xlator_t     *link_subvol   = NULL;
-        int           ret           = -1;
-        int32_t       fd_count      = 0;
-        dht_conf_t   *conf          = NULL;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-        dict_t       *dict_req      = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-
-        local  = frame->local;
-        loc    = &local->loc;
-        conf   = this->private;
-
-        prev   = cookie;
-        subvol = prev;
-
-        gf_msg_debug (this->name, 0,
-                      "returned with op_ret %d and op_errno %d (%s) "
-                      "from subvol %s", op_ret, op_errno, loc->path,
-                      subvol->name);
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno != ENOENT)
-                                local->op_errno = op_errno;
-                        goto unlock;
-                }
+static int
+dht_lookup_everywhere_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xattr,
+                          struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    int is_linkfile = 0;
+    int is_dir = 0;
+    loc_t *loc = NULL;
+    xlator_t *link_subvol = NULL;
+    int ret = -1;
+    int32_t fd_count = 0;
+    dht_conf_t *conf = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    dict_t *dict_req = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+
+    local = frame->local;
+    loc = &local->loc;
+    conf = this->private;
+
+    prev = cookie;
+
+    gf_msg_debug(this->name, 0,
+                 "returned with op_ret %d and op_errno %d (%s) "
+                 "from subvol %s",
+                 op_ret, op_errno, loc->path, prev->name);
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if (op_errno != ENOENT)
+                local->op_errno = op_errno;
+            if (op_errno == ENODATA)
+                local->gfid_missing = _gf_true;
+            goto unlock;
+        }
 
-                if (gf_uuid_is_null (local->gfid))
-                        gf_uuid_copy (local->gfid, buf->ia_gfid);
+        if (gf_uuid_is_null(local->gfid))
+            gf_uuid_copy(local->gfid, buf->ia_gfid);
 
-                gf_uuid_unparse(local->gfid, gfid);
+        gf_uuid_unparse(local->gfid, gfid);
 
-                if (gf_uuid_compare (local->gfid, buf->ia_gfid)) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_GFID_MISMATCH,
-                                "%s: gfid differs on subvolume %s,"
-                                " gfid local = %s, gfid node = %s",
-                                loc->path, prev->name, gfid,
-                                uuid_utoa(buf->ia_gfid));
-                }
+        if (gf_uuid_compare(local->gfid, buf->ia_gfid)) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+                   "%s: gfid differs on subvolume %s,"
+                   " gfid local = %s, gfid node = %s",
+                   loc->path, prev->name, gfid, uuid_utoa(buf->ia_gfid));
+        }
 
-                is_linkfile = check_is_linkfile (inode, buf, xattr,
-                                                 conf->link_xattr_name);
-
-                if (is_linkfile) {
-                        link_subvol = dht_linkfile_subvol (this, inode, buf,
-                                                           xattr);
-                        gf_msg_debug (this->name, 0,
-                                      "found on %s linkfile %s (-> %s)",
-                                      subvol->name, loc->path,
-                                      link_subvol ? link_subvol->name : "''");
-                        goto unlock;
-                }
+        is_linkfile = check_is_linkfile(inode, buf, xattr,
+                                        conf->link_xattr_name);
 
-                is_dir = check_is_dir (inode, buf, xattr);
+        if (is_linkfile) {
+            link_subvol = dht_linkfile_subvol(this, inode, buf, xattr);
+            gf_msg_debug(this->name, 0, "found on %s linkfile %s (-> %s)",
+                         prev->name, loc->path,
+                         link_subvol ? link_subvol->name : "''");
+            goto unlock;
+        }
 
-                /* non linkfile GFID takes precedence but don't overwrite
-                 gfid if we have already found a cached file*/
-                if (!local->cached_subvol)
-                        gf_uuid_copy (local->gfid, buf->ia_gfid);
+        is_dir = check_is_dir(inode, buf, xattr);
 
-                if (is_dir) {
-                        local->dir_count++;
+        /* non linkfile GFID takes precedence but don't overwrite
+         gfid if we have already found a cached file*/
+        if (!local->cached_subvol)
+            gf_uuid_copy(local->gfid, buf->ia_gfid);
 
-                        gf_msg_debug (this->name, 0,
-                                      "found on %s directory %s",
-                                      subvol->name, loc->path);
-                } else {
-                        local->file_count++;
-
-                        gf_msg_debug (this->name, 0,
-                                      "found cached file on %s for %s",
-                                      subvol->name, loc->path);
-
-                        if (!local->cached_subvol) {
-                                /* found one file */
-                                dht_iatt_merge (this, &local->stbuf, buf,
-                                                subvol);
-                                local->xattr = dict_ref (xattr);
-                                local->cached_subvol = subvol;
-
-                                gf_msg_debug (this->name, 0,
-                                              "storing cached on %s file"
-                                              " %s", subvol->name, loc->path);
-
-                                dht_iatt_merge (this, &local->postparent,
-                                                postparent, subvol);
-
-                                gf_uuid_copy (local->skip_unlink.cached_gfid,
-                                           buf->ia_gfid);
-                        } else {
-                                /* This is where we need 'rename' both entries logic */
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_FILE_ON_MULT_SUBVOL,
-                                        "multiple subvolumes (%s and %s) have "
-                                        "file %s (preferably rename the file "
-                                        "in the backend,and do a fresh lookup)",
-                                        local->cached_subvol->name,
-                                        subvol->name, local->loc.path);
-                        }
-                }
+        if (is_dir) {
+            local->dir_count++;
+
+            gf_msg_debug(this->name, 0, "found on %s directory %s", prev->name,
+                         loc->path);
+        } else {
+            local->file_count++;
+
+            gf_msg_debug(this->name, 0, "found cached file on %s for %s",
+                         prev->name, loc->path);
+
+            if (!local->cached_subvol) {
+                /* found one file */
+                dht_iatt_merge(this, &local->stbuf, buf);
+
+                local->xattr = dict_ref(xattr);
+                local->cached_subvol = prev;
+
+                gf_msg_debug(this->name, 0,
+                             "storing cached on %s file"
+                             " %s",
+                             prev->name, loc->path);
+
+                dht_iatt_merge(this, &local->postparent, postparent);
+
+                gf_uuid_copy(local->skip_unlink.cached_gfid, buf->ia_gfid);
+            } else {
+                /* This is where we need 'rename' both entries logic */
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       DHT_MSG_FILE_ON_MULT_SUBVOL,
+                       "multiple subvolumes (%s and %s) have "
+                       "file %s (preferably rename the file "
+                       "in the backend,and do a fresh lookup)",
+                       local->cached_subvol->name, prev->name, local->loc.path);
+            }
         }
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 
-        if (is_linkfile) {
-                ret = dict_get_int32 (xattr, GLUSTERFS_OPEN_FD_COUNT, &fd_count);
+    if (is_linkfile) {
+        ret = dict_get_int32(xattr, GLUSTERFS_OPEN_FD_COUNT, &fd_count);
 
-                /*  Any linkto file found on the non-hashed subvolume should
-                 *  be unlinked (performed in the "else if" block below)
-                 *
-                 *  But if a linkto file is found on hashed subvolume, it may be
-                 *  pointing to valid cached node. So unlinking of linkto
-                 *  file on hashed subvolume is skipped and inside
-                 *  dht_lookup_everywhere_done, checks are performed. If this
-                 *  linkto file is found as stale linkto file, it is deleted
-                 *  otherwise unlink is skipped.
+        /*  Any linkto file found on the non-hashed subvolume should
+         *  be unlinked (performed in the "else if" block below)
+         *
+         *  But if a linkto file is found on hashed subvolume, it may be
+         *  pointing to valid cached node. So unlinking of linkto
+         *  file on hashed subvolume is skipped and inside
+         *  dht_lookup_everywhere_done, checks are performed. If this
+         *  linkto file is found as stale linkto file, it is deleted
+         *  otherwise unlink is skipped.
+         */
+
+        if (local->hashed_subvol && local->hashed_subvol == prev) {
+            local->skip_unlink.handle_valid_link = _gf_true;
+            local->skip_unlink.opend_fd_count = fd_count;
+            local->skip_unlink.hash_links_to = link_subvol;
+            gf_uuid_copy(local->skip_unlink.hashed_gfid, buf->ia_gfid);
+
+            gf_msg_debug(this->name, 0,
+                         "Found"
+                         " one linkto file on hashed subvol %s "
+                         "for %s: Skipping unlinking till "
+                         "everywhere_done",
+                         prev->name, loc->path);
+
+        } else if (!ret && (fd_count == 0)) {
+            dict_req = dict_new();
+
+            ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(dict_req);
+
+            if (ret) {
+                /* Skip unlinking for dict_failure
+                 *File is found as a linkto file on non-hashed,
+                 *subvolume. In the current implementation,
+                 *finding a linkto-file on non-hashed does not
+                 *always implies that it is stale. So deletion
+                 *of file should be done only when both fd is
+                 *closed and linkto-xattr is set. In case of
+                 *dict_set failure, avoid skipping of file.
+                 *NOTE: dht_frame_return should get called for
+                 *      this block.
                  */
 
-                if (local->hashed_subvol && local->hashed_subvol == subvol) {
-
-                        local->skip_unlink.handle_valid_link = _gf_true;
-                        local->skip_unlink.opend_fd_count = fd_count;
-                        local->skip_unlink.hash_links_to = link_subvol;
-                        gf_uuid_copy (local->skip_unlink.hashed_gfid,
-                                   buf->ia_gfid);
-
-                        gf_msg_debug (this->name, 0, "Found"
-                                      " one linkto file on hashed subvol %s "
-                                      "for %s: Skipping unlinking till "
-                                      "everywhere_done", subvol->name,
-                                      loc->path);
-
-                } else if (!ret && (fd_count == 0)) {
-
-                        dict_req = dict_new ();
-
-                        ret = dht_fill_dict_to_avoid_unlink_of_migrating_file
-                              (dict_req);
-
-                        if (ret) {
-
-                                /* Skip unlinking for dict_failure
-                                 *File is found as a linkto file on non-hashed,
-                                 *subvolume. In the current implementation,
-                                 *finding a linkto-file on non-hashed does not
-                                 *always implies that it is stale. So deletion
-                                 *of file should be done only when both fd is
-                                 *closed and linkto-xattr is set. In case of
-                                 *dict_set failure, avoid skipping of file.
-                                 *NOTE: dht_frame_return should get called for
-                                 *      this block.
-                                 */
-
-                                dict_unref (dict_req);
-
-                        } else {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_SUBVOL_INFO,
-                                        "attempting deletion of stale linkfile "
-                                        "%s on %s (hashed subvol is %s)",
-                                        loc->path, subvol->name,
-                                        (local->hashed_subvol?
-                                        local->hashed_subvol->name : "<null>"));
-                                /* *
-                                 * These stale files may be created using root
-                                 * user. Hence deletion will work only with
-                                 * root.
-                                 */
-                                FRAME_SU_DO (frame, dht_local_t);
-                                STACK_WIND (frame, dht_lookup_unlink_cbk,
-                                            subvol, subvol->fops->unlink, loc,
-                                            0, dict_req);
-
-                                dict_unref (dict_req);
-
-                                return 0;
-                        }
-                }
-        }
+                dict_unref(dict_req);
+
+            } else {
+                gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                       "attempting deletion of stale linkfile "
+                       "%s on %s (hashed subvol is %s)",
+                       loc->path, prev->name,
+                       (local->hashed_subvol ? local->hashed_subvol->name
+                                             : "<null>"));
+                /* *
+                 * These stale files may be created using root
+                 * user. Hence deletion will work only with
+                 * root.
+                 */
+                FRAME_SU_DO(frame, dht_local_t);
+                STACK_WIND(frame, dht_lookup_unlink_cbk, prev,
+                           prev->fops->unlink, loc, 0, dict_req);
+
+                dict_unref(dict_req);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                dht_lookup_everywhere_done (frame, this);
+                return 0;
+            }
         }
+    }
+
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_lookup_everywhere_done(frame, this);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_lookup_everywhere(call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-        dht_conf_t     *conf = NULL;
-        dht_local_t    *local = NULL;
-        int             i = 0;
-        int             call_cnt = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int i = 0;
+    int call_cnt = 0;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", loc, out);
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+    GF_VALIDATE_OR_GOTO("dht", loc, out);
 
-        conf = this->private;
-        local = frame->local;
+    conf = this->private;
+    local = frame->local;
 
-        call_cnt = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
 
-        if (!local->inode)
-                local->inode = inode_ref (loc->inode);
+    if (!local->inode)
+        local->inode = inode_ref(loc->inode);
 
-        gf_msg_debug (this->name, 0,
-                      "winding lookup call to %d subvols", call_cnt);
+    gf_msg_debug(this->name, 0, "winding lookup call to %d subvols", call_cnt);
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND_COOKIE (frame, dht_lookup_everywhere_cbk,
-                                   conf->subvolumes[i], conf->subvolumes[i],
-                                   conf->subvolumes[i]->fops->lookup,
-                                   loc, local->xattr_req);
-        }
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_lookup_everywhere_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->lookup, loc,
+                          local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 out:
-        DHT_STACK_UNWIND (lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
+    DHT_STACK_UNWIND(lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
 err:
-        return -1;
+    return -1;
 }
 
-
 int
-dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int op_ret, int op_errno,
-                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                         struct iatt *postparent)
-{
-        xlator_t     *prev          = NULL;
-        dht_local_t  *local         = NULL;
-        xlator_t     *subvol        = NULL;
-        loc_t        *loc           = NULL;
-        dht_conf_t   *conf          = NULL;
-        int           ret           = 0;
-        char          gfid[GF_UUID_BUF_SIZE]     = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, unwind);
-
-        prev   = cookie;
-        subvol = prev;
-        conf   = this->private;
-        local  = frame->local;
-        loc    = &local->loc;
-
-        gf_uuid_unparse(loc->gfid, gfid);
-
-        if (op_ret == -1) {
-                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                        DHT_MSG_LINK_FILE_LOOKUP_INFO,
-                        "Lookup of %s on %s (following linkfile) failed "
-                        ",gfid = %s", local->loc.path, subvol->name, gfid);
-
-                /* If cached subvol returned ENOTCONN, do not do
-                lookup_everywhere. We need to make sure linkfile does not get
-                removed, which can take away the namespace, and subvol is
-                anyways down. */
-
-                local->cached_subvol = NULL;
-                if (op_errno != ENOTCONN)
-                        goto err;
-                else
-                        goto unwind;
-        }
-
-        if (check_is_dir (inode, stbuf, xattr)) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LINK_FILE_LOOKUP_INFO,
-                        "Lookup of %s on %s (following linkfile) reached dir,"
-                        " gfid = %s", local->loc.path, subvol->name, gfid);
-                goto err;
-        }
-
-        if (check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LINK_FILE_LOOKUP_INFO,
-                        "lookup of %s on %s (following linkfile) reached link,"
-                        "gfid = %s", local->loc.path, subvol->name, gfid);
-                goto err;
-        }
-
-        if (gf_uuid_compare (local->gfid, stbuf->ia_gfid)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_GFID_MISMATCH,
-                        "%s: gfid different on data file on %s,"
-                        " gfid local = %s, gfid node = %s ",
-                        local->loc.path, subvol->name, gfid,
-                        uuid_utoa(stbuf->ia_gfid));
-                goto err;
-        }
-
-        if ((stbuf->ia_nlink == 1)
-            && (conf && conf->unhashed_sticky_bit)) {
-                stbuf->ia_prot.sticky = 1;
-        }
-
-        ret = dht_layout_preset (this, prev, inode);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LAYOUT_PRESET_FAILED,
-                        "Failed to set layout for subvolume %s,"
-                        "gfid = %s", prev->name, gfid);
-                op_ret   = -1;
-                op_errno = EINVAL;
-        }
-
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
+dht_lookup_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent)
+{
+    xlator_t *prev = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    loc_t *loc = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, unwind);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, unwind);
+    GF_VALIDATE_OR_GOTO("dht", this->private, unwind);
+    GF_VALIDATE_OR_GOTO("dht", cookie, unwind);
+
+    prev = cookie;
+    subvol = prev;
+    conf = this->private;
+    local = frame->local;
+    loc = &local->loc;
+
+    gf_uuid_unparse(loc->gfid, gfid);
+
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+               "Lookup of %s on %s (following linkfile) failed "
+               ",gfid = %s",
+               local->loc.path, subvol->name, gfid);
+
+        /* If cached subvol returned ENOTCONN, do not do
+        lookup_everywhere. We need to make sure linkfile does not get
+        removed, which can take away the namespace, and subvol is
+        anyways down. */
+
+        local->cached_subvol = NULL;
+        if (op_errno != ENOTCONN)
+            goto err;
+        else
+            goto unwind;
+    }
+
+    if (check_is_dir(inode, stbuf, xattr)) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+               "Lookup of %s on %s (following linkfile) reached dir,"
+               " gfid = %s",
+               local->loc.path, subvol->name, gfid);
+        goto err;
+    }
+
+    if (check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name)) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+               "lookup of %s on %s (following linkfile) reached link,"
+               "gfid = %s",
+               local->loc.path, subvol->name, gfid);
+        goto err;
+    }
+
+    if (gf_uuid_compare(local->gfid, stbuf->ia_gfid)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "%s: gfid different on data file on %s,"
+               " gfid local = %s, gfid node = %s ",
+               local->loc.path, subvol->name, gfid, uuid_utoa(stbuf->ia_gfid));
+        goto err;
+    }
+
+    if ((stbuf->ia_nlink == 1) && (conf && conf->unhashed_sticky_bit)) {
+        stbuf->ia_prot.sticky = 1;
+    }
+
+    ret = dht_layout_preset(this, prev, inode);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_PRESET_FAILED,
+               "Failed to set layout for subvolume %s,"
+               "gfid = %s",
+               prev->name, gfid);
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
+
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
 
 unwind:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        dht_set_fixed_dir_stat (postparent);
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
-                          postparent);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     postparent);
 
-        return 0;
+    return 0;
 
 err:
-        dht_lookup_everywhere (frame, this, loc);
+    dht_lookup_everywhere(frame, this, loc);
 out:
-        return 0;
-}
-
-
-int
-dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        int           call_cnt = 0;
-        int           i = 0;
-        dht_conf_t   *conf = NULL;
-        dht_local_t  *local = NULL;
-        int           ret = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", loc, unwind);
-
-        conf = this->private;
-        local = frame->local;
-
-        call_cnt        = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
-
-        local->layout = dht_layout_new (this, conf->subvolume_cnt);
-        if (!local->layout) {
-                goto unwind;
-        }
-
-        if (local->xattr != NULL) {
-                dict_unref (local->xattr);
-                local->xattr = NULL;
-        }
-
-        if (!gf_uuid_is_null (local->gfid)) {
-                ret = dict_set_gfuuid (local->xattr_req, "gfid-req",
-                                       local->gfid, true);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "%s: Failed to set dictionary value:"
-                                " key = gfid-req", local->loc.path);
-        }
-
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND_COOKIE (frame, dht_lookup_dir_cbk,
-                                   conf->subvolumes[i], conf->subvolumes[i],
-                                   conf->subvolumes[i]->fops->lookup,
-                                   &local->loc, local->xattr_req);
-        }
-        return 0;
-unwind:
-        DHT_STACK_UNWIND (lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
-out:
-        return 0;
-
+    return 0;
 }
 
 /* Code to get hashed subvol based on inode and loc
@@ -3075,7471 +2990,7612 @@ out:
    hashed subvol based on loc.
 */
 
-xlator_t *
-dht_inode_get_hashed_subvol (inode_t *inode, xlator_t *this, loc_t *loc)
-{
-        char                    *path           = NULL;
-        loc_t                    populate_loc   = {0, };
-        char                    *name           = NULL;
-        xlator_t                *hash_subvol    = NULL;
-
-        if (!inode)
-                return hash_subvol;
-
-        if (loc && loc->parent && loc->path) {
-                if (!loc->name) {
-                        name = strrchr (loc->path, '/');
-                        if (name) {
-                                loc->name = name + 1;
-                        } else {
-                                goto out;
-                        }
-                }
-                hash_subvol = dht_subvol_get_hashed (this, loc);
-                goto out;
+static gf_boolean_t
+dht_should_lookup_everywhere(xlator_t *this, dht_conf_t *conf, loc_t *loc)
+{
+    dht_layout_t *parent_layout = NULL;
+    int ret = 0;
+    gf_boolean_t lookup_everywhere = _gf_true;
+
+    /* lookup-optimize supersedes lookup-unhashed settings.
+     * If it is set, do not process search_unhashed
+     * If lookup-optimize if enabled, lookup everywhere if:
+     *  - this is the rebalance daemon.
+     *  - loc->parent is unavailable.
+     *  - parent_layout is unavailable
+     *  - parent_layout->commit_hash != conf->vol_commit_hash
+     */
+
+    if (conf->lookup_optimize) {
+        if (!conf->defrag && loc->parent) {
+            ret = dht_inode_ctx_layout_get(loc->parent, this, &parent_layout);
+            if (!ret && parent_layout &&
+                (parent_layout->commit_hash == conf->vol_commit_hash)) {
+                lookup_everywhere = _gf_false;
+            }
         }
+        goto out;
+    } else {
+        if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) {
+            if (loc->parent) {
+                ret = dht_inode_ctx_layout_get(loc->parent, this,
+                                               &parent_layout);
+                if (ret || !parent_layout ||
+                    (!parent_layout->search_unhashed)) {
+                    lookup_everywhere = _gf_false;
+                }
+            } else {
+                lookup_everywhere = _gf_false;
+            }
 
-        if (!gf_uuid_is_null (inode->gfid)) {
-                populate_loc.inode = inode_ref (inode);
-                populate_loc.parent = inode_parent (populate_loc.inode,
-                                                    NULL, NULL);
-                inode_path (populate_loc.inode, NULL, &path);
-
-                if (!path)
-                        goto out;
-
-                populate_loc.path = path;
-                if (!populate_loc.name && populate_loc.path) {
-                        name = strrchr (populate_loc.path, '/');
-                        if (name) {
-                                populate_loc.name = name + 1;
-
-                        } else {
-                                goto out;
-                        }
-                }
-                hash_subvol = dht_subvol_get_hashed (this, &populate_loc);
+            goto out;
         }
+    }
 out:
-        if (populate_loc.inode)
-                loc_wipe (&populate_loc);
-        return hash_subvol;
+    return lookup_everywhere;
 }
 
-
 int
-dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno,
-                inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                struct iatt *postparent)
-{
-        char          is_linkfile   = 0;
-        char          is_dir        = 0;
-        xlator_t     *subvol        = NULL;
-        dht_conf_t   *conf          = NULL;
-        dht_local_t  *local         = NULL;
-        loc_t        *loc           = NULL;
-        xlator_t     *prev          = NULL;
-        int           ret           = 0;
-        dht_layout_t *parent_layout = NULL;
-        uint32_t      vol_commit_hash = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-
-        conf  = this->private;
-
-        prev  = cookie;
-        local = frame->local;
-        loc   = &local->loc;
-
-        /* This is required for handling stale linkfile deletion,
-         * or any more call which happens from this 'loc'.
-         */
-        if (!op_ret && gf_uuid_is_null (local->gfid))
-                memcpy (local->gfid, stbuf->ia_gfid, 16);
-
-        gf_msg_debug (this->name, op_errno,
-                      "fresh_lookup returned for %s with op_ret %d",
-                      loc->path, op_ret);
-
-        if (!conf->vch_forced) {
-                ret = dict_get_uint32 (xattr, conf->commithash_xattr_name,
-                                       &vol_commit_hash);
-                if (ret == 0) {
-                        conf->vol_commit_hash = vol_commit_hash;
-                }
-        }
-
-        if (ENTRY_MISSING (op_ret, op_errno)) {
-
-                if (1 == conf->subvolume_cnt) {
-                        /* No need to lookup again */
-                        goto out;
-                }
-
-                gf_msg_debug (this->name, 0,
-                              "Entry %s missing on subvol %s",
-                              loc->path, prev->name);
-
-                /* lookup-optimize supercedes lookup-unhashed settings,
-                 *   - so if it is set, do not process search_unhashed
-                 *   - except, in the case of rebalance deamon, we want to
-                 *     force the lookup_everywhere behavior */
-                if (!conf->defrag && conf->lookup_optimize && loc->parent) {
-                        ret = dht_inode_ctx_layout_get (loc->parent, this,
-                                                        &parent_layout);
-                        if (ret || !parent_layout ||
-                            (parent_layout->commit_hash !=
-                             conf->vol_commit_hash)) {
-                                gf_msg_debug (this->name, 0,
-                                              "hashes don't match (ret - %d,"
-                                              " parent_layout - %p, parent_hash - %x,"
-                                              " vol_hash - %x), do global lookup",
-                                              ret, parent_layout,
-                                              (parent_layout ?
-                                              parent_layout->commit_hash : -1),
-                                              conf->vol_commit_hash);
-                                local->op_errno = ENOENT;
-                                dht_lookup_everywhere (frame, this, loc);
-                                return 0;
-                        }
-                } else {
-                        if (conf->search_unhashed ==
-                            GF_DHT_LOOKUP_UNHASHED_ON) {
-                                local->op_errno = ENOENT;
-                                dht_lookup_everywhere (frame, this, loc);
-                                return 0;
-                        }
+dht_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+               struct iatt *postparent)
+{
+    char is_linkfile = 0;
+    char is_dir = 0;
+    xlator_t *subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    xlator_t *prev = NULL;
+    int ret = 0;
+    uint32_t vol_commit_hash = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+
+    conf = this->private;
+
+    prev = cookie;
+    local = frame->local;
+    loc = &local->loc;
+
+    gf_msg_debug(this->name, op_errno,
+                 "%s: fresh_lookup on %s returned with op_ret %d", loc->path,
+                 prev->name, op_ret);
+
+    if (op_ret == -1) {
+        if (ENTRY_MISSING(op_ret, op_errno)) {
+            if (1 == conf->subvolume_cnt) {
+                /* No need to lookup again */
+                goto out;
+            }
 
-                        if ((conf->search_unhashed ==
-                            GF_DHT_LOOKUP_UNHASHED_AUTO) &&
-                            (loc->parent)) {
-                                ret = dht_inode_ctx_layout_get (loc->parent,
-                                                                this,
-                                                                &parent_layout);
-                                if (ret || !parent_layout)
-                                        goto out;
-                                if (parent_layout->search_unhashed) {
-                                        local->op_errno = ENOENT;
-                                        dht_lookup_everywhere (frame, this,
-                                                               loc);
-                                        return 0;
-                                }
-                        }
-                }
-        }
+            gf_msg_debug(this->name, 0, "Entry %s missing on subvol %s",
+                         loc->path, prev->name);
 
-        if (op_ret == 0) {
-                is_dir      = check_is_dir (inode, stbuf, xattr);
-                if (is_dir) {
-                        local->inode = inode_ref (inode);
-                        local->xattr = dict_ref (xattr);
-                }
-        }
+            if (dht_should_lookup_everywhere(this, conf, loc)) {
+                local->op_errno = ENOENT;
+                dht_lookup_everywhere(frame, this, loc);
+                return 0;
+            }
 
-        if (is_dir || (op_ret == -1 && op_errno == ENOTCONN)) {
-                dht_lookup_directory (frame, this, &local->loc);
+        } else {
+            /* posix returns ENODATA if the gfid is not set but the client and
+             * server protocol layers do not send the stbuf. We need to
+             * heal this so check if this is a directory on the other subvols.
+             */
+            if ((op_errno == ENOTCONN) || (op_errno == ENODATA)) {
+                dht_lookup_directory(frame, this, &local->loc);
                 return 0;
+            }
         }
-
-        if (op_ret == -1) {
-                gf_msg_debug (this->name, op_errno,
-                              "Lookup of %s for subvolume"
-                              " %s failed", loc->path,
-                              prev->name);
-                goto out;
+        gf_msg_debug(this->name, op_errno, "%s: Lookup on subvolume %s failed",
+                     loc->path, prev->name);
+        goto out;
+    }
+
+    /* Lookup succeeded - op_ret = 0 */
+
+    /* This is required for handling stale linkfile deletion,
+     * or any more call which happens from this 'loc'.
+     */
+    if (gf_uuid_is_null(local->gfid)) {
+        /*This is set from the first successful response*/
+        memcpy(local->gfid, stbuf->ia_gfid, 16);
+    }
+
+    if (!conf->vch_forced) {
+        /* Update the commit hash in conf if it is found */
+        ret = dict_get_uint32(xattr, conf->commithash_xattr_name,
+                              &vol_commit_hash);
+        if (ret == 0) {
+            conf->vol_commit_hash = vol_commit_hash;
         }
+    }
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                         conf->link_xattr_name);
-
-        if (!is_linkfile) {
-                /* non-directory and not a linkfile */
-
-                ret = dht_layout_preset (this, prev, inode);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_LAYOUT_PRESET_FAILED,
-                                "could not set pre-set layout for subvolume %s",
-                                prev->name);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        goto out;
-                }
-                goto out;
-        }
+    is_dir = check_is_dir(inode, stbuf, xattr);
+    if (is_dir) {
+        /* A directory is present on all subvols, send the lookup to
+         * all subvols now */
+        local->inode = inode_ref(inode);
+        local->xattr = dict_ref(xattr);
+        dht_lookup_directory(frame, this, &local->loc);
+        return 0;
+    }
 
-        subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
-        if (!subvol) {
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
 
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_SUBVOL_INFO, "linkfile not having link "
-                        "subvol for %s", loc->path);
+    if (!is_linkfile) {
+        /* non-directory and not a linkto file. This is a data file
+         * Update the layout to point to the cached subvol
+         */
 
-                gf_msg_debug (this->name, 0,
-                              "linkfile not having link subvolume. path=%s",
-                              loc->path);
-                dht_lookup_everywhere (frame, this, loc);
-                return 0;
+        ret = dht_layout_preset(this, prev, inode);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_PRESET_FAILED,
+                   "%s: could not set pre-set layout for subvolume %s",
+                   loc->path, prev->name);
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto out;
         }
+        goto out;
+    }
 
-        gf_msg_debug (this->name, 0,
-                      "Calling lookup on linkto target %s for path %s",
-                      subvol->name, loc->path);
-
-        STACK_WIND_COOKIE (frame, dht_lookup_linkfile_cbk, subvol,
-                           subvol, subvol->fops->lookup,
-                           &local->loc, local->xattr_req);
-
+    /* This is a linkto file. Get the value of the target subvol from the
+     * linkto xattr and lookup there to see if the file exists
+     */
+    subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "%s: No link subvol for linkto", loc->path);
+        dht_lookup_everywhere(frame, this, loc);
         return 0;
+    }
 
-out:
-        /*
-         * FIXME: postparent->ia_size and postparent->st_blocks do not have
-         * correct values. since, postparent corresponds to a directory these
-         * two members should have values equal to sum of corresponding values
-         * from each of the subvolume. See dht_iatt_merge for reference.
-         */
+    gf_msg_debug(this->name, 0, "%s: Calling lookup on linkto target %s",
+                 loc->path, subvol->name);
 
-        if (!op_ret && local && local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
+    STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                      subvol->fops->lookup, &local->loc, local->xattr_req);
 
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        dht_set_fixed_dir_stat (postparent);
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
-                          postparent);
+    return 0;
+
+out:
+    /*
+     * FIXME: postparent->ia_size and postparent->st_blocks do not have
+     * correct values. since, postparent corresponds to a directory these
+     * two members should have values equal to sum of corresponding values
+     * from each of the subvolume. See dht_iatt_merge for reference.
+     */
+
+    if (!op_ret && local && local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     postparent);
 err:
-        return 0;
+    return 0;
 }
 
-/* For directories, check if acl xattrs have been requested (by the acl xlator),
- * if not, request for them. These xattrs are needed for dht dir self-heal to
- * perform proper self-healing of dirs
+/* For directories, check if acl xattrs have been requested (by the acl
+ * xlator), if not, request for them. These xattrs are needed for dht dir
+ * self-heal to perform proper self-healing of dirs
  */
-void
-dht_check_and_set_acl_xattr_req (inode_t *inode, dict_t *xattr_req)
+static void
+dht_check_and_set_acl_xattr_req(xlator_t *this, dict_t *xattr_req)
 {
-        int     ret = 0;
-
-        GF_ASSERT (inode);
-        GF_ASSERT (xattr_req);
+    int ret = 0;
 
-        if (inode->ia_type != IA_IFDIR)
-                return;
+    GF_ASSERT(xattr_req);
 
-        if (!dict_get (xattr_req, POSIX_ACL_ACCESS_XATTR)) {
-                ret = dict_set_int8 (xattr_req, POSIX_ACL_ACCESS_XATTR, 0);
-                if (ret)
-                        gf_msg (THIS->name, GF_LOG_WARNING, -ret,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s",
-                                POSIX_ACL_ACCESS_XATTR);
-        }
+    if (!dict_get(xattr_req, POSIX_ACL_ACCESS_XATTR)) {
+        ret = dict_set_int8(xattr_req, POSIX_ACL_ACCESS_XATTR, 0);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary value:key = %s",
+                   POSIX_ACL_ACCESS_XATTR);
+    }
 
-        if (!dict_get (xattr_req, POSIX_ACL_DEFAULT_XATTR)) {
-                ret = dict_set_int8 (xattr_req, POSIX_ACL_DEFAULT_XATTR, 0);
-                if (ret)
-                        gf_msg (THIS->name, GF_LOG_WARNING, -ret,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s",
-                                POSIX_ACL_DEFAULT_XATTR);
-        }
+    if (!dict_get(xattr_req, POSIX_ACL_DEFAULT_XATTR)) {
+        ret = dict_set_int8(xattr_req, POSIX_ACL_DEFAULT_XATTR, 0);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary value:key = %s",
+                   POSIX_ACL_DEFAULT_XATTR);
+    }
 
-        return;
+    return;
 }
 
-int
-dht_lookup (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, dict_t *xattr_req)
-{
-        xlator_t     *subvol = NULL;
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *mds_subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret    = -1;
-        int           op_errno = -1;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-        int           gen = 0;
-        loc_t         new_loc = {0,};
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        conf = this->private;
-        if (!conf)
-                goto err;
+/* for directories, we need the following info:
+ * the layout : trusted.glusterfs.dht
+ * the mds information : trusted.glusterfs.dht.mds
+ * the acl info: See above
+ */
+static int
+dht_set_dir_xattr_req(xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    int ret = -EINVAL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    if (!conf) {
+        goto err;
+    }
+
+    if (!xattr_req) {
+        goto err;
+    }
+
+    /* Xattr to get the layout for a directory
+     */
+    ret = dict_set_uint32(xattr_req, conf->xattr_name, 4 * 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->xattr_name, loc->path);
+        goto err;
+    }
+
+    /*Non-fatal failure */
+    ret = dict_set_uint32(xattr_req, conf->mds_xattr_key, 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->mds_xattr_key, loc->path);
+    }
+
+    dht_check_and_set_acl_xattr_req(this, xattr_req);
+    ret = 0;
+err:
+    return ret;
+}
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+/* If the hashed subvol is present, send the lookup to only that subvol first.
+ * If no hashed subvol, send a lookup to all subvols and proceed based on the
+ * responses.
+ */
+static int
+dht_do_fresh_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    xlator_t *hashed_subvol = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    int call_cnt = 0;
+    int i = 0;
+
+    conf = this->private;
+    if (!conf) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local = frame->local;
+    if (!local) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* Since we don't know whether this is a file or a directory,
+     * request all xattrs*/
+    ret = dht_set_file_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    ret = dht_set_dir_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    /* Fuse sets a random value in gfid-req. If the gfid is missing
+     * on one or more subvols, posix will set the gfid to this value,
+     * causing GFID mismatches for directories. Remove the value fuse
+     * has sent before sending the lookup.
+     */
+    ret = dict_get_gfuuid(local->xattr_req, "gfid-req", &local->gfid_req);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "%s: No gfid-req available", loc->path);
+    } else {
+        dict_del(local->xattr_req, "gfid-req");
+    }
+    /* This should have been set in dht_lookup */
+    hashed_subvol = local->hashed_subvol;
+
+    if (!hashed_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "%s: no subvolume in layout for path, "
+                     "checking on all the subvols to see if "
+                     "it is a directory",
+                     loc->path);
 
-        ret = dht_filter_loc_subvol_key (this, loc, &new_loc,
-                                         &hashed_subvol);
-        if (ret) {
-                loc_wipe (&local->loc);
-                ret = loc_dup (&new_loc, &local->loc);
-
-                /* we no more need 'new_loc' entries */
-                loc_wipe (&new_loc);
-
-                /* check if loc_dup() is successful */
-                if (ret == -1) {
-                        op_errno = errno;
-                        gf_msg_debug (this->name, errno,
-                                      "copying location failed for path=%s",
-                                      loc->path);
-                        goto err;
-                }
-        }
+        call_cnt = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
 
-        if (xattr_req) {
-                local->xattr_req = dict_ref (xattr_req);
-        } else {
-                local->xattr_req = dict_new ();
+        /* Allocate a layout. This will be populated and saved in
+         * the dht inode_ctx on successful lookup
+         */
+        local->layout = dht_layout_new(this, conf->subvolume_cnt);
+        if (!local->layout) {
+            op_errno = ENOMEM;
+            goto err;
         }
 
-        ret = dict_set_uint32 (local->xattr_req, conf->mds_xattr_key, 4);
+        gf_msg_debug(this->name, 0,
+                     "%s: Found null hashed subvol. Calling lookup"
+                     " on all nodes.",
+                     loc->path);
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value:key = %s for "
-                        "path %s", conf->mds_xattr_key, loc->path);
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
         }
+        return 0;
+    }
 
+    /* if the hashed_subvol is non-null, send the lookup there first so
+     * as to see whether we have a file or a directory */
+    gf_msg_debug(this->name, 0, "%s: Calling fresh lookup on %s", loc->path,
+                 hashed_subvol->name);
 
-        /* Nameless lookup */
-
-        if (gf_uuid_is_null (loc->pargfid) && !gf_uuid_is_null (loc->gfid) &&
-            !__is_root_gfid (loc->inode->gfid)) {
-                local->cached_subvol = NULL;
-                dht_discover (frame, this, loc);
-                return 0;
-        }
+    STACK_WIND_COOKIE(frame, dht_lookup_cbk, hashed_subvol, hashed_subvol,
+                      hashed_subvol->fops->lookup, loc, local->xattr_req);
+    return 0;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
 
-        if (__is_root_gfid(loc->gfid)) {
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->commithash_xattr_name,
-                                       sizeof(uint32_t));
+static int
+dht_do_revalidate(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+    int gen = 0;
+
+    conf = this->private;
+    if (!conf) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local = frame->local;
+    if (!local) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0,
+                     "path = %s. No layout found in the inode ctx.", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* Generation number has changed. This layout may be stale. */
+    if (layout->gen && (layout->gen < conf->gen)) {
+        gen = layout->gen;
+        dht_layout_unref(this, local->layout);
+        local->layout = NULL;
+        local->cached_subvol = NULL;
+
+        gf_msg_debug(this->name, 0,
+                     "path = %s. In memory layout may be stale."
+                     "(layout->gen (%d) is less than "
+                     "conf->gen (%d)). Calling fresh lookup.",
+                     loc->path, gen, conf->gen);
+
+        dht_do_fresh_lookup(frame, this, loc);
+        return 0;
+    }
+
+    local->inode = inode_ref(loc->inode);
+
+    /* Since we don't know whether this has changed,
+     * request all xattrs*/
+    ret = dht_set_file_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    ret = dht_set_dir_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    if (IA_ISDIR(local->inode->ia_type)) {
+        ret = dht_inode_ctx_mdsvol_get(local->inode, this, &mds_subvol);
+        if (ret || !mds_subvol) {
+            gf_msg_debug(this->name, 0, "path = %s. No mds subvol in inode ctx",
+                         local->loc.path);
         }
+        local->mds_subvol = mds_subvol;
+        local->call_cnt = conf->subvolume_cnt;
 
-        if (!hashed_subvol)
-                hashed_subvol = dht_subvol_get_hashed (this, loc);
-        local->hashed_subvol = hashed_subvol;
-
-
-        /* The entry has been looked up before and has an inode_ctx set
+        /* local->call_cnt will change as responses are processed. Always use a
+         * local copy to loop through the STACK_WIND calls
          */
-        if (is_revalidate (loc)) {
-                layout = local->layout;
-                if (!layout) {
-                        gf_msg_debug (this->name, 0,
-                                      "Revalidate lookup without cache."
-                                      " path=%s", loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                if (layout->gen && (layout->gen < conf->gen)) {
-                        gf_msg_trace (this->name, 0,
-                                      "incomplete layout failure for path=%s",
-                                      loc->path);
-
-                        gen = layout->gen;
-                        dht_layout_unref (this, local->layout);
-                        local->layout = NULL;
-                        local->cached_subvol = NULL;
-
-                        gf_msg_debug(this->name, 0,
-                                     "Called revalidate lookup for %s, "
-                                     "but layout->gen (%d) is less than "
-                                     "conf->gen (%d), calling fresh_lookup",
-                                     loc->path, gen, conf->gen);
-
-                        goto do_fresh_lookup;
-                }
-
-                local->inode = inode_ref (loc->inode);
-
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", conf->xattr_name, loc->path);
-                        goto err;
-                }
-
-                /* need it in case file is not found on cached file
-                 * on revalidate path and we may encounter linkto files on
-                 * with dht_lookup_everywhere*/
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->link_xattr_name, 256);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", conf->link_xattr_name, loc->path);
-                        goto err;
-                }
-                if (IA_ISDIR (local->inode->ia_type)) {
-                        ret = dht_inode_ctx_mdsvol_get (local->inode, this,
-                                                        &mds_subvol);
-                        if (ret || !mds_subvol) {
-                                gf_msg_debug (this->name, 0,
-                                              "Failed to get mds subvol for path %s",
-                                              local->loc.path);
-                        }
-                        local->mds_subvol = mds_subvol;
-                        local->call_cnt = call_cnt = conf->subvolume_cnt;
-                        for (i = 0; i < call_cnt; i++) {
-                                STACK_WIND_COOKIE (frame, dht_revalidate_cbk,
-                                                   conf->subvolumes[i],
-                                                   conf->subvolumes[i],
-                                                   conf->subvolumes[i]->fops->lookup,
-                                                   loc, local->xattr_req);
-                        }
-                        return 0;
-                }
-
-                call_cnt = local->call_cnt = layout->cnt;
-
-                /* need it for self-healing linkfiles which is
-                   'in-migration' state */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       GLUSTERFS_OPEN_FD_COUNT, 4);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", GLUSTERFS_OPEN_FD_COUNT, loc->path);
-                        goto err;
-                }
-                /* need it for dir self-heal */
-                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
-
-                for (i = 0; i < call_cnt; i++) {
-                        subvol = layout->list[i].xlator;
-
-                        gf_msg_debug (this->name, 0, "calling "
-                                      "revalidate lookup for %s at %s",
-                                      loc->path, subvol->name);
-
-                        STACK_WIND_COOKIE (frame, dht_revalidate_cbk, subvol,
-                                           subvol, subvol->fops->lookup,
-                                           &local->loc, local->xattr_req);
-
-                }
-        } else {
-        do_fresh_lookup:
-                /* TODO: remove the hard-coding */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", conf->xattr_name, loc->path);
-                        goto err;
-                }
 
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->link_xattr_name, 256);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", conf->link_xattr_name, loc->path);
-                        goto err;
-                }
-                /* need it for self-healing linkfiles which is
-                   'in-migration' state */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       GLUSTERFS_OPEN_FD_COUNT, 4);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", GLUSTERFS_OPEN_FD_COUNT, loc->path);
-                        goto err;
-                }
-                /* need it for dir self-heal */
-                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
+        call_cnt = local->call_cnt;
 
-                if (!hashed_subvol) {
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_revalidate_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, loc,
+                              local->xattr_req);
+        }
+        return 0;
+    }
 
-                        gf_msg_debug (this->name, 0,
-                                      "no subvolume in layout for path=%s, "
-                                      "checking on all the subvols to see if "
-                                      "it is a directory", loc->path);
+    /* If not a dir, this should be 1 */
+    local->call_cnt = layout->cnt;
+    call_cnt = local->call_cnt;
 
-                        call_cnt        = conf->subvolume_cnt;
-                        local->call_cnt = call_cnt;
+    for (i = 0; i < call_cnt; i++) {
+        subvol = layout->list[i].xlator;
 
-                        local->layout = dht_layout_new (this,
-                                                        conf->subvolume_cnt);
-                        if (!local->layout) {
-                                op_errno = ENOMEM;
-                                goto err;
-                        }
+        gf_msg_debug(this->name, 0,
+                     "path = %s. Calling "
+                     "revalidate lookup on %s",
+                     loc->path, subvol->name);
 
-                        gf_msg_debug (this->name, 0,
-                                      "Found null hashed subvol. Calling lookup"
-                                      " on all nodes.");
+        STACK_WIND_COOKIE(frame, dht_revalidate_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, local->xattr_req);
+    }
+    return 0;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
 
-                        for (i = 0; i < call_cnt; i++) {
-                                STACK_WIND_COOKIE (frame, dht_lookup_dir_cbk,
-                                                   conf->subvolumes[i],
-                                                   conf->subvolumes[i],
-                                                   conf->subvolumes[i]->fops->lookup,
-                                                   &local->loc, local->xattr_req);
-                        }
-                        return 0;
-                }
+/* Depending on the input, decide if this is a:
+ * fresh-lookup: loc->name is provided but no dht inode ctx
+ * revalidation: loc->name is provided, dht inode ctx is present
+ * discover: gfid based nameless lookup.
+ */
 
-                gf_msg_debug (this->name, 0, "Calling fresh lookup for %s on"
-                              " %s", loc->path, hashed_subvol->name);
+int
+dht_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    xlator_t *hashed_subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    loc_t new_loc = {
+        0,
+    };
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    conf = this->private;
+    if (!conf)
+        goto err;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_LOOKUP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dht_filter_loc_subvol_key(this, loc, &new_loc, &hashed_subvol);
+    if (ret) {
+        loc_wipe(&local->loc);
+        ret = loc_dup(&new_loc, &local->loc);
+
+        /* we no longer need 'new_loc' entries */
+        loc_wipe(&new_loc);
+
+        /* check if loc_dup() is successful */
+        if (ret == -1) {
+            op_errno = errno;
+            gf_msg_debug(this->name, errno,
+                         "copying location failed for path=%s", loc->path);
+            goto err;
+        }
+    }
+
+    if (xattr_req) {
+        local->xattr_req = dict_ref(xattr_req);
+    } else {
+        local->xattr_req = dict_new();
+    }
+
+    /* Nameless lookup */
+
+    /* This is usually sent by NFS. Lookups are done based on the gfid and
+     * no name information is available. Without the name, dht cannot calculate
+     * the hash and has to send a lookup to all subvols.
+     */
+    if (gf_uuid_is_null(loc->pargfid) && !gf_uuid_is_null(loc->gfid) &&
+        !__is_root_gfid(loc->inode->gfid)) {
+        local->cached_subvol = NULL;
+        dht_do_discover(frame, this, loc);
+        return 0;
+    }
+
+    if (loc_is_root(loc)) {
+        /* Request the DHT commit hash xattr (trusted.glusterfs.dht.commithash)
+         * set on the brick root.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->commithash_xattr_name,
+                              sizeof(uint32_t));
+    }
 
-                STACK_WIND_COOKIE (frame, dht_lookup_cbk, hashed_subvol,
-                                   hashed_subvol, hashed_subvol->fops->lookup,
-                                   loc, local->xattr_req);
-        }
+    if (!hashed_subvol)
+        hashed_subvol = dht_subvol_get_hashed(this, loc);
+    local->hashed_subvol = hashed_subvol;
 
+    if (is_revalidate(loc)) {
+        /* The entry has been looked up before and has a dht inode_ctx
+         */
+        dht_do_revalidate(frame, this, loc);
+        return 0;
+    } else {
+        /* Entry has not been looked up before
+         */
+        dht_do_fresh_lookup(frame, this, loc);
         return 0;
+    }
 
+    return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
-        return 0;
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
-int
-dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
+static int
+dht_unlink_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t     *local          = NULL;
-        xlator_t        *prev           = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
 
-        local = frame->local;
-        prev  = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if ((op_ret == -1) && !((op_errno == ENOENT) ||
-                                        (op_errno == ENOTCONN))) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "Unlink link: subvolume %s"
-                                      " returned -1",
-                                      prev->name);
-                        goto unlock;
-                }
+    local = frame->local;
+    prev = cookie;
 
-                local->op_ret = 0;
+    LOCK(&frame->lock);
+    {
+        if ((op_ret == -1) &&
+            !((op_errno == ENOENT) || (op_errno == ENOTCONN))) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno,
+                         "Unlink link: subvolume %s returned -1", prev->name);
+            goto post_unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        dht_set_fixed_dir_stat (&local->preparent);
-        dht_set_fixed_dir_stat (&local->postparent);
-        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, xdata);
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
+    DHT_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+static int
+dht_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *preparent, struct iatt *postparent,
+               dict_t *xdata)
 {
-        dht_local_t     *local          = NULL;
-        xlator_t        *prev           = NULL;
-        xlator_t        *hashed_subvol  = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *hashed_subvol = NULL;
 
-        local = frame->local;
-        prev  = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno != ENOENT) {
-                                local->op_ret   = -1;
-                                local->op_errno = op_errno;
-                        } else {
-                                local->op_ret = 0;
-                        }
-                        gf_msg_debug (this->name, op_errno,
-                                      "Unlink: subvolume %s returned -1",
-                                       prev->name);
-                        goto unlock;
-                }
+    local = frame->local;
+    prev = cookie;
 
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if (op_errno != ENOENT) {
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+            } else {
                 local->op_ret = 0;
-
-                local->postparent = *postparent;
-                local->preparent = *preparent;
-
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->preparent, 0);
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+            }
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno,
+                         "Unlink: subvolume %s returned -1", prev->name);
+            goto post_unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        if (!local->op_ret) {
-                hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
-                if (hashed_subvol && hashed_subvol != local->cached_subvol) {
-                        /*
-                         * If hashed and cached are different, then we need
-                         * to unlink linkfile from hashed subvol if data
-                         * file is deleted successfully
-                         */
-                         STACK_WIND_COOKIE (frame, dht_unlink_linkfile_cbk,
-                                            hashed_subvol, hashed_subvol,
-                                            hashed_subvol->fops->unlink,
-                                            &local->loc, local->flags, xdata);
-                         return 0;
-                }
-        }
+        local->op_ret = 0;
 
-        dht_set_fixed_dir_stat (&local->preparent);
-        dht_set_fixed_dir_stat (&local->postparent);
-        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, xdata);
+        local->postparent = *postparent;
+        local->preparent = *preparent;
 
-        return 0;
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->preparent, 0);
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
+        }
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!local->op_ret) {
+        hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
+        if (hashed_subvol && hashed_subvol != local->cached_subvol) {
+            /*
+             * If hashed and cached are different, then we need
+             * to unlink linkfile from hashed subvol if data
+             * file is deleted successfully
+             */
+            STACK_WIND_COOKIE(frame, dht_unlink_linkfile_cbk, hashed_subvol,
+                              hashed_subvol, hashed_subvol->fops->unlink,
+                              &local->loc, local->flags, xdata);
+            return 0;
+        }
+    }
+
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
+    DHT_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, xdata);
+
+    return 0;
 }
 
 static int
-dht_common_setxattr_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno,
-                         dict_t *xdata)
+dht_common_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-         DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
-         return 0;
+    DHT_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
-
 static int
-dht_fix_layout_setxattr_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             dict_t *xdata)
+dht_fix_layout_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t   *local   = NULL;
-        dht_layout_t  *layout  = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
 
-        if (op_ret == 0) {
+    if (op_ret == 0) {
+        /* update the layout in the inode ctx */
+        local = frame->local;
+        layout = local->selfheal.layout;
 
-                /* update the layout in the inode ctx */
-                local = frame->local;
-                layout = local->selfheal.layout;
+        dht_layout_set(this, local->loc.inode, layout);
+    }
 
-                dht_layout_set (this, local->loc.inode, layout);
-        }
-
-         DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
-         return 0;
+    DHT_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
-int
-dht_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_err_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+            int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        xlator_t     *prev = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
 
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "subvolume %s returned -1",
-                                      prev->name);
-                        goto unlock;
-                }
+    local = frame->local;
+    prev = cookie;
 
-                local->op_ret = 0;
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+            goto post_unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
-                                  local->op_errno, NULL);
-        }
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if ((local->fop == GF_FOP_SETXATTR) ||
+            (local->fop == GF_FOP_FSETXATTR)) {
+            DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                             NULL);
+            /* 'local' itself may not be valid after this */
+            goto out;
+        }
+        if ((local->fop == GF_FOP_REMOVEXATTR) ||
+            (local->fop == GF_FOP_FREMOVEXATTR)) {
+            DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                             NULL);
+        }
+    }
 
-        return 0;
+out:
+    return 0;
 }
 
 /* Set the value[] of key into dict after convert from
    host byte order to network byte order
 */
-int32_t dht_dict_set_array (dict_t *dict, char *key, int32_t value[],
-                            int32_t size)
+int32_t
+dht_dict_set_array(dict_t *dict, char *key, int32_t value[], int32_t size)
 {
-        int         ret = -1;
-        int32_t   *ptr = NULL;
-        int32_t     vindex;
+    int ret = -1;
+    int32_t *ptr = NULL;
+    int32_t vindex;
 
-        if (value == NULL) {
-                return -EINVAL;
-        }
+    if (value == NULL) {
+        return -EINVAL;
+    }
 
-        ptr = GF_MALLOC(sizeof(int32_t) * size, gf_common_mt_char);
-        if (ptr == NULL) {
-                return -ENOMEM;
-        }
-        for (vindex = 0; vindex < size; vindex++) {
-                ptr[vindex] = hton32(value[vindex]);
-        }
-        ret = dict_set_bin(dict, key, ptr, sizeof(int32_t) * size);
-        if (ret)
-                GF_FREE (ptr);
-        return ret;
+    ptr = GF_MALLOC(sizeof(int32_t) * size, gf_common_mt_char);
+    if (ptr == NULL) {
+        return -ENOMEM;
+    }
+    for (vindex = 0; vindex < size; vindex++) {
+        ptr[vindex] = hton32(value[vindex]);
+    }
+    ret = dict_set_bin(dict, key, ptr, sizeof(int32_t) * size);
+    if (ret)
+        GF_FREE(ptr);
+    return ret;
 }
 
-int
-dht_common_mds_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+static int
+dht_common_mds_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        call_frame_t  *prev  = cookie;
+    dht_local_t *local = NULL;
+    call_frame_t *prev = cookie;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret)
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->this->name);
+    if (op_ret)
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->this->name);
 
-        if (local->fop == GF_FOP_SETXATTR) {
-                DHT_STACK_UNWIND (setxattr, frame, 0, op_errno, local->xdata);
-        } else {
-                DHT_STACK_UNWIND (fsetxattr, frame, 0, op_errno, local->xdata);
-        }
-        return 0;
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, 0, op_errno, local->xdata);
+        /* 'local' itself may not be valid after this */
+        goto out;
+    }
+
+    if (local->fop == GF_FOP_FSETXATTR) {
+        DHT_STACK_UNWIND(fsetxattr, frame, 0, op_errno, local->xdata);
+        /* 'local' itself may not be valid after this */
+        goto out;
+    }
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, 0, op_errno, NULL);
+        /* 'local' itself may not be valid after this */
+        goto out;
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        DHT_STACK_UNWIND(fremovexattr, frame, 0, op_errno, NULL);
+    }
+
+out:
+    return 0;
 }
 
 /* Code to wind a xattrop call to add 1 on current mds internal xattr
    value
 */
-int
-dht_setxattr_non_mds_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_setxattr_non_mds_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        int           ret           = 0;
-        dict_t        *xattrop      = NULL;
-        int32_t       addone[1]     = {1};
-        call_frame_t  *prev         = NULL;
-        dht_conf_t    *conf         = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    int ret = 0;
+    dict_t *xattrop = NULL;
+    int32_t addone[1] = {1};
+    call_frame_t *prev = NULL;
+    dht_conf_t *conf = NULL;
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret && !local->op_ret) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->this->name);
+            goto post_unlock;
+        }
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+
+    if (is_last_call(this_call_cnt)) {
+        if (!local->op_ret) {
+            xattrop = dict_new();
+            if (!xattrop) {
+                gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, 0,
+                       "dictionary creation failed");
+                ret = -1;
+                goto out;
+            }
+            ret = dht_dict_set_array(xattrop, conf->mds_xattr_key, addone, 1);
+            if (ret != 0) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                       "dictionary set array failed ");
+                ret = -1;
+                goto out;
+            }
+            if ((local->fop == GF_FOP_SETXATTR) ||
+                (local->fop == GF_FOP_REMOVEXATTR)) {
+                STACK_WIND(frame, dht_common_mds_xattrop_cbk, local->mds_subvol,
+                           local->mds_subvol->fops->xattrop, &local->loc,
+                           GF_XATTROP_ADD_ARRAY, xattrop, NULL);
+            } else {
+                STACK_WIND(frame, dht_common_mds_xattrop_cbk, local->mds_subvol,
+                           local->mds_subvol->fops->fxattrop, local->fd,
+                           GF_XATTROP_ADD_ARRAY, xattrop, NULL);
+            }
+        } else {
+            if (local->fop == GF_FOP_SETXATTR) {
+                DHT_STACK_UNWIND(setxattr, frame, 0, 0, local->xdata);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
 
-        local = frame->local;
-        prev = cookie;
-        conf = this->private;
-
-        LOCK (&frame->lock);
-        {
-             if (op_ret && !local->op_ret) {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                         gf_msg_debug (this->name, op_errno,
-                                       "subvolume %s returned -1",
-                                       prev->this->name);
-             }
-        }
-        UNLOCK (&frame->lock);
-        this_call_cnt = dht_frame_return (frame);
-
-        if (is_last_call (this_call_cnt)) {
-                if (!local->op_ret) {
-                        xattrop = dict_new ();
-                        if (!xattrop) {
-                                gf_msg (this->name, GF_LOG_ERROR,
-                                        DHT_MSG_NO_MEMORY, 0,
-                                        "dictionary creation failed");
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = dht_dict_set_array (xattrop,
-                                                  conf->mds_xattr_key,
-                                                  addone, 1);
-                        if (ret != 0) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "dictionary set array failed ");
-                                ret = -1;
-                                goto out;
-                        }
-                        if (local->fop == GF_FOP_SETXATTR) {
-                                STACK_WIND (frame, dht_common_mds_xattrop_cbk,
-                                            local->mds_subvol,
-                                            local->mds_subvol->fops->xattrop,
-                                            &local->loc, GF_XATTROP_ADD_ARRAY,
-                                            xattrop, NULL);
-                        } else {
-                                STACK_WIND (frame, dht_common_mds_xattrop_cbk,
-                                            local->mds_subvol,
-                                            local->mds_subvol->fops->fxattrop,
-                                            local->fd, GF_XATTROP_ADD_ARRAY,
-                                            xattrop, NULL);
-                        }
-                } else  {
-                        if (local->fop == GF_FOP_SETXATTR)
-                                DHT_STACK_UNWIND (setxattr, frame, 0, 0, local->xdata);
-                        else
-                                DHT_STACK_UNWIND (fsetxattr, frame, 0, 0, local->xdata);
-                }
+            if (local->fop == GF_FOP_FSETXATTR) {
+                DHT_STACK_UNWIND(fsetxattr, frame, 0, 0, local->xdata);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
+
+            if (local->fop == GF_FOP_REMOVEXATTR) {
+                DHT_STACK_UNWIND(removexattr, frame, 0, 0, NULL);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
+
+            if (local->fop == GF_FOP_FREMOVEXATTR) {
+                DHT_STACK_UNWIND(fremovexattr, frame, 0, 0, NULL);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
         }
+    }
 out:
-        if (xattrop)
-                dict_unref (xattrop);
-        if (ret) {
-                if (local->fop == GF_FOP_SETXATTR)
-                        DHT_STACK_UNWIND (setxattr, frame, 0, 0, local->xdata);
-                else
-                        DHT_STACK_UNWIND (fsetxattr, frame, 0, 0, local->xdata);
+    if (ret) {
+        if (local->fop == GF_FOP_SETXATTR) {
+            DHT_STACK_UNWIND(setxattr, frame, 0, 0, local->xdata);
+            /* 'local' itself may not be valid after this */
+            goto just_return;
         }
-        return 0;
-}
 
+        if (local->fop == GF_FOP_FSETXATTR) {
+            DHT_STACK_UNWIND(fsetxattr, frame, 0, 0, local->xdata);
+            /* 'local' itself may not be valid after this */
+            goto just_return;
+        }
 
-int
-dht_setxattr_mds_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, dict_t *xdata)
+        if (local->fop == GF_FOP_REMOVEXATTR) {
+            DHT_STACK_UNWIND(removexattr, frame, 0, 0, NULL);
+            /* 'local' itself may not be valid after this */
+            goto just_return;
+        }
+
+        if (local->fop == GF_FOP_FREMOVEXATTR) {
+            DHT_STACK_UNWIND(fremovexattr, frame, 0, 0, NULL);
+        }
+    }
+just_return:
+    if (xattrop)
+        dict_unref(xattrop);
+    return 0;
+}
+
+static int
+dht_setxattr_mds_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf  = NULL;
-        call_frame_t *prev = NULL;
-        xlator_t     *mds_subvol = NULL;
-        int i = 0;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    call_frame_t *prev = NULL;
+    xlator_t *mds_subvol = NULL;
+    int i = 0;
 
-        local = frame->local;
-        prev = cookie;
-        conf = this->private;
-        mds_subvol = local->mds_subvol;
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+    mds_subvol = local->mds_subvol;
 
-        if (op_ret == -1) {
-                local->op_ret  = op_ret;
-                local->op_errno = op_errno;
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->this->name);
-                goto out;
+    if (op_ret == -1) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->this->name);
+        goto out;
+    }
+
+    local->op_ret = 0;
+    local->call_cnt = conf->subvolume_cnt - 1;
+    local->xdata = dict_ref(xdata);
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (mds_subvol && (mds_subvol == conf->subvolumes[i]))
+            continue;
+        if (local->fop == GF_FOP_SETXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->setxattr, &local->loc,
+                       local->xattr, local->flags, local->xattr_req);
         }
 
-        local->op_ret = 0;
-        local->call_cnt = conf->subvolume_cnt - 1;
-        local->xdata    = dict_ref (xdata);
+        if (local->fop == GF_FOP_FSETXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->fsetxattr, local->fd,
+                       local->xattr, local->flags, local->xattr_req);
+        }
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (mds_subvol && (mds_subvol == conf->subvolumes[i]))
-                        continue;
-                if (local->fop == GF_FOP_SETXATTR) {
-                        STACK_WIND (frame, dht_setxattr_non_mds_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->setxattr,
-                                    &local->loc, local->xattr,
-                                    local->flags, local->xattr_req);
-                } else {
-                        STACK_WIND (frame, dht_setxattr_non_mds_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->fsetxattr,
-                                    local->fd, local->xattr,
-                                    local->flags, local->xattr_req);
-                }
+        if (local->fop == GF_FOP_REMOVEXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->removexattr, &local->loc,
+                       local->key, local->xattr_req);
         }
 
-        return 0;
-out:
-        if (local->fop == GF_FOP_SETXATTR) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
-                                  local->op_errno, xdata);
-        } else {
-                DHT_STACK_UNWIND (fsetxattr, frame, local->op_ret,
-                                  local->op_errno, xdata);
+        if (local->fop == GF_FOP_FREMOVEXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->fremovexattr, local->fd,
+                       local->key, local->xattr_req);
         }
+    }
 
-        return 0;
+    return 0;
+out:
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FSETXATTR) {
+        DHT_STACK_UNWIND(fsetxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        DHT_STACK_UNWIND(fremovexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+    }
+
+just_return:
+    return 0;
 }
 
-int
-dht_xattrop_mds_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+static int
+dht_xattrop_mds_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    call_frame_t *prev = NULL;
 
-        local = frame->local;
-        prev = cookie;
+    local = frame->local;
+    prev = cookie;
 
-        if (op_ret == -1) {
-                local->op_errno = op_errno;
-                local->op_ret   = op_ret;
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->this->name);
-                goto out;
-        }
-
-        if (local->fop == GF_FOP_SETXATTR) {
-                STACK_WIND (frame, dht_setxattr_mds_cbk,
-                            local->mds_subvol,
-                            local->mds_subvol->fops->setxattr,
-                            &local->loc, local->xattr,
-                            local->flags, local->xattr_req);
-        } else {
-                STACK_WIND (frame, dht_setxattr_mds_cbk,
-                            local->mds_subvol,
-                            local->mds_subvol->fops->fsetxattr,
-                            local->fd, local->xattr,
-                            local->flags, local->xattr_req);
-        }
-        return 0;
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        local->op_ret = op_ret;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->this->name);
+        goto out;
+    }
+
+    if (local->fop == GF_FOP_SETXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->setxattr, &local->loc, local->xattr,
+                   local->flags, local->xattr_req);
+    }
+
+    if (local->fop == GF_FOP_FSETXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->fsetxattr, local->fd, local->xattr,
+                   local->flags, local->xattr_req);
+    }
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->removexattr, &local->loc,
+                   local->key, local->xattr_req);
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->fremovexattr, local->fd, local->key,
+                   local->xattr_req);
+    }
+
+    return 0;
 out:
-        if (local->fop == GF_FOP_SETXATTR)
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
-                                  local->op_errno, xdata);
-        else
-                DHT_STACK_UNWIND (fsetxattr, frame, local->op_ret,
-                                  local->op_errno, xdata);
-        return 0;
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FSETXATTR) {
+        DHT_STACK_UNWIND(fsetxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        DHT_STACK_UNWIND(fremovexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+    }
+
+just_return:
+    return 0;
 }
 
 static void
-fill_layout_info (dht_layout_t *layout, char *buf)
+fill_layout_info(dht_layout_t *layout, char *buf)
 {
-        int i = 0;
-        char tmp_buf[128] = {0,};
+    int i = 0;
+    char tmp_buf[128] = {
+        0,
+    };
 
-        for (i = 0; i < layout->cnt; i++) {
-                snprintf (tmp_buf, 128, "(%s %u %u)",
-                          layout->list[i].xlator->name,
-                          layout->list[i].start,
-                          layout->list[i].stop);
-                if (i)
-                        strcat (buf, " ");
-                strcat (buf, tmp_buf);
-        }
+    for (i = 0; i < layout->cnt; i++) {
+        snprintf(tmp_buf, sizeof(tmp_buf), "(%s %u %u)",
+                 layout->list[i].xlator->name, layout->list[i].start,
+                 layout->list[i].stop);
+        if (i)
+            strcat(buf, " ");
+        strcat(buf, tmp_buf);
+    }
 }
 
-void
-dht_fill_pathinfo_xattr (xlator_t *this, dht_local_t *local,
-                         char *xattr_buf, int32_t alloc_len,
-                         int flag, char *layout_buf)
-{
-        if (flag && local->xattr_val)
-                snprintf (xattr_buf, alloc_len,
-                          "((<"DHT_PATHINFO_HEADER"%s> %s) (%s-layout %s))",
-                          this->name, local->xattr_val, this->name,
-                          layout_buf);
-        else if (local->xattr_val)
-                snprintf (xattr_buf, alloc_len,
-                          "(<"DHT_PATHINFO_HEADER"%s> %s)",
-                          this->name, local->xattr_val);
-        else if (flag)
-                snprintf (xattr_buf, alloc_len, "(%s-layout %s)",
-                          this->name, layout_buf);
+static void
+dht_fill_pathinfo_xattr(xlator_t *this, dht_local_t *local, char *xattr_buf,
+                        int32_t alloc_len, int flag, char *layout_buf)
+{
+    if (flag) {
+        if (local->xattr_val) {
+            snprintf(xattr_buf, alloc_len,
+                     "((<" DHT_PATHINFO_HEADER "%s> %s) (%s-layout %s))",
+                     this->name, local->xattr_val, this->name, layout_buf);
+        } else {
+            snprintf(xattr_buf, alloc_len, "(%s-layout %s)", this->name,
+                     layout_buf);
+        }
+    } else if (local->xattr_val) {
+        snprintf(xattr_buf, alloc_len, "(<" DHT_PATHINFO_HEADER "%s> %s)",
+                 this->name, local->xattr_val);
+    } else {
+        xattr_buf[0] = '\0';
+    }
 }
 
-int
-dht_vgetxattr_alloc_and_fill (dht_local_t *local, dict_t *xattr, xlator_t *this,
-                              int op_errno)
+static int
+dht_vgetxattr_alloc_and_fill(dht_local_t *local, dict_t *xattr, xlator_t *this,
+                             int op_errno)
 {
-        int      ret       = -1;
-        char    *value     = NULL;
-        int32_t  plen      = 0;
+    int ret = -1;
+    char *value = NULL;
 
-        ret = dict_get_str (xattr, local->xsel, &value);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        DHT_MSG_GET_XATTR_FAILED,
-                        "Subvolume %s returned -1", this->name);
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-                goto out;
-        }
+    ret = dict_get_str(xattr, local->xsel, &value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_GET_XATTR_FAILED,
+               "Subvolume %s returned -1", this->name);
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto out;
+    }
 
-        local->alloc_len += strlen(value);
+    local->alloc_len += strlen(value);
 
+    if (!local->xattr_val) {
+        local->alloc_len += (SLEN(DHT_PATHINFO_HEADER) + 10);
+        local->xattr_val = GF_MALLOC(local->alloc_len, gf_common_mt_char);
         if (!local->xattr_val) {
-                local->alloc_len += (strlen (DHT_PATHINFO_HEADER) + 10);
-                local->xattr_val = GF_CALLOC (local->alloc_len, sizeof (char),
-                                              gf_common_mt_char);
-                if (!local->xattr_val) {
-                        ret = -1;
-                        goto out;
-                }
+            ret = -1;
+            goto out;
         }
+        local->xattr_val[0] = '\0';
+    }
 
-        if (local->xattr_val) {
-                plen = strlen (local->xattr_val);
-                if (plen) {
-                        /* extra byte(s) for \0 to be safe */
-                        local->alloc_len += (plen + 2);
-                        local->xattr_val = GF_REALLOC (local->xattr_val,
-                                                       local->alloc_len);
-                        if (!local->xattr_val) {
-                                ret = -1;
-                                goto out;
-                        }
-                }
-
-                (void) strcat (local->xattr_val, value);
-                (void) strcat (local->xattr_val, " ");
-                local->op_ret = 0;
+    int plen = strlen(local->xattr_val);
+    if (plen) {
+        /* extra byte(s) for \0 to be safe */
+        local->alloc_len += (plen + 2);
+        local->xattr_val = GF_REALLOC(local->xattr_val, local->alloc_len);
+        if (!local->xattr_val) {
+            ret = -1;
+            goto out;
         }
+    }
 
-        ret = 0;
+    (void)strcat(local->xattr_val, value);
+    (void)strcat(local->xattr_val, " ");
+    local->op_ret = 0;
 
- out:
-        return ret;
+    ret = 0;
+
+out:
+    return ret;
 }
 
-int
-dht_vgetxattr_fill_and_set (dht_local_t *local, dict_t **dict, xlator_t *this,
-                            gf_boolean_t flag)
+static int
+dht_vgetxattr_fill_and_set(dht_local_t *local, dict_t **dict, xlator_t *this,
+                           gf_boolean_t flag)
 {
-        int   ret             = -1;
-        char *xattr_buf       = NULL;
-        char layout_buf[8192] = {0,};
-
-        if (flag)
-                fill_layout_info (local->layout, layout_buf);
+    int ret = -1;
+    char *xattr_buf = NULL;
+    char layout_buf[8192] = {
+        0,
+    };
 
-        *dict = dict_new ();
-        if (!*dict)
-                goto out;
+    if (flag)
+        fill_layout_info(local->layout, layout_buf);
 
-        local->xattr_val[strlen (local->xattr_val) - 1] = '\0';
+    *dict = dict_new();
+    if (!*dict)
+        goto out;
 
-        /* we would need max this many bytes to create xattr string
-         * extra 40 bytes is just an estimated amount of additional
-         * space required as we include translator name and some
-         * spaces, brackets etc. when forming the pathinfo string.
-         *
-         * For node-uuid we just don't have all the pretty formatting,
-         * but since this is a generic routine for pathinfo & node-uuid
-         * we dont have conditional space allocation and try to be
-         * generic
-         */
-        local->alloc_len += (2 * strlen (this->name))
-                + strlen (layout_buf)
-                + 40;
-        xattr_buf = GF_CALLOC (local->alloc_len, sizeof (char),
-                               gf_common_mt_char);
-        if (!xattr_buf)
-                goto out;
+    local->xattr_val[strlen(local->xattr_val) - 1] = '\0';
+
+    /* we would need max this many bytes to create xattr string
+     * extra 40 bytes is just an estimated amount of additional
+     * space required as we include translator name and some
+     * spaces, brackets etc. when forming the pathinfo string.
+     *
+     * For node-uuid we just don't have all the pretty formatting,
+     * but since this is a generic routine for pathinfo & node-uuid
+     * we don't have conditional space allocation and try to be
+     * generic
+     */
+    local->alloc_len += (2 * strlen(this->name)) + strlen(layout_buf) + 40;
+    xattr_buf = GF_MALLOC(local->alloc_len, gf_common_mt_char);
+    if (!xattr_buf)
+        goto out;
 
-        if (XATTR_IS_PATHINFO (local->xsel)) {
-                (void) dht_fill_pathinfo_xattr (this, local, xattr_buf,
-                                                local->alloc_len, flag,
-                                                layout_buf);
-        } else if ((XATTR_IS_NODE_UUID (local->xsel))
-                   || (XATTR_IS_NODE_UUID_LIST (local->xsel))) {
-                (void) snprintf (xattr_buf, local->alloc_len, "%s",
-                                 local->xattr_val);
-        } else {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_GET_XATTR_FAILED,
-                        "Unknown local->xsel (%s)", local->xsel);
-                GF_FREE (xattr_buf);
-                goto out;
-        }
+    if (XATTR_IS_PATHINFO(local->xsel)) {
+        (void)dht_fill_pathinfo_xattr(this, local, xattr_buf, local->alloc_len,
+                                      flag, layout_buf);
+    } else if ((XATTR_IS_NODE_UUID(local->xsel)) ||
+               (XATTR_IS_NODE_UUID_LIST(local->xsel))) {
+        (void)snprintf(xattr_buf, local->alloc_len, "%s", local->xattr_val);
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GET_XATTR_FAILED,
+               "Unknown local->xsel (%s)", local->xsel);
+        GF_FREE(xattr_buf);
+        goto out;
+    }
 
-        ret = dict_set_dynstr (*dict, local->xsel, xattr_buf);
-        if (ret)
-                GF_FREE (xattr_buf);
-        GF_FREE (local->xattr_val);
+    ret = dict_set_dynstr(*dict, local->xsel, xattr_buf);
+    if (ret)
+        GF_FREE(xattr_buf);
+    GF_FREE(local->xattr_val);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
+static int
+dht_find_local_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, dict_t *xattr,
+                          dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
+    int ret = 0;
+    char *uuid_str = NULL;
+    char *uuid_list = NULL;
+    char *next_uuid_str = NULL;
+    char *saveptr = NULL;
+    uuid_t node_uuid = {
+        0,
+    };
+    char *uuid_list_copy = NULL;
+    int count = 0;
+    int i = 0;
+    int index = 0;
+    int found = 0;
+    nodeuuid_info_t *tmp_ptr = NULL;
+
+    VALIDATE_OR_GOTO(frame, out);
+    VALIDATE_OR_GOTO(frame->local, out);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    VALIDATE_OR_GOTO(conf->defrag, out);
+
+    gf_msg_debug(this->name, 0, "subvol %s returned", prev->name);
+
+    LOCK(&frame->lock);
+    {
+        this_call_cnt = --local->call_cnt;
+        if (op_ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            if (op_errno == ENODATA)
+                gf_msg_debug(this->name, 0, "failed to get node-uuid");
+            else
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_GET_XATTR_FAILED, "failed to get node-uuid");
+            goto post_unlock;
+        }
 
-int
-dht_find_local_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr,
-                           dict_t *xdata)
-{
-        dht_local_t  *local         = NULL;
-        dht_conf_t   *conf          = NULL;
-        xlator_t     *prev          = NULL;
-        int           this_call_cnt = 0;
-        int           ret           = 0;
-        char         *uuid_str      = NULL;
-        char         *uuid_list     = NULL;
-        char         *next_uuid_str = NULL;
-        char         *saveptr       = NULL;
-        uuid_t        node_uuid     = {0,};
-        char         *uuid_list_copy = NULL;
-        int           count          = 0;
-        int           i              = 0;
-        int           index          = 0;
-        int           found          = 0;
-        nodeuuid_info_t *tmp_ptr     = NULL;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
+        ret = dict_get_str(xattr, local->xsel, &uuid_list);
 
-        local = frame->local;
-        prev = cookie;
-        conf = this->private;
-
-        VALIDATE_OR_GOTO (conf->defrag, out);
-
-        gf_msg_debug (this->name, 0, "subvol %s returned", prev->name);
-
-        LOCK (&frame->lock);
-        {
-                this_call_cnt = --local->call_cnt;
-                if (op_ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                DHT_MSG_GET_XATTR_FAILED,
-                                "getxattr err for dir");
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                        goto unlock;
-                }
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_GET_FAILED,
+                   "Failed to get %s", local->xsel);
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
+            goto unlock;
+        }
 
-                ret = dict_get_str (xattr, local->xsel, &uuid_list);
+        /* As DHT will not know details of its child xlators
+         * we need to parse this twice to get the count first
+         * and allocate memory later.
+         */
+        count = 0;
+        index = conf->local_subvols_cnt;
 
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_GET_FAILED,
-                                "Failed to get %s", local->xsel);
-                        local->op_ret = -1;
-                        local->op_errno = EINVAL;
-                        goto unlock;
-                }
+        uuid_list_copy = gf_strdup(uuid_list);
+        if (!uuid_list_copy)
+            goto unlock;
 
-                /* As DHT will not know details of its child xlators
-                 * we need to parse this twice to get the count first
-                 * and allocate memory later.
-                 */
-                count = 0;
-                index = conf->local_subvols_cnt;
-
-                uuid_list_copy = gf_strdup (uuid_list);
-
-                for (uuid_str = strtok_r (uuid_list, " ", &saveptr);
-                     uuid_str;
-                     uuid_str = next_uuid_str) {
-
-                        next_uuid_str = strtok_r (NULL, " ", &saveptr);
-                        if (gf_uuid_parse (uuid_str, node_uuid)) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_UUID_PARSE_ERROR,
-                                        "Failed to parse uuid"
-                                        " for %s", prev->name);
-                                local->op_ret = -1;
-                                local->op_errno = EINVAL;
-                                goto unlock;
-                        }
-
-                        count++;
-                        if (gf_uuid_compare (node_uuid, conf->defrag->node_uuid)) {
-                                gf_msg_debug (this->name, 0, "subvol %s does not"
-                                              "belong to this node",
-                                              prev->name);
-                        } else {
-
-                                /* handle multiple bricks of the same replica
-                                 * on the same node */
-                                if (found)
-                                        continue;
-                                conf->local_subvols[(conf->local_subvols_cnt)++]
-                                                = prev;
-                                found = 1;
-                                gf_msg_debug (this->name, 0, "subvol %s belongs to"
-                                              " this node", prev->name);
-                        }
-                }
+        for (uuid_str = strtok_r(uuid_list, " ", &saveptr); uuid_str;
+             uuid_str = next_uuid_str) {
+            next_uuid_str = strtok_r(NULL, " ", &saveptr);
+            if (gf_uuid_parse(uuid_str, node_uuid)) {
+                local->op_ret = -1;
+                local->op_errno = EINVAL;
+                UNLOCK(&frame->lock);
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UUID_PARSE_ERROR,
+                       "Failed to parse uuid for %s", prev->name);
+                goto post_unlock;
+            }
 
-                if (!found) {
-                        local->op_ret = 0;
-                        goto unlock;
-                }
+            count++;
+            if (gf_uuid_compare(node_uuid, conf->defrag->node_uuid)) {
+                gf_msg_debug(this->name, 0,
+                             "subvol %s does not"
+                             "belong to this node",
+                             prev->name);
+            } else {
+                /* handle multiple bricks of the same replica
+                 * on the same node */
+                if (found)
+                    continue;
+                conf->local_subvols[(conf->local_subvols_cnt)++] = prev;
+                found = 1;
+                gf_msg_debug(this->name, 0,
+                             "subvol %s belongs to"
+                             " this node",
+                             prev->name);
+            }
+        }
 
-                conf->local_nodeuuids[index].count = count;
-                conf->local_nodeuuids[index].elements
-                               = GF_CALLOC (count, sizeof (nodeuuid_info_t), 1);
+        if (!found) {
+            local->op_ret = 0;
+            goto unlock;
+        }
 
-                /* The node-uuids are guaranteed to be returned in the same
-                 * order as the bricks
-                 * A null node-uuid is returned for a brick that is down.
-                 */
+        conf->local_nodeuuids[index].count = count;
+        conf->local_nodeuuids[index].elements = GF_CALLOC(
+            count, sizeof(nodeuuid_info_t), 1);
 
-                saveptr = NULL;
-                i = 0;
+        /* The node-uuids are guaranteed to be returned in the same
+         * order as the bricks
+         * A null node-uuid is returned for a brick that is down.
+         */
 
-                for (uuid_str = strtok_r (uuid_list_copy, " ", &saveptr);
-                     uuid_str;
-                     uuid_str = next_uuid_str) {
+        saveptr = NULL;
+        i = 0;
 
-                        next_uuid_str = strtok_r (NULL, " ", &saveptr);
-                        tmp_ptr = &(conf->local_nodeuuids[index].elements[i]);
-                        gf_uuid_parse (uuid_str, tmp_ptr->uuid);
+        for (uuid_str = strtok_r(uuid_list_copy, " ", &saveptr); uuid_str;
+             uuid_str = next_uuid_str) {
+            next_uuid_str = strtok_r(NULL, " ", &saveptr);
+            tmp_ptr = &(conf->local_nodeuuids[index].elements[i]);
+            gf_uuid_parse(uuid_str, tmp_ptr->uuid);
 
-                        if (!gf_uuid_compare (tmp_ptr->uuid,
-                                              conf->defrag->node_uuid)) {
-                                tmp_ptr->info = REBAL_NODEUUID_MINE;
-                        }
-                        i++;
-                        tmp_ptr = NULL;
-                }
+            if (!gf_uuid_compare(tmp_ptr->uuid, conf->defrag->node_uuid)) {
+                tmp_ptr->info = REBAL_NODEUUID_MINE;
+            }
+            i++;
+            tmp_ptr = NULL;
         }
+    }
 
-        local->op_ret = 0;
- unlock:
-        UNLOCK (&frame->lock);
-
-        if (!is_last_call (this_call_cnt))
-                goto out;
+    local->op_ret = 0;
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!is_last_call(this_call_cnt))
+        goto out;
 
-        if (local->op_ret == -1) {
-                goto unwind;
-        }
+    if (local->op_ret == -1) {
+        goto unwind;
+    }
 
-        DHT_STACK_UNWIND (getxattr, frame, 0, 0, xattr, xdata);
-        goto out;
+    DHT_STACK_UNWIND(getxattr, frame, 0, 0, xattr, xdata);
+    goto out;
 
- unwind:
+unwind:
 
-        GF_FREE (conf->local_nodeuuids[index].elements);
-        conf->local_nodeuuids[index].elements = NULL;
+    GF_FREE(conf->local_nodeuuids[index].elements);
+    conf->local_nodeuuids[index].elements = NULL;
 
-        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno, NULL, xdata);
- out:
-        GF_FREE (uuid_list_copy);
-        return 0;
+    DHT_STACK_UNWIND(getxattr, frame, -1, local->op_errno, NULL, xdata);
+out:
+    GF_FREE(uuid_list_copy);
+    return 0;
 }
 
-int
-dht_vgetxattr_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+static int
+dht_vgetxattr_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        int          ret           = 0;
-        dht_local_t *local         = NULL;
-        int          this_call_cnt = 0;
-        dict_t      *dict          = NULL;
+    int ret = 0;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    dict_t *dict = NULL;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
+    VALIDATE_OR_GOTO(frame, out);
+    VALIDATE_OR_GOTO(frame->local, out);
 
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                this_call_cnt = --local->call_cnt;
-                if (op_ret < 0) {
-                        if (op_errno != ENOTCONN) {
-                                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                        DHT_MSG_GET_XATTR_FAILED,
-                                        "getxattr err for dir");
-                                local->op_ret = -1;
-                                local->op_errno = op_errno;
-                        }
+    local = frame->local;
 
-                        goto unlock;
-                }
+    LOCK(&frame->lock);
+    {
+        this_call_cnt = --local->call_cnt;
+        if (op_ret < 0) {
+            if (op_errno != ENOTCONN) {
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                UNLOCK(&frame->lock);
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_GET_XATTR_FAILED, "getxattr err for dir");
+                goto post_unlock;
+            }
 
-                ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
-                                                    op_errno);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "alloc or fill failure");
+            goto unlock;
         }
- unlock:
-        UNLOCK (&frame->lock);
 
-        if (!is_last_call (this_call_cnt))
-                goto out;
+        ret = dht_vgetxattr_alloc_and_fill(local, xattr, this, op_errno);
+        if (ret) {
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_DICT_SET_FAILED,
+                   "alloc or fill failure");
+            goto post_unlock;
+        }
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!is_last_call(this_call_cnt))
+        goto out;
 
-        /* -- last call: do patch ups -- */
+    /* -- last call: do patch ups -- */
 
-        if (local->op_ret == -1) {
-                goto unwind;
-        }
+    if (local->op_ret == -1) {
+        goto unwind;
+    }
 
-        ret = dht_vgetxattr_fill_and_set (local, &dict, this, _gf_true);
-        if (ret)
-                goto unwind;
+    ret = dht_vgetxattr_fill_and_set(local, &dict, this, _gf_true);
+    if (ret)
+        goto unwind;
 
-        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
-        goto cleanup;
+    DHT_STACK_UNWIND(getxattr, frame, 0, 0, dict, xdata);
+    goto cleanup;
 
- unwind:
-        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno, NULL, NULL);
- cleanup:
-        if (dict)
-                dict_unref (dict);
- out:
-        return 0;
+unwind:
+    DHT_STACK_UNWIND(getxattr, frame, -1, local->op_errno, NULL, NULL);
+cleanup:
+    if (dict)
+        dict_unref(dict);
+out:
+    return 0;
 }
 
-int
-dht_vgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+static int
+dht_vgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        dht_local_t  *local         = NULL;
-        int           ret           = 0;
-        dict_t       *dict          = NULL;
-        xlator_t     *prev          = NULL;
-        gf_boolean_t  flag          = _gf_true;
+    dht_local_t *local = NULL;
+    int ret = 0;
+    dict_t *dict = NULL;
+    xlator_t *prev = NULL;
+    gf_boolean_t flag = _gf_true;
 
-        local = frame->local;
-        prev = cookie;
+    local = frame->local;
+    prev = cookie;
 
-        if (op_ret < 0) {
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        DHT_MSG_GET_XATTR_FAILED,
-                        "vgetxattr: Subvolume %s returned -1",
-                         prev->name);
-                goto unwind;
-        }
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_GET_XATTR_FAILED,
+               "vgetxattr: Subvolume %s returned -1", prev->name);
+        goto unwind;
+    }
 
-        ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
-                                            op_errno);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_NO_MEMORY,
-                        "Allocation or fill failure");
-                goto unwind;
-        }
+    ret = dht_vgetxattr_alloc_and_fill(local, xattr, this, op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+               "Allocation or fill failure");
+        goto unwind;
+    }
 
-        flag = (local->layout->cnt > 1) ? _gf_true : _gf_false;
+    flag = (local->layout->cnt > 1) ? _gf_true : _gf_false;
 
-        ret = dht_vgetxattr_fill_and_set (local, &dict, this, flag);
-        if (ret)
-                goto unwind;
+    ret = dht_vgetxattr_fill_and_set(local, &dict, this, flag);
+    if (ret)
+        goto unwind;
 
-        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
-        goto cleanup;
+    DHT_STACK_UNWIND(getxattr, frame, 0, 0, dict, xdata);
+    goto cleanup;
 
- unwind:
-        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno,
-                          NULL, NULL);
- cleanup:
-        if (dict)
-                dict_unref (dict);
+unwind:
+    DHT_STACK_UNWIND(getxattr, frame, -1, local->op_errno, NULL, NULL);
+cleanup:
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_linkinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr,
-                           dict_t *xdata)
+static int
+dht_linkinfo_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, dict_t *xattr,
+                          dict_t *xdata)
 {
-        int   ret   = 0;
-        char *value = NULL;
+    int ret = 0;
+    char *value = NULL;
 
-        if (op_ret != -1) {
-                ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &value);
-                if (!ret) {
-                        ret = dict_set_str (xattr, GF_XATTR_LINKINFO_KEY, value);
-                        if (!ret)
-                                gf_msg_trace (this->name, 0,
-                                              "failed to set linkinfo");
-                }
+    if (op_ret != -1) {
+        ret = dict_get_str(xattr, GF_XATTR_PATHINFO_KEY, &value);
+        if (!ret) {
+            ret = dict_set_str(xattr, GF_XATTR_LINKINFO_KEY, value);
+            if (!ret)
+                gf_msg_trace(this->name, 0, "failed to set linkinfo");
         }
+    }
 
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
+    DHT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_mds_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+static int
+dht_mds_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        dht_local_t     *local = NULL;
-        dht_conf_t      *conf = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(frame->local, err);
+    VALIDATE_OR_GOTO(this->private, err);
 
-        conf = this->private;
-        local = frame->local;
+    conf = this->private;
+    local = frame->local;
 
-        if (!xattr || (op_ret == -1)) {
-                local->op_ret = op_ret;
-                goto out;
-        }
-        if (dict_get (xattr, conf->xattr_name)) {
-                dict_del (xattr, conf->xattr_name);
-        }
-        local->op_ret = 0;
+    if (!xattr || (op_ret == -1)) {
+        local->op_ret = op_ret;
+        goto out;
+    }
+    dict_del(xattr, conf->xattr_name);
+    local->op_ret = 0;
 
-        if (!local->xattr) {
-                local->xattr = dict_copy_with_ref (xattr, NULL);
-        }
+    if (!local->xattr) {
+        local->xattr = dict_copy_with_ref(xattr, NULL);
+    }
 
 out:
-        DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
-                          local->xattr, xdata);
-        return 0;
+    DHT_STACK_UNWIND(getxattr, frame, local->op_ret, op_errno, local->xattr,
+                     xdata);
+    return 0;
+err:
+    DHT_STACK_UNWIND(getxattr, frame, -1, EINVAL, NULL, NULL);
+    return 0;
 }
 
-
 int
-dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+dht_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        int             this_call_cnt = 0;
-        dht_local_t     *local = NULL;
-        dht_conf_t      *conf = NULL;
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(frame->local, err);
+    VALIDATE_OR_GOTO(this->private, err);
 
-        conf = this->private;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (!xattr || (op_ret == -1)) {
-                        local->op_ret = op_ret;
-                        goto unlock;
-                }
+    conf = this->private;
+    local = frame->local;
 
-                if (dict_get (xattr, conf->xattr_name)) {
-                        dict_del (xattr, conf->xattr_name);
-                }
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto err;
+        return 0;
+    }
 
-                if (frame->root->pid >= 0) {
-                        GF_REMOVE_INTERNAL_XATTR
-                                ("trusted.glusterfs.quota*", xattr);
-                        GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
-                }
+    LOCK(&frame->lock);
+    {
+        if (!xattr || (op_ret == -1)) {
+            local->op_ret = op_ret;
+            goto unlock;
+        }
 
-                local->op_ret = 0;
+        dict_del(xattr, conf->xattr_name);
+        dict_del(xattr, conf->mds_xattr_key);
 
-                if (!local->xattr) {
-                        local->xattr = dict_copy_with_ref (xattr, NULL);
-                } else {
-                        dht_aggregate_xattr (local->xattr, xattr);
-                }
+        dict_del(xattr, conf->commithash_xattr_name);
 
+        if (frame->root->pid >= 0) {
+            GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+            GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
-out:
-        if (is_last_call (this_call_cnt)) {
+        local->op_ret = 0;
 
-                /* If we have a valid xattr received from any one of the
-                 * subvolume, let's return it */
-                if (local->xattr) {
-                        local->op_ret = 0;
-                }
+        if (!local->xattr) {
+            local->xattr = dict_copy_with_ref(xattr, NULL);
+        } else {
+            dht_aggregate_xattr(local->xattr, xattr);
+        }
 
-                DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
-                                  local->xattr, NULL);
+        if (!local->xdata) {
+            local->xdata = dict_ref(xdata);
+        } else if ((local->inode && IA_ISDIR(local->inode->ia_type)) ||
+                   (local->fd && IA_ISDIR(local->fd->inode->ia_type))) {
+            dht_aggregate_xattr(local->xdata, xdata);
         }
-        return 0;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        /* If we have a valid xattr received from any one of the
+         * subvolume, let's return it */
+        if (local->xattr) {
+            local->op_ret = 0;
+        }
+
+        DHT_STACK_UNWIND(getxattr, frame, local->op_ret, op_errno, local->xattr,
+                         local->xdata);
+    }
+    return 0;
+err:
+    DHT_STACK_UNWIND(getxattr, frame, -1, EINVAL, NULL, NULL);
+    return 0;
 }
 
-int32_t
-dht_getxattr_unwind (call_frame_t *frame,
-                     int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+static int32_t
+dht_getxattr_unwind(call_frame_t *frame, int op_ret, int op_errno, dict_t *dict,
+                    dict_t *xdata)
 {
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    DHT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
-int
-dht_getxattr_get_real_filename_cbk (call_frame_t *frame, void *cookie,
-                                    xlator_t *this, int op_ret, int op_errno,
-                                    dict_t *xattr, dict_t *xdata)
+static int
+dht_getxattr_get_real_filename_cbk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int op_ret, int op_errno,
+                                   dict_t *xattr, dict_t *xdata)
 {
-        int             this_call_cnt = 0;
-        dht_local_t     *local = NULL;
-
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->op_errno == ENODATA ||
-                    local->op_errno == EOPNOTSUPP) {
-                        /* Nothing to do here, we have already found
-                         * a subvol which does not have the get_real_filename
-                         * optimization. If condition is for simple logic.
-                         */
-                        goto unlock;
-                }
+    local = frame->local;
 
-                if (op_ret == -1) {
-
-                        if (op_errno == ENODATA || op_errno == EOPNOTSUPP) {
-                                /* This subvol does not have the optimization.
-                                 * Better let the user know we don't support it.
-                                 * Remove previous results if any.
-                                 */
-
-                                if (local->xattr) {
-                                        dict_unref (local->xattr);
-                                        local->xattr = NULL;
-                                }
-
-                                if (local->xattr_req) {
-                                        dict_unref (local->xattr_req);
-                                        local->xattr_req = NULL;
-                                }
-
-                                local->op_ret = op_ret;
-                                local->op_errno = op_errno;
-                                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                                        DHT_MSG_UPGRADE_BRICKS, "At least "
-                                        "one of the bricks does not support "
-                                        "this operation. Please upgrade all "
-                                        "bricks.");
-                                goto unlock;
-                        }
-
-                        if (op_errno == ENOENT) {
-                                /* Do nothing, our defaults are set to this.
-                                 */
-                                goto unlock;
-                        }
+    LOCK(&frame->lock);
+    {
+        if (local->op_errno == EOPNOTSUPP) {
+            /* Nothing to do here, we have already found
+             * a subvol which does not have the get_real_filename
+             * optimization. If condition is for simple logic.
+             */
+            goto unlock;
+        }
 
-                        /* This is a place holder for every other error
-                         * case. I am not sure of how to interpret
-                         * ENOTCONN etc. As of now, choosing to ignore
-                         * down subvol and return a good result(if any)
-                         * from other subvol.
-                         */
-                        gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                                DHT_MSG_GET_XATTR_FAILED,
-                                "Failed to get real filename.");
-                        goto unlock;
+        if (op_ret == -1) {
+            if (op_errno == EOPNOTSUPP) {
+                /* This subvol does not have the optimization.
+                 * Better let the user know we don't support it.
+                 * Remove previous results if any.
+                 */
 
+                if (local->xattr) {
+                    dict_unref(local->xattr);
+                    local->xattr = NULL;
                 }
 
-
-                /* This subvol has the required file.
-                 * There could be other subvols which have returned
-                 * success already, choosing to return the latest good
-                 * result.
-                 */
-                if (local->xattr)
-                        dict_unref (local->xattr);
-                local->xattr = dict_ref (xattr);
-
                 if (local->xattr_req) {
-                        dict_unref (local->xattr_req);
-                        local->xattr_req = NULL;
+                    dict_unref(local->xattr_req);
+                    local->xattr_req = NULL;
                 }
-                if (xdata)
-                        local->xattr_req = dict_ref (xdata);
 
                 local->op_ret = op_ret;
-                local->op_errno = 0;
-                gf_msg_debug (this->name, 0, "Found a matching "
-                              "file.");
-        }
-unlock:
-        UNLOCK (&frame->lock);
+                local->op_errno = op_errno;
+                UNLOCK(&frame->lock);
+                gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                       DHT_MSG_UPGRADE_BRICKS,
+                       "At least "
+                       "one of the bricks does not support "
+                       "this operation. Please upgrade all "
+                       "bricks.");
+                goto post_unlock;
+            }
 
+            if (op_errno == ENOATTR) {
+                /* Do nothing, our defaults are set to this.
+                 */
+                goto unlock;
+            }
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (getxattr, frame, local->op_ret,
-                                  local->op_errno, local->xattr,
-                                  local->xattr_req);
+            /* This is a place holder for every other error
+             * case. I am not sure of how to interpret
+             * ENOTCONN etc. As of now, choosing to ignore
+             * down subvol and return a good result(if any)
+             * from other subvol.
+             */
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                   DHT_MSG_GET_XATTR_FAILED, "Failed to get real filename.");
+            goto post_unlock;
+        }
+
+        /* This subvol has the required file.
+         * There could be other subvols which have returned
+         * success already, choosing to return the latest good
+         * result.
+         */
+        if (local->xattr)
+            dict_unref(local->xattr);
+        local->xattr = dict_ref(xattr);
+
+        if (local->xattr_req) {
+            dict_unref(local->xattr_req);
+            local->xattr_req = NULL;
         }
+        if (xdata)
+            local->xattr_req = dict_ref(xdata);
 
-        return 0;
-}
+        local->op_ret = op_ret;
+        local->op_errno = 0;
+        UNLOCK(&frame->lock);
+        gf_msg_debug(this->name, 0, "Found a matching file.");
+        goto post_unlock;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         local->xattr, local->xattr_req);
+    }
 
+    return 0;
+}
 
-int
-dht_getxattr_get_real_filename (call_frame_t *frame, xlator_t *this,
-                                loc_t *loc, const char *key, dict_t *xdata)
+static int
+dht_getxattr_get_real_filename(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                               const char *key, dict_t *xdata)
 {
-        dht_local_t     *local = NULL;
-        int              i = 0;
-        dht_layout_t    *layout = NULL;
-        int              cnt = 0;
-        xlator_t        *subvol = NULL;
+    dht_local_t *local = NULL;
+    int i = 0;
+    dht_layout_t *layout = NULL;
+    int cnt = 0;
+    xlator_t *subvol = NULL;
 
+    local = frame->local;
+    layout = local->layout;
 
-        local = frame->local;
-        layout = local->layout;
+    cnt = local->call_cnt = layout->cnt;
 
-        cnt = local->call_cnt = layout->cnt;
+    local->op_ret = -1;
+    local->op_errno = ENOATTR;
 
-        local->op_ret = -1;
-        local->op_errno = ENOENT;
-
-        for (i = 0; i < cnt; i++) {
-                subvol = layout->list[i].xlator;
-                STACK_WIND (frame, dht_getxattr_get_real_filename_cbk,
-                            subvol, subvol->fops->getxattr,
-                            loc, key, xdata);
-        }
+    for (i = 0; i < cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND(frame, dht_getxattr_get_real_filename_cbk, subvol,
+                   subvol->fops->getxattr, loc, key, xdata);
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_marker_populate_args (call_frame_t *frame, int type, int *gauge,
-                          xlator_t **subvols)
+static int
+dht_marker_populate_args(call_frame_t *frame, int type, int *gauge,
+                         xlator_t **subvols)
 {
-        dht_local_t *local = NULL;
-        int         i = 0;
-        dht_layout_t *layout = NULL;
+    dht_local_t *local = NULL;
+    int i = 0;
+    dht_layout_t *layout = NULL;
 
-        local  = frame->local;
-        layout = local->layout;
+    local = frame->local;
+    layout = local->layout;
 
-        for (i = 0; i < layout->cnt; i++)
-                subvols[i] = layout->list[i].xlator;
+    for (i = 0; i < layout->cnt; i++)
+        subvols[i] = layout->list[i].xlator;
 
-        return layout->cnt;
+    return layout->cnt;
 }
 
+static int
+dht_is_debug_xattr_key(const char **array, char *key)
+{
+    int i = 0;
+
+    for (i = 0; array[i]; i++) {
+        if (fnmatch(array[i], key, FNM_NOESCAPE) == 0)
+            return i;
+    }
+
+    return -1;
+}
 
 /* Note we already have frame->local initialised here*/
 
-int
-dht_handle_debug_getxattr (call_frame_t *frame, xlator_t *this,
-                           loc_t *loc, const char *key)
+static int
+dht_handle_debug_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                          const char *key)
 {
-        dht_local_t  *local         = NULL;
-        int           ret           = -1;
-        int           op_errno      = ENODATA;
-        ia_type_t     type          = 0;
-        char         *value         = NULL;
-
-        local = frame->local;
-        if (!key) {
-                op_errno = EINVAL;
-                goto out;
-        }
+    dht_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENODATA;
+    char *value = NULL;
+    loc_t file_loc = {0};
+    const char *name = NULL;
 
-        if (gf_get_index_by_elem (dht_dbg_vxattrs, (char *)key) == -1) {
-                goto out;
-        }
+    local = frame->local;
 
-        /* getxattr does not set the parent inode*/
-        if (!loc->parent) {
-                   loc->parent =
-                                inode_parent(local->loc.inode, NULL, NULL);
-        }
-        if (!loc->parent) {
-                op_errno = EINVAL;
-                goto out;
-        }
+    if (dht_is_debug_xattr_key(dht_dbg_vxattrs, (char *)key) == -1) {
+        goto out;
+    }
 
-        gf_uuid_copy (loc->pargfid, loc->parent->gfid);
+    local->xattr = dict_new();
+    if (!local->xattr) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-        if (!loc->name && loc->path) {
-                loc->name = strrchr (loc->path, '/');
-                if (loc->name) {
-                        ++(loc->name);
-                }
+    if (strncmp(key, DHT_DBG_HASHED_SUBVOL_KEY,
+                SLEN(DHT_DBG_HASHED_SUBVOL_KEY)) == 0) {
+        name = key + strlen(DHT_DBG_HASHED_SUBVOL_KEY);
+        if (strlen(name) == 0) {
+            op_errno = EINVAL;
+            goto out;
         }
 
-        local->xattr = dict_new ();
-        if (!local->xattr) {
-                op_errno = ENOMEM;
-                goto out;
+        ret = dht_build_child_loc(this, &file_loc, loc, (char *)name);
+        if (ret) {
+            op_errno = ENOMEM;
+            goto out;
         }
 
-        type = loc->inode->ia_type;
-
-        if (strcmp (key, DHT_DBG_HASHED_SUBVOL_KEY) == 0) {
-                if (IA_ISDIR (type)) {
-                        op_errno = EINVAL;
-                        goto out;
-                }
-
-                local->hashed_subvol = dht_subvol_get_hashed (this, loc);
-                if (local->hashed_subvol == NULL) {
-                        op_errno = ENODATA;
-                        goto out;
-                }
+        local->hashed_subvol = dht_subvol_get_hashed(this, &file_loc);
+        if (local->hashed_subvol == NULL) {
+            op_errno = ENODATA;
+            goto out;
+        }
 
-                value = gf_strdup (local->hashed_subvol->name);
-                if (!value) {
-                        op_errno = ENOMEM;
-                        goto out;
-                }
+        value = gf_strdup(local->hashed_subvol->name);
+        if (!value) {
+            op_errno = ENOMEM;
+            goto out;
+        }
 
-                ret = dict_set_dynstr (local->xattr,  (char *)key, value);
-                if (ret < 0) {
-                        op_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
-                ret = 0;
-                goto out;
+        ret = dict_set_dynstr(local->xattr, (char *)key, value);
+        if (ret < 0) {
+            op_errno = -ret;
+            ret = -1;
+            goto out;
         }
+        ret = 0;
+        goto out;
+    }
 
 out:
-        DHT_STACK_UNWIND (getxattr, frame, ret, op_errno, local->xattr, NULL);
-        return 0;
+    loc_wipe(&file_loc);
+    DHT_STACK_UNWIND(getxattr, frame, ret, op_errno, local->xattr, NULL);
+    return 0;
 }
 
+/* Virtual Xattr which returns 1 if all subvols are up,
+   else returns 0. Geo-rep then uses this virtual xattr
+   after a fresh mount and starts the I/O.
+*/
+
+enum dht_vxattr_subvol {
+    DHT_VXATTR_SUBVOLS_UP = 1,
+    DHT_VXATTR_SUBVOLS_DOWN = 0,
+};
 
 int
-dht_getxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, const char *key, dict_t *xdata)
-#define DHT_IS_DIR(layout)  (layout->cnt > 1)
-{
-
-        xlator_t     *subvol        = NULL;
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *mds_subvol    = NULL;
-        xlator_t     *cached_subvol = NULL;
-        dht_conf_t   *conf          = NULL;
-        dht_local_t  *local         = NULL;
-        dht_layout_t *layout        = NULL;
-        int           op_errno      = -1;
-        int           i             = 0;
-        int           cnt           = 0;
-        char         *node_uuid_key = NULL;
-        int           ret           = -1;
-
-        GF_CHECK_XATTR_KEY_AND_GOTO (key, IO_THREADS_QUEUE_SIZE_KEY,
-                                     op_errno, err);
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf   = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_GETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
+dht_vgetxattr_subvol_status(call_frame_t *frame, xlator_t *this,
+                            const char *key)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENODATA;
+    int value = DHT_VXATTR_SUBVOLS_UP;
+    int i = 0;
+    dht_conf_t *conf = NULL;
 
-                goto err;
-        }
+    conf = this->private;
+    local = frame->local;
 
-        layout = local->layout;
-        if (!layout) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LAYOUT_NULL,
-                        "Layout is NULL");
-                op_errno = ENOENT;
-                goto err;
-        }
+    if (!key) {
+        op_errno = EINVAL;
+        goto out;
+    }
+    local->xattr = dict_new();
+    if (!local->xattr) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->subvolume_status[i]) {
+            value = DHT_VXATTR_SUBVOLS_DOWN;
+            gf_msg_debug(this->name, 0, "subvol %s is down ",
+                         conf->subvolumes[i]->name);
+            break;
+        }
+    }
+    ret = dict_set_int8(local->xattr, (char *)key, value);
+    if (ret < 0) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 
-        if (key) {
-                local->key = gf_strdup (key);
-                if (!local->key) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-        }
+out:
+    DHT_STACK_UNWIND(getxattr, frame, ret, op_errno, local->xattr, NULL);
+    return 0;
+}
 
-        if (key &&
-            (strncmp (key, conf->mds_xattr_key, strlen(key)) == 0)) {
-                op_errno = ENOTSUP;
-                goto err;
+int
+dht_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *key,
+             dict_t *xdata)
+#define DHT_IS_DIR(layout) (layout->cnt > 1)
+{
+    xlator_t *subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int op_errno = -1;
+    int i = 0;
+    int cnt = 0;
+    char *node_uuid_key = NULL;
+    int ret = -1;
+
+    GF_CHECK_XATTR_KEY_AND_GOTO(key, IO_THREADS_QUEUE_SIZE_KEY, op_errno, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_GETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LAYOUT_NULL,
+               "Layout is NULL");
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    /* skip over code which is irrelevant without a valid key */
+    if (!key)
+        goto no_key;
+
+    local->key = gf_strdup(key);
+    if (!local->key) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (strncmp(key, conf->mds_xattr_key, strlen(key)) == 0) {
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    if (strncmp(key, DHT_SUBVOL_STATUS_KEY, SLEN(DHT_SUBVOL_STATUS_KEY)) == 0) {
+        dht_vgetxattr_subvol_status(frame, this, key);
+        return 0;
+    }
+
+    /* skip over code which is irrelevant if !DHT_IS_DIR(layout) */
+    if (!DHT_IS_DIR(layout))
+        goto no_dht_is_dir;
+
+    if ((strncmp(key, GF_XATTR_GET_REAL_FILENAME_KEY,
+                 SLEN(GF_XATTR_GET_REAL_FILENAME_KEY)) == 0) &&
+        DHT_IS_DIR(layout)) {
+        dht_getxattr_get_real_filename(frame, this, loc, key, xdata);
+        return 0;
+    }
+
+    if (!strcmp(key, GF_REBAL_FIND_LOCAL_SUBVOL)) {
+        ret = gf_asprintf(&node_uuid_key, "%s", GF_XATTR_LIST_NODE_UUIDS_KEY);
+        if (ret == -1 || !node_uuid_key) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+                   "Failed to copy node uuid key");
+            op_errno = ENOMEM;
+            goto err;
+        }
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", node_uuid_key);
+        cnt = local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_find_local_subvol_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->getxattr, loc,
+                              node_uuid_key, xdata);
         }
+        if (node_uuid_key)
+            GF_FREE(node_uuid_key);
+        return 0;
+    }
 
-        if (key &&
-            (strncmp (key, GF_XATTR_GET_REAL_FILENAME_KEY,
-                      strlen (GF_XATTR_GET_REAL_FILENAME_KEY)) == 0)
-            && DHT_IS_DIR(layout)) {
-                dht_getxattr_get_real_filename (frame, this, loc, key, xdata);
-                return 0;
+    if (!strcmp(key, GF_REBAL_OLD_FIND_LOCAL_SUBVOL)) {
+        ret = gf_asprintf(&node_uuid_key, "%s", GF_XATTR_NODE_UUID_KEY);
+        if (ret == -1 || !node_uuid_key) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+                   "Failed to copy node uuid key");
+            op_errno = ENOMEM;
+            goto err;
         }
-
-        if (key && DHT_IS_DIR(layout) &&
-           (!strcmp (key, GF_REBAL_FIND_LOCAL_SUBVOL))) {
-                ret = gf_asprintf
-                           (&node_uuid_key, "%s", GF_XATTR_LIST_NODE_UUIDS_KEY);
-                if (ret == -1 || !node_uuid_key) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_NO_MEMORY,
-                                "Failed to copy key");
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                (void) strncpy (local->xsel, node_uuid_key, 256);
-                cnt = local->call_cnt = conf->subvolume_cnt;
-                for (i = 0; i < cnt; i++) {
-                        STACK_WIND_COOKIE (frame, dht_find_local_subvol_cbk,
-                                           conf->subvolumes[i],
-                                           conf->subvolumes[i],
-                                           conf->subvolumes[i]->fops->getxattr,
-                                           loc, node_uuid_key, xdata);
-                }
-                if (node_uuid_key)
-                        GF_FREE (node_uuid_key);
-                return 0;
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", node_uuid_key);
+        cnt = local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_find_local_subvol_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->getxattr, loc,
+                              node_uuid_key, xdata);
+        }
+        if (node_uuid_key)
+            GF_FREE(node_uuid_key);
+        return 0;
+    }
+
+    /* for file use cached subvolume (obviously!): see if {}
+     * below
+     * for directory:
+     *  wind to all subvolumes and exclude subvolumes which
+     *  return ENOTCONN (in callback)
+     *
+     * NOTE: Don't trust inode here, as that may not be valid
+     *       (until inode_link() happens)
+     */
+
+    if (XATTR_IS_PATHINFO(key) || (strcmp(key, GF_XATTR_NODE_UUID_KEY) == 0) ||
+        (strcmp(key, GF_XATTR_LIST_NODE_UUIDS_KEY) == 0)) {
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", key);
+        cnt = local->call_cnt = layout->cnt;
+        for (i = 0; i < cnt; i++) {
+            subvol = layout->list[i].xlator;
+            STACK_WIND(frame, dht_vgetxattr_dir_cbk, subvol,
+                       subvol->fops->getxattr, loc, key, xdata);
         }
+        return 0;
+    }
 
-        if (key && DHT_IS_DIR(layout) &&
-           (!strcmp (key, GF_REBAL_OLD_FIND_LOCAL_SUBVOL))) {
-                ret = gf_asprintf
-                           (&node_uuid_key, "%s", GF_XATTR_NODE_UUID_KEY);
-                if (ret == -1 || !node_uuid_key) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_NO_MEMORY,
-                                "Failed to copy key");
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                (void) strncpy (local->xsel, node_uuid_key, 256);
-                cnt = local->call_cnt = conf->subvolume_cnt;
-                for (i = 0; i < cnt; i++) {
-                        STACK_WIND_COOKIE (frame, dht_find_local_subvol_cbk,
-                                           conf->subvolumes[i],
-                                           conf->subvolumes[i],
-                                           conf->subvolumes[i]->fops->getxattr,
-                                           loc, node_uuid_key, xdata);
-                }
-                if (node_uuid_key)
-                        GF_FREE (node_uuid_key);
-                return 0;
-        }
+no_dht_is_dir:
+    /* node-uuid or pathinfo for files */
+    if (XATTR_IS_PATHINFO(key) || (strcmp(key, GF_XATTR_NODE_UUID_KEY) == 0)) {
+        cached_subvol = local->cached_subvol;
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", key);
+        local->call_cnt = 1;
+        STACK_WIND_COOKIE(frame, dht_vgetxattr_cbk, cached_subvol,
+                          cached_subvol, cached_subvol->fops->getxattr, loc,
+                          key, xdata);
 
-        /* for file use cached subvolume (obviously!): see if {}
-         * below
-         * for directory:
-         *  wind to all subvolumes and exclude subvolumes which
-         *  return ENOTCONN (in callback)
-         *
-         * NOTE: Don't trust inode here, as that may not be valid
-         *       (until inode_link() happens)
-         */
+        return 0;
+    }
 
-        if (key && DHT_IS_DIR(layout) &&
-            (XATTR_IS_PATHINFO (key)
-             || (strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0)
-             || (strcmp (key, GF_XATTR_LIST_NODE_UUIDS_KEY) == 0))) {
-                (void) strncpy (local->xsel, key, 256);
-                cnt = local->call_cnt = layout->cnt;
-                for (i = 0; i < cnt; i++) {
-                        subvol = layout->list[i].xlator;
-                        STACK_WIND (frame, dht_vgetxattr_dir_cbk,
-                                    subvol, subvol->fops->getxattr,
-                                    loc, key, xdata);
-                }
-                return 0;
+    if (strcmp(key, GF_XATTR_LINKINFO_KEY) == 0) {
+        hashed_subvol = dht_subvol_get_hashed(this, loc);
+        if (!hashed_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get hashed subvol for %s", loc->path);
+            op_errno = EINVAL;
+            goto err;
         }
 
-        /* node-uuid or pathinfo for files */
-        if (key && ((strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0)
-                    || XATTR_IS_PATHINFO (key))) {
-                cached_subvol = local->cached_subvol;
-                (void) strncpy (local->xsel, key, 256);
-
-                local->call_cnt = 1;
-                STACK_WIND_COOKIE (frame, dht_vgetxattr_cbk, cached_subvol,
-                                   cached_subvol, cached_subvol->fops->getxattr,
-                                   loc, key, xdata);
+        cached_subvol = dht_subvol_get_cached(this, loc->inode);
+        if (!cached_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_CACHED_SUBVOL_GET_FAILED,
+                   "Failed to get cached subvol for %s", loc->path);
+            op_errno = EINVAL;
+            goto err;
+        }
 
-                return 0;
+        if (hashed_subvol == cached_subvol) {
+            op_errno = ENODATA;
+            goto err;
         }
 
-        if (key && (strcmp (key, GF_XATTR_LINKINFO_KEY) == 0)) {
+        STACK_WIND(frame, dht_linkinfo_getxattr_cbk, hashed_subvol,
+                   hashed_subvol->fops->getxattr, loc, GF_XATTR_PATHINFO_KEY,
+                   xdata);
+        return 0;
+    }
 
-                hashed_subvol = dht_subvol_get_hashed (this, loc);
-                if (!hashed_subvol) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                                "Failed to get hashed subvol for %s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    if (dht_is_debug_xattr_key(dht_dbg_vxattrs, (char *)key) >= 0) {
+        dht_handle_debug_getxattr(frame, this, loc, key);
+        return 0;
+    }
 
-                cached_subvol = dht_subvol_get_cached (this, loc->inode);
-                if (!cached_subvol) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_CACHED_SUBVOL_GET_FAILED,
-                                "Failed to get cached subvol for %s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
+no_key:
+    if (cluster_handle_marker_getxattr(frame, loc, key, conf->vol_uuid,
+                                       dht_getxattr_unwind,
+                                       dht_marker_populate_args) == 0)
+        return 0;
 
-                if (hashed_subvol == cached_subvol) {
-                        op_errno = ENODATA;
-                        goto err;
+    if (DHT_IS_DIR(layout)) {
+        local->call_cnt = conf->subvolume_cnt;
+        cnt = conf->subvolume_cnt;
+        ret = dht_inode_ctx_mdsvol_get(loc->inode, this, &mds_subvol);
+        if (!mds_subvol) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Cannot determine MDS, fetching xattr %s randomly"
+                   " from a subvol for path %s ",
+                   key, loc->path);
+        } else {
+            /* TODO need to handle it, As of now we are
+               choosing availability instead of chossing
+               consistencty, in case of mds_subvol is
+               down winding a getxattr call on other subvol
+               and return xattr
+            */
+            local->mds_subvol = mds_subvol;
+            for (i = 0; i < cnt; i++) {
+                if (conf->subvolumes[i] == mds_subvol) {
+                    if (!conf->subvolume_status[i]) {
+                        gf_msg(this->name, GF_LOG_INFO, 0,
+                               DHT_MSG_HASHED_SUBVOL_DOWN,
+                               "MDS %s is down for path"
+                               " path %s so fetching xattr "
+                               "%s randomly from a subvol ",
+                               local->mds_subvol->name, loc->path, key);
+                        ret = 1;
+                    }
                 }
-
-                STACK_WIND (frame, dht_linkinfo_getxattr_cbk, hashed_subvol,
-                            hashed_subvol->fops->getxattr, loc,
-                            GF_XATTR_PATHINFO_KEY, xdata);
-                return 0;
-        }
-
-        if (key && (gf_get_index_by_elem (dht_dbg_vxattrs, (char *)key) >= 0)) {
-                dht_handle_debug_getxattr (frame, this, loc, key);
-                return 0;
+            }
         }
 
-        if (cluster_handle_marker_getxattr (frame, loc, key, conf->vol_uuid,
-                                            dht_getxattr_unwind,
-                                            dht_marker_populate_args) == 0)
-                return 0;
-
-        if (DHT_IS_DIR(layout)) {
-                local->call_cnt = conf->subvolume_cnt;
-                cnt = conf->subvolume_cnt;
-                ret = dht_inode_ctx_mdsvol_get (loc->inode, this, &mds_subvol);
-                if (!mds_subvol) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                                "Cannot determine MDS, fetching xattr %s randomly"
-                                " from a subvol for path %s ", key, loc->path);
-                } else {
-                        /* TODO need to handle it, As of now we are
-                           choosing availability instead of chossing
-                           consistencty, in case of mds_subvol is
-                           down winding a getxattr call on other subvol
-                           and return xattr
-                        */
-                        local->mds_subvol = mds_subvol;
-                        for (i = 0; i < cnt; i++) {
-                                if (conf->subvolumes[i] == mds_subvol) {
-                                        if (!conf->subvolume_status[i]) {
-                                                gf_msg (this->name,
-                                                        GF_LOG_INFO, 0,
-                                                        DHT_MSG_HASHED_SUBVOL_DOWN,
-                                                        "MDS %s is down for path"
-                                                        " path %s so fetching xattr "
-                                                        "%s randomly from a subvol ",
-                                                        local->mds_subvol->name,
-                                                        loc->path, key);
-                                                ret = 1;
-                                        }
-                                }
-                        }
-                }
-
-                if (!ret && key && local->mds_subvol && dht_match_xattr (key)) {
-                        STACK_WIND (frame, dht_mds_getxattr_cbk,
-                                    local->mds_subvol,
-                                    local->mds_subvol->fops->getxattr,
-                                    loc, key, xdata);
+        if (!ret && key && local->mds_subvol && dht_match_xattr(key)) {
+            STACK_WIND(frame, dht_mds_getxattr_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->getxattr, loc, key, xdata);
 
-                        return 0;
-                }
-        } else {
-                cnt = local->call_cnt  = 1;
+            return 0;
         }
+    } else {
+        cnt = local->call_cnt = 1;
+    }
 
-        for (i = 0; i < cnt; i++) {
-                subvol = layout->list[i].xlator;
-                STACK_WIND (frame, dht_getxattr_cbk,
-                            subvol, subvol->fops->getxattr,
-                            loc, key, xdata);
-        }
-        return 0;
+    for (i = 0; i < cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND(frame, dht_getxattr_cbk, subvol, subvol->fops->getxattr, loc,
+                   key, xdata);
+    }
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(getxattr, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 #undef DHT_IS_DIR
 
 int
-dht_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *key, dict_t *xdata)
-{
-        xlator_t     *subvol        = NULL;
-        dht_local_t  *local         = NULL;
-        dht_layout_t *layout        = NULL;
-        int           op_errno      = -1;
-        int           i             = 0;
-        int           cnt           = 0;
-        xlator_t      *mds_subvol = NULL;
-        int           ret           = -1;
-        dht_conf_t    *conf         = NULL;
-        char           gfid[GF_UUID_BUF_SIZE] = {0};
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FGETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-
-                goto err;
+dht_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+              dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int op_errno = -1;
+    int i = 0;
+    int cnt = 0;
+    xlator_t *mds_subvol = NULL;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(fd->inode, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FGETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LAYOUT_NULL,
+               "Layout is NULL");
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    if (key) {
+        local->key = gf_strdup(key);
+        if (!local->key) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+    }
+
+    gf_uuid_unparse(fd->inode->gfid, gfid);
+
+    if ((fd->inode->ia_type == IA_IFDIR) && key &&
+        (strncmp(key, GF_XATTR_LOCKINFO_KEY, SLEN(GF_XATTR_LOCKINFO_KEY)) !=
+         0)) {
+        local->call_cnt = conf->subvolume_cnt;
+        cnt = conf->subvolume_cnt;
+        ret = dht_inode_ctx_mdsvol_get(fd->inode, this, &mds_subvol);
+
+        if (!mds_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "cannot determine MDS, fetching xattr %s "
+                   " randomly from a subvol for gfid %s ",
+                   key, gfid);
+        } else {
+            /* TODO need to handle it, As of now we are
+               choosing availability instead of chossing
+               consistencty, in case of hashed_subvol is
+               down winding a getxattr call on other subvol
+               and return xattr
+            */
+            local->mds_subvol = mds_subvol;
+            for (i = 0; i < cnt; i++) {
+                if (conf->subvolumes[i] == mds_subvol) {
+                    if (!conf->subvolume_status[i]) {
+                        gf_msg(this->name, GF_LOG_WARNING, 0,
+                               DHT_MSG_HASHED_SUBVOL_DOWN,
+                               "MDS subvolume %s is down"
+                               " for gfid %s so fetching xattr "
+                               " %s randomly from a subvol ",
+                               local->mds_subvol->name, gfid, key);
+                        ret = 1;
+                    }
+                }
+            }
         }
 
-        layout = local->layout;
-        if (!layout) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LAYOUT_NULL,
-                        "Layout is NULL");
-                op_errno = ENOENT;
-                goto err;
-        }
+        if (!ret && key && local->mds_subvol && dht_match_xattr(key)) {
+            STACK_WIND(frame, dht_mds_getxattr_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->fgetxattr, fd, key, NULL);
 
-        if (key) {
-                local->key = gf_strdup (key);
-                if (!local->key) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
+            return 0;
         }
 
-        if (fd->inode)
-                gf_uuid_unparse(fd->inode->gfid, gfid);
+    } else {
+        cnt = local->call_cnt = 1;
+    }
 
-        if ((fd->inode->ia_type == IA_IFDIR)
-            && key
-            && (strncmp (key, GF_XATTR_LOCKINFO_KEY,
-                         strlen (GF_XATTR_LOCKINFO_KEY)) != 0)) {
-                local->call_cnt = conf->subvolume_cnt;
-                cnt             = conf->subvolume_cnt;
-                ret = dht_inode_ctx_mdsvol_get (fd->inode, this, &mds_subvol);
+    for (i = 0; i < cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND(frame, dht_getxattr_cbk, subvol, subvol->fops->fgetxattr, fd,
+                   key, NULL);
+    }
+    return 0;
 
-                if (!mds_subvol) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                                "cannot determine MDS, fetching xattr %s "
-                                " randomly from a subvol for gfid %s ",
-                                key, gfid);
-                } else {
-                        /* TODO need to handle it, As of now we are
-                           choosing availability instead of chossing
-                           consistencty, in case of hashed_subvol is
-                           down winding a getxattr call on other subvol
-                           and return xattr
-                        */
-                        local->mds_subvol = mds_subvol;
-                        for (i = 0; i < cnt; i++) {
-                                if (conf->subvolumes[i] == mds_subvol) {
-                                        if (!conf->subvolume_status[i]) {
-                                                gf_msg (this->name,
-                                                        GF_LOG_WARNING, 0,
-                                                        DHT_MSG_HASHED_SUBVOL_DOWN,
-                                                        "MDS subvolume %s is down"
-                                                        " for gfid %s so fetching xattr "
-                                                        " %s randomly from a subvol ",
-                                                        local->mds_subvol->name,
-                                                        gfid, key);
-                                                ret = 1;
-                                        }
-                                }
-                        }
-                }
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
 
-                if (!ret && key && local->mds_subvol &&
-                    dht_match_xattr (key)) {
-                        STACK_WIND (frame, dht_mds_getxattr_cbk,
-                                    local->mds_subvol,
-                                    local->mds_subvol->fops->fgetxattr,
-                                    fd, key, NULL);
+    return 0;
+}
 
-                        return 0;
-                }
+static int
+dht_setxattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+{
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        } else {
-                cnt = local->call_cnt  = 1;
-        }
+    if (!frame || !frame->local)
+        goto err;
 
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        for (i = 0; i < cnt; i++) {
-                subvol = layout->list[i].xlator;
-                STACK_WIND (frame, dht_getxattr_cbk,
-                            subvol, subvol->fops->fgetxattr,
-                            fd, key, NULL);
-        }
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                         local->rebalance.xdata);
         return 0;
+    }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
+    if (subvol == NULL)
+        goto err;
 
-        return 0;
+    local->call_cnt = 2; /* This is the second attempt */
+
+    if (local->fop == GF_FOP_SETXATTR) {
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->setxattr, &local->loc,
+                          local->rebalance.xattr, local->rebalance.flags,
+                          local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->fsetxattr, local->fd,
+                          local->rebalance.xattr, local->rebalance.flags,
+                          local->xattr_req);
+    }
+
+    return 0;
+
+err:
+    DHT_STACK_UNWIND(setxattr, frame, (local ? local->op_ret : -1), op_errno,
+                     NULL);
+    return 0;
 }
 
 int
-dht_file_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, dict_t *xdata)
+dht_file_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata)
 {
-        int           ret     = -1;
-        dht_local_t  *local   = NULL;
-        xlator_t     *prev    = NULL;
-        struct iatt  *stbuf   = NULL;
-        inode_t      *inode   = NULL;
-        xlator_t     *subvol1 = NULL, *subvol2 = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    struct iatt *stbuf = NULL;
+    inode_t *inode = NULL;
+    xlator_t *subvol1 = NULL, *subvol2 = NULL;
 
-        local = frame->local;
-        prev = cookie;
+    local = frame->local;
+    prev = cookie;
 
-        local->op_errno = op_errno;
+    local->op_errno = op_errno;
 
-        if ((local->fop == GF_FOP_FSETXATTR) &&
-            op_ret == -1 && (op_errno == EBADF) && !(local->fd_checked)) {
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto out;
-                return 0;
-        }
-
-        if ((op_ret == -1) && !dht_inode_missing (op_errno)) {
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1.",
-                              prev->name);
-                goto out;
-        }
+    if ((local->fop == GF_FOP_FSETXATTR) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        if (local->call_cnt != 1)
-                goto out;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1.",
+                     prev->name);
+        goto out;
+    }
 
-        ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf);
+    if (local->call_cnt != 1)
+        goto out;
 
-        if ((!op_ret) && !stbuf) {
-                goto out;
-        }
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
 
-        local->op_ret = op_ret;
-        local->rebalance.target_op_fn = dht_setxattr2;
-        if (xdata)
-                local->rebalance.xdata = dict_ref (xdata);
+    if ((!op_ret) && !stbuf) {
+        goto out;
+    }
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+    local->op_ret = op_ret;
+    local->rebalance.target_op_fn = dht_setxattr2;
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
 
-        /* Phase 1 of migration */
-        if (IS_DHT_MIGRATION_PHASE1 (stbuf)) {
-                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-                ret = dht_inode_ctx_get_mig_info (this, inode,
-                                                  &subvol1, &subvol2);
-                if (!dht_mig_info_is_invalid (local->cached_subvol,
-                                              subvol1, subvol2)) {
-                        dht_setxattr2 (this, subvol2, frame, 0);
-                        return 0;
-                }
+    /* Phase 1 of migration */
+    if (IS_DHT_MIGRATION_PHASE1(stbuf)) {
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
 
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        ret = dht_inode_ctx_get_mig_info(this, inode, &subvol1, &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            dht_setxattr2(this, subvol2, frame, 0);
+            return 0;
         }
 
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
 out:
 
-        if (local->fop == GF_FOP_SETXATTR) {
-                DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, NULL);
-        } else {
-                DHT_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
-        }
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    } else {
+        DHT_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
+    }
 
-        return 0;
+    return 0;
 }
 
 /* Function is call by dict_foreach_fnmatch if key is match with
    user.* and set boolean flag to true
 */
 static int
-dht_is_user_xattr (dict_t *this, char *key, data_t *value, void *data)
+dht_is_user_xattr(dict_t *this, char *key, data_t *value, void *data)
 {
-        gf_boolean_t *user_xattr_found = data;
-        *user_xattr_found = _gf_true;
-        return 0;
+    gf_boolean_t *user_xattr_found = data;
+    *user_xattr_found = _gf_true;
+    return 0;
 }
 
+/* Common code to wind a (f)(set|remove)xattr call to set xattr on directory
+ */
+static int
+dht_dir_common_set_remove_xattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                                fd_t *fd, dict_t *xattr, int flags,
+                                dict_t *xdata, int *op_errno)
+
+{
+    dict_t *xattrop = NULL;
+    int32_t subone[1] = {-1};
+    gf_boolean_t uxattr_key_found = _gf_false;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *travvol = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int i = 0;
+    int call_cnt = 0;
+    dht_local_t *local = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char **xattrs_to_heal;
+
+    conf = this->private;
+    local = frame->local;
+    call_cnt = conf->subvolume_cnt;
+    local->flags = flags;
+    xattrs_to_heal = get_xattrs_to_heal();
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        gf_uuid_unparse(local->gfid, gfid_local);
+    }
 
-/* Common code to wind a (f)setxattr call to set xattr on directory
-*/
-int
-dht_dir_common_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                         fd_t *fd, dict_t *xattr, int flags, dict_t *xdata,
-                         int *op_errno)
-
-{
-        dict_t       *xattrop             = NULL;
-        int32_t       subone[1]            = {-1};
-        gf_boolean_t  uxattr_key_found     = _gf_false;
-        xlator_t     *mds_subvol          = NULL;
-        xlator_t     *travvol              = NULL;
-        dht_conf_t   *conf                = NULL;
-        int           ret                  = -1;
-        int           i                    = 0;
-        int           call_cnt             = 0;
-        dht_local_t  *local                = NULL;
-        char          gfid_local[GF_UUID_BUF_SIZE] = {0};
-
-        conf = this->private;
-        local    = frame->local;
-        call_cnt = conf->subvolume_cnt;
-        local->flags = flags;
-
-        if (local->gfid)
-                gf_uuid_unparse(local->gfid, gfid_local);
-
+    if ((local->fop == GF_FOP_SETXATTR) || (local->fop == GF_FOP_FSETXATTR)) {
         /* Check if any user xattr present in xattr
-        */
-        dict_foreach_fnmatch (xattr, "user*", dht_is_user_xattr,
-                              &uxattr_key_found);
+         */
+        dict_foreach_fnmatch(xattr, "user*", dht_is_user_xattr,
+                             &uxattr_key_found);
 
         /* Check if any custom key xattr present in dict xattr
            and start index from 1 because user xattr already
            checked in previous line
         */
         for (i = 1; xattrs_to_heal[i]; i++)
-                if (dict_get (xattr, xattrs_to_heal[i]))
-                        uxattr_key_found = _gf_true;
+            if (dict_get(xattr, xattrs_to_heal[i]))
+                uxattr_key_found = _gf_true;
+    }
 
-        /* If there is no custom key xattr present or gfid is root
-           or call_cnt is 1 then wind a (f)setxattr call on all subvols
-        */
-        if (!uxattr_key_found || __is_root_gfid (local->gfid) || call_cnt == 1) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        travvol = conf->subvolumes[i];
-                        if (fd) {
-                                STACK_WIND_COOKIE (frame, dht_err_cbk,
-                                                   travvol, travvol,
-                                                   travvol->fops->fsetxattr,
-                                                   fd, xattr, flags, xdata);
-                        } else {
-                                STACK_WIND_COOKIE (frame, dht_err_cbk,
-                                                   travvol, travvol,
-                                                   travvol->fops->setxattr,
-                                                   loc, xattr, flags, xdata);
-                        }
-                }
-
-                return 0;
-        }
-
-        /* Calculate hash subvol based on inode and parent inode
+    if ((local->fop == GF_FOP_REMOVEXATTR) ||
+        (local->fop == GF_FOP_FREMOVEXATTR)) {
+        /* Check if any custom key xattr present in local->key
          */
-        if (fd) {
-                ret = dht_inode_ctx_mdsvol_get (fd->inode, this, &mds_subvol);
-        } else {
-                ret = dht_inode_ctx_mdsvol_get (loc->inode, this, &mds_subvol);
-        }
-        if (ret || !mds_subvol) {
+        for (i = 0; xattrs_to_heal[i]; i++)
+            if (strstr(local->key, xattrs_to_heal[i]))
+                uxattr_key_found = _gf_true;
+    }
+
+    /* If there is no custom key xattr present or gfid is root
+       or call_cnt is 1 then wind a (f)setxattr call on all subvols
+    */
+    if (!uxattr_key_found || __is_root_gfid(local->gfid) || call_cnt == 1) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            travvol = conf->subvolumes[i];
+            if ((local->fop == GF_FOP_SETXATTR) ||
+                (local->fop == GF_FOP_FSETXATTR)) {
                 if (fd) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                                "Failed to get mds subvol for fd %p"
-                                "gfid is %s ", fd, gfid_local);
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->fsetxattr, fd, xattr,
+                                      flags, xdata);
                 } else {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                                "Failed to get mds subvol for path %s"
-                                "gfid is %s ", loc->path, gfid_local);
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->setxattr, loc, xattr,
+                                      flags, xdata);
                 }
-                (*op_errno) = ENOENT;
-                goto err;
-        }
-
-        local->mds_subvol = mds_subvol;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] ==  mds_subvol) {
-                        if (!conf->subvolume_status[i]) {
-                                gf_msg (this->name, GF_LOG_WARNING,
-                                        0, DHT_MSG_HASHED_SUBVOL_DOWN,
-                                        "MDS subvol is down for path "
-                                        " %s gfid is %s Unable to set xattr " ,
-                                        local->loc.path, gfid_local);
-                                (*op_errno) = ENOTCONN;
-                                goto err;
-                        }
-                }
-        }
+            }
 
-        if (uxattr_key_found) {
-                xattrop = dict_new ();
-                if (!xattrop) {
-                        gf_msg (this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY,
-                                0, "dictionary creation failed for path %s "
-                                "for gfid is %s ", local->loc.path, gfid_local);
-                        (*op_errno) = ENOMEM;
-                        goto err;
-                }
-                local->xattr = dict_ref (xattr);
-                /* Subtract current MDS xattr value to -1 , value of MDS
-                   xattr represents no. of times xattr modification failed
-                   on non MDS subvols.
-                */
-                ret = dht_dict_set_array (xattrop, conf->mds_xattr_key, subone, 1);
-                if (ret != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
-                                "dictionary set array failed for path %s "
-                                "for gfid is %s ", local->loc.path, gfid_local);
-                        if (xattrop)
-                                dict_unref (xattrop);
-                        (*op_errno) = ret;
-                        goto err;
-                }
-                /* Wind a xattrop call to use ref counting approach
-                   update mds xattr to -1 before update xattr on
-                   hashed subvol and update mds xattr to +1 after update
-                   xattr on all non hashed subvol
-                */
+            if ((local->fop == GF_FOP_REMOVEXATTR) ||
+                (local->fop == GF_FOP_FREMOVEXATTR)) {
                 if (fd) {
-                        STACK_WIND (frame, dht_xattrop_mds_cbk,
-                                    local->mds_subvol,
-                                    local->mds_subvol->fops->fxattrop,
-                                     fd, GF_XATTROP_ADD_ARRAY, xattrop, NULL);
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->fremovexattr, fd,
+                                      local->key, local->xattr_req);
                 } else {
-                        STACK_WIND (frame, dht_xattrop_mds_cbk,
-                                    local->mds_subvol,
-                                    local->mds_subvol->fops->xattrop,
-                                    loc, GF_XATTROP_ADD_ARRAY,
-                                    xattrop, NULL);
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->removexattr, loc,
+                                      local->key, local->xattr_req);
                 }
-                if (xattrop)
-                        dict_unref (xattrop);
+            }
         }
 
         return 0;
-err:
-        return -1;
-}
-
-
-int
-dht_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *xattr, int flags, dict_t *xdata)
-{
-        xlator_t     *subvol   = NULL;
-        dht_local_t  *local    = NULL;
-        int           op_errno = EINVAL;
-        dht_conf_t   *conf     = NULL;
-        dht_layout_t *layout   = NULL;
-        int           ret      = -1;
-        int           call_cnt = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        if (!conf->defrag)
-                GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
-                                           op_errno, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    }
 
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for fd=%p", fd);
-                op_errno = EINVAL;
+    /* Calculate hash subvol based on inode and parent inode
+     */
+    if (fd) {
+        ret = dht_inode_ctx_mdsvol_get(fd->inode, this, &mds_subvol);
+    } else {
+        ret = dht_inode_ctx_mdsvol_get(loc->inode, this, &mds_subvol);
+    }
+    if (ret || !mds_subvol) {
+        if (fd) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get mds subvol for fd %p"
+                   "gfid is %s ",
+                   fd, gfid_local);
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "%s: Failed to get mds subvol. (gfid is %s)", loc->path,
+                   gfid_local);
+        }
+        (*op_errno) = ENOENT;
+        goto err;
+    }
+
+    local->mds_subvol = mds_subvol;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == mds_subvol) {
+            if (!conf->subvolume_status[i]) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       DHT_MSG_HASHED_SUBVOL_DOWN,
+                       "MDS subvol is down for path "
+                       " %s gfid is %s Unable to set xattr ",
+                       local->loc.path, gfid_local);
+                (*op_errno) = ENOTCONN;
                 goto err;
+            }
         }
-
-        local->xattr_req = xdata ? dict_ref (xdata) : dict_new ();
-        local->call_cnt = call_cnt = layout->cnt;
-
-        if (IA_ISDIR (fd->inode->ia_type)) {
-                local->hashed_subvol = NULL;
-                ret = dht_dir_common_setxattr (frame, this, NULL, fd,
-                                               xattr, flags, xdata, &op_errno);
-                if (ret)
-                        goto err;
+    }
+
+    if (uxattr_key_found) {
+        xattrop = dict_new();
+        if (!xattrop) {
+            gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, 0,
+                   "dictionary creation failed for path %s "
+                   "for gfid is %s ",
+                   local->loc.path, gfid_local);
+            (*op_errno) = ENOMEM;
+            goto err;
+        }
+        local->xattr = dict_ref(xattr);
+        /* Subtract current MDS xattr value to -1 , value of MDS
+           xattr represents no. of times xattr modification failed
+           on non MDS subvols.
+        */
+        ret = dht_dict_set_array(xattrop, conf->mds_xattr_key, subone, 1);
+        if (ret != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "dictionary set array failed for path %s "
+                   "for gfid is %s ",
+                   local->loc.path, gfid_local);
+            if (xattrop)
+                dict_unref(xattrop);
+            (*op_errno) = ret;
+            goto err;
+        }
+        /* Wind a xattrop call to use ref counting approach
+           update mds xattr to -1 before update xattr on
+           hashed subvol and update mds xattr to +1 after update
+           xattr on all non hashed subvol
+        */
+        if (fd) {
+            STACK_WIND(frame, dht_xattrop_mds_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->fxattrop, fd,
+                       GF_XATTROP_ADD_ARRAY, xattrop, NULL);
         } else {
-
-                local->call_cnt = 1;
-                local->rebalance.xattr = dict_ref (xattr);
-                local->rebalance.flags = flags;
-
-                ret = dict_set_int8 (local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
-                if (ret) {
-                        gf_msg_debug (this->name, 0,
-                                      "Failed to set dictionary key %s for fd=%p",
-                                      DHT_IATT_IN_XDATA_KEY, fd);
-                }
-
-                STACK_WIND_COOKIE (frame, dht_file_setxattr_cbk, subvol,
-                                   subvol, subvol->fops->fsetxattr, fd, xattr,
-                                   flags, local->xattr_req);
+            STACK_WIND(frame, dht_xattrop_mds_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->xattrop, loc,
+                       GF_XATTROP_ADD_ARRAY, xattrop, NULL);
         }
-        return 0;
+        if (xattrop)
+            dict_unref(xattrop);
+    }
 
+    return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
-
-        return 0;
+    return -1;
 }
 
-
 int
-dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr,
-                           dict_t *xdata)
-{
-        int           i     = -1;
-        int           ret   = -1;
-        char         *value = NULL;
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf  = NULL;
-        xlator_t     *prev  = NULL;
-        int           this_call_cnt = 0;
-
-        local = frame->local;
-        prev = cookie;
-        conf = this->private;
-
-        if (op_ret == -1)
-                goto out;
-
-
-        ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &value);
+dht_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr,
+              int flags, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(fd->inode, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    if (!conf->defrag)
+        GF_IF_INTERNAL_XATTR_GOTO(conf->wild_xattr_name, xattr, op_errno, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+    local->call_cnt = call_cnt = layout->cnt;
+
+    if (IA_ISDIR(fd->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, NULL, fd, xattr,
+                                              flags, xdata, &op_errno);
         if (ret)
-                goto out;
-
-        if (!strcmp (value, local->key)) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->subvolumes[i] == prev)
-                                conf->decommissioned_bricks[i] = prev;
-                }
-        }
+            goto err;
+    } else {
+        local->call_cnt = 1;
+        local->rebalance.xattr = dict_ref(xattr);
+        local->rebalance.flags = flags;
 
-out:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, ENOTSUP, NULL);
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set dictionary key %s for fd=%p",
+                         DHT_IATT_IN_XDATA_KEY, fd);
         }
-        return 0;
-
-}
 
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->fsetxattr, fd, xattr, flags,
+                          local->xattr_req);
+    }
+    return 0;
 
-int
-dht_setxattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
-{
-        dht_local_t  *local  = NULL;
-        int          op_errno = EINVAL;
-
-        if (!frame || !frame->local)
-                goto err;
-
-        local = frame->local;
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
-                                  local->op_errno, local->rebalance.xdata);
-                return 0;
-        }
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
 
-        if (subvol == NULL)
-                goto err;
+    return 0;
+}
 
+static int
+dht_checking_pathinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, dict_t *xattr,
+                          dict_t *xdata)
+{
+    int i = -1;
+    int ret = -1;
+    char *value = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    if (op_ret == -1)
+        goto out;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    ret = dict_get_str(xattr, GF_XATTR_PATHINFO_KEY, &value);
+    if (ret)
+        goto out;
 
-        if (local->fop == GF_FOP_SETXATTR) {
-                STACK_WIND_COOKIE (frame, dht_file_setxattr_cbk, subvol,
-                                   subvol, subvol->fops->setxattr, &local->loc,
-                                   local->rebalance.xattr,
-                                   local->rebalance.flags, local->xattr_req);
-        } else {
-                STACK_WIND_COOKIE (frame, dht_file_setxattr_cbk, subvol,
-                                   subvol, subvol->fops->fsetxattr, local->fd,
-                                   local->rebalance.xattr,
-                                   local->rebalance.flags, local->xattr_req);
+    if (!strcmp(value, local->key)) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvolumes[i] == prev)
+                conf->decommissioned_bricks[i] = prev;
         }
+    }
 
-        return 0;
-
-err:
-        DHT_STACK_UNWIND (setxattr, frame, (local ? local->op_ret : -1),
-                          op_errno, NULL);
-        return 0;
+out:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, ENOTSUP, NULL);
+    }
+    return 0;
 }
 
-int
-dht_nuke_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+static int
+dht_nuke_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
-int
-dht_nuke_dir (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *tmp)
+static int
+dht_nuke_dir(call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *tmp)
 {
-        if (!IA_ISDIR(loc->inode->ia_type)) {
-                DHT_STACK_UNWIND (setxattr, frame, -1, ENOTSUP, NULL);
-                return 0;
-        }
+    if (!IA_ISDIR(loc->inode->ia_type)) {
+        DHT_STACK_UNWIND(setxattr, frame, -1, ENOTSUP, NULL);
+        return 0;
+    }
 
-        /* Setxattr didn't need the parent, but rmdir does. */
-        loc->parent = inode_parent (loc->inode, NULL, NULL);
-        if (!loc->parent) {
-                DHT_STACK_UNWIND (setxattr, frame, -1, ENOENT, NULL);
-                return 0;
-        }
-        gf_uuid_copy (loc->pargfid, loc->parent->gfid);
+    /* Setxattr didn't need the parent, but rmdir does. */
+    loc->parent = inode_parent(loc->inode, NULL, NULL);
+    if (!loc->parent) {
+        DHT_STACK_UNWIND(setxattr, frame, -1, ENOENT, NULL);
+        return 0;
+    }
+    gf_uuid_copy(loc->pargfid, loc->parent->gfid);
 
-        if (!loc->name && loc->path) {
-                loc->name = strrchr (loc->path, '/');
-                if (loc->name) {
-                        ++(loc->name);
-                }
+    if (!loc->name && loc->path) {
+        loc->name = strrchr(loc->path, '/');
+        if (loc->name) {
+            ++(loc->name);
         }
+    }
 
-        /*
-         * We do this instead of calling dht_rmdir_do directly for two reasons.
-         * The first is that we want to reuse all of the initialization that
-         * dht_rmdir does, so if it ever changes we'll just follow along.  The
-         * second (i.e. why we don't use STACK_WIND_TAIL) is so that we don't
-         * obscure the fact that we came in via this path instead of a genuine
-         * rmdir.  That makes debugging just a tiny bit easier.
-         */
-        STACK_WIND (frame, dht_nuke_dir_cbk, this, this->fops->rmdir,
-                    loc, 1, NULL);
+    /*
+     * We do this instead of calling dht_rmdir_do directly for two reasons.
+     * The first is that we want to reuse all of the initialization that
+     * dht_rmdir does, so if it ever changes we'll just follow along.  The
+     * second (i.e. why we don't use STACK_WIND_TAIL) is so that we don't
+     * obscure the fact that we came in via this path instead of a genuine
+     * rmdir.  That makes debugging just a tiny bit easier.
+     */
+    STACK_WIND(frame, dht_nuke_dir_cbk, this, this->fops->rmdir, loc, 1, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr, int flags, dict_t *xdata)
-{
-        xlator_t     *subvol   = NULL;
-        dht_local_t  *local    = NULL;
-        dht_conf_t   *conf     = NULL;
-        dht_methods_t *methods = NULL;
-        dht_layout_t *layout   = NULL;
-        int           i        = 0;
-        int           op_errno = EINVAL;
-        int           ret      = -1;
-        data_t       *tmp      = NULL;
-        uint32_t      dir_spread = 0;
-        char          value[4096] = {0,};
-        gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
-        int           call_cnt = 0;
-        uint32_t      new_hash = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        conf   = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, err);
-
-        methods = &(conf->methods);
-
-        /* Rebalance daemon is allowed to set internal keys */
-        if (!conf->defrag)
-                GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
-                                           op_errno, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_SETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s",
-                              loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        local->call_cnt = call_cnt = layout->cnt;
-        tmp = dict_get (xattr, conf->mds_xattr_key);
-        if (tmp) {
-                op_errno = ENOTSUP;
-                goto err;
-        }
-
-        tmp = dict_get (xattr, GF_XATTR_FILE_MIGRATE_KEY);
-        if (tmp) {
-
-                if (IA_ISDIR (loc->inode->ia_type)) {
-                        op_errno = ENOTSUP;
-                        goto err;
-                }
-
-                /* TODO: need to interpret the 'value' for more meaning
-                   (ie, 'target' subvolume given there, etc) */
-                memcpy (value, tmp->data, tmp->len);
-                if (strcmp (value, "force") == 0)
-                       forced_rebalance =
-                                GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
-
-                if (conf->decommission_in_progress)
-                        forced_rebalance = GF_DHT_MIGRATE_HARDLINK;
-
-                if (!loc->path) {
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                if (!local->loc.name)
-                        local->loc.name = strrchr (local->loc.path, '/')+1;
-
-                if (!local->loc.parent)
-                        local->loc.parent =
-                                inode_parent(local->loc.inode, NULL, NULL);
-
-                if ((!local->loc.name) || (!local->loc.parent)) {
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                if (gf_uuid_is_null (local->loc.pargfid))
-                        gf_uuid_copy (local->loc.pargfid, local->loc.parent->gfid);
-
-                methods->migration_get_dst_subvol(this, local);
-
-                if (!local->rebalance.target_node) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                                "Failed to get hashed subvol for %s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                local->rebalance.from_subvol = local->cached_subvol;
-
-                if (local->rebalance.target_node == local->rebalance.from_subvol) {
-                        op_errno = EEXIST;
-                        goto err;
-                }
-                if (local->rebalance.target_node) {
-                        local->flags = forced_rebalance;
-
-                        /* Flag to suggest its a tiering migration
-                         * The reason for this dic key-value is that
-                         * promotions and demotions are multithreaded
-                         * so the original frame from gf_defrag_start()
-                         * is not carried. A new frame will be created when
-                         * we do syncop_setxattr(). This doesnot have the
-                         * frame->root->pid of the original frame. So we pass
-                         * this dic key-value when we do syncop_setxattr() to do
-                         * data migration and set the frame->root->pid to
-                         * GF_CLIENT_PID_TIER_DEFRAG in dht_setxattr() just before
-                         * calling dht_start_rebalance_task() */
-                        tmp = dict_get (xattr, TIERING_MIGRATION_KEY);
-                        if (tmp)
-                                frame->root->pid = GF_CLIENT_PID_TIER_DEFRAG;
-                        else
-                                frame->root->pid = GF_CLIENT_PID_DEFRAG;
-
-                        ret = dht_start_rebalance_task (this, frame);
-                        if (!ret)
-                                return 0;
-
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_REBALANCE_START_FAILED,
-                                "%s: failed to create a new rebalance synctask",
-                                loc->path);
-                }
-                op_errno = EINVAL;
-                goto err;
-
+dht_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr,
+             int flags, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int op_errno = EINVAL;
+    int ret = -1;
+    data_t *tmp = NULL;
+    uint32_t dir_spread = 0;
+    char value[4096] = {
+        0,
+    };
+    gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
+    int call_cnt = 0;
+    uint32_t new_hash = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, err);
+
+    methods = &(conf->methods);
+
+    /* Rebalance daemon is allowed to set internal keys */
+    if (!conf->defrag)
+        GF_IF_INTERNAL_XATTR_GOTO(conf->wild_xattr_name, xattr, op_errno, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_SETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local->call_cnt = call_cnt = layout->cnt;
+    tmp = dict_get(xattr, conf->mds_xattr_key);
+    if (tmp) {
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    tmp = dict_get(xattr, GF_XATTR_FILE_MIGRATE_KEY);
+    if (tmp) {
+        if (IA_ISDIR(loc->inode->ia_type)) {
+            op_errno = ENOTSUP;
+            goto err;
+        }
+
+        /* TODO: need to interpret the 'value' for more meaning
+           (ie, 'target' subvolume given there, etc) */
+        memcpy(value, tmp->data, tmp->len);
+        if (strcmp(value, "force") == 0)
+            forced_rebalance = GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
+
+        if (conf->decommission_in_progress)
+            forced_rebalance = GF_DHT_MIGRATE_HARDLINK;
+
+        if (!loc->path) {
+            op_errno = EINVAL;
+            goto err;
+        }
+
+        if (!local->loc.name)
+            local->loc.name = strrchr(local->loc.path, '/') + 1;
+
+        if (!local->loc.parent)
+            local->loc.parent = inode_parent(local->loc.inode, NULL, NULL);
+
+        if ((!local->loc.name) || (!local->loc.parent)) {
+            op_errno = EINVAL;
+            goto err;
+        }
+
+        if (gf_uuid_is_null(local->loc.pargfid))
+            gf_uuid_copy(local->loc.pargfid, local->loc.parent->gfid);
+
+        methods->migration_get_dst_subvol(this, local);
+
+        if (!local->rebalance.target_node) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get hashed subvol for %s", loc->path);
+            op_errno = EINVAL;
+            goto err;
+        }
+
+        local->rebalance.from_subvol = local->cached_subvol;
+
+        if (local->rebalance.target_node == local->rebalance.from_subvol) {
+            op_errno = EEXIST;
+            goto err;
         }
+        if (local->rebalance.target_node) {
+            local->flags = forced_rebalance;
 
-        tmp = dict_get (xattr, "decommission-brick");
-        if (tmp) {
-                /* This operation should happen only on '/' */
-                if (!__is_root_gfid (loc->inode->gfid)) {
-                        op_errno = ENOTSUP;
-                        goto err;
-                }
+            frame->root->pid = GF_CLIENT_PID_DEFRAG;
 
-                memcpy (value, tmp->data, ((tmp->len < 4095) ? tmp->len : 4095));
-                local->key = gf_strdup (value);
-                local->call_cnt = conf->subvolume_cnt;
-
-                for (i = 0 ; i < conf->subvolume_cnt; i++) {
-                        /* Get the pathinfo, and then compare */
-                        STACK_WIND_COOKIE (frame, dht_checking_pathinfo_cbk,
-                                           conf->subvolumes[i], conf->subvolumes[i],
-                                           conf->subvolumes[i]->fops->getxattr,
-                                           loc, GF_XATTR_PATHINFO_KEY, NULL);
-                }
+            ret = dht_start_rebalance_task(this, frame);
+            if (!ret)
                 return 0;
-        }
-
-        tmp = dict_get (xattr, GF_XATTR_FIX_LAYOUT_KEY);
-        if (tmp) {
-                ret = dict_get_uint32(xattr, "new-commit-hash", &new_hash);
-                if (ret == 0) {
-                        gf_msg_debug (this->name, 0,
-                                      "updating commit hash for %s from %u to %u",
-                                      uuid_utoa(loc->gfid),
-                                      layout->commit_hash, new_hash);
-                        layout->commit_hash = new_hash;
-
-                        ret = dht_update_commit_hash_for_layout (frame);
-                        if (ret) {
-                                op_errno = ENOTCONN;
-                                goto err;
-                        }
-                        return ret;
-                }
-
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_FIX_LAYOUT_INFO,
-                        "fixing the layout of %s", loc->path);
-
-                ret = dht_fix_directory_layout (frame,
-                                                dht_fix_layout_setxattr_cbk,
-                                                layout);
-                if (ret) {
-                        op_errno = ENOTCONN;
-                        goto err;
-                }
-                return ret;
-        }
 
-        tmp = dict_get (xattr, "distribute.directory-spread-count");
-        if (tmp) {
-                /* Setxattr value is packed as 'binary', not string */
-                memcpy (value, tmp->data, ((tmp->len < 4095)?tmp->len:4095));
-                ret = gf_string2uint32 (value, &dir_spread);
-                if (!ret && ((dir_spread <= conf->subvolume_cnt) &&
-                             (dir_spread > 0))) {
-                        layout->spread_cnt = dir_spread;
-
-                        ret = dht_fix_directory_layout (frame,
-                                                        dht_common_setxattr_cbk,
-                                                        layout);
-                        if (ret) {
-                                op_errno = ENOTCONN;
-                                goto err;
-                        }
-                        return ret;
-                }
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_OPERATION_NOT_SUP,
-                        "wrong 'directory-spread-count' value (%s)", value);
-                op_errno = ENOTSUP;
-                goto err;
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_START_FAILED,
+                   "%s: failed to create a new rebalance synctask", loc->path);
         }
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        tmp = dict_get (xattr, "glusterfs.dht.nuke");
-        if (tmp) {
-                return dht_nuke_dir (frame, this, loc, tmp);
+    tmp = dict_get(xattr, "decommission-brick");
+    if (tmp) {
+        /* This operation should happen only on '/' */
+        if (!__is_root_gfid(loc->inode->gfid)) {
+            op_errno = ENOTSUP;
+            goto err;
         }
-        local->xattr_req = xdata ? dict_ref (xdata) : dict_new ();
 
-        if (IA_ISDIR (loc->inode->ia_type)) {
-                local->hashed_subvol = NULL;
-                ret = dht_dir_common_setxattr (frame, this, loc, NULL,
-                                               xattr, flags, xdata, &op_errno);
-                if (ret)
-                        goto err;
-        } else {
-
-                local->rebalance.xattr = dict_ref (xattr);
-                local->rebalance.flags = flags;
-                local->call_cnt = 1;
-
-                ret = dict_set_int8 (local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+        memcpy(value, tmp->data, min(tmp->len, 4095));
+        local->key = gf_strdup(value);
+        local->call_cnt = conf->subvolume_cnt;
 
-                STACK_WIND_COOKIE (frame, dht_file_setxattr_cbk, subvol,
-                                   subvol, subvol->fops->setxattr, loc, xattr,
-                                   flags, local->xattr_req);
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            /* Get the pathinfo, and then compare */
+            STACK_WIND_COOKIE(frame, dht_checking_pathinfo_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->getxattr, loc,
+                              GF_XATTR_PATHINFO_KEY, NULL);
         }
-
         return 0;
+    }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-
-        return 0;
-}
-
-
-
-
-int
-dht_file_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int op_ret, int op_errno, dict_t *xdata)
-{
-        int           ret     = -1;
-        dht_local_t  *local   = NULL;
-        xlator_t     *prev    = NULL;
-        struct iatt  *stbuf   = NULL;
-        inode_t      *inode   = NULL;
-        xlator_t     *subvol1 = NULL, *subvol2 = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        local->op_errno = op_errno;
-
-        if ((local->fop == GF_FOP_FREMOVEXATTR) &&
-            (op_ret == -1) && (op_errno == EBADF) && !(local->fd_checked)) {
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto out;
-                return 0;
-        }
-
-        if ((op_ret == -1) && !dht_inode_missing (op_errno)) {
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->name);
-                goto out;
+    tmp = dict_get(xattr, GF_XATTR_FIX_LAYOUT_KEY);
+    if (tmp) {
+        ret = dict_get_uint32(xattr, "new-commit-hash", &new_hash);
+        if (ret == 0) {
+            gf_msg_debug(this->name, 0,
+                         "updating commit hash for %s from %u to %u",
+                         uuid_utoa(loc->gfid), layout->commit_hash, new_hash);
+            layout->commit_hash = new_hash;
+
+            ret = dht_update_commit_hash_for_layout(frame);
+            if (ret) {
+                op_errno = ENOTCONN;
+                goto err;
+            }
+            return ret;
         }
 
-        if (local->call_cnt != 1)
-                goto out;
-
-        ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf);
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_FIX_LAYOUT_INFO,
+               "fixing the layout of %s", loc->path);
 
-        if ((!op_ret) && !stbuf) {
-                goto out;
+        ret = dht_fix_directory_layout(frame, dht_fix_layout_setxattr_cbk,
+                                       layout);
+        if (ret) {
+            op_errno = ENOTCONN;
+            goto err;
         }
+        return ret;
+    }
+
+    tmp = dict_get(xattr, "distribute.directory-spread-count");
+    if (tmp) {
+        /* Setxattr value is packed as 'binary', not string */
+        memcpy(value, tmp->data, min(tmp->len, 4095));
+        ret = gf_string2uint32(value, &dir_spread);
+        if (!ret && ((dir_spread <= conf->subvolume_cnt) && (dir_spread > 0))) {
+            layout->spread_cnt = dir_spread;
+
+            ret = dht_fix_directory_layout(frame, dht_common_setxattr_cbk,
+                                           layout);
+            if (ret) {
+                op_errno = ENOTCONN;
+                goto err;
+            }
+            return ret;
+        }
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_OPERATION_NOT_SUP,
+               "wrong 'directory-spread-count' value (%s)", value);
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    tmp = dict_get(xattr, "glusterfs.dht.nuke");
+    if (tmp) {
+        return dht_nuke_dir(frame, this, loc, tmp);
+    }
+    local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+
+    if (IA_ISDIR(loc->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, loc, NULL, xattr,
+                                              flags, xdata, &op_errno);
+        if (ret)
+            goto err;
+    } else {
+        local->rebalance.xattr = dict_ref(xattr);
+        local->rebalance.flags = flags;
+        local->call_cnt = 1;
 
-        local->op_ret = 0;
-
-        local->rebalance.target_op_fn = dht_removexattr2;
-        if (xdata)
-                local->rebalance.xdata = dict_ref (xdata);
-
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
 
-        /* Phase 1 of migration */
-        if (IS_DHT_MIGRATION_PHASE1 (stbuf)) {
-                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->setxattr, loc, xattr, flags,
+                          local->xattr_req);
+    }
 
-                ret = dht_inode_ctx_get_mig_info (this, inode,
-                                                  &subvol1, &subvol2);
-                if (!dht_mig_info_is_invalid (local->cached_subvol,
-                                              subvol1, subvol2)) {
-                        dht_removexattr2 (this, subvol2, frame, 0);
-                        return 0;
-                }
+    return 0;
 
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
-
-out:
-        if (local->fop == GF_FOP_REMOVEXATTR) {
-                DHT_STACK_UNWIND (removexattr, frame, op_ret, op_errno, NULL);
-        } else {
-                DHT_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL);
-        }
-        return 0;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
 
+    return 0;
 }
 
-int
-dht_removexattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame,
-                  int ret)
+static int
+dht_removexattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int          op_errno = EINVAL;
-
-        if (!frame || !frame->local)
-                goto err;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        local = frame->local;
-        op_errno = local->op_errno;
+    if (!frame || !frame->local)
+        goto err;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        if (we_are_not_migrating (ret)) {
+    local->call_cnt = 2; /* This is the second attempt */
 
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
-                                  local->op_errno, local->rebalance.xdata);
-                return 0;
-        }
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto err;
+    if (subvol == NULL)
+        goto err;
 
-        if (local->fop == GF_FOP_REMOVEXATTR) {
-                STACK_WIND_COOKIE (frame, dht_file_removexattr_cbk, subvol,
-                                   subvol, subvol->fops->removexattr,
-                                   &local->loc, local->key, local->xattr_req);
-        } else {
-                STACK_WIND_COOKIE (frame, dht_file_removexattr_cbk, subvol,
-                                   subvol, subvol->fops->fremovexattr,
-                                   local->fd, local->key, local->xattr_req);
-        }
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->removexattr, &local->loc, local->key,
+                          local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->fremovexattr, local->fd, local->key,
+                          local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
-        return 0;
+    DHT_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-
 int
-dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, dict_t *xdata)
+dht_file_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        xlator_t     *prev = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    struct iatt *stbuf = NULL;
+    inode_t *inode = NULL;
+    xlator_t *subvol1 = NULL, *subvol2 = NULL;
 
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "subvolume %s returned -1",
-                                      prev->name);
-                        goto unlock;
-                }
+    local = frame->local;
+    prev = cookie;
 
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
-                                  local->op_errno, NULL);
-        }
+    local->op_errno = op_errno;
 
+    if ((local->fop == GF_FOP_FREMOVEXATTR) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
         return 0;
-}
+    }
 
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
 
-int
-dht_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *key, dict_t *xdata)
-{
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           call_cnt = 0;
-        dht_conf_t   *conf = NULL;
-        int i;
-        int           ret = 0;
+    if (local->call_cnt != 1)
+        goto out;
 
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (this->private, err);
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
 
-        conf = this->private;
+    if ((!op_ret) && !stbuf) {
+        goto out;
+    }
 
-        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
+    local->op_ret = 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
+    local->rebalance.target_op_fn = dht_removexattr2;
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_REMOVEXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    /* Phase 1 of migration */
+    if (IS_DHT_MIGRATION_PHASE1(stbuf)) {
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
 
-        layout = local->layout;
-        if (!local->layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
+        ret = dht_inode_ctx_get_mig_info(this, inode, &subvol1, &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            dht_removexattr2(this, subvol2, frame, 0);
+            return 0;
         }
-        local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();
 
-        local->call_cnt = call_cnt = layout->cnt;
-        local->key = gf_strdup (key);
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-        if (key &&
-            (strncmp (key, conf->mds_xattr_key, strlen(key)) == 0)) {
-                op_errno = ENOTSUP;
-                goto err;
-        }
-
-        if (IA_ISDIR (loc->inode->ia_type)) {
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND_COOKIE (frame, dht_removexattr_cbk,
-                                           layout->list[i].xlator,
-                                           layout->list[i].xlator,
-                                           layout->list[i].xlator->fops->removexattr,
-                                           loc, key, local->xattr_req);
-                }
-
-        } else {
+out:
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
+    } else {
+        DHT_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, xdata);
+    }
+    return 0;
+}
 
-                local->call_cnt = 1;
-                ret = dict_set_int8 (local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED, "Failed to "
-                                "set dictionary key %s for %s",
-                                DHT_IATT_IN_XDATA_KEY, loc->path);
-                }
+int
+dht_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *key, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int call_cnt = 0;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
+
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    GF_IF_NATIVE_XATTR_GOTO(conf->wild_xattr_name, key, op_errno, err);
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_REMOVEXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!local->layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+
+    local->call_cnt = call_cnt = layout->cnt;
+    local->key = gf_strdup(key);
+
+    if (key && (strncmp(key, conf->mds_xattr_key, strlen(key)) == 0)) {
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    if (IA_ISDIR(loc->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, loc, NULL, NULL, 0,
+                                              local->xattr_req, &op_errno);
+        if (ret)
+            goto err;
 
-                STACK_WIND_COOKIE (frame, dht_file_removexattr_cbk, subvol,
-                                   subvol, subvol->fops->removexattr, loc, key,
-                                   local->xattr_req);
+    } else {
+        local->call_cnt = 1;
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to "
+                   "set dictionary key %s for %s",
+                   DHT_IATT_IN_XDATA_KEY, loc->path);
         }
 
-        return 0;
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->removexattr, loc, key,
+                          local->xattr_req);
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *key, dict_t *xdata)
+dht_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+                 dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           call_cnt = 0;
-        dht_conf_t   *conf = 0;
-        int           ret = 0;
-
-        int i;
-
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
-
-        VALIDATE_OR_GOTO (frame, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FREMOVEXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for inode=%s",
-                              uuid_utoa (fd->inode->gfid));
-                op_errno = EINVAL;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int call_cnt = 0;
+    dht_conf_t *conf = 0;
+    int ret = 0;
+
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    GF_IF_NATIVE_XATTR_GOTO(conf->wild_xattr_name, key, op_errno, err);
+
+    VALIDATE_OR_GOTO(frame, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FREMOVEXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for inode=%s",
+                     uuid_utoa(fd->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!local->layout) {
+        gf_msg_debug(this->name, 0, "no layout for inode=%s",
+                     uuid_utoa(fd->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+    local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+
+    local->call_cnt = call_cnt = layout->cnt;
+    local->key = gf_strdup(key);
+
+    if (IA_ISDIR(fd->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, NULL, fd, NULL, 0,
+                                              local->xattr_req, &op_errno);
+        if (ret)
+            goto err;
 
-        layout = local->layout;
-        if (!local->layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for inode=%s",
-                              uuid_utoa (fd->inode->gfid));
-                op_errno = EINVAL;
-                goto err;
+    } else {
+        local->call_cnt = 1;
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to "
+                   "set dictionary key %s for fd=%p",
+                   DHT_IATT_IN_XDATA_KEY, fd);
         }
-        local->xattr_req = xdata ? dict_ref (xdata) : dict_new ();
-
-        local->call_cnt = call_cnt = layout->cnt;
-        local->key = gf_strdup (key);
 
-        if (IA_ISDIR (fd->inode->ia_type)) {
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND_COOKIE (frame, dht_removexattr_cbk,
-                                           layout->list[i].xlator,
-                                           layout->list[i].xlator,
-                                           layout->list[i].xlator->fops->fremovexattr,
-                                           fd, key, local->xattr_req);
-                }
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->fremovexattr, fd, key,
+                          local->xattr_req);
+    }
 
-        } else {
-
-                local->call_cnt = 1;
-                ret = dict_set_int8 (local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED, "Failed to "
-                                "set dictionary key %s for fd=%p",
-                                DHT_IATT_IN_XDATA_KEY, fd);
-                }
-
-                STACK_WIND_COOKIE (frame, dht_file_removexattr_cbk, subvol,
-                                   subvol, subvol->fops->fremovexattr, fd, key,
-                                   local->xattr_req);
-        }
-
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+dht_fd_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+           int op_errno, fd_t *fd, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        xlator_t     *prev = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
 
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "subvolume %s returned -1",
-                                      prev->name);
-                        goto unlock;
-                }
+    local = frame->local;
+    prev = cookie;
 
-                local->op_ret = 0;
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+            goto post_unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-                                  local->fd, NULL);
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_UNWIND(open, frame, local->op_ret, local->op_errno, local->fd,
+                         NULL);
 
-        return 0;
+    return 0;
 }
 
 /*
  * dht_normalize_stats -
  */
-void
-dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
-                     unsigned long frsize)
+static void
+dht_normalize_stats(struct statvfs *buf, unsigned long bsize,
+                    unsigned long frsize)
 {
-        double factor = 0;
+    double factor = 0;
 
-        if (buf->f_bsize != bsize) {
-                buf->f_bsize = bsize;
-        }
+    if (buf->f_bsize != bsize) {
+        buf->f_bsize = bsize;
+    }
 
-        if (buf->f_frsize != frsize) {
-                factor = ((double) buf->f_frsize) / frsize;
-                buf->f_frsize = frsize;
-                buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
-                buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
-                buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
-
-        }
+    if (buf->f_frsize != frsize) {
+        factor = ((double)buf->f_frsize) / frsize;
+        buf->f_frsize = frsize;
+        buf->f_blocks = (fsblkcnt_t)(factor * buf->f_blocks);
+        buf->f_bfree = (fsblkcnt_t)(factor * buf->f_bfree);
+        buf->f_bavail = (fsblkcnt_t)(factor * buf->f_bavail);
+    }
 }
 
-int
-dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct statvfs *statvfs,
-                dict_t *xdata)
-{
-
-        gf_boolean_t            event              = _gf_false;
-        qdstatfs_action_t       action             = qdstatfs_action_OFF;
-        dht_local_t *           local              = NULL;
-        int                     this_call_cnt      = 0;
-        int                     bsize              = 0;
-        int                     frsize             = 0;
-        GF_UNUSED int           ret                = 0;
-        unsigned long           new_usage          = 0;
-        unsigned long           cur_usage          = 0;
-
-        local = frame->local;
-        GF_ASSERT (local);
-
-        if (xdata)
-                ret = dict_get_int8 (xdata, "quota-deem-statfs",
-                                     (int8_t *)&event);
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        goto unlock;
-                }
-                if (!statvfs) {
-                        op_errno = EINVAL;
-                        local->op_ret = -1;
-                        goto unlock;
-                }
-                local->op_ret = 0;
-
-                if (local->quota_deem_statfs) {
-                        if (event == _gf_true) {
-                                action = qdstatfs_action_COMPARE;
-                        } else {
-                                action = qdstatfs_action_NEGLECT;
-                        }
-                } else {
-                        if (event == _gf_true) {
-                                action = qdstatfs_action_REPLACE;
-                                local->quota_deem_statfs = _gf_true;
-                        }
-                }
-
-                if (local->quota_deem_statfs) {
-                        switch (action) {
-                        case qdstatfs_action_NEGLECT:
-                                goto unlock;
-
-                        case qdstatfs_action_REPLACE:
-                                local->statvfs = *statvfs;
-                                goto unlock;
-
-                        case qdstatfs_action_COMPARE:
-                                new_usage = statvfs->f_blocks -
-                                             statvfs->f_bfree;
-                                cur_usage = local->statvfs.f_blocks -
-                                             local->statvfs.f_bfree;
+static int
+dht_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct statvfs *statvfs, dict_t *xdata)
+{
+    gf_boolean_t event = _gf_false;
+    qdstatfs_action_t action = qdstatfs_action_OFF;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    int bsize = 0;
+    int frsize = 0;
+    GF_UNUSED int ret = 0;
+    unsigned long new_usage = 0;
+    unsigned long cur_usage = 0;
+
+    local = frame->local;
+    GF_ASSERT(local);
+
+    if (xdata)
+        ret = dict_get_int8(xdata, "quota-deem-statfs", (int8_t *)&event);
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            goto unlock;
+        }
+        if (!statvfs) {
+            op_errno = EINVAL;
+            local->op_ret = -1;
+            goto unlock;
+        }
+        local->op_ret = 0;
 
-                                /* Take the max of the usage from subvols */
-                                if (new_usage >= cur_usage)
-                                        local->statvfs = *statvfs;
-                                goto unlock;
+        if (local->quota_deem_statfs) {
+            if (event == _gf_true) {
+                action = qdstatfs_action_COMPARE;
+            } else {
+                action = qdstatfs_action_NEGLECT;
+            }
+        } else {
+            if (event == _gf_true) {
+                action = qdstatfs_action_REPLACE;
+                local->quota_deem_statfs = _gf_true;
+            }
+        }
 
-                        default:
-                                break;
-                        }
-                }
+        if (local->quota_deem_statfs) {
+            switch (action) {
+                case qdstatfs_action_NEGLECT:
+                    goto unlock;
 
-                if (local->statvfs.f_bsize != 0) {
-                        bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
-                        frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
-                        dht_normalize_stats(&local->statvfs, bsize, frsize);
-                        dht_normalize_stats(statvfs, bsize, frsize);
-                } else {
-                        local->statvfs.f_bsize    = statvfs->f_bsize;
-                        local->statvfs.f_frsize   = statvfs->f_frsize;
-                }
+                case qdstatfs_action_REPLACE:
+                    local->statvfs = *statvfs;
+                    goto unlock;
 
-                local->statvfs.f_blocks  += statvfs->f_blocks;
-                local->statvfs.f_bfree   += statvfs->f_bfree;
-                local->statvfs.f_bavail  += statvfs->f_bavail;
-                local->statvfs.f_files   += statvfs->f_files;
-                local->statvfs.f_ffree   += statvfs->f_ffree;
-                local->statvfs.f_favail  += statvfs->f_favail;
-                local->statvfs.f_fsid     = statvfs->f_fsid;
-                local->statvfs.f_flag     = statvfs->f_flag;
-                local->statvfs.f_namemax  = statvfs->f_namemax;
+                case qdstatfs_action_COMPARE:
+                    new_usage = statvfs->f_blocks - statvfs->f_bfree;
+                    cur_usage = local->statvfs.f_blocks -
+                                local->statvfs.f_bfree;
 
+                    /* Take the max of the usage from subvols */
+                    if (new_usage >= cur_usage)
+                        local->statvfs = *statvfs;
+                    goto unlock;
 
+                default:
+                    break;
+            }
         }
-unlock:
-        UNLOCK (&frame->lock);
 
+        if (local->statvfs.f_bsize != 0) {
+            bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
+            frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
+            dht_normalize_stats(&local->statvfs, bsize, frsize);
+            dht_normalize_stats(statvfs, bsize, frsize);
+        } else {
+            local->statvfs.f_bsize = statvfs->f_bsize;
+            local->statvfs.f_frsize = statvfs->f_frsize;
+        }
+
+        local->statvfs.f_blocks += statvfs->f_blocks;
+        local->statvfs.f_bfree += statvfs->f_bfree;
+        local->statvfs.f_bavail += statvfs->f_bavail;
+        local->statvfs.f_files += statvfs->f_files;
+        local->statvfs.f_ffree += statvfs->f_ffree;
+        local->statvfs.f_favail += statvfs->f_favail;
+        local->statvfs.f_fsid = statvfs->f_fsid;
+        local->statvfs.f_flag = statvfs->f_flag;
+        local->statvfs.f_namemax = statvfs->f_namemax;
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->statvfs, xdata);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_UNWIND(statfs, frame, local->op_ret, local->op_errno,
+                         &local->statvfs, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
-        inode_t          *inode         = NULL;
-        inode_table_t    *itable        = NULL;
-        uuid_t            root_gfid     = {0, };
-        loc_t         newloc = {0, };
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, NULL, NULL, GF_FOP_STATFS);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+dht_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
+    inode_t *inode = NULL;
+    inode_table_t *itable = NULL;
+    static uuid_t root_gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    loc_t newloc = {
+        0,
+    };
 
-        if (loc->inode && !IA_ISDIR (loc->inode->ia_type)) {
-                itable = loc->inode->table;
-                if (!itable) {
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(this->private, err);
 
-                loc = &local->loc2;
-                root_gfid[15] = 1;
+    conf = this->private;
 
-                inode = inode_find (itable, root_gfid);
-                if (!inode) {
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    local = dht_local_init(frame, NULL, NULL, GF_FOP_STATFS);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-                dht_build_root_loc (inode, &newloc);
-                loc = &newloc;
+    if (loc->inode && !IA_ISDIR(loc->inode->ia_type)) {
+        itable = loc->inode->table;
+        if (!itable) {
+            op_errno = EINVAL;
+            goto err;
         }
 
-        local->call_cnt = conf->subvolume_cnt;
+        loc = &local->loc2;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_statfs_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->statfs, loc,
-                            xdata);
+        inode = inode_find(itable, root_gfid);
+        if (!inode) {
+            op_errno = EINVAL;
+            goto err;
         }
-        return 0;
+
+        dht_build_root_loc(inode, &newloc);
+        loc = &newloc;
+    }
+
+    local->call_cnt = conf->subvolume_cnt;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND(frame, dht_statfs_cbk, conf->subvolumes[i],
+                   conf->subvolumes[i]->fops->statfs, loc, xdata);
+    }
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-             dict_t *xdata)
+dht_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
 {
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
-        int           ret = 0;
-        gf_boolean_t  new_xdata = _gf_false;
-        xlator_t    **subvolumes = NULL;
-        int           call_count = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, loc, fd, GF_FOP_OPENDIR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->first_up_subvol = dht_first_up_subvol (this);
-
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int ret = 0;
+    gf_boolean_t new_xdata = _gf_false;
+    xlator_t **subvolumes = NULL;
+    int call_count = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_OPENDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    local->first_up_subvol = dht_first_up_subvol(this);
+
+    if (!xdata) {
+        xdata = dict_new();
         if (!xdata) {
-                xdata = dict_new ();
-                if (!xdata) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                new_xdata = _gf_true;
+            op_errno = ENOMEM;
+            goto err;
+        }
+        new_xdata = _gf_true;
+    }
+
+    ret = dict_set_uint32(xdata, conf->link_xattr_name, 256);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value : key = %s",
+               conf->link_xattr_name);
+
+    /* dht_readdirp will wind to all subvols so open has to be sent to
+     * all subvols whether or not conf->local_subvols is set */
+
+    call_count = local->call_cnt = conf->subvolume_cnt;
+    subvolumes = conf->subvolumes;
+
+    /* In case of parallel-readdir, the readdir-ahead will be loaded
+     * below dht, in this case, if we want to enable or disable SKIP_DIRs
+     * it has to be done in opendir, so that prefetching logic in
+     * readdir-ahead, honors it */
+    for (i = 0; i < call_count; i++) {
+        if (conf->readdir_optimize == _gf_true) {
+            if (subvolumes[i] != local->first_up_subvol) {
+                ret = dict_set_int32(xdata, GF_READDIR_SKIP_DIRS, 1);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                           "Failed to set dictionary"
+                           " value :key = %s, ret:%d",
+                           GF_READDIR_SKIP_DIRS, ret);
+            }
         }
 
-        ret = dict_set_uint32 (xdata, conf->link_xattr_name, 256);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value : key = %s",
-                        conf->link_xattr_name);
-
-        /* dht_readdirp will wind to all subvols so open has to be sent to
-         * all subvols whether or not conf->local_subvols is set */
-
-        call_count = local->call_cnt = conf->subvolume_cnt;
-        subvolumes = conf->subvolumes;
-
-        /* In case of parallel-readdir, the readdir-ahead will be loaded
-         * below dht, in this case, if we want to enable or disable SKIP_DIRs
-         * it has to be done in opendir, so that prefetching logic in
-         * readdir-ahead, honors it */
-        for (i = 0; i < call_count; i++) {
-                if (conf->readdir_optimize == _gf_true) {
-                        if (subvolumes[i] != local->first_up_subvol) {
-                                ret = dict_set_int32 (xdata,
-                                                      GF_READDIR_SKIP_DIRS, 1);
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                DHT_MSG_DICT_SET_FAILED,
-                                                "Failed to set dictionary"
-                                                " value :key = %s, ret:%d",
-                                                GF_READDIR_SKIP_DIRS, ret);
-                                }
-                }
+        STACK_WIND_COOKIE(frame, dht_fd_cbk, subvolumes[i], subvolumes[i],
+                          subvolumes[i]->fops->opendir, loc, fd, xdata);
+        dict_del(xdata, GF_READDIR_SKIP_DIRS);
+    }
 
-                STACK_WIND_COOKIE (frame, dht_fd_cbk,
-                                   subvolumes[i],
-                                   subvolumes[i],
-                                   subvolumes[i]->fops->opendir,
-                                   loc, fd, xdata);
-                dict_del (xdata, GF_READDIR_SKIP_DIRS);
-        }
+    if (new_xdata)
+        dict_unref(xdata);
 
-        if (new_xdata)
-                dict_unref (xdata);
-
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(opendir, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* dht_readdirp_cbk creates a new dentry and dentry->inode is not assigned.
-   This functions assigns an inode if all of the following conditions are true:
+   This functions assigns an inode if all of the following conditions are
+   true:
 
-   * DHT has only one child. In this case the entire layout is present on this
-     single child and hence we can set complete layout in inode.
-   * backend has complete layout and there are no anomalies in it and from this
-    information layout can be constructed and set in inode.
+   * DHT has only one child. In this case the entire layout is present on
+   this single child and hence we can set complete layout in inode.
+   * backend has complete layout and there are no anomalies in it and from
+   this information layout can be constructed and set in inode.
 */
 
-void
-dht_populate_inode_for_dentry (xlator_t *this, xlator_t *subvol,
-                               gf_dirent_t *entry, gf_dirent_t *orig_entry)
+static void
+dht_populate_inode_for_dentry(xlator_t *this, xlator_t *subvol,
+                              gf_dirent_t *entry, gf_dirent_t *orig_entry)
 {
-        dht_layout_t *layout = NULL;
-        int           ret    = 0;
-        loc_t         loc    = {0, };
-
-        if (gf_uuid_is_null (orig_entry->d_stat.ia_gfid)) {
-                /* this skips the '..' entry for the root of the volume */
-                return;
-        }
+    dht_layout_t *layout = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
 
-        gf_uuid_copy (loc.gfid, orig_entry->d_stat.ia_gfid);
-        loc.inode = inode_ref (orig_entry->inode);
+    if (gf_uuid_is_null(orig_entry->d_stat.ia_gfid)) {
+        /* this skips the '..' entry for the root of the volume */
+        return;
+    }
 
-        if (is_revalidate (&loc)) {
-                goto out;
-        }
+    gf_uuid_copy(loc.gfid, orig_entry->d_stat.ia_gfid);
+    loc.inode = inode_ref(orig_entry->inode);
 
-        layout = dht_layout_new (this, 1);
-        if (!layout)
-                goto out;
+    if (is_revalidate(&loc)) {
+        goto out;
+    }
 
-        ret = dht_layout_merge (this, layout, subvol, 0, 0, orig_entry->dict);
-        if (!ret) {
-                ret = dht_layout_normalize (this, &loc, layout);
-                if (ret == 0) {
-                        dht_layout_set (this, orig_entry->inode, layout);
-                        entry->inode = inode_ref (orig_entry->inode);
-                        layout = NULL;
-                }
+    layout = dht_layout_new(this, 1);
+    if (!layout)
+        goto out;
 
+    ret = dht_layout_merge(this, layout, subvol, 0, 0, orig_entry->dict);
+    if (!ret) {
+        ret = dht_layout_normalize(this, &loc, layout);
+        if (ret == 0) {
+            dht_layout_set(this, orig_entry->inode, layout);
+            entry->inode = inode_ref(orig_entry->inode);
+            layout = NULL;
         }
+    }
 
-        if (layout)
-                dht_layout_unref (this, layout);
+    if (layout)
+        dht_layout_unref(this, layout);
 
 out:
-        loc_wipe (&loc);
-        return;
+    loc_wipe(&loc);
+    return;
 }
 
-int
-dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-                  int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
-{
-        dht_local_t             *local = NULL;
-        gf_dirent_t             *orig_entry = NULL;
-        gf_dirent_t             *entry = NULL;
-        xlator_t                *prev = NULL;
-        xlator_t                *next_subvol = NULL;
-        off_t                    next_offset = 0;
-        int                      count = 0;
-        dht_layout_t            *layout = NULL;
-        dht_conf_t              *conf   = NULL;
-        dht_methods_t           *methods = NULL;
-        xlator_t                *subvol = 0;
-        xlator_t                *hashed_subvol = 0;
-        int                      ret    = 0;
-        int                      readdir_optimize = 0;
-        inode_table_t           *itable = NULL;
-        inode_t                 *inode = NULL;
-
-        prev = cookie;
-        local = frame->local;
-        itable = local->fd ? local->fd->inode->table : NULL;
-
-        conf  = this->private;
-        GF_VALIDATE_OR_GOTO(this->name, conf, unwind);
+/* Posix returns op_errno = ENOENT to indicate that there are no more
+ * entries
+ */
+static int
+dht_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *orig_entry = NULL;
+    gf_dirent_t *entry = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *next_subvol = NULL;
+    off_t next_offset = 0;
+    int count = 0;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
+    xlator_t *subvol = 0;
+    xlator_t *hashed_subvol = 0;
+    int ret = 0;
+    int readdir_optimize = 0;
+    inode_table_t *itable = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t skip_hashed_check = _gf_false;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    prev = cookie;
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO(this->name, local->fd, unwind);
+
+    itable = local->fd->inode->table;
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, unwind);
+
+    methods = &(conf->methods);
+
+    if (op_ret <= 0) {
+        goto done;
+    }
+
+    /* Why aren't we skipping DHT entirely in case of a single subvol?
+     * Because if this was a larger volume earlier and all but one subvol
+     * was removed, there might be stale linkto files on the subvol.
+     */
+    if (conf->subvolume_cnt == 1) {
+        /* return all directory and file entries except
+         * linkto files for a single child DHT
+         */
+        skip_hashed_check = _gf_true;
+    }
 
-        methods = &(conf->methods);
+    if (!local->layout)
+        local->layout = dht_layout_get(this, local->fd->inode);
 
-        local->op_errno = op_errno;
+    layout = local->layout;
 
-        if (op_ret < 0)
-                goto done;
+    /* This will skip the entries on the subvol without a layout,
+     * hence preventing the crash but rmdir might fail with
+     * "directory not empty" errors*/
 
-        if (local->op_ret < 0)
-                local->op_ret = 0;
+    if (layout == NULL)
+        goto done;
 
-        if (!local->layout)
-                local->layout = dht_layout_get (this, local->fd->inode);
+    if (conf->readdir_optimize == _gf_true)
+        readdir_optimize = 1;
 
-        layout = local->layout;
+    gf_msg_debug(this->name, 0, "Processing entries from %s", prev->name);
 
-        /* We have seen crashes in while running "rm -rf" on tier volumes
-           when the layout was NULL on the hot tier. This will skip the
-           entries on the subvol without a layout, hence preventing the crash
-           but rmdir might fail with "directory not empty" errors*/
+    list_for_each_entry(orig_entry, (&orig_entries->list), list)
+    {
+        next_offset = orig_entry->d_off;
 
-        if (layout == NULL)
-                goto done;
+        gf_msg_debug(this->name, 0, "%s: entry = %s, type = %d", prev->name,
+                     orig_entry->d_name, orig_entry->d_type);
 
-        if (conf->readdir_optimize == _gf_true)
-                readdir_optimize = 1;
+        if (IA_ISINVAL(orig_entry->d_stat.ia_type)) {
+            /*stat failed somewhere- display this entry but the data may
+             * be inaccurate.
+             */
+            gf_msg_debug(this->name, EINVAL, "Invalid stat for %s (gfid %s)",
+                         orig_entry->d_name,
+                         uuid_utoa(orig_entry->d_stat.ia_gfid));
+        }
 
-        gf_msg_debug (this->name, 0, "Processing entries from %s",
-                      prev->name);
+        if (check_is_linkfile(NULL, (&orig_entry->d_stat), orig_entry->dict,
+                              conf->link_xattr_name)) {
+            gf_msg_debug(this->name, 0, "%s: %s is a linkto file", prev->name,
+                         orig_entry->d_name);
+            continue;
+        }
 
-        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
-                next_offset = orig_entry->d_off;
+        if (skip_hashed_check) {
+            goto list;
+        }
 
-                gf_msg_debug (this->name, 0, "%s: entry = %s, type = %d",
-                              prev->name, orig_entry->d_name,
-                              orig_entry->d_type);
+        if (check_is_dir(NULL, (&orig_entry->d_stat), NULL)) {
+            /*Directory entries filtering :
+             * a) If rebalance is running, pick from first_up_subvol
+             * b) (rebalance not running)hashed subvolume is NULL or
+             * down then filter in first_up_subvolume. Other wise the
+             * corresponding hashed subvolume will take care of the
+             * directory entry.
+             */
+            if (readdir_optimize) {
+                if (prev == local->first_up_subvol)
+                    goto list;
+                else
+                    continue;
+            }
 
-                if (IA_ISINVAL(orig_entry->d_stat.ia_type)) {
-                        /*stat failed somewhere- ignore this entry*/
-                        gf_msg_debug (this->name, EINVAL,
-                                      "Invalid stat, ignoring entry "
-                                      "%s gfid %s", orig_entry->d_name,
-                                      uuid_utoa (orig_entry->d_stat.ia_gfid));
-                        continue;
-                }
+            hashed_subvol = methods->layout_search(this, layout,
+                                                   orig_entry->d_name);
 
-                if (check_is_dir (NULL, (&orig_entry->d_stat), NULL)) {
+            if (prev == hashed_subvol)
+                goto list;
+            if ((hashed_subvol && dht_subvol_status(conf, hashed_subvol)) ||
+                (prev != local->first_up_subvol))
+                continue;
 
-                /*Directory entries filtering :
-                 * a) If rebalance is running, pick from first_up_subvol
-                 * b) (rebalance not running)hashed subvolume is NULL or
-                 * down then filter in first_up_subvolume. Other wise the
-                 * corresponding hashed subvolume will take care of the
-                 * directory entry.
-                 */
-                        if (readdir_optimize) {
-                                if (prev == local->first_up_subvol)
-                                        goto list;
-                                else
-                                        continue;
+            goto list;
+        }
 
-                        }
+    list:
+        entry = gf_dirent_for_name(orig_entry->d_name);
+        if (!entry) {
+            goto unwind;
+        }
 
-                        hashed_subvol = methods->layout_search (this, layout,
-                                                         orig_entry->d_name);
+        /* Do this if conf->search_unhashed is set to "auto" */
+        if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) {
+            subvol = methods->layout_search(this, layout, orig_entry->d_name);
+            if (!subvol || (subvol != prev)) {
+                /* TODO: Count the number of entries which need
+                   linkfile to prove its existence in fs */
+                layout->search_unhashed++;
+            }
+        }
 
-                        if (prev == hashed_subvol)
-                                goto list;
-                        if ((hashed_subvol
-                             && dht_subvol_status (conf, hashed_subvol))
-                            || (prev != local->first_up_subvol))
-                                continue;
+        entry->d_off = orig_entry->d_off;
+        entry->d_stat = orig_entry->d_stat;
+        entry->d_ino = orig_entry->d_ino;
+        entry->d_type = orig_entry->d_type;
+        entry->d_len = orig_entry->d_len;
 
-                        goto list;
-                }
+        if (orig_entry->dict)
+            entry->dict = dict_ref(orig_entry->dict);
 
-                if (check_is_linkfile (NULL, (&orig_entry->d_stat),
-                                       orig_entry->dict,
-                                       conf->link_xattr_name)) {
-                        gf_msg_debug (this->name, 0, "%s: %s is a linkto file",
-                                      prev->name, orig_entry->d_name);
-                        continue;
-                }
+        /* making sure we set the inode ctx right with layout,
+           currently possible only for non-directories, so for
+           directories don't set entry inodes */
+        if (IA_ISDIR(entry->d_stat.ia_type)) {
+            entry->d_stat.ia_blocks = DHT_DIR_STAT_BLOCKS;
+            entry->d_stat.ia_size = DHT_DIR_STAT_SIZE;
+            if (orig_entry->inode) {
+                dht_inode_ctx_time_update(orig_entry->inode, this,
+                                          &entry->d_stat, 1);
 
-list:
-                entry = gf_dirent_for_name (orig_entry->d_name);
-                if (!entry) {
-                        goto unwind;
+                if (conf->subvolume_cnt == 1) {
+                    dht_populate_inode_for_dentry(this, prev, entry,
+                                                  orig_entry);
                 }
+            }
+        } else {
+            if (orig_entry->dict &&
+                dict_get(orig_entry->dict, conf->link_xattr_name)) {
+                /* Strip out the S and T flags set by rebalance*/
+                DHT_STRIP_PHASE1_FLAGS(&entry->d_stat);
+            }
 
-                /* Do this if conf->search_unhashed is set to "auto" */
-                if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) {
-                        subvol = methods->layout_search (this, layout,
-                                                         orig_entry->d_name);
-                        if (!subvol || (subvol != prev)) {
-                                /* TODO: Count the number of entries which need
-                                   linkfile to prove its existence in fs */
-                                layout->search_unhashed++;
-                        }
+            if (orig_entry->inode) {
+                ret = dht_layout_preset(this, prev, orig_entry->inode);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_WARNING, 0,
+                           DHT_MSG_LAYOUT_SET_FAILED,
+                           "failed to link the layout "
+                           "in inode for %s",
+                           orig_entry->d_name);
+
+                entry->inode = inode_ref(orig_entry->inode);
+            } else if (itable) {
+                /*
+                 * orig_entry->inode might be null if any upper
+                 * layer xlators below client set to null, to
+                 * force a lookup on the inode even if the inode
+                 * is present in the inode table. In that case
+                 * we just update the ctx to make sure we didn't
+                 * missed anything.
+                 */
+                inode = inode_find(itable, orig_entry->d_stat.ia_gfid);
+                if (inode) {
+                    ret = dht_layout_preset(this, prev, inode);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_WARNING, 0,
+                               DHT_MSG_LAYOUT_SET_FAILED,
+                               "failed to link the layout"
+                               " in inode for %s",
+                               orig_entry->d_name);
+                    inode_unref(inode);
+                    inode = NULL;
                 }
+            }
+        }
 
-                entry->d_off  = orig_entry->d_off;
-                entry->d_stat = orig_entry->d_stat;
-                entry->d_ino  = orig_entry->d_ino;
-                entry->d_type = orig_entry->d_type;
-                entry->d_len  = orig_entry->d_len;
-
-                if (orig_entry->dict)
-                        entry->dict = dict_ref (orig_entry->dict);
-
-                /* making sure we set the inode ctx right with layout,
-                   currently possible only for non-directories, so for
-                   directories don't set entry inodes */
-                if (IA_ISDIR(entry->d_stat.ia_type)) {
-                        entry->d_stat.ia_blocks = DHT_DIR_STAT_BLOCKS;
-                        entry->d_stat.ia_size = DHT_DIR_STAT_SIZE;
-                        if (orig_entry->inode) {
-                                dht_inode_ctx_time_update (orig_entry->inode,
-                                                           this, &entry->d_stat,
-                                                           1);
-
-                                if (conf->subvolume_cnt == 1) {
-                                        dht_populate_inode_for_dentry (this,
-                                                                       prev,
-                                                                       entry,
-                                                                       orig_entry);
-                                }
+        gf_msg_debug(this->name, 0, "%s: Adding entry = %s", prev->name,
+                     entry->d_name);
 
-                        }
-                } else {
-                        if (orig_entry->inode) {
-                                ret = dht_layout_preset (this, prev,
-                                                         orig_entry->inode);
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                                DHT_MSG_LAYOUT_SET_FAILED,
-                                                "failed to link the layout "
-                                                "in inode for %s",
-                                                orig_entry->d_name);
-
-                                entry->inode = inode_ref (orig_entry->inode);
-                        } else if (itable) {
-                                /*
-                                 * orig_entry->inode might be null if any upper
-                                 * layer xlators below client set to null, to
-                                 * force a lookup on the inode even if the inode
-                                 * is present in the inode table. In that case
-                                 * we just update the ctx to make sure we didn't
-                                 * missed anything.
-                                 */
-                                inode = inode_find (itable,
-                                                    orig_entry->d_stat.ia_gfid);
-                                if (inode) {
-                                        ret = dht_layout_preset
-                                                            (this, prev,
-                                                             inode);
-                                        if (ret)
-                                                gf_msg (this->name,
-                                                     GF_LOG_WARNING, 0,
-                                                     DHT_MSG_LAYOUT_SET_FAILED,
-                                                     "failed to link the layout"
-                                                     " in inode for %s",
-                                                     orig_entry->d_name);
-                                        inode_unref (inode);
-                                        inode = NULL;
-                                }
-                        }
-                }
+        list_add_tail(&entry->list, &entries.list);
+        count++;
+    }
 
-                gf_msg_debug (this->name, 0, "%s: Adding entry = %s",
-                              prev->name, entry->d_name);
+done:
 
-                list_add_tail (&entry->list, &local->entries.list);
-                local->filled += gf_dirent_size (entry->d_name);
-                count++;
-                local->op_ret++;
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     * Possible values:
+     * op_ret == 0 and op_errno != 0
+     *   if op_errno != ENOENT : Error.Unwind.
+     *   if op_errno == ENOENT : There are no more entries on this subvol.
+     *                           Move to the next one.
+     * op_ret > 0 and count == 0 :
+     *    The subvol returned entries to dht but all were stripped out.
+     *    For example, if they were linkto files or dirs where
+     *    hashed_subvol != prev. Try to get some entries by winding
+     *    to the next subvol. This can be dangerous if parallel readdir
+     *    is enabled as it grows the stack.
+     *
+     * op_ret > 0 and count > 0:
+     *   We found some entries. Unwind even if the buffer is not full.
+     *
+     */
+
+    op_ret = count;
+    if (count == 0) {
+        /* non-zero next_offset means that
+         * EOF is not yet hit on the current subvol
+         */
+        if ((next_offset == 0) || (op_errno == ENOENT)) {
+            next_offset = 0;
+            next_subvol = dht_subvol_next(this, prev);
+        } else {
+            next_subvol = prev;
         }
 
-done:
-        if ((op_ret == 0) && op_errno != ENOENT) {
-                /* remaining buffer size is not enough to hold even one
-                 * dentry
-                 */
-                goto unwind;
+        if (!next_subvol) {
+            goto unwind;
         }
 
-        if ((count == 0) || (local && (local->filled < local->size))) {
-                if ((next_offset == 0) || (op_errno == ENOENT)) {
-                        next_offset = 0;
-                        next_subvol = dht_subvol_next (this, prev);
-                } else {
-                        next_subvol = prev;
-                }
-
-                if (!next_subvol) {
-                        goto unwind;
-                }
-
-                if (conf->readdir_optimize == _gf_true) {
-                        if (next_subvol != local->first_up_subvol) {
-                                ret = dict_set_int32 (local->xattr,
-                                                      GF_READDIR_SKIP_DIRS, 1);
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                DHT_MSG_DICT_SET_FAILED,
-                                                "Failed to set dictionary value"
-                                                ":key = %s",
-                                                GF_READDIR_SKIP_DIRS );
-                        } else {
-                                 dict_del (local->xattr,
-                                           GF_READDIR_SKIP_DIRS);
-                        }
-                }
-
-                STACK_WIND_COOKIE (frame, dht_readdirp_cbk, next_subvol,
-                                   next_subvol, next_subvol->fops->readdirp,
-                                   local->fd, (local->size - local->filled),
-                                   next_offset, local->xattr);
-                return 0;
+        if (conf->readdir_optimize == _gf_true) {
+            if (next_subvol != local->first_up_subvol) {
+                ret = dict_set_int32(local->xattr, GF_READDIR_SKIP_DIRS, 1);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                           "Failed to set dictionary value"
+                           ":key = %s",
+                           GF_READDIR_SKIP_DIRS);
+            } else {
+                dict_del(local->xattr, GF_READDIR_SKIP_DIRS);
+            }
         }
 
+        STACK_WIND_COOKIE(frame, dht_readdirp_cbk, next_subvol, next_subvol,
+                          next_subvol->fops->readdirp, local->fd, local->size,
+                          next_offset, local->xattr);
+        return 0;
+    }
+
 unwind:
-        /* We need to ensure that only the last subvolume's end-of-directory
-         * notification is respected so that directory reading does not stop
-         * before all subvolumes have been read. That could happen because the
-         * posix for each subvolume sends a ENOENT on end-of-directory but in
-         * distribute we're not concerned only with a posix's view of the
-         * directory but the aggregated namespace' view of the directory.
-         */
-        if ((local->op_ret >= 0) && (prev != dht_last_up_subvol (this)))
-                local->op_errno = 0;
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     */
+    if (op_ret < 0)
+        op_ret = 0;
 
+    if (prev != dht_last_up_subvol(this))
+        op_errno = 0;
 
-        DHT_STACK_UNWIND (readdirp, frame, local->op_ret, local->op_errno,
-                          &local->entries, NULL);
+    DHT_STACK_UNWIND(readdirp, frame, op_ret, op_errno, &entries, NULL);
 
-        return 0;
+    gf_dirent_free(&entries);
+    return 0;
 }
 
-
-int
-dht_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries,
-                 dict_t *xdata)
+static int
+dht_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        gf_dirent_t  *orig_entry = NULL;
-        gf_dirent_t  *entry = NULL;
-        xlator_t     *prev = NULL;
-        xlator_t     *next_subvol = NULL;
-        off_t         next_offset = 0;
-        int           count = 0;
-        dht_layout_t *layout = 0;
-        xlator_t     *subvol = 0;
-        dht_conf_t   *conf = NULL;
-        dht_methods_t *methods = NULL;
-
-        prev = cookie;
-        local = frame->local;
+    dht_local_t *local = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *orig_entry = NULL;
+    gf_dirent_t *entry = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *next_subvol = NULL;
+    off_t next_offset = 0;
+    int count = 0;
+    dht_layout_t *layout = 0;
+    xlator_t *subvol = 0;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
+    gf_boolean_t skip_hashed_check = _gf_false;
 
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, done);
+    INIT_LIST_HEAD(&entries.list);
 
-        methods = &(conf->methods);
+    prev = cookie;
+    local = frame->local;
 
-        local->op_errno = op_errno;
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, done);
 
-        if (op_ret < 0) {
-                goto done;
-        }
+    methods = &(conf->methods);
 
-        if (local->op_ret < 0)
-                local->op_ret = 0;
+    if (op_ret <= 0)
+        goto done;
 
-        if (!local->layout)
-                local->layout = dht_layout_get (this, local->fd->inode);
+    if (!local->layout)
+        local->layout = dht_layout_get(this, local->fd->inode);
 
-        layout = local->layout;
+    layout = local->layout;
 
-        gf_msg_debug (this->name, 0, "Processing entries from %s",
-                      prev->name);
+    gf_msg_debug(this->name, 0, "Processing entries from %s", prev->name);
 
-        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
-                next_offset = orig_entry->d_off;
+    if (conf->subvolume_cnt == 1) {
+        /*return everything*/
+        skip_hashed_check = _gf_true;
+        count = op_ret;
+        goto done;
+    }
 
-                gf_msg_debug (this->name, 0, "%s: entry = %s, type = %d",
-                              prev->name, orig_entry->d_name,
-                              orig_entry->d_type);
+    list_for_each_entry(orig_entry, (&orig_entries->list), list)
+    {
+        next_offset = orig_entry->d_off;
 
-                subvol = methods->layout_search (this, layout,
-                                                 orig_entry->d_name);
+        gf_msg_debug(this->name, 0, "%s: entry = %s, type = %d", prev->name,
+                     orig_entry->d_name, orig_entry->d_type);
 
-                if (!subvol || (subvol == prev)) {
-                        entry = gf_dirent_for_name (orig_entry->d_name);
-                        if (!entry) {
-                                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                        DHT_MSG_NO_MEMORY,
-                                        "Memory allocation failed ");
-                                goto unwind;
-                        }
+        subvol = methods->layout_search(this, layout, orig_entry->d_name);
 
-                        entry->d_off  = orig_entry->d_off;
-                        entry->d_ino  = orig_entry->d_ino;
-                        entry->d_type = orig_entry->d_type;
-                        entry->d_len  = orig_entry->d_len;
+        if (!subvol || (subvol == prev)) {
+            entry = gf_dirent_for_name(orig_entry->d_name);
+            if (!entry) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                       "Memory allocation failed ");
+                goto unwind;
+            }
 
-                        gf_msg_debug (this->name, 0, "%s: Adding = entry %s",
-                                      prev->name, entry->d_name);
+            entry->d_off = orig_entry->d_off;
+            entry->d_ino = orig_entry->d_ino;
+            entry->d_type = orig_entry->d_type;
+            entry->d_len = orig_entry->d_len;
 
-                        list_add_tail (&entry->list, &local->entries.list);
-                        count++;
-                        local->filled += gf_dirent_size (entry->d_name);
-                        local->op_ret++;
-                }
-        }
+            gf_msg_debug(this->name, 0, "%s: Adding = entry %s", prev->name,
+                         entry->d_name);
 
+            list_add_tail(&entry->list, &entries.list);
+            count++;
+        }
+    }
 done:
-        if ((op_ret == 0) && op_errno != ENOENT) {
-                /* remaining buffer size is not enough to hold even one
-                 * dentry
-                 */
-                goto unwind;
+    op_ret = count;
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     */
+    if (count == 0) {
+        if ((next_offset == 0) || (op_errno == ENOENT)) {
+            next_offset = 0;
+            next_subvol = dht_subvol_next(this, prev);
+        } else {
+            next_subvol = prev;
         }
 
-        if ((count == 0) || (local && (local->filled < local->size))) {
-                if ((op_ret <= 0) || (op_errno == ENOENT)) {
-                        next_subvol = dht_subvol_next (this, prev);
-                } else {
-                        next_subvol = prev;
-                }
-
-                if (!next_subvol) {
-                        goto unwind;
-                }
-
-                STACK_WIND_COOKIE (frame, dht_readdir_cbk, next_subvol,
-                                   next_subvol, next_subvol->fops->readdir,
-                                   local->fd, (local->size - local->filled),
-                                   next_offset, NULL);
-                return 0;
+        if (!next_subvol) {
+            goto unwind;
         }
 
-unwind:
-        /* We need to ensure that only the last subvolume's end-of-directory
-         * notification is respected so that directory reading does not stop
-         * before all subvolumes have been read. That could happen because the
-         * posix for each subvolume sends a ENOENT on end-of-directory but in
-         * distribute we're not concerned only with a posix's view of the
-         * directory but the aggregated namespace' view of the directory.
-         */
-        if ((local->op_ret >= 0) && (prev != dht_last_up_subvol (this)))
-                local->op_errno = 0;
-
-
-        DHT_STACK_UNWIND (readdir, frame, local->op_ret, local->op_errno,
-                          &local->entries, NULL);
-
+        STACK_WIND_COOKIE(frame, dht_readdir_cbk, next_subvol, next_subvol,
+                          next_subvol->fops->readdir, local->fd, local->size,
+                          next_offset, NULL);
         return 0;
-}
+    }
 
+unwind:
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     */
 
-int
-dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t yoff, int whichop, dict_t *dict)
-{
-        dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-        xlator_t     *xvol = NULL;
-        int           ret = 0;
-        dht_conf_t   *conf = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, NULL, NULL, whichop);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->fd = fd_ref (fd);
-        local->size = size;
-        local->xattr_req = (dict)? dict_ref (dict) : NULL;
-        local->first_up_subvol = dht_first_up_subvol (this);
-        local->op_ret = -1;
-
-        dht_deitransform (this, yoff, &xvol);
-
-        /* TODO: do proper readdir */
-        if (whichop == GF_FOP_READDIRP) {
-                if (dict)
-                        local->xattr = dict_ref (dict);
-                else
-                        local->xattr = dict_new ();
-
-                if (local->xattr) {
-                        ret = dict_set_uint32 (local->xattr,
-                                               conf->link_xattr_name, 256);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "Failed to set dictionary value"
-                                        " : key = %s",
-                                        conf->link_xattr_name);
-
-                        if (conf->readdir_optimize == _gf_true) {
-                                if (xvol != local->first_up_subvol) {
-                                        ret = dict_set_int32 (local->xattr,
-                                                       GF_READDIR_SKIP_DIRS, 1);
-                                        if (ret)
-                                                gf_msg (this->name,
-                                                        GF_LOG_ERROR, 0,
-                                                        DHT_MSG_DICT_SET_FAILED,
-                                                        "Failed to set "
-                                                        "dictionary value: "
-                                                        "key = %s",
-                                                        GF_READDIR_SKIP_DIRS);
-                                } else {
-                                        dict_del (local->xattr,
-                                                  GF_READDIR_SKIP_DIRS);
-                                }
-                        }
+    if (prev != dht_last_up_subvol(this))
+        op_errno = 0;
 
-                        if (conf->subvolume_cnt == 1) {
-                                ret = dict_set_uint32 (local->xattr,
-                                                       conf->xattr_name, 4 * 4);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_WARNING,
-                                                ENOMEM,
-                                                DHT_MSG_DICT_SET_FAILED,
-                                                "Failed to set dictionary "
-                                                "value:key = %s ",
-                                                conf->xattr_name);
-                                }
-                        }
+    if (!skip_hashed_check) {
+        DHT_STACK_UNWIND(readdir, frame, op_ret, op_errno, &entries, NULL);
+        gf_dirent_free(&entries);
 
+    } else {
+        DHT_STACK_UNWIND(readdir, frame, op_ret, op_errno, orig_entries, NULL);
+    }
+    return 0;
+}
 
+static int
+dht_do_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t yoff, int whichop, dict_t *dict)
+{
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    xlator_t *xvol = NULL;
+    int ret = 0;
+    dht_conf_t *conf = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, NULL, NULL, whichop);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->fd = fd_ref(fd);
+    local->size = size;
+    local->xattr_req = (dict) ? dict_ref(dict) : NULL;
+    local->first_up_subvol = dht_first_up_subvol(this);
+    local->op_ret = -1;
+
+    dht_deitransform(this, yoff, &xvol);
+
+    /* TODO: do proper readdir */
+    if (whichop == GF_FOP_READDIRP) {
+        if (dict)
+            local->xattr = dict_ref(dict);
+        else
+            local->xattr = dict_new();
+
+        if (local->xattr) {
+            ret = dict_set_uint32(local->xattr, conf->link_xattr_name, 256);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                       "Failed to set dictionary value"
+                       " : key = %s",
+                       conf->link_xattr_name);
+
+            if (conf->readdir_optimize == _gf_true) {
+                if (xvol != local->first_up_subvol) {
+                    ret = dict_set_int32(local->xattr, GF_READDIR_SKIP_DIRS, 1);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               DHT_MSG_DICT_SET_FAILED,
+                               "Failed to set "
+                               "dictionary value: "
+                               "key = %s",
+                               GF_READDIR_SKIP_DIRS);
+                } else {
+                    dict_del(local->xattr, GF_READDIR_SKIP_DIRS);
                 }
+            }
 
-                STACK_WIND_COOKIE (frame, dht_readdirp_cbk, xvol, xvol,
-                                   xvol->fops->readdirp, fd, size, yoff,
-                                   local->xattr);
-        } else {
-                STACK_WIND_COOKIE (frame, dht_readdir_cbk, xvol, xvol,
-                                   xvol->fops->readdir, fd, size, yoff,
-                                   local->xattr);
+            if (conf->subvolume_cnt == 1) {
+                ret = dict_set_uint32(local->xattr, conf->xattr_name, 4 * 4);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+                           DHT_MSG_DICT_SET_FAILED,
+                           "Failed to set dictionary "
+                           "value:key = %s ",
+                           conf->xattr_name);
+                }
+            }
         }
 
-        return 0;
+        STACK_WIND_COOKIE(frame, dht_readdirp_cbk, xvol, xvol,
+                          xvol->fops->readdirp, fd, size, yoff, local->xattr);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_readdir_cbk, xvol, xvol,
+                          xvol->fops->readdir, fd, size, yoff, local->xattr);
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(readdir, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t yoff, dict_t *xdata)
+dht_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t yoff, dict_t *xdata)
 {
-        int          op = GF_FOP_READDIR;
-        dht_conf_t  *conf = NULL;
-        int          i = 0;
+    int op = GF_FOP_READDIR;
+    dht_conf_t *conf = NULL;
+    int i = 0;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->subvolume_status[i]) {
-                        op = GF_FOP_READDIRP;
-                        break;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->subvolume_status[i]) {
+            op = GF_FOP_READDIRP;
+            break;
         }
+    }
 
-        if (conf->use_readdirp)
-                op = GF_FOP_READDIRP;
+    if (conf->use_readdirp)
+        op = GF_FOP_READDIRP;
 
 out:
-        dht_do_readdir (frame, this, fd, size, yoff, op, 0);
-        return 0;
+    dht_do_readdir(frame, this, fd, size, yoff, op, 0);
+    return 0;
 }
 
 int
-dht_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t yoff, dict_t *dict)
+dht_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t yoff, dict_t *dict)
 {
-        dht_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIRP, dict);
-        return 0;
+    dht_do_readdir(frame, this, fd, size, yoff, GF_FOP_READDIRP, dict);
+    return 0;
 }
 
-
-
-int
-dht_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
 
+    local = frame->local;
 
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1)
-                        local->op_errno = op_errno;
-
-                if (op_ret == 0)
-                        local->op_ret = 0;
-        }
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                  local->op_errno, xdata);
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1)
+            local->op_errno = op_errno;
+        else if (op_ret == 0)
+            local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_UNWIND(fsyncdir, frame, local->op_ret, local->op_errno,
+                         xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int datasync, dict_t *xdata)
+dht_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+             dict_t *xdata)
 {
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (this->private, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(this->private, err);
 
-        conf = this->private;
+    conf = this->private;
 
-        local = dht_local_init (frame, NULL, NULL, GF_FOP_FSYNCDIR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = dht_local_init(frame, NULL, NULL, GF_FOP_FSYNCDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        local->fd = fd_ref (fd);
-        local->call_cnt = conf->subvolume_cnt;
+    local->fd = fd_ref(fd);
+    local->call_cnt = conf->subvolume_cnt;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_fsyncdir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->fsyncdir,
-                            fd, datasync, xdata);
-        }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND(frame, dht_fsyncdir_cbk, conf->subvolumes[i],
+                   conf->subvolumes[i]->fops->fsyncdir, fd, datasync, xdata);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsyncdir, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno,
-                 inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
+dht_newfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, inode_t *inode, struct iatt *stbuf,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        xlator_t     *prev = NULL;
-        int           ret = -1;
-        dht_local_t  *local = NULL;
-
-
-        if (op_ret == -1)
-                goto out;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        prev = cookie;
-
-        if (local->loc.parent) {
-
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           preparent, 0);
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
+    if (op_ret == -1)
+        goto out;
 
-        ret = dht_layout_preset (this, prev, inode);
-        if (ret < 0) {
-                gf_msg_debug (this->name, EINVAL,
-                              "could not set pre-set layout for subvolume %s",
-                              prev? prev->name: NULL);
-                op_ret   = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-        if (local->linked == _gf_true)
-                dht_linkfile_attr_heal (frame, this);
+    local = frame->local;
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    prev = cookie;
+
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, preparent, 0);
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
+    ret = dht_layout_preset(this, prev, inode);
+    if (ret < 0) {
+        gf_msg_debug(this->name, EINVAL,
+                     "could not set pre-set layout for subvolume %s",
+                     prev ? prev->name : NULL);
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+    if (local->linked == _gf_true)
+        dht_linkfile_attr_heal(frame, this);
 out:
-        /*
-         * FIXME: ia_size and st_blocks of preparent and postparent do not have
-         * correct values. since, preparent and postparent buffers correspond
-         * to a directory these two members should have values equal to sum of
-         * corresponding values from each of the subvolume.
-         * See dht_iatt_merge for reference.
-         */
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        dht_set_fixed_dir_stat (postparent);
-        dht_set_fixed_dir_stat (preparent);
-
-        if (local && local->lock[0].layout.parent_layout.locks) {
-                /* store op_errno for failure case*/
-                local->op_errno = op_errno;
-                local->refresh_layout_unlock (frame, this, op_ret, 1);
+    /*
+     * FIXME: ia_size and st_blocks of preparent and postparent do not have
+     * correct values. since, preparent and postparent buffers correspond
+     * to a directory these two members should have values equal to sum of
+     * corresponding values from each of the subvolume.
+     * See dht_iatt_merge for reference.
+     */
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(postparent);
+    dht_set_fixed_dir_stat(preparent);
+
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        /* store op_errno for failure case*/
+        local->op_errno = op_errno;
+        local->refresh_layout_unlock(frame, this, op_ret, 1);
 
-                if (op_ret == 0) {
-                        DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno,
-                                          inode, stbuf, preparent, postparent,
-                                          xdata);
-                }
-        } else {
-                DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode,
-                                  stbuf, preparent, postparent, xdata);
+        if (op_ret == 0) {
+            DHT_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, stbuf,
+                             preparent, postparent, xdata);
         }
+    } else {
+        DHT_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, stbuf,
+                         preparent, postparent, xdata);
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_mknod_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this,
-                               int32_t op_ret, int32_t op_errno,
-                               inode_t *inode, struct iatt *stbuf,
-                               struct iatt *preparent, struct iatt *postparent,
-                               dict_t *xdata)
+static int
+dht_mknod_linkfile_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, inode_t *inode,
+                              struct iatt *stbuf, struct iatt *preparent,
+                              struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t     *local          = NULL;
-        xlator_t        *cached_subvol  = NULL;
-        dht_conf_t      *conf           = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (!local || !local->cached_subvol) {
-                op_errno = EINVAL;
-                goto err;
-        }
+    if (!local || !local->cached_subvol) {
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        if (op_ret == -1) {
-                local->op_errno = op_errno;
-                goto err;
-        }
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        goto err;
+    }
 
-        conf = this->private;
-        if (!conf) {
-                local->op_errno =  EINVAL;
-                op_errno = EINVAL;
-                goto err;
-        }
+    conf = this->private;
+    if (!conf) {
+        local->op_errno = EINVAL;
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        cached_subvol = local->cached_subvol;
+    cached_subvol = local->cached_subvol;
 
-        if (local->params) {
-                 dict_del (local->params, conf->link_xattr_name);
-                 dict_del (local->params, GLUSTERFS_INTERNAL_FOP_KEY);
-        }
+    if (local->params) {
+        dict_del(local->params, conf->link_xattr_name);
+        dict_del(local->params, GLUSTERFS_INTERNAL_FOP_KEY);
+    }
 
-        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)cached_subvol,
-                           cached_subvol, cached_subvol->fops->mknod,
-                           &local->loc, local->mode, local->rdev, local->umask,
-                           local->params);
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)cached_subvol,
+                      cached_subvol, cached_subvol->fops->mknod, &local->loc,
+                      local->mode, local->rdev, local->umask, local->params);
 
-        return 0;
+    return 0;
 err:
-        if (local && local->lock[0].layout.parent_layout.locks) {
-                local->refresh_layout_unlock (frame, this, -1, 1);
-        } else {
-                DHT_STACK_UNWIND (mknod, frame, -1,
-                                  op_errno, NULL, NULL, NULL,
-                                  NULL, NULL);
-        }
-        return 0;
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    } else {
+        DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                         NULL);
+    }
+    return 0;
 }
 
-int
-dht_mknod_wind_to_avail_subvol (call_frame_t *frame, xlator_t *this,
-                                 xlator_t *subvol, loc_t *loc, dev_t rdev,
-                                 mode_t mode, mode_t umask, dict_t *params)
+static int
+dht_mknod_wind_to_avail_subvol(call_frame_t *frame, xlator_t *this,
+                               xlator_t *subvol, loc_t *loc, dev_t rdev,
+                               mode_t mode, mode_t umask, dict_t *params)
 {
-        dht_local_t     *local          = NULL;
-        xlator_t        *avail_subvol   = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *avail_subvol = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (!dht_is_subvol_filled (this, subvol)) {
-                gf_msg_debug (this->name, 0,
-                              "creating %s on %s", loc->path,
-                              subvol->name);
+    if (!dht_is_subvol_filled(this, subvol)) {
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
 
-                STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol,
-                                   subvol, subvol->fops->mknod, loc, mode,
-                                   rdev, umask, params);
-        } else {
-                avail_subvol = dht_free_disk_available_subvol (this, subvol, local);
-
-                if (avail_subvol != subvol) {
-                        local->params = dict_ref (params);
-                        local->rdev = rdev;
-                        local->mode = mode;
-                        local->umask = umask;
-                        local->cached_subvol = avail_subvol;
-                        local->hashed_subvol = subvol;
+        STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                          subvol->fops->mknod, loc, mode, rdev, umask, params);
+    } else {
+        avail_subvol = dht_free_disk_available_subvol(this, subvol, local);
 
-                        gf_msg_debug (this->name, 0,
-                                      "creating %s on %s (link at %s)", loc->path,
-                                      avail_subvol->name, subvol->name);
+        if (avail_subvol != subvol) {
+            local->params = dict_ref(params);
+            local->rdev = rdev;
+            local->mode = mode;
+            local->umask = umask;
+            local->cached_subvol = avail_subvol;
+            local->hashed_subvol = subvol;
 
-                        dht_linkfile_create (frame,
-                                             dht_mknod_linkfile_create_cbk,
-                                             this, avail_subvol, subvol, loc);
+            gf_msg_debug(this->name, 0, "creating %s on %s (link at %s)",
+                         loc->path, avail_subvol->name, subvol->name);
 
-                        goto out;
-                }
+            dht_linkfile_create(frame, dht_mknod_linkfile_create_cbk, this,
+                                avail_subvol, subvol, loc);
 
-                gf_msg_debug (this->name, 0,
-                              "creating %s on %s", loc->path, subvol->name);
+            goto out;
+        }
 
-                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
-                                   (void *)subvol, subvol,
-                                   subvol->fops->mknod, loc, mode,
-                                   rdev, umask, params);
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
 
-        }
+        STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                          subvol->fops->mknod, loc, mode, rdev, umask, params);
+    }
 out:
-        return 0;
+    return 0;
 }
 
-int32_t
-dht_mknod_do (call_frame_t *frame)
+static int32_t
+dht_mknod_do(call_frame_t *frame)
 {
-        dht_local_t     *local          = NULL;
-        dht_layout_t    *refreshed      = NULL;
-        xlator_t        *subvol         = NULL;
-        xlator_t        *this           = NULL;
-        dht_conf_t      *conf           = NULL;
-        dht_methods_t   *methods        = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *refreshed = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        this = THIS;
+    this = THIS;
 
-        conf = this->private;
+    conf = this->private;
 
-        GF_VALIDATE_OR_GOTO (this->name, conf, err);
+    GF_VALIDATE_OR_GOTO(this->name, conf, err);
 
-        methods = &(conf->methods);
+    methods = &(conf->methods);
 
-        /* We don't need parent_loc anymore */
-        loc_wipe (&local->loc);
+    /* We don't need parent_loc anymore */
+    loc_wipe(&local->loc);
 
-        loc_copy (&local->loc, &local->loc2);
+    loc_copy(&local->loc, &local->loc2);
 
-        loc_wipe (&local->loc2);
+    loc_wipe(&local->loc2);
 
-        refreshed = local->selfheal.refreshed_layout;
+    refreshed = local->selfheal.refreshed_layout;
 
-        subvol = methods->layout_search (this, refreshed, local->loc.name);
+    subvol = methods->layout_search(this, refreshed, local->loc.name);
 
-        if (!subvol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_HASHED_SUBVOL_GET_FAILED, "no subvolume in "
-                        "layout for path=%s", local->loc.path);
-                local->op_errno = ENOENT;
-                goto err;
-        }
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "no subvolume in "
+               "layout for path=%s",
+               local->loc.path);
+        local->op_errno = ENOENT;
+        goto err;
+    }
 
-        dht_mknod_wind_to_avail_subvol (frame, this, subvol, &local->loc,
-                                         local->rdev, local->mode,
-                                         local->umask, local->params);
-        return 0;
+    dht_mknod_wind_to_avail_subvol(frame, this, subvol, &local->loc,
+                                   local->rdev, local->mode, local->umask,
+                                   local->params);
+    return 0;
 err:
-        local->refresh_layout_unlock (frame, this, -1, 1);
+    local->refresh_layout_unlock(frame, this, -1, 1);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-dht_mknod_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int32_t
+dht_mknod_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
-int32_t
-dht_mknod_finish (call_frame_t *frame, xlator_t *this, int op_ret,
-                  int invoke_cbk)
+static int32_t
+dht_mknod_finish(call_frame_t *frame, xlator_t *this, int op_ret,
+                 int invoke_cbk)
 {
-        dht_local_t  *local      = NULL, *lock_local = NULL;
-        call_frame_t *lock_frame = NULL;
-        int           lock_count = 0;
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
 
-        local = frame->local;
-        lock_count = dht_lock_count (local->lock[0].layout.parent_layout.locks,
-                                     local->lock[0].layout.parent_layout.lk_count);
-        if (lock_count == 0)
-                goto done;
+    local = frame->local;
+    lock_count = dht_lock_count(local->lock[0].layout.parent_layout.locks,
+                                local->lock[0].layout.parent_layout.lk_count);
+    if (lock_count == 0)
+        goto done;
 
-        lock_frame = copy_frame (frame);
-        if (lock_frame == NULL) {
-                goto done;
-        }
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        goto done;
+    }
 
-        lock_local = dht_local_init (lock_frame, &local->loc, NULL,
-                                     lock_frame->root->op);
-        if (lock_local == NULL) {
-                goto done;
-        }
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL) {
+        goto done;
+    }
 
-        lock_local->lock[0].layout.parent_layout.locks = local->lock[0].layout.parent_layout.locks;
-        lock_local->lock[0].layout.parent_layout.lk_count = local->lock[0].layout.parent_layout.lk_count;
+    lock_local->lock[0]
+        .layout.parent_layout.locks = local->lock[0].layout.parent_layout.locks;
+    lock_local->lock[0].layout.parent_layout.lk_count =
+        local->lock[0].layout.parent_layout.lk_count;
 
-        local->lock[0].layout.parent_layout.locks = NULL;
-        local->lock[0].layout.parent_layout.lk_count = 0;
+    local->lock[0].layout.parent_layout.locks = NULL;
+    local->lock[0].layout.parent_layout.lk_count = 0;
 
-        dht_unlock_inodelk (lock_frame,
-                            lock_local->lock[0].layout.parent_layout.locks,
-                            lock_local->lock[0].layout.parent_layout.lk_count,
-                            dht_mknod_unlock_cbk);
-        lock_frame = NULL;
+    dht_unlock_inodelk(lock_frame,
+                       lock_local->lock[0].layout.parent_layout.locks,
+                       lock_local->lock[0].layout.parent_layout.lk_count,
+                       dht_mknod_unlock_cbk);
+    lock_frame = NULL;
 
 done:
-        if (lock_frame != NULL) {
-                DHT_STACK_DESTROY (lock_frame);
-        }
-
-        if (op_ret == 0)
-                return 0;
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
 
-        DHT_STACK_UNWIND (mknod, frame, op_ret, local->op_errno, NULL, NULL,
-                          NULL, NULL, NULL);
+    if (op_ret == 0)
         return 0;
+
+    DHT_STACK_UNWIND(mknod, frame, op_ret, local->op_errno, NULL, NULL, NULL,
+                     NULL, NULL);
+    return 0;
 }
 
-int32_t
-dht_mknod_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int32_t
+dht_mknod_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t     *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (!local) {
-                goto err;
-        }
+    if (!local) {
+        goto err;
+    }
 
-        if (op_ret < 0) {
-                gf_msg ("DHT", GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
-                        "mknod lock failed for file: %s", local->loc2.name);
+    if (op_ret < 0) {
+        gf_msg("DHT", GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+               "mknod lock failed for file: %s", local->loc2.name);
 
-                local->op_errno = op_errno;
+        local->op_errno = op_errno;
 
-                goto err;
-        }
+        goto err;
+    }
 
-        local->refresh_layout_unlock = dht_mknod_finish;
+    local->refresh_layout_unlock = dht_mknod_finish;
 
-        local->refresh_layout_done = dht_mknod_do;
+    local->refresh_layout_done = dht_mknod_do;
 
-        dht_refresh_layout (frame);
+    dht_refresh_layout(frame);
 
-        return 0;
+    return 0;
 err:
-        dht_mknod_finish (frame, this, -1, 0);
-        return 0;
+    if (local)
+        dht_mknod_finish(frame, this, -1, 0);
+    else
+        DHT_STACK_UNWIND(mknod, frame, -1, EINVAL, NULL, NULL, NULL, NULL,
+                         NULL);
+    return 0;
 }
 
-int32_t
-dht_mknod_lock (call_frame_t *frame, xlator_t *subvol)
+static int32_t
+dht_mknod_lock(call_frame_t *frame, xlator_t *subvol)
 {
-        dht_local_t     *local          = NULL;
-        int              count  = 1,    ret = -1;
-        dht_lock_t     **lk_array       = NULL;
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1;
+    dht_lock_t **lk_array = NULL;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO (frame->this->name, frame->local, err);
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
 
-        local = frame->local;
+    local = frame->local;
 
-        lk_array = GF_CALLOC (count, sizeof (*lk_array), gf_common_mt_pointer);
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
 
-        if (lk_array == NULL)
-                goto err;
+    if (lk_array == NULL)
+        goto err;
 
-        lk_array[0] = dht_lock_new (frame->this, subvol, &local->loc, F_RDLCK,
-                                    DHT_LAYOUT_HEAL_DOMAIN, NULL,
-                                    IGNORE_ENOENT_ESTALE);
+    lk_array[0] = dht_lock_new(frame->this, subvol, &local->loc, F_RDLCK,
+                               DHT_LAYOUT_HEAL_DOMAIN, NULL,
+                               IGNORE_ENOENT_ESTALE);
 
-        if (lk_array[0] == NULL)
-                goto err;
+    if (lk_array[0] == NULL)
+        goto err;
 
-        local->lock[0].layout.parent_layout.locks = lk_array;
-        local->lock[0].layout.parent_layout.lk_count = count;
+    local->lock[0].layout.parent_layout.locks = lk_array;
+    local->lock[0].layout.parent_layout.lk_count = count;
 
-        ret = dht_blocking_inodelk (frame, lk_array, count, dht_mknod_lock_cbk);
+    ret = dht_blocking_inodelk(frame, lk_array, count, dht_mknod_lock_cbk);
 
-        if (ret < 0) {
-                local->lock[0].layout.parent_layout.locks = NULL;
-                local->lock[0].layout.parent_layout.lk_count = 0;
-                goto err;
-        }
+    if (ret < 0) {
+        local->lock[0].layout.parent_layout.locks = NULL;
+        local->lock[0].layout.parent_layout.lk_count = 0;
+        goto err;
+    }
 
-        return 0;
+    return 0;
 err:
-        if (lk_array != NULL) {
-                dht_lock_array_free (lk_array, count);
-                GF_FREE (lk_array);
-        }
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
 
-        return -1;
+    return -1;
 }
 
-int
-dht_refresh_parent_layout_resume (call_frame_t *frame, xlator_t *this, int ret,
-                                  int invoke_cbk)
+static int
+dht_refresh_parent_layout_resume(call_frame_t *frame, xlator_t *this, int ret,
+                                 int invoke_cbk)
 {
-        dht_local_t  *local        = NULL, *parent_local = NULL;
-        call_stub_t  *stub         = NULL;
-        call_frame_t *parent_frame = NULL;
+    dht_local_t *local = NULL, *parent_local = NULL;
+    call_stub_t *stub = NULL;
+    call_frame_t *parent_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        stub = local->stub;
-        local->stub = NULL;
+    stub = local->stub;
+    local->stub = NULL;
 
-        parent_frame = stub->frame;
-        parent_local = parent_frame->local;
+    parent_frame = stub->frame;
+    parent_local = parent_frame->local;
 
-        if (ret < 0) {
-                parent_local->op_ret = -1;
-                parent_local->op_errno = local->op_errno
-                        ? local->op_errno : EIO;
-        } else {
-                parent_local->op_ret = 0;
-        }
+    if (ret < 0) {
+        parent_local->op_ret = -1;
+        parent_local->op_errno = local->op_errno ? local->op_errno : EIO;
+    } else {
+        parent_local->op_ret = 0;
+    }
 
-        call_resume (stub);
+    call_resume(stub);
 
-        DHT_STACK_DESTROY (frame);
+    DHT_STACK_DESTROY(frame);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_refresh_parent_layout_done (call_frame_t *frame)
+static int
+dht_refresh_parent_layout_done(call_frame_t *frame)
 {
-        dht_local_t *local = NULL;
-        int          ret   = 0;
+    dht_local_t *local = NULL;
+    int ret = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret < 0) {
-                ret = -1;
-                goto resume;
-        }
+    if (local->op_ret < 0) {
+        ret = -1;
+        goto resume;
+    }
 
-        dht_layout_set (frame->this, local->loc.inode,
-                        local->selfheal.refreshed_layout);
+    dht_layout_set(frame->this, local->loc.inode,
+                   local->selfheal.refreshed_layout);
 
 resume:
-        dht_refresh_parent_layout_resume (frame, frame->this, ret, 1);
-        return 0;
+    dht_refresh_parent_layout_resume(frame, frame->this, ret, 1);
+    return 0;
 }
 
-
-int
-dht_handle_parent_layout_change (xlator_t *this, call_stub_t *stub)
+static int
+dht_handle_parent_layout_change(xlator_t *this, call_stub_t *stub)
 {
-        call_frame_t *refresh_frame = NULL, *frame = NULL;
-        dht_local_t  *refresh_local = NULL, *local = NULL;
+    call_frame_t *refresh_frame = NULL, *frame = NULL;
+    dht_local_t *refresh_local = NULL, *local = NULL;
 
-        frame = stub->frame;
-        local = frame->local;
+    frame = stub->frame;
+    local = frame->local;
 
-        refresh_frame = copy_frame (frame);
-        refresh_local = dht_local_init (refresh_frame, NULL, NULL,
-                                        stub->fop);
+    refresh_frame = copy_frame(frame);
+    if (!refresh_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "mem allocation failed for refresh_frame");
+        return -1;
+    }
 
-        refresh_local->loc.inode = inode_ref (local->loc.parent);
-        gf_uuid_copy (refresh_local->loc.gfid, local->loc.parent->gfid);
+    refresh_local = dht_local_init(refresh_frame, NULL, NULL, stub->fop);
+    if (!refresh_local) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "mem allocation failed for refresh_local");
+        return -1;
+    }
 
-        refresh_local->stub = stub;
+    refresh_local->loc.inode = inode_ref(local->loc.parent);
+    gf_uuid_copy(refresh_local->loc.gfid, local->loc.parent->gfid);
 
-        refresh_local->refresh_layout_unlock = dht_refresh_parent_layout_resume;
-        refresh_local->refresh_layout_done = dht_refresh_parent_layout_done;
+    refresh_local->stub = stub;
 
-        dht_refresh_layout (refresh_frame);
-        return 0;
+    refresh_local->refresh_layout_unlock = dht_refresh_parent_layout_resume;
+    refresh_local->refresh_layout_done = dht_refresh_parent_layout_done;
+
+    dht_refresh_layout(refresh_frame);
+    return 0;
 }
 
-int32_t
-dht_call_mkdir_stub (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int32_t
+dht_call_mkdir_stub(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        call_stub_t *stub  = NULL;
+    dht_local_t *local = NULL;
+    call_stub_t *stub = NULL;
 
-        local = frame->local;
-        stub = local->stub;
-        local->stub = NULL;
+    local = frame->local;
+    stub = local->stub;
+    local->stub = NULL;
 
-        if (op_ret < 0) {
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-        } else {
-                local->op_ret = 0;
-        }
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    } else {
+        local->op_ret = 0;
+    }
 
-        call_resume (stub);
+    call_resume(stub);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-dht_guard_parent_layout_and_namespace (xlator_t *subvol, call_stub_t *stub)
+static int32_t
+dht_guard_parent_layout_and_namespace(xlator_t *subvol, call_stub_t *stub)
 {
-        dht_local_t   *local                  = NULL;
-        int            ret                    = -1;
-        loc_t         *loc                    = NULL;
-        xlator_t      *hashed_subvol          = NULL, *this = NULL;;
-        call_frame_t  *frame                  = NULL;
-        char          pgfid[GF_UUID_BUF_SIZE] = {0};
-        int32_t       *parent_disk_layout     = NULL;
-        dht_layout_t  *parent_layout          = NULL;
-        dht_conf_t    *conf                   = NULL;
+    dht_local_t *local = NULL;
+    int ret = -1;
+    loc_t *loc = NULL;
+    xlator_t *hashed_subvol = NULL, *this = NULL;
+    ;
+    call_frame_t *frame = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t *parent_disk_layout = NULL;
+    dht_layout_t *parent_layout = NULL;
+    dht_conf_t *conf = NULL;
 
-        GF_VALIDATE_OR_GOTO ("dht", stub, err);
+    GF_VALIDATE_OR_GOTO("dht", stub, err);
 
-        frame = stub->frame;
-        this = frame->this;
+    frame = stub->frame;
+    this = frame->this;
 
-        conf = this->private;
+    conf = this->private;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->stub = stub;
+    local->stub = stub;
 
-        /* TODO: recheck whether we should lock on src or dst if we do similar
-         * stale layout checks for rename.
-         */
-        loc = &stub->args.loc;
+    /* TODO: recheck whether we should lock on src or dst if we do similar
+     * stale layout checks for rename.
+     */
+    loc = &stub->args.loc;
 
-        gf_uuid_unparse (loc->parent->gfid, pgfid);
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
 
+    if (local->params == NULL) {
+        local->params = dict_new();
         if (local->params == NULL) {
-                local->params = dict_new ();
-                if (local->params == NULL) {
-                        local->op_errno = ENOMEM;
-                        gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                                DHT_MSG_PARENT_LAYOUT_CHANGED,
-                                "%s (%s/%s) (path: %s): "
-                                "dict allocation failed",
-                                gf_fop_list[stub->fop],
-                                pgfid, loc->name, loc->path);
-                        goto err;
-                }
-        }
-
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        if (hashed_subvol == NULL) {
-                local->op_errno = EINVAL;
-
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "%s (%s/%s) (path: %s): "
-                        "hashed subvolume not found", gf_fop_list[stub->fop],
-                        pgfid, loc->name, loc->path);
-                goto err;
-        }
-
-        parent_layout = dht_layout_get (this, loc->parent);
-
-        ret = dht_disk_layout_extract_for_subvol (this, parent_layout,
-                                                  hashed_subvol,
-                                                  &parent_disk_layout);
-        if (ret == -1) {
-                local->op_errno = EINVAL;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "%s (%s/%s) (path: %s): "
-                        "extracting in-memory layout of parent failed. ",
-                        gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
-                goto err;
-        }
-
-        memcpy ((void *)local->parent_disk_layout, (void *)parent_disk_layout,
-                sizeof (local->parent_disk_layout));
-
-        dht_layout_unref (this, parent_layout);
-        parent_layout = NULL;
-
-        ret = dict_set_str (local->params, GF_PREOP_PARENT_KEY,
-                            conf->xattr_name);
-        if (ret < 0) {
-                local->op_errno = -ret;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "%s (%s/%s) (path: %s): "
-                        "setting %s key in params dictionary failed. ",
-                        gf_fop_list[stub->fop], pgfid, loc->name, loc->path,
-                        GF_PREOP_PARENT_KEY);
-                goto err;
-        }
-
-        ret = dict_set_bin (local->params, conf->xattr_name, parent_disk_layout,
-                            4 * 4);
-        if (ret < 0) {
-                local->op_errno = -ret;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "%s (%s/%s) (path: %s): "
-                        "setting parent-layout in params dictionary failed. ",
-                        gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
-                goto err;
-        }
-
-        parent_disk_layout = NULL;
-        local->hashed_subvol = hashed_subvol;
-
-        local->current = &local->lock[0];
-        ret = dht_protect_namespace (frame, loc, hashed_subvol,
-                                     &local->current->ns, dht_call_mkdir_stub);
-        if (ret < 0)
-                goto err;
-
-        return 0;
+            local->op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+                   DHT_MSG_PARENT_LAYOUT_CHANGED,
+                   "%s (%s/%s) (path: %s): "
+                   "dict allocation failed",
+                   gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+            goto err;
+        }
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        local->op_errno = EINVAL;
+
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "hashed subvolume not found",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    parent_layout = dht_layout_get(this, loc->parent);
+
+    ret = dht_disk_layout_extract_for_subvol(this, parent_layout, hashed_subvol,
+                                             &parent_disk_layout);
+    if (ret == -1) {
+        local->op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "extracting in-memory layout of parent failed. ",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    memcpy((void *)local->parent_disk_layout, (void *)parent_disk_layout,
+           sizeof(local->parent_disk_layout));
+
+    dht_layout_unref(this, parent_layout);
+    parent_layout = NULL;
+
+    ret = dict_set_str(local->params, GF_PREOP_PARENT_KEY, conf->xattr_name);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting %s key in params dictionary failed. ",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path,
+               GF_PREOP_PARENT_KEY);
+        goto err;
+    }
+
+    ret = dict_set_bin(local->params, conf->xattr_name, parent_disk_layout,
+                       4 * 4);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting parent-layout in params dictionary failed. ",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    parent_disk_layout = NULL;
+    local->hashed_subvol = hashed_subvol;
+
+    local->current = &local->lock[0];
+    ret = dht_protect_namespace(frame, loc, hashed_subvol, &local->current->ns,
+                                dht_call_mkdir_stub);
+    if (ret < 0)
+        goto err;
+
+    return 0;
 err:
 
-        if (parent_disk_layout != NULL)
-                GF_FREE (parent_disk_layout);
+    if (parent_disk_layout != NULL)
+        GF_FREE(parent_disk_layout);
 
-        if (parent_layout != NULL)
-                dht_layout_unref (this, parent_layout);
+    if (parent_layout != NULL)
+        dht_layout_unref(this, parent_layout);
 
-        return -1;
+    return -1;
 }
 
 int
-dht_mknod (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
-{
-        xlator_t       *subvol     = NULL;
-        int             op_errno   = -1;
-        int             i          = 0;
-        int             ret        = 0;
-        dht_local_t    *local      = NULL;
-        dht_conf_t     *conf       = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = EIO;
-                goto err;
-        }
-
-        /* Post remove-brick, the client layout may not be in sync with
-        * disk layout because of lack of lookup. Hence,a mknod call
-        * may fall on the decommissioned brick.  Hence, if the
-        * hashed_subvol is part of decommissioned bricks  list, do a
-        * lookup on parent dir. If a fix-layout is already done by the
-        * remove-brick process, the parent directory layout will be in
-        * sync with that of the disk. If fix-layout is still ending
-        * on the parent directory, we can let the file get created on
-        * the decommissioned brick which will be eventually migrated to
-        * non-decommissioned brick based on the new layout.
-        */
+dht_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *params)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    int i = 0;
+    int ret = 0;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKNOD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = EIO;
+        goto err;
+    }
+
+    /* Post remove-brick, the client layout may not be in sync with
+     * disk layout because of lack of lookup. Hence,a mknod call
+     * may fall on the decommissioned brick.  Hence, if the
+     * hashed_subvol is part of decommissioned bricks  list, do a
+     * lookup on parent dir. If a fix-layout is already done by the
+     * remove-brick process, the parent directory layout will be in
+     * sync with that of the disk. If fix-layout is still ending
+     * on the parent directory, we can let the file get created on
+     * the decommissioned brick which will be eventually migrated to
+     * non-decommissioned brick based on the new layout.
+     */
+
+    if (conf->decommission_subvols_cnt) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->decommissioned_bricks[i] &&
+                conf->decommissioned_bricks[i] == subvol) {
+                gf_msg_debug(this->name, 0,
+                             "hashed subvol:%s is "
+                             "part of decommission brick list for "
+                             "file: %s",
+                             subvol->name, loc->path);
+
+                /* dht_refresh_layout needs directory info in
+                 * local->loc. Hence, storing the parent_loc in
+                 * local->loc and storing the create context in
+                 * local->loc2. We will restore this information
+                 * in dht_creation do */
+
+                ret = loc_copy(&local->loc2, &local->loc);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                           "loc_copy failed %s", loc->path);
 
-        if (conf->decommission_subvols_cnt) {
-            for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->decommissioned_bricks[i] &&
-                        conf->decommissioned_bricks[i] == subvol) {
-
-                        gf_msg_debug (this->name, 0, "hashed subvol:%s is "
-                                      "part of decommission brick list for "
-                                      "file: %s", subvol->name, loc->path);
-
-                        /* dht_refresh_layout needs directory info in
-                         * local->loc. Hence, storing the parent_loc in
-                         * local->loc and storing the create context in
-                         * local->loc2. We will restore this information
-                         * in dht_creation do */
-
-                        ret = loc_copy (&local->loc2, &local->loc);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                        DHT_MSG_NO_MEMORY,
-                                        "loc_copy failed %s", loc->path);
-
-                                goto err;
-                        }
+                    goto err;
+                }
 
-                        local->params = dict_ref (params);
-                        local->rdev = rdev;
-                        local->mode = mode;
-                        local->umask = umask;
+                local->params = dict_ref(params);
+                local->rdev = rdev;
+                local->mode = mode;
+                local->umask = umask;
 
-                        loc_wipe (&local->loc);
+                loc_wipe(&local->loc);
 
-                        ret = dht_build_parent_loc (this, &local->loc, loc,
-                                                                 &op_errno);
+                ret = dht_build_parent_loc(this, &local->loc, loc, &op_errno);
 
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                        DHT_MSG_LOC_FAILED,
-                                        "parent loc build failed");
-                                goto err;
-                        }
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_LOC_FAILED,
+                           "parent loc build failed");
+                    goto err;
+                }
 
-                        ret = dht_mknod_lock (frame, subvol);
+                ret = dht_mknod_lock(frame, subvol);
 
-                        if (ret < 0) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_INODE_LK_ERROR,
-                                        "locking parent failed");
-                                goto err;
-                        }
+                if (ret < 0) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+                           "locking parent failed");
+                    goto err;
+                }
 
-                        goto done;
-               }
+                goto done;
             }
         }
+    }
 
-        dht_mknod_wind_to_avail_subvol (frame, this, subvol, loc, rdev, mode,
-                                        umask, params);
+    dht_mknod_wind_to_avail_subvol(frame, this, subvol, loc, rdev, mode, umask,
+                                   params);
 
 done:
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_symlink (call_frame_t *frame, xlator_t *this,
-             const char *linkname, loc_t *loc, mode_t umask, dict_t *params)
+dht_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+            loc_t *loc, mode_t umask, dict_t *params)
 {
-        xlator_t    *subvol = NULL;
-        int          op_errno = -1;
-        dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_SYMLINK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = dht_local_init(frame, loc, NULL, GF_FOP_SYMLINK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = EIO;
-                goto err;
-        }
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = EIO;
+        goto err;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
-                           subvol->fops->symlink, linkname, loc, umask,
-                           params);
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                      subvol->fops->symlink, linkname, loc, umask, params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (link, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-            dict_t *xdata)
+dht_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata)
 {
-        xlator_t    *cached_subvol = NULL;
-        int          op_errno = -1;
-        dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_UNLINK);
-        if (!local) {
-                op_errno = ENOMEM;
+    local = dht_local_init(frame, loc, NULL, GF_FOP_UNLINK);
+    if (!local) {
+        op_errno = ENOMEM;
 
-                goto err;
-        }
+        goto err;
+    }
 
-        cached_subvol = local->cached_subvol;
-        if (!cached_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    cached_subvol = local->cached_subvol;
+    if (!cached_subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        local->flags = xflag;
-        STACK_WIND_COOKIE (frame, dht_unlink_cbk, cached_subvol, cached_subvol,
-                           cached_subvol->fops->unlink, loc, xflag, xdata);
+    local->flags = xflag;
+    STACK_WIND_COOKIE(frame, dht_unlink_cbk, cached_subvol, cached_subvol,
+                      cached_subvol->fops->unlink, loc, xflag, xdata);
 
-        return 0;
+    return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(unlink, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno,
-              inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        int           ret = -1;
-        gf_boolean_t  stbuf_merged = _gf_false;
-        xlator_t     *subvol = NULL;
-        call_frame_t *cleanup_frame = NULL;
-        dht_local_t  *cleanup_local = NULL;
+static int
+dht_remove_stale_linkto_cbk(int ret, call_frame_t *sync_frame, void *data)
+{
+    DHT_STACK_DESTROY(sync_frame);
+    return 0;
+}
 
-        local = frame->local;
+static int
+dht_remove_stale_linkto(void *data)
+{
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata_in = NULL;
+    int ret = 0;
 
-        if (op_ret == -1) {
-                /* Remove the linkto if exists */
-                if (local->linked) {
-                        cleanup_frame = create_frame (this, this->ctx->pool);
-                        if (cleanup_frame) {
-                                cleanup_local = dht_local_init (cleanup_frame,
-                                                                &local->loc2,
-                                                                NULL, 0);
-                                if (!cleanup_local || !local->link_subvol) {
-                                        DHT_STACK_DESTROY (cleanup_frame);
-                                        goto out;
-                                }
-                                cleanup_local->link_subvol = local->link_subvol;
-                                FRAME_SU_DO (cleanup_frame, dht_local_t);
-                                ret = synctask_new (this->ctx->env,
-                                                    dht_remove_stale_linkto,
-                                                    dht_remove_stale_linkto_cbk,
-                                                    cleanup_frame,
-                                                    cleanup_frame);
-                        }
-                }
-                /* No continuation on DHT inode missing errors, as we should
-                 * then have a good stbuf that states P2 happened. We would
-                 * get inode missing if, the file completed migrated between
-                 * the lookup and the link call */
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("dht", data, out);
 
-        /* Update parent on success, even if P1/2 checks are positve.
-         * The second call on success will further update the parent */
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           preparent, 0);
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
-
-        /* Update linkto attrs, if this is the first call and non-P2,
-         * if we detect P2 then we need to trust the attrs from the
-         * second call, not the first */
-        if (local->linked == _gf_true &&
-            ((local->call_cnt == 1 && !IS_DHT_MIGRATION_PHASE2 (stbuf))
-             || (local->call_cnt != 1 &&
-                 IS_DHT_MIGRATION_PHASE2 (&local->stbuf)))) {
-                dht_iatt_merge (this, &local->stbuf, stbuf, NULL);
-                stbuf_merged = _gf_true;
-                dht_linkfile_attr_heal (frame, this);
-        }
-
-        /* No further P1/2 checks if we are in the second iteration of
-         * the call */
-        if (local->call_cnt != 1) {
-                goto out;
-        } else {
-                /* Preserve the return values, in case the migration decides
-                 * to recreate the link on the same subvol that the current
-                 * hased for the link was created on. */
-                dht_iatt_merge (this, &local->preparent,
-                                preparent, NULL);
-                dht_iatt_merge (this, &local->postparent,
-                                postparent, NULL);
-                if (!stbuf_merged) {
-                        dht_iatt_merge (this, &local->stbuf,
-                                        stbuf, NULL);
-                        stbuf_merged = _gf_true;
-                }
+    frame = data;
+    local = frame->local;
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", local, out);
+    GF_VALIDATE_OR_GOTO("dht", local->link_subvol, out);
 
-                local->inode = inode_ref (inode);
-        }
+    xdata_in = dict_new();
+    if (!xdata_in)
+        goto out;
 
-        local->op_ret = op_ret;
-        local->op_errno = op_errno;
-        local->rebalance.target_op_fn = dht_link2;
-        dht_set_local_rebalance (this, local, stbuf, preparent,
-                                 postparent, xdata);
-
-        /* Check if the rebalance phase2 is true */
-        if (IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                ret = dht_inode_ctx_get_mig_info (this, local->loc.inode, NULL,
-                                                  &subvol);
-                if (!subvol) {
-                        /* Phase 2 of migration */
-                        ret = dht_rebalance_complete_check (this, frame);
-                        if (!ret)
-                                return 0;
-                } else {
-                        dht_link2 (this, subvol, frame, 0);
-                        return 0;
-                }
-        }
+    ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(xdata_in);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, 0,
+               "Failed to set keys for stale linkto"
+               "deletion on path %s",
+               local->loc.path);
+        goto out;
+    }
+
+    ret = syncop_unlink(local->link_subvol, &local->loc, xdata_in, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, 0,
+               "Removal of linkto failed"
+               " on path %s at subvol %s",
+               local->loc.path, local->link_subvol->name);
+    }
+out:
+    if (xdata_in)
+        dict_unref(xdata_in);
+    return ret;
+}
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (stbuf)) {
-                ret = dht_inode_ctx_get_mig_info (this, local->loc.inode, NULL,
-                                                  &subvol);
-                if (subvol) {
-                        dht_link2 (this, subvol, frame, 0);
-                        return 0;
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+static int
+dht_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, inode_t *inode, struct iatt *stbuf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    gf_boolean_t stbuf_merged = _gf_false;
+    xlator_t *subvol = NULL;
+    call_frame_t *cleanup_frame = NULL;
+    dht_local_t *cleanup_local = NULL;
+
+    local = frame->local;
+
+    if (op_ret == -1) {
+        /* Remove the linkto if exists */
+        if (local->linked) {
+            cleanup_frame = create_frame(this, this->ctx->pool);
+            if (cleanup_frame) {
+                cleanup_local = dht_local_init(cleanup_frame, &local->loc2,
+                                               NULL, 0);
+                if (!cleanup_local || !local->link_subvol) {
+                    DHT_STACK_DESTROY(cleanup_frame);
+                    goto out;
+                }
+                cleanup_local->link_subvol = local->link_subvol;
+                FRAME_SU_DO(cleanup_frame, dht_local_t);
+                ret = synctask_new(this->ctx->env, dht_remove_stale_linkto,
+                                   dht_remove_stale_linkto_cbk, cleanup_frame,
+                                   cleanup_frame);
+            }
         }
+        /* No continuation on DHT inode missing errors, as we should
+         * then have a good stbuf that states P2 happened. We would
+         * get inode missing if, the file completed migrated between
+         * the lookup and the link call */
+        goto out;
+    }
+
+    /* Update parent on success, even if P1/2 checks are positive.
+     * The second call on success will further update the parent */
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, preparent, 0);
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
+    /* Update linkto attrs, if this is the first call and non-P2,
+     * if we detect P2 then we need to trust the attrs from the
+     * second call, not the first */
+    if (local->linked == _gf_true &&
+        ((local->call_cnt == 1 && !IS_DHT_MIGRATION_PHASE2(stbuf)) ||
+         (local->call_cnt != 1 && IS_DHT_MIGRATION_PHASE2(&local->stbuf)))) {
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        stbuf_merged = _gf_true;
+        dht_linkfile_attr_heal(frame, this);
+    }
+
+    /* No further P1/2 checks if we are in the second iteration of
+     * the call */
+    if (local->call_cnt != 1) {
+        goto out;
+    } else {
+        /* Preserve the return values, in case the migration decides
+         * to recreate the link on the same subvol that the current
+         * hased for the link was created on. */
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+        if (!stbuf_merged) {
+            dht_iatt_merge(this, &local->stbuf, stbuf);
+            stbuf_merged = _gf_true;
+        }
+
+        local->inode = inode_ref(inode);
+    }
+
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+    local->rebalance.target_op_fn = dht_link2;
+    dht_set_local_rebalance(this, local, stbuf, preparent, postparent, xdata);
+
+    /* Check if the rebalance phase2 is true */
+    if (IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        ret = dht_inode_ctx_get_mig_info(this, local->loc.inode, NULL, &subvol);
+        if (!subvol) {
+            /* Phase 2 of migration */
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            dht_link2(this, subvol, frame, 0);
+            return 0;
+        }
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(stbuf)) {
+        ret = dht_inode_ctx_get_mig_info(this, local->loc.inode, NULL, &subvol);
+        if (subvol) {
+            dht_link2(this, subvol, frame, 0);
+            return 0;
+        }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
 
-        dht_set_fixed_dir_stat (preparent);
-        dht_set_fixed_dir_stat (postparent);
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf,
-                          preparent, postparent, NULL);
+    dht_set_fixed_dir_stat(preparent);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_link2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_link2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local  = NULL;
-        int          op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        local = frame->local;
-        if (!local)
-                goto err;
+    local = frame->local;
+    if (!local)
+        goto err;
 
-        op_errno = local->op_errno;
+    op_errno = local->op_errno;
 
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                dht_set_fixed_dir_stat (&local->preparent);
-                dht_set_fixed_dir_stat (&local->postparent);
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        dht_set_fixed_dir_stat(&local->preparent);
+        dht_set_fixed_dir_stat(&local->postparent);
 
-                DHT_STACK_UNWIND (link, frame, local->op_ret, op_errno,
-                                  local->inode,
-                                  &local->stbuf, &local->preparent,
-                                  &local->postparent, NULL);
-                return 0;
-        }
+        DHT_STACK_UNWIND(link, frame, local->op_ret, op_errno, local->inode,
+                         &local->stbuf, &local->preparent, &local->postparent,
+                         NULL);
+        return 0;
+    }
 
-        if (subvol == NULL) {
-                op_errno = EINVAL;
-                goto err;
-        }
+    if (subvol == NULL) {
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        /* Second call to create link file could result in EEXIST as the
-         * first call created the linkto in the currently
-         * migrating subvol, which could be the new hashed subvol */
-        if (local->link_subvol == subvol) {
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                dht_set_fixed_dir_stat (&local->preparent);
-                dht_set_fixed_dir_stat (&local->postparent);
-                DHT_STACK_UNWIND (link, frame, 0, 0, local->inode,
-                                  &local->stbuf, &local->preparent,
-                                  &local->postparent, NULL);
+    /* Second call to create link file could result in EEXIST as the
+     * first call created the linkto in the currently
+     * migrating subvol, which could be the new hashed subvol */
+    if (local->link_subvol == subvol) {
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->preparent);
+        dht_set_fixed_dir_stat(&local->postparent);
+        DHT_STACK_UNWIND(link, frame, 0, 0, local->inode, &local->stbuf,
+                         &local->preparent, &local->postparent, NULL);
 
-                return 0;
-        }
+        return 0;
+    }
 
-        local->call_cnt = 2;
+    local->call_cnt = 2;
 
-        STACK_WIND (frame, dht_link_cbk, subvol, subvol->fops->link,
-                    &local->loc, &local->loc2, local->xattr_req);
+    STACK_WIND(frame, dht_link_cbk, subvol, subvol->fops->link, &local->loc,
+               &local->loc2, local->xattr_req);
 
-        return 0;
+    return 0;
 err:
-        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
+    DHT_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       inode_t *inode, struct iatt *stbuf,
-                       struct iatt *preparent, struct iatt *postparent,
-                       dict_t *xdata)
+static int
+dht_link_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *stbuf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *srcvol = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *srcvol = NULL;
 
-        if (op_ret == -1)
-                goto err;
+    if (op_ret == -1)
+        goto err;
 
-        local = frame->local;
-        srcvol = local->linkfile.srcvol;
+    local = frame->local;
+    srcvol = local->linkfile.srcvol;
 
-        STACK_WIND (frame, dht_link_cbk, srcvol, srcvol->fops->link,
-                    &local->loc, &local->loc2, local->xattr_req);
+    STACK_WIND(frame, dht_link_cbk, srcvol, srcvol->fops->link, &local->loc,
+               &local->loc2, local->xattr_req);
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        dht_set_fixed_dir_stat (preparent);
-        dht_set_fixed_dir_stat (postparent);
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent, xdata);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(preparent);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        xlator_t    *cached_subvol = NULL;
-        xlator_t    *hashed_subvol = NULL;
-        int          op_errno = -1;
-        int          ret = -1;
-        dht_local_t *local = NULL;
+dht_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata)
+{
+    xlator_t *cached_subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    int op_errno = -1;
+    int ret = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(oldloc, err);
+    VALIDATE_OR_GOTO(newloc, err);
+
+    local = dht_local_init(frame, oldloc, NULL, GF_FOP_LINK);
+    if (!local) {
+        op_errno = ENOMEM;
+
+        goto err;
+    }
+    local->call_cnt = 1;
+
+    cached_subvol = local->cached_subvol;
+    if (!cached_subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     oldloc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, newloc);
+    if (!hashed_subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     newloc->path);
+        op_errno = EIO;
+        goto err;
+    }
+
+    ret = loc_copy(&local->loc2, newloc);
+    if (ret == -1) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (hashed_subvol != cached_subvol) {
+        gf_uuid_copy(local->gfid, oldloc->inode->gfid);
+        dht_linkfile_create(frame, dht_link_linkfile_cbk, this, cached_subvol,
+                            hashed_subvol, newloc);
+    } else {
+        STACK_WIND(frame, dht_link_cbk, cached_subvol,
+                   cached_subvol->fops->link, oldloc, newloc, xdata);
+    }
+
+    return 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (newloc, err);
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        local = dht_local_init (frame, oldloc, NULL, GF_FOP_LINK);
-        if (!local) {
-                op_errno = ENOMEM;
+    return 0;
+}
 
-                goto err;
-        }
-        local->call_cnt = 1;
+int
+dht_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, fd_t *fd, inode_t *inode, struct iatt *stbuf,
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    xlator_t *prev = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    gf_boolean_t parent_layout_changed = _gf_false;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    xlator_t *subvol = NULL;
 
-        cached_subvol = local->cached_subvol;
-        if (!cached_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", oldloc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+    local = frame->local;
 
-        hashed_subvol = dht_subvol_get_hashed (this, newloc);
-        if (!hashed_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              newloc->path);
-                op_errno = EIO;
-                goto err;
-        }
+    local = frame->local;
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        ret = loc_copy (&local->loc2, newloc);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        parent_layout_changed = (xdata &&
+                                 dict_get(xdata, GF_PREOP_CHECK_FAILED))
+                                    ? _gf_true
+                                    : _gf_false;
+
+        if (parent_layout_changed) {
+            if (local && local->lock[0].layout.parent_layout.locks) {
+                /* Returning failure as the layout could not be fixed even under
+                 * the lock */
+                goto out;
+            }
 
-        if (hashed_subvol != cached_subvol) {
-                gf_uuid_copy (local->gfid, oldloc->inode->gfid);
-                dht_linkfile_create (frame, dht_link_linkfile_cbk, this,
-                                     cached_subvol, hashed_subvol, newloc);
-        } else {
-                STACK_WIND (frame, dht_link_cbk,
-                            cached_subvol, cached_subvol->fops->link,
-                            oldloc, newloc, xdata);
-        }
+            gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_PARENT_LAYOUT_CHANGED,
+                   "create (%s/%s) (path: %s): parent layout "
+                   "changed. Attempting a layout refresh and then a "
+                   "retry",
+                   pgfid, local->loc.name, local->loc.path);
 
-        return 0;
+            /*
+              dht_refresh_layout needs directory info in local->loc.Hence,
+              storing the parent_loc in local->loc and storing the create
+              context in local->loc2. We will restore this information in
+              dht_creation_do.
+             */
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+            loc_wipe(&local->loc2);
 
-        return 0;
-}
+            ret = loc_copy(&local->loc2, &local->loc);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                       "loc_copy failed %s", local->loc.path);
 
+                goto out;
+            }
 
-int
-dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno,
-                fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        xlator_t     *prev = NULL;
-        int           ret = -1;
-        dht_local_t  *local = NULL;
+            loc_wipe(&local->loc);
 
-        local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+            ret = dht_build_parent_loc(this, &local->loc, &local->loc2,
+                                       &op_errno);
 
-        if (op_ret == -1)
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_LOC_FAILED,
+                       "parent loc build failed");
                 goto out;
+            }
 
-        prev = cookie;
-
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           preparent, 0);
+            subvol = dht_subvol_get_hashed(this, &local->loc2);
 
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
+            ret = dht_create_lock(frame, subvol);
+            if (ret < 0) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+                       "locking parent failed");
+                goto out;
+            }
 
-        ret = dht_fd_ctx_set (this, fd, prev);
-        if (ret != 0) {
-                gf_msg_debug (this->name, 0, "Possible fd leak. "
-                              "Could not set fd ctx for subvol %s",
-                              prev->name);
+            return 0;
         }
 
+        goto out;
+    }
+
+    prev = cookie;
+
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, preparent, 0);
+
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
+    ret = dht_fd_ctx_set(this, fd, prev);
+    if (ret != 0) {
+        gf_msg_debug(this->name, 0,
+                     "Possible fd leak. "
+                     "Could not set fd ctx for subvol %s",
+                     prev->name);
+    }
+
+    ret = dht_layout_preset(this, prev, inode);
+    if (ret != 0) {
+        gf_msg_debug(this->name, 0, "could not set preset layout for subvol %s",
+                     prev->name);
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        ret = dht_layout_preset (this, prev, inode);
-        if (ret != 0) {
-                gf_msg_debug (this->name, 0,
-                              "could not set preset layout for subvol %s",
-                              prev->name);
-                op_ret   = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        local->op_errno = op_errno;
+    local->op_errno = op_errno;
 
-        if (local->linked == _gf_true) {
-                local->stbuf = *stbuf;
-                dht_linkfile_attr_heal (frame, this);
-        }
+    if (local->linked == _gf_true) {
+        local->stbuf = *stbuf;
+        dht_linkfile_attr_heal(frame, this);
+    }
 out:
 
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        dht_set_fixed_dir_stat (preparent);
-        dht_set_fixed_dir_stat (postparent);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(preparent);
+    dht_set_fixed_dir_stat(postparent);
 
-        if (local && local->lock[0].layout.parent_layout.locks) {
-                /* store op_errno for failure case*/
-                local->op_errno = op_errno;
-                local->refresh_layout_unlock (frame, this, op_ret, 1);
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        /* store op_errno for failure case*/
+        local->op_errno = op_errno;
+        local->refresh_layout_unlock(frame, this, op_ret, 1);
 
-                if (op_ret == 0) {
-                        DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd,
-                                          inode, stbuf, preparent, postparent,
-                                          xdata);
-                }
-        } else {
-                DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode,
-                                  stbuf, preparent, postparent, xdata);
+        if (op_ret == 0) {
+            DHT_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                             preparent, postparent, xdata);
         }
-        return 0;
+    } else {
+        DHT_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                         preparent, postparent, xdata);
+    }
+    return 0;
 }
 
-int
-dht_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent,
-                                dict_t *xdata)
+static int
+dht_create_linkfile_create_cbk(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               inode_t *inode, struct iatt *stbuf,
+                               struct iatt *preparent, struct iatt *postparent,
+                               dict_t *xdata)
 {
-        dht_local_t     *local             = NULL;
-        xlator_t        *cached_subvol     = NULL;
-        dht_conf_t      *conf              = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
 
-        local = frame->local;
-        if (!local) {
-                op_errno = EINVAL;
-                goto err;
-        }
+    local = frame->local;
+    if (!local) {
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        if (op_ret == -1) {
-                local->op_errno = op_errno;
-                goto err;
-        }
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        goto err;
+    }
 
-        conf = this->private;
-        if (!conf) {
-                local->op_errno = EINVAL;
-                op_errno = EINVAL;
-                goto err;
-        }
+    conf = this->private;
+    if (!conf) {
+        local->op_errno = EINVAL;
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        cached_subvol = local->cached_subvol;
+    cached_subvol = local->cached_subvol;
 
-        if (local->params) {
-                dict_del (local->params, conf->link_xattr_name);
-                dict_del (local->params, GLUSTERFS_INTERNAL_FOP_KEY);
-        }
+    if (local->params) {
+        dict_del(local->params, conf->link_xattr_name);
+        dict_del(local->params, GLUSTERFS_INTERNAL_FOP_KEY);
+    }
 
-        STACK_WIND_COOKIE (frame, dht_create_cbk, cached_subvol,
-                           cached_subvol, cached_subvol->fops->create,
-                           &local->loc, local->flags, local->mode,
-                           local->umask, local->fd, local->params);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, cached_subvol, cached_subvol,
+                      cached_subvol->fops->create, &local->loc, local->flags,
+                      local->mode, local->umask, local->fd, local->params);
 
-        return 0;
+    return 0;
 err:
-        if (local && local->lock[0].layout.parent_layout.locks) {
-                local->refresh_layout_unlock (frame, this, -1, 1);
-        } else {
-                DHT_STACK_UNWIND (create, frame, -1,
-                                  op_errno, NULL, NULL, NULL,
-                                  NULL, NULL, NULL);
-        }
-        return 0;
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    } else {
+        DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                         NULL, NULL);
+    }
+    return 0;
 }
 
-int
-dht_create_wind_to_avail_subvol (call_frame_t *frame, xlator_t *this,
-                                 xlator_t *subvol, loc_t *loc, int32_t flags,
-                                 mode_t mode, mode_t umask, fd_t *fd,
-                                 dict_t *params)
+static int
+dht_create_wind_to_avail_subvol(call_frame_t *frame, xlator_t *this,
+                                xlator_t *subvol, loc_t *loc, int32_t flags,
+                                mode_t mode, mode_t umask, fd_t *fd,
+                                dict_t *params)
 {
-        dht_local_t     *local          = NULL;
-        xlator_t        *avail_subvol   = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *avail_subvol = NULL;
 
-        local = frame->local;
-
-        if (!dht_is_subvol_filled (this, subvol)) {
-                gf_msg_debug (this->name, 0,
-                              "creating %s on %s", loc->path,
-                              subvol->name);
+    local = frame->local;
 
-                STACK_WIND_COOKIE (frame, dht_create_cbk, subvol,
-                                   subvol, subvol->fops->create,
-                                   loc, flags, mode, umask, fd, params);
+    if (!dht_is_subvol_filled(this, subvol)) {
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
 
-        } else {
-                avail_subvol = dht_free_disk_available_subvol (this, subvol, local);
+        dht_set_parent_layout_in_dict(loc, this, local);
 
-                if (avail_subvol != subvol) {
-                        local->params = dict_ref (params);
-                        local->flags = flags;
-                        local->mode = mode;
-                        local->umask = umask;
-                        local->cached_subvol = avail_subvol;
-                        local->hashed_subvol = subvol;
+        STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                          subvol->fops->create, loc, flags, mode, umask, fd,
+                          params);
 
-                        gf_msg_debug (this->name, 0,
-                                      "creating %s on %s (link at %s)", loc->path,
-                                      avail_subvol->name, subvol->name);
+    } else {
+        avail_subvol = dht_free_disk_available_subvol(this, subvol, local);
 
-                        dht_linkfile_create (frame, dht_create_linkfile_create_cbk,
-                                             this, avail_subvol, subvol, loc);
+        if (avail_subvol != subvol) {
+            local->cached_subvol = avail_subvol;
+            local->hashed_subvol = subvol;
 
-                        goto out;
-                }
+            gf_msg_debug(this->name, 0, "creating %s on %s (link at %s)",
+                         loc->path, avail_subvol->name, subvol->name);
 
-                gf_msg_debug (this->name, 0,
-                              "creating %s on %s", loc->path, subvol->name);
+            dht_linkfile_create(frame, dht_create_linkfile_create_cbk, this,
+                                avail_subvol, subvol, loc);
 
-                STACK_WIND_COOKIE (frame, dht_create_cbk, subvol,
-                                   subvol, subvol->fops->create,
-                                   loc, flags, mode, umask, fd, params);
+            goto out;
         }
+
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
+
+        dht_set_parent_layout_in_dict(loc, this, local);
+
+        STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                          subvol->fops->create, loc, flags, mode, umask, fd,
+                          params);
+    }
 out:
-        return 0;
+    return 0;
 }
 
 int
-dht_build_parent_loc (xlator_t *this, loc_t *parent, loc_t *child,
-                      int32_t *op_errno)
+dht_build_parent_loc(xlator_t *this, loc_t *parent, loc_t *child,
+                     int32_t *op_errno)
 {
-        inode_table_t   *table = NULL;
-        int     ret = -1;
+    inode_table_t *table = NULL;
+    int ret = -1;
 
-        if (!parent || !child) {
-                if (op_errno)
-                        *op_errno = EINVAL;
-                goto out;
-        }
+    if (!parent || !child) {
+        if (op_errno)
+            *op_errno = EINVAL;
+        goto out;
+    }
 
-        if (child->parent) {
-                parent->inode = inode_ref (child->parent);
-                if (!parent->inode) {
-                        if (op_errno)
-                                *op_errno = EINVAL;
-                        goto out;
-                }
+    if (child->parent) {
+        parent->inode = inode_ref(child->parent);
+        if (!parent->inode) {
+            if (op_errno)
+                *op_errno = EINVAL;
+            goto out;
+        }
 
-                gf_uuid_copy (parent->gfid, child->pargfid);
+        gf_uuid_copy(parent->gfid, child->pargfid);
 
-                ret = 0;
+        ret = 0;
 
-                goto out;
-        } else {
-                if (gf_uuid_is_null (child->pargfid)) {
-                        if (op_errno)
-                                *op_errno = EINVAL;
-                        goto out;
-                }
+        goto out;
+    } else {
+        if (gf_uuid_is_null(child->pargfid)) {
+            if (op_errno)
+                *op_errno = EINVAL;
+            goto out;
+        }
 
-                table = this->itable;
+        table = this->itable;
 
-                if (!table) {
-                        if (op_errno) {
-                                *op_errno = EINVAL;
-                                goto out;
-                        }
-                }
+        if (!table) {
+            if (op_errno) {
+                *op_errno = EINVAL;
+                goto out;
+            }
+        }
 
-                parent->inode = inode_find (table, child->pargfid);
+        parent->inode = inode_find(table, child->pargfid);
 
-                if (!parent->inode) {
-                         if (op_errno) {
-                                *op_errno = EINVAL;
-                                goto out;
-                        }
-                }
+        if (!parent->inode) {
+            if (op_errno) {
+                *op_errno = EINVAL;
+                goto out;
+            }
+        }
 
-                gf_uuid_copy (parent->gfid, child->pargfid);
+        gf_uuid_copy(parent->gfid, child->pargfid);
 
-                ret = 0;
-        }
+        ret = 0;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-
-int32_t
-dht_create_do (call_frame_t *frame)
+static int32_t
+dht_create_do(call_frame_t *frame)
 {
-        dht_local_t     *local          = NULL;
-        dht_layout_t    *refreshed      = NULL;
-        xlator_t        *subvol         = NULL;
-        xlator_t        *this           = NULL;
-        dht_conf_t      *conf           = NULL;
-        dht_methods_t   *methods        = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *refreshed = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        this = THIS;
+    this = THIS;
 
-        conf = this->private;
+    conf = this->private;
 
-        GF_VALIDATE_OR_GOTO (this->name, conf, err);
+    GF_VALIDATE_OR_GOTO(this->name, conf, err);
 
-        methods = &(conf->methods);
+    methods = &(conf->methods);
 
-        /* We don't need parent_loc anymore */
-        loc_wipe (&local->loc);
+    /* We don't need parent_loc anymore */
+    loc_wipe(&local->loc);
 
-        loc_copy (&local->loc, &local->loc2);
+    loc_copy(&local->loc, &local->loc2);
 
-        loc_wipe (&local->loc2);
+    loc_wipe(&local->loc2);
 
-        refreshed = local->selfheal.refreshed_layout;
+    refreshed = local->selfheal.refreshed_layout;
 
-        subvol = methods->layout_search (this, refreshed, local->loc.name);
+    subvol = methods->layout_search(this, refreshed, local->loc.name);
 
-        if (!subvol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_HASHED_SUBVOL_GET_FAILED, "no subvolume in "
-                        "layout for path=%s", local->loc.path);
-                local->op_errno = ENOENT;
-                goto err;
-        }
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "no subvolume in "
+               "layout for path=%s",
+               local->loc.path);
+        local->op_errno = ENOENT;
+        goto err;
+    }
 
-        dht_create_wind_to_avail_subvol (frame, this, subvol, &local->loc,
-                                         local->flags, local->mode,
-                                         local->umask, local->fd, local->params);
-        return 0;
+    dht_create_wind_to_avail_subvol(frame, this, subvol, &local->loc,
+                                    local->flags, local->mode, local->umask,
+                                    local->fd, local->params);
+    return 0;
 err:
-        local->refresh_layout_unlock (frame, this, -1, 1);
+    local->refresh_layout_unlock(frame, this, -1, 1);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-dht_create_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int32_t
+dht_create_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
-int32_t
-dht_create_finish (call_frame_t *frame, xlator_t *this, int op_ret,
-                   int invoke_cbk)
+static int32_t
+dht_create_finish(call_frame_t *frame, xlator_t *this, int op_ret,
+                  int invoke_cbk)
 {
-        dht_local_t  *local      = NULL, *lock_local = NULL;
-        call_frame_t *lock_frame = NULL;
-        int           lock_count = 0;
-
-        local = frame->local;
-        lock_count = dht_lock_count (local->lock[0].layout.parent_layout.locks,
-                                     local->lock[0].layout.parent_layout.lk_count);
-        if (lock_count == 0)
-                goto done;
-
-        lock_frame = copy_frame (frame);
-        if (lock_frame == NULL) {
-                goto done;
-        }
-
-        lock_local = dht_local_init (lock_frame, &local->loc, NULL,
-                                     lock_frame->root->op);
-        if (lock_local == NULL) {
-                goto done;
-        }
-
-        lock_local->lock[0].layout.parent_layout.locks = local->lock[0].layout.parent_layout.locks;
-        lock_local->lock[0].layout.parent_layout.lk_count = local->lock[0].layout.parent_layout.lk_count;
-
-        local->lock[0].layout.parent_layout.locks = NULL;
-	local->lock[0].layout.parent_layout.lk_count = 0;
-
-        dht_unlock_inodelk (lock_frame,
-                            lock_local->lock[0].layout.parent_layout.locks,
-                            lock_local->lock[0].layout.parent_layout.lk_count,
-                            dht_create_unlock_cbk);
-        lock_frame = NULL;
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
+
+    local = frame->local;
+    lock_count = dht_lock_count(local->lock[0].layout.parent_layout.locks,
+                                local->lock[0].layout.parent_layout.lk_count);
+    if (lock_count == 0)
+        goto done;
+
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        goto done;
+    }
+
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL) {
+        goto done;
+    }
+
+    lock_local->lock[0]
+        .layout.parent_layout.locks = local->lock[0].layout.parent_layout.locks;
+    lock_local->lock[0].layout.parent_layout.lk_count =
+        local->lock[0].layout.parent_layout.lk_count;
+
+    local->lock[0].layout.parent_layout.locks = NULL;
+    local->lock[0].layout.parent_layout.lk_count = 0;
+
+    dht_unlock_inodelk(lock_frame,
+                       lock_local->lock[0].layout.parent_layout.locks,
+                       lock_local->lock[0].layout.parent_layout.lk_count,
+                       dht_create_unlock_cbk);
+    lock_frame = NULL;
 
 done:
-        if (lock_frame != NULL) {
-                DHT_STACK_DESTROY (lock_frame);
-        }
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
 
-        if (op_ret == 0)
-                return 0;
-
-        DHT_STACK_UNWIND (create, frame, op_ret, local->op_errno, NULL, NULL,
-                          NULL, NULL, NULL, NULL);
+    if (op_ret == 0)
         return 0;
+
+    DHT_STACK_UNWIND(create, frame, op_ret, local->op_errno, NULL, NULL, NULL,
+                     NULL, NULL, NULL);
+    return 0;
 }
 
-int32_t
-dht_create_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int32_t
+dht_create_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t     *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (!local) {
-                goto err;
-        }
+    if (!local) {
+        goto err;
+    }
 
-        if (op_ret < 0) {
-                gf_msg ("DHT", GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
-                        "Create lock failed for file: %s", local->loc2.name);
+    if (op_ret < 0) {
+        gf_msg("DHT", GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+               "Create lock failed for file: %s", local->loc2.name);
 
-                local->op_errno = op_errno;
+        local->op_errno = op_errno;
 
-                goto err;
-        }
+        goto err;
+    }
 
-        local->refresh_layout_unlock = dht_create_finish;
+    local->refresh_layout_unlock = dht_create_finish;
 
-        local->refresh_layout_done = dht_create_do;
+    local->refresh_layout_done = dht_create_do;
 
-        dht_refresh_layout (frame);
+    dht_refresh_layout(frame);
 
-        return 0;
+    return 0;
 err:
-        dht_create_finish (frame, this, -1, 0);
-        return 0;
+    if (local)
+        dht_create_finish(frame, this, -1, 0);
+    else
+        DHT_STACK_UNWIND(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL,
+                         NULL, NULL);
+    return 0;
 }
 
 int32_t
-dht_create_lock (call_frame_t *frame, xlator_t *subvol)
+dht_create_lock(call_frame_t *frame, xlator_t *subvol)
 {
-        dht_local_t     *local          = NULL;
-        int              count  = 1,    ret = -1;
-        dht_lock_t     **lk_array       = NULL;
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1;
+    dht_lock_t **lk_array = NULL;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO (frame->this->name, frame->local, err);
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
 
-        local = frame->local;
+    local = frame->local;
 
-        lk_array = GF_CALLOC (count, sizeof (*lk_array), gf_common_mt_pointer);
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
 
-        if (lk_array == NULL)
-                goto err;
+    if (lk_array == NULL)
+        goto err;
 
-        lk_array[0] = dht_lock_new (frame->this, subvol, &local->loc, F_RDLCK,
-                                    DHT_LAYOUT_HEAL_DOMAIN, NULL,
-                                    IGNORE_ENOENT_ESTALE);
+    lk_array[0] = dht_lock_new(frame->this, subvol, &local->loc, F_RDLCK,
+                               DHT_LAYOUT_HEAL_DOMAIN, NULL,
+                               IGNORE_ENOENT_ESTALE);
 
-        if (lk_array[0] == NULL)
-                goto err;
+    if (lk_array[0] == NULL)
+        goto err;
 
-        local->lock[0].layout.parent_layout.locks = lk_array;
-        local->lock[0].layout.parent_layout.lk_count = count;
+    local->lock[0].layout.parent_layout.locks = lk_array;
+    local->lock[0].layout.parent_layout.lk_count = count;
 
-        ret = dht_blocking_inodelk (frame, lk_array, count,
-                                    dht_create_lock_cbk);
+    ret = dht_blocking_inodelk(frame, lk_array, count, dht_create_lock_cbk);
 
-        if (ret < 0) {
-                local->lock[0].layout.parent_layout.locks = NULL;
-                local->lock[0].layout.parent_layout.lk_count = 0;
-                goto err;
-        }
+    if (ret < 0) {
+        local->lock[0].layout.parent_layout.locks = NULL;
+        local->lock[0].layout.parent_layout.lk_count = 0;
+        goto err;
+    }
 
-        return 0;
+    return 0;
 err:
-        if (lk_array != NULL) {
-                dht_lock_array_free (lk_array, count);
-                GF_FREE (lk_array);
-        }
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
 
-        return -1;
+    return -1;
 }
 
 int
-dht_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *params)
-{
-        int             op_errno           = -1;
-        xlator_t       *subvol             = NULL;
-        dht_local_t    *local              = NULL;
-        int             i                  = 0;
-        dht_conf_t     *conf               = NULL;
-        int             ret                = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf = this->private;
+dht_set_parent_layout_in_dict(loc_t *loc, xlator_t *this, dht_local_t *local)
+{
+    dht_conf_t *conf = this->private;
+    dht_layout_t *parent_layout = NULL;
+    int *parent_disk_layout = NULL;
+    xlator_t *hashed_subvol = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    parent_layout = dht_layout_get(this, loc->parent);
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+
+    ret = dht_disk_layout_extract_for_subvol(this, parent_layout, hashed_subvol,
+                                             &parent_disk_layout);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "extracting in-memory layout of parent failed. ",
+               gf_fop_list[local->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    ret = dict_set_str_sizen(local->params, GF_PREOP_PARENT_KEY,
+                             conf->xattr_name);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting %s key in params dictionary failed. ",
+               gf_fop_list[local->fop], pgfid, loc->name, loc->path,
+               GF_PREOP_PARENT_KEY);
+        goto err;
+    }
+
+    ret = dict_set_bin(local->params, conf->xattr_name, parent_disk_layout,
+                       4 * 4);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting parent-layout in params dictionary failed. ",
+               gf_fop_list[local->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
 
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+err:
+    dht_layout_unref(this, parent_layout);
+    return ret;
+}
 
-        if (dht_filter_loc_subvol_key (this, loc, &local->loc,
-                                       &subvol)) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_SUBVOL_INFO,
-                        "creating %s on %s (got create on %s)",
-                        local->loc.path, subvol->name, loc->path);
-                STACK_WIND_COOKIE (frame, dht_create_cbk, subvol,
-                                   subvol, subvol->fops->create, &local->loc,
-                                   flags, mode, umask, fd, params);
-                goto done;
-        }
+int
+dht_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *params)
+{
+    int op_errno = -1;
+    xlator_t *subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    dht_local_t *local = NULL;
+    int i = 0;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_CREATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->params = dict_ref(params);
+    local->flags = flags;
+    local->mode = mode;
+    local->umask = umask;
+
+    if (dht_filter_loc_subvol_key(this, loc, &local->loc, &subvol)) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "creating %s on %s (got create on %s)", local->loc.path,
+               subvol->name, loc->path);
+
+        /* Since lookup-optimize is enabled by default, we need
+         * to create the linkto file if required.
+         * Note this does not check for decommisioned bricks
+         * and min-free-disk limits as this is a debugging tool
+         * and not expected to be used in production.
+         */
+        hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
 
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                        "no subvolume in layout for path=%s",
-                        loc->path);
+        if (hashed_subvol && (hashed_subvol != subvol)) {
+            /* Create the linkto file and then the data file */
+            local->cached_subvol = subvol;
+            local->hashed_subvol = hashed_subvol;
 
-                op_errno = EIO;
-                goto err;
+            dht_linkfile_create(frame, dht_create_linkfile_create_cbk, this,
+                                subvol, hashed_subvol, &local->loc);
+            goto done;
         }
+        /* We either don't have a hashed subvol or the hashed subvol is
+         * the same as the one specified. No need to create the linkto
+         * file as we expect a lookup everywhere if there are problems
+         * with the parent layout
+         */
 
-       /* Post remove-brick, the client layout may not be in sync with
-        * disk layout because of lack of lookup. Hence,a create call
-        * may fall on the decommissioned brick.  Hence, if the
-        * hashed_subvol is part of decommissioned bricks  list, do a
-        * lookup on parent dir. If a fix-layout is already done by the
-        * remove-brick process, the parent directory layout will be in
-        * sync with that of the disk. If fix-layout is still ending
-        * on the parent directory, we can let the file get created on
-        * the decommissioned brick which will be eventually migrated to
-        * non-decommissioned brick based on the new layout.
-        */
-
-        if (conf->decommission_subvols_cnt) {
-            for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->decommissioned_bricks[i] &&
-                        conf->decommissioned_bricks[i] == subvol) {
-
-                        gf_msg_debug (this->name, 0, "hashed subvol:%s is "
-                                      "part of decommission brick list for "
-                                      "file: %s", subvol->name, loc->path);
-
-                        /* dht_refresh_layout needs directory info in
-                         * local->loc. Hence, storing the parent_loc in
-                         * local->loc and storing the create context in
-                         * local->loc2. We will restore this information
-                         * in dht_creation do */
-
-                        ret = loc_copy (&local->loc2, &local->loc);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                        DHT_MSG_NO_MEMORY,
-                                        "loc_copy failed %s", loc->path);
-
-                                goto err;
-                        }
+        dht_set_parent_layout_in_dict(loc, this, local);
+
+        STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                          subvol->fops->create, &local->loc, flags, mode, umask,
+                          fd, params);
+        goto done;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "no subvolume in layout for path=%s", loc->path);
+
+        op_errno = EIO;
+        goto err;
+    }
+
+    /* Post remove-brick, the client layout may not be in sync with
+     * disk layout because of lack of lookup. Hence,a create call
+     * may fall on the decommissioned brick.  Hence, if the
+     * hashed_subvol is part of decommissioned bricks  list, do a
+     * lookup on parent dir. If a fix-layout is already done by the
+     * remove-brick process, the parent directory layout will be in
+     * sync with that of the disk. If fix-layout is still ending
+     * on the parent directory, we can let the file get created on
+     * the decommissioned brick which will be eventually migrated to
+     * non-decommissioned brick based on the new layout.
+     */
+
+    if (conf->decommission_subvols_cnt) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->decommissioned_bricks[i] &&
+                conf->decommissioned_bricks[i] == subvol) {
+                gf_msg_debug(this->name, 0,
+                             "hashed subvol:%s is "
+                             "part of decommission brick list for "
+                             "file: %s",
+                             subvol->name, loc->path);
+
+                /* dht_refresh_layout needs directory info in
+                 * local->loc. Hence, storing the parent_loc in
+                 * local->loc and storing the create context in
+                 * local->loc2. We will restore this information
+                 * in dht_creation do */
+
+                ret = loc_copy(&local->loc2, &local->loc);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                           "loc_copy failed %s", loc->path);
 
-                        local->params = dict_ref (params);
-                        local->flags = flags;
-                        local->mode = mode;
-                        local->umask = umask;
+                    goto err;
+                }
 
-                        loc_wipe (&local->loc);
+                loc_wipe(&local->loc);
 
-                        ret = dht_build_parent_loc (this, &local->loc, loc,
-                                                                 &op_errno);
+                ret = dht_build_parent_loc(this, &local->loc, loc, &op_errno);
 
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                        DHT_MSG_LOC_FAILED,
-                                        "parent loc build failed");
-                                goto err;
-                        }
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_LOC_FAILED,
+                           "parent loc build failed");
+                    goto err;
+                }
 
-                        ret = dht_create_lock (frame, subvol);
+                ret = dht_create_lock(frame, subvol);
 
-                        if (ret < 0) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_INODE_LK_ERROR,
-                                        "locking parent failed");
-                                goto err;
-                        }
+                if (ret < 0) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+                           "locking parent failed");
+                    goto err;
+                }
 
-                        goto done;
-               }
+                goto done;
             }
         }
+    }
 
-
-        dht_create_wind_to_avail_subvol (frame, this, subvol, loc, flags, mode,
-                                         umask, fd, params);
+    dht_create_wind_to_avail_subvol(frame, this, subvol, loc, flags, mode,
+                                    umask, fd, params);
 done:
-        return 0;
+    return 0;
 
 err:
 
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
-                        xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int
+dht_mkdir_selfheal_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t   *local = NULL;
-        dht_layout_t  *layout = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
 
-        local = frame->local;
-        layout = local->selfheal.layout;
+    local = frame->local;
+    layout = local->selfheal.layout;
 
-        FRAME_SU_UNDO (frame, dht_local_t);
-        dht_set_fixed_dir_stat (&local->preparent);
-        dht_set_fixed_dir_stat (&local->postparent);
+    FRAME_SU_UNDO(frame, dht_local_t);
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-        if (op_ret == 0) {
-                dht_layout_set (this, local->inode, layout);
+    if (op_ret == 0) {
+        dht_layout_set(this, local->inode, layout);
 
-                dht_inode_ctx_time_update (local->inode, this,
-                                           &local->stbuf, 1);
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->preparent, 0);
+        dht_inode_ctx_time_update(local->inode, this, &local->stbuf, 1);
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->preparent, 0);
 
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
         }
+    }
 
-        DHT_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
-                          local->inode, &local->stbuf, &local->preparent,
-                          &local->postparent, NULL);
+    DHT_STACK_UNWIND(mkdir, frame, op_ret, op_errno, local->inode,
+                     &local->stbuf, &local->preparent, &local->postparent,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
-               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        int           ret = -1;
-        gf_boolean_t subvol_filled = _gf_false;
-        gf_boolean_t dir_exists = _gf_false;
-        xlator_t     *prev = NULL;
-        dht_layout_t *layout = NULL;
-
-        local = frame->local;
-        prev  = cookie;
-        layout = local->layout;
-
-        subvol_filled = dht_is_subvol_filled (this, prev);
-
-        LOCK (&frame->lock);
-        {
-                if (subvol_filled && (op_ret != -1)) {
-                        ret = dht_layout_merge (this, layout, prev,
-                                                -1, ENOSPC, NULL);
-                } else {
-                        if (op_ret == -1 && op_errno == EEXIST) {
-                                /* Very likely just a race between mkdir and
-                                   self-heal (from lookup of a concurrent mkdir
-                                   attempt).
-                                   Ignore error for now. layout setting will
-                                   anyways fail if this was a different (old)
-                                   pre-existing different directory.
-                                */
-                                op_ret = 0;
-                                dir_exists = _gf_true;
-                        }
-                        ret = dht_layout_merge (this, layout, prev,
-                                                op_ret, op_errno, NULL);
-                }
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_LAYOUT_MERGE_FAILED,
-                                "%s: failed to merge layouts for subvol %s",
-                                local->loc.path, prev->name);
-
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        goto unlock;
-                }
-
-                if (dir_exists)
-                        goto unlock;
-
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev);
-                dht_iatt_merge (this, &local->preparent, preparent, prev);
-                dht_iatt_merge (this, &local->postparent, postparent, prev);
+static int
+dht_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, inode_t *inode, struct iatt *stbuf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    int ret = -1;
+    gf_boolean_t subvol_filled = _gf_false;
+    gf_boolean_t dir_exists = _gf_false;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+
+    local = frame->local;
+    prev = cookie;
+    layout = local->layout;
+
+    subvol_filled = dht_is_subvol_filled(this, prev);
+
+    LOCK(&frame->lock);
+    {
+        if (subvol_filled && (op_ret != -1)) {
+            ret = dht_layout_merge(this, layout, prev, -1, ENOSPC, NULL);
+        } else {
+            if (op_ret == -1 && op_errno == EEXIST) {
+                /* Very likely just a race between mkdir and
+                   self-heal (from lookup of a concurrent mkdir
+                   attempt).
+                   Ignore error for now. layout setting will
+                   anyways fail if this was a different (old)
+                   pre-existing different directory.
+                */
+                op_ret = 0;
+                dir_exists = _gf_true;
+            }
+            ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, NULL);
         }
-unlock:
-        UNLOCK (&frame->lock);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                   "%s: failed to merge layouts for subvol %s", local->loc.path,
+                   prev->name);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                /*Unlock entrylk and inodelk once mkdir is done on all subvols*/
-                dht_unlock_namespace (frame, &local->lock[0]);
-                FRAME_SU_DO (frame, dht_local_t);
-                dht_selfheal_new_directory (frame, dht_mkdir_selfheal_cbk,
-                                            layout);
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            goto unlock;
         }
 
-        return 0;
-}
+        if (dir_exists)
+            goto unlock;
 
-int
-dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int op_ret, int op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata);
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-int
-dht_mkdir_helper (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
-{
-        dht_local_t  *local                   = NULL;
-        dht_conf_t   *conf                    = NULL;
-        int           op_errno                = -1, ret = -1;
-        xlator_t     *hashed_subvol           = NULL;
-        int32_t      *parent_disk_layout      = NULL;
-        dht_layout_t *parent_layout           = NULL;
-        char          pgfid[GF_UUID_BUF_SIZE] = {0};
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        gf_uuid_unparse (loc->parent->gfid, pgfid);
-
-        conf = this->private;
-        local = frame->local;
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        /*Unlock entrylk and inodelk once mkdir is done on all subvols*/
+        dht_unlock_namespace(frame, &local->lock[0]);
+        FRAME_SU_DO(frame, dht_local_t);
+        dht_selfheal_new_directory(frame, dht_mkdir_selfheal_cbk, layout);
+    }
 
-        if (local->op_ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): refreshing parent layout "
-                        "failed.", pgfid, loc->name,
-                        loc->path);
+    return 0;
+}
 
-                op_errno = local->op_errno;
-                goto err;
-        }
+static int
+dht_mkdir_hashed_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, inode_t *inode,
+                     struct iatt *stbuf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata);
 
-        local->op_ret = -1;
+static int
+dht_mkdir_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                 mode_t umask, dict_t *params)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1, ret = -1;
+    xlator_t *hashed_subvol = NULL;
+    int32_t *parent_disk_layout = NULL;
+    dht_layout_t *parent_layout = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    conf = this->private;
+    local = frame->local;
+
+    if (local->op_ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): refreshing parent layout "
+               "failed.",
+               pgfid, loc->name, loc->path);
 
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        if (hashed_subvol == NULL) {
-                gf_msg_debug (this->name, 0,
-                              "mkdir (%s/%s) (path: %s): hashed subvol not "
-                              "found", pgfid, loc->name, loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+        op_errno = local->op_errno;
+        goto err;
+    }
+
+    local->op_ret = -1;
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "mkdir (%s/%s) (path: %s): hashed subvol not "
+                     "found",
+                     pgfid, loc->name, loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    local->hashed_subvol = hashed_subvol;
+
+    parent_layout = dht_layout_get(this, loc->parent);
+
+    ret = dht_disk_layout_extract_for_subvol(this, parent_layout, hashed_subvol,
+                                             &parent_disk_layout);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, EIO, DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "extracting in-memory layout of parent failed. ",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    if (memcmp(local->parent_disk_layout, parent_disk_layout,
+               sizeof(local->parent_disk_layout)) == 0) {
+        gf_msg(this->name, GF_LOG_WARNING, EIO, DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): loop detected. "
+               "parent layout didn't change even though "
+               "previous attempt of mkdir failed because of "
+               "in-memory layout not matching with that on disk.",
+               pgfid, loc->name, loc->path);
+        op_errno = EIO;
+        goto err;
+    }
+
+    memcpy((void *)local->parent_disk_layout, (void *)parent_disk_layout,
+           sizeof(local->parent_disk_layout));
+
+    dht_layout_unref(this, parent_layout);
+    parent_layout = NULL;
+
+    ret = dict_set_str(params, GF_PREOP_PARENT_KEY, conf->xattr_name);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "setting %s key in params dictionary failed. ",
+               pgfid, loc->name, loc->path, GF_PREOP_PARENT_KEY);
+        goto err;
+    }
+
+    ret = dict_set_bin(params, conf->xattr_name, parent_disk_layout, 4 * 4);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "setting parent-layout in params dictionary failed. "
+               "mkdir (%s/%s) (path: %s)",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    parent_disk_layout = NULL;
+
+    STACK_WIND_COOKIE(frame, dht_mkdir_hashed_cbk, hashed_subvol, hashed_subvol,
+                      hashed_subvol->fops->mkdir, loc, mode, umask, params);
+
+    return 0;
 
-        local->hashed_subvol = hashed_subvol;
+err:
+    dht_unlock_namespace(frame, &local->lock[0]);
 
-        parent_layout = dht_layout_get (this, loc->parent);
+    op_errno = local ? local->op_errno : op_errno;
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        ret = dht_disk_layout_extract_for_subvol (this, parent_layout,
-                                                  hashed_subvol,
-                                                  &parent_disk_layout);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, EIO,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): "
-                        "extracting in-memory layout of parent failed. ",
-                        pgfid, loc->name, loc->path);
-                goto err;
-        }
+    if (parent_disk_layout != NULL)
+        GF_FREE(parent_disk_layout);
 
-        if (memcmp (local->parent_disk_layout, parent_disk_layout,
-                    sizeof (local->parent_disk_layout)) == 0) {
-                gf_msg (this->name, GF_LOG_WARNING, EIO,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): loop detected. "
-                        "parent layout didn't change even though "
-                        "previous attempt of mkdir failed because of "
-                        "in-memory layout not matching with that on disk.",
-                        pgfid, loc->name, loc->path);
-                op_errno = EIO;
-                goto err;
-        }
+    if (parent_layout != NULL)
+        dht_layout_unref(this, parent_layout);
 
-        memcpy ((void *)local->parent_disk_layout, (void *)parent_disk_layout,
-                sizeof (local->parent_disk_layout));
+    return 0;
+}
 
-        dht_layout_unref (this, parent_layout);
-        parent_layout = NULL;
+static int
+dht_mkdir_hashed_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, inode_t *inode,
+                     struct iatt *stbuf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *hashed_subvol = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    gf_boolean_t parent_layout_changed = _gf_false;
+    call_stub_t *stub = NULL;
+
+    local = frame->local;
+    prev = cookie;
+    layout = local->layout;
+    conf = this->private;
+    hashed_subvol = local->hashed_subvol;
+
+    gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+
+    if (gf_uuid_is_null(local->loc.gfid) && !op_ret)
+        gf_uuid_copy(local->loc.gfid, stbuf->ia_gfid);
+
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
 
-        ret = dict_set_str (params, GF_PREOP_PARENT_KEY, conf->xattr_name);
-        if (ret < 0) {
-                local->op_errno = -ret;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): "
-                        "setting %s key in params dictionary failed. ",
-                        pgfid, loc->name, loc->path, GF_PREOP_PARENT_KEY);
+        parent_layout_changed = (xdata &&
+                                 dict_get(xdata, GF_PREOP_CHECK_FAILED))
+                                    ? 1
+                                    : 0;
+        if (parent_layout_changed) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_PARENT_LAYOUT_CHANGED,
+                   "mkdir (%s/%s) (path: %s): parent layout "
+                   "changed. Attempting a refresh and then a "
+                   "retry",
+                   pgfid, local->loc.name, local->loc.path);
+
+            stub = fop_mkdir_stub(frame, dht_mkdir_helper, &local->loc,
+                                  local->mode, local->umask, local->params);
+            if (stub == NULL) {
                 goto err;
-        }
+            }
 
-        ret = dict_set_bin (params, conf->xattr_name, parent_disk_layout,
-                            4 * 4);
-        if (ret < 0) {
-                local->op_errno = -ret;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "setting parent-layout in params dictionary failed. "
-                        "mkdir (%s/%s) (path: %s)", pgfid, loc->name,
-                        loc->path);
+            ret = dht_handle_parent_layout_change(this, stub);
+            if (ret) {
                 goto err;
-        }
-
-        parent_disk_layout = NULL;
-
-        STACK_WIND_COOKIE (frame, dht_mkdir_hashed_cbk, hashed_subvol,
-                           hashed_subvol, hashed_subvol->fops->mkdir,
-                           loc, mode, umask, params);
-
-        return 0;
+            }
 
+            stub = NULL;
+
+            return 0;
+        }
+
+        goto err;
+    }
+
+    dict_del(local->params, GF_PREOP_PARENT_KEY);
+    dict_del(local->params, conf->xattr_name);
+
+    if (dht_is_subvol_filled(this, hashed_subvol))
+        ret = dht_layout_merge(this, layout, prev, -1, ENOSPC, NULL);
+    else
+        ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, NULL);
+
+    /* TODO: we may have to return from the function
+       if layout merge fails. For now, lets just log an error */
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+               "%s: failed to merge layouts for subvol %s", local->loc.path,
+               prev->name);
+
+    local->op_ret = 0;
+
+    dht_iatt_merge(this, &local->stbuf, stbuf);
+    dht_iatt_merge(this, &local->preparent, preparent);
+    dht_iatt_merge(this, &local->postparent, postparent);
+
+    local->call_cnt = conf->subvolume_cnt - 1;
+    /* Delete internal mds xattr from params dict to avoid store
+      internal mds xattr on other subvols
+    */
+    dict_del(local->params, conf->mds_xattr_key);
+
+    if (gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, stbuf->ia_gfid);
+
+    /* Set hashed subvol as a mds subvol on inode ctx */
+    /*if (!local->inode)
+            local->inode  = inode_ref (inode);
+    */
+    ret = dht_inode_ctx_mdsvol_set(local->inode, this, hashed_subvol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+               "Failed to set hashed subvol for %s on inode vol is %s",
+               local->loc.path, hashed_subvol->name);
+    }
+
+    if (local->call_cnt == 0) {
+        /*Unlock namespace lock once mkdir is done on all subvols*/
+        dht_unlock_namespace(frame, &local->lock[0]);
+        FRAME_SU_DO(frame, dht_local_t);
+        dht_selfheal_directory(frame, dht_mkdir_selfheal_cbk, &local->loc,
+                               layout);
+        return 0;
+    }
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == hashed_subvol)
+            continue;
+        STACK_WIND_COOKIE(frame, dht_mkdir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i], conf->subvolumes[i]->fops->mkdir,
+                          &local->loc, local->mode, local->umask,
+                          local->params);
+    }
+
+    return 0;
 err:
-        dht_unlock_namespace (frame, &local->lock[0]);
+    if (local->op_ret != 0) {
+        dht_unlock_namespace(frame, &local->lock[0]);
+    }
 
-        op_errno = local ? local->op_errno : op_errno;
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        if (parent_disk_layout != NULL)
-                GF_FREE (parent_disk_layout);
+    return 0;
+}
 
-        if (parent_layout != NULL)
-                dht_layout_unref (this, parent_layout);
+static int
+dht_mkdir_guard_parent_layout_cbk(call_frame_t *frame, xlator_t *this,
+                                  loc_t *loc, mode_t mode, mode_t umask,
+                                  dict_t *params)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = 0;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = -1;
+    int32_t zero[1] = {0};
+
+    local = frame->local;
+    conf = this->private;
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "Acquiring lock on parent to guard against "
+               "layout-change failed.",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    local->op_ret = -1;
+    /* Add internal MDS xattr on disk for hashed subvol
+     */
+    ret = dht_dict_set_array(params, conf->mds_xattr_key, zero, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->mds_xattr_key, loc->path);
+    }
+
+    STACK_WIND_COOKIE(frame, dht_mkdir_hashed_cbk, local->hashed_subvol,
+                      local->hashed_subvol, local->hashed_subvol->fops->mkdir,
+                      loc, mode, umask, params);
+
+    return 0;
+err:
+    DHT_STACK_UNWIND(mkdir, frame, -1, local->op_errno, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int op_ret, int op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
-{
-        dht_local_t  *local                   = NULL;
-        int           ret                     = -1;
-        xlator_t     *prev                    = NULL;
-        dht_layout_t *layout                  = NULL;
-        dht_conf_t   *conf                    = NULL;
-        int           i                       = 0;
-        xlator_t     *hashed_subvol           = NULL;
-        char          pgfid[GF_UUID_BUF_SIZE] = {0};
-        gf_boolean_t  parent_layout_changed   = _gf_false;
-        call_stub_t  *stub                    = NULL;
-
-        VALIDATE_OR_GOTO (this->private, err);
-
-        local = frame->local;
-        prev  = cookie;
-        layout = local->layout;
-        conf = this->private;
-        hashed_subvol = local->hashed_subvol;
-
-        gf_uuid_unparse (local->loc.parent->gfid, pgfid);
-
-        if (gf_uuid_is_null (local->loc.gfid) && !op_ret)
-                gf_uuid_copy (local->loc.gfid, stbuf->ia_gfid);
-
-        if (op_ret == -1) {
-                local->op_errno = op_errno;
-
-                parent_layout_changed = (xdata && dict_get (xdata, GF_PREOP_CHECK_FAILED))
-                        ? 1 : 0;
-                if (parent_layout_changed) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_PARENT_LAYOUT_CHANGED,
-                                "mkdir (%s/%s) (path: %s): parent layout "
-                                "changed. Attempting a refresh and then a "
-                                "retry", pgfid, local->loc.name,
-                                local->loc.path);
-
-                        stub = fop_mkdir_stub (frame, dht_mkdir_helper,
-                                               &local->loc, local->mode,
-                                               local->umask, local->params);
-                        if (stub == NULL) {
-                                goto err;
-                        }
-
-                        dht_handle_parent_layout_change (this, stub);
-                        stub = NULL;
-
-                        return 0;
-                }
-
-                goto err;
-        }
-
-        dict_del (local->params, GF_PREOP_PARENT_KEY);
-        dict_del (local->params, conf->xattr_name);
-
-        if (dht_is_subvol_filled (this, hashed_subvol))
-                ret = dht_layout_merge (this, layout, prev,
-                                        -1, ENOSPC, NULL);
-        else
-                ret = dht_layout_merge (this, layout, prev,
-                                        op_ret, op_errno, NULL);
-
-        /* TODO: we may have to return from the function
-           if layout merge fails. For now, lets just log an error */
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LAYOUT_MERGE_FAILED,
-                        "%s: failed to merge layouts for subvol %s",
-                        local->loc.path, prev->name);
-
-        local->op_ret = 0;
-
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev);
-        dht_iatt_merge (this, &local->preparent, preparent, prev);
-        dht_iatt_merge (this, &local->postparent, postparent, prev);
-
-        local->call_cnt = conf->subvolume_cnt - 1;
-        /* Delete internal mds xattr from params dict to avoid store
-          internal mds xattr on other subvols
-        */
-        dict_del (local->params, conf->mds_xattr_key);
+dht_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *params)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = EINVAL, ret = -1;
+    xlator_t *hashed_subvol = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    call_stub_t *stub = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    conf = this->private;
+
+    if (!params || !dict_get(params, "gfid-req")) {
+        op_errno = EPERM;
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, op_errno,
+                         DHT_MSG_GFID_NULL,
+                         "mkdir: %s is received "
+                         "without gfid-req %p",
+                         loc->path, params);
+        goto err;
+    }
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        gf_msg_debug(this->name, 0, "hashed subvol not found for %s",
+                     loc->path);
+        local->op_errno = EIO;
+        goto err;
+    }
+
+    local->hashed_subvol = hashed_subvol;
+    local->mode = mode;
+    local->umask = umask;
+    if (params)
+        local->params = dict_ref(params);
+
+    local->inode = inode_ref(loc->inode);
+
+    local->layout = dht_layout_new(this, conf->subvolume_cnt);
+    if (!local->layout) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /* set the newly created directory hash to the commit hash
+     * if the configuration option is set. If configuration option
+     * is not set, the older clients may still be connecting to the
+     * volume and hence we need to preserve the 1 in disk[0] part of the
+     * layout xattr */
+    if (conf->lookup_optimize)
+        local->layout->commit_hash = conf->vol_commit_hash;
+    else
+        local->layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
+
+    stub = fop_mkdir_stub(frame, dht_mkdir_guard_parent_layout_cbk, loc, mode,
+                          umask, params);
+    if (stub == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "creating stub failed.",
+               pgfid, loc->name, loc->path);
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dht_guard_parent_layout_and_namespace(this, stub);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s) cannot wind lock request to "
+               "guard parent layout",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    return 0;
 
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, stbuf->ia_gfid);
+err:
+    op_errno = local ? local->op_errno : op_errno;
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        /* Set hashed subvol as a mds subvol on inode ctx */
-        /*if (!local->inode)
-                local->inode  = inode_ref (inode);
-        */
-        ret = dht_inode_ctx_mdsvol_set (local->inode, this, hashed_subvol);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
-                        "Failed to set hashed subvol for %s on inode vol is %s",
-                        local->loc.path, hashed_subvol->name);
-        }
+    return 0;
+}
 
-        if (local->call_cnt == 0) {
-                /*Unlock namespace lock once mkdir is done on all subvols*/
-                dht_unlock_namespace (frame, &local->lock[0]);
-                FRAME_SU_DO (frame, dht_local_t);
-                dht_selfheal_directory (frame, dht_mkdir_selfheal_cbk,
-                                        &local->loc, layout);
-                return 0;
-        }
+static int
+dht_rmdir_selfheal_cbk(call_frame_t *heal_frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_local_t *heal_local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == hashed_subvol)
-                        continue;
-                STACK_WIND_COOKIE (frame, dht_mkdir_cbk, conf->subvolumes[i],
-                                   conf->subvolumes[i],
-                                   conf->subvolumes[i]->fops->mkdir,
-                                   &local->loc, local->mode, local->umask,
-                                   local->params);
-        }
+    heal_local = heal_frame->local;
+    main_frame = heal_local->main_frame;
+    local = main_frame->local;
 
-        return 0;
-err:
-        if (local->op_ret != 0) {
-                dht_unlock_namespace (frame, &local->lock[0]);
-        }
+    DHT_STACK_DESTROY(heal_frame);
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
+    DHT_STACK_UNWIND(rmdir, main_frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_mkdir_guard_parent_layout_cbk (call_frame_t *frame, xlator_t *this,
-                                   loc_t *loc, mode_t mode, mode_t umask,
-                                   dict_t *params)
-{
-        dht_local_t *local                    = NULL;
-        dht_conf_t  *conf                     = 0;
-        char          pgfid[GF_UUID_BUF_SIZE] = {0};
-        int          ret                      = -1;
-        int32_t      zero[1]                  = {0};
-
-        local = frame->local;
-        conf  = this->private;
-
-        gf_uuid_unparse (loc->parent->gfid, pgfid);
+static int
+dht_rmdir_hashed_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int op_ret, int op_errno, struct iatt *preparent,
+                            struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_local_t *heal_local = NULL;
+    call_frame_t *heal_frame = NULL;
+    dht_conf_t *conf = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            local->op_ret = -1;
+            if (conf->subvolume_cnt != 1) {
+                if (op_errno != ENOENT && op_errno != EACCES &&
+                    op_errno != ESTALE) {
+                    local->need_selfheal = 1;
+                }
+            }
 
-        if (local->op_ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): "
-                        "Acquiring lock on parent to guard against "
-                        "layout-change failed.", pgfid, loc->name, loc->path);
-                goto err;
+            gf_msg_debug(this->name, op_errno,
+                         "rmdir on %s for %s failed "
+                         "(gfid = %s)",
+                         prev->name, local->loc.path, gfid);
+            goto unlock;
         }
 
-        local->op_ret = -1;
-        /* Add internal MDS xattr on disk for hashed subvol
-        */
-        ret = dht_dict_set_array (params, conf->mds_xattr_key, zero, 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value:key = %s for "
-                        "path %s", conf->mds_xattr_key, loc->path);
-        }
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        STACK_WIND_COOKIE (frame, dht_mkdir_hashed_cbk, local->hashed_subvol,
-                           local->hashed_subvol,
-                           local->hashed_subvol->fops->mkdir,
-                           loc, mode, umask, params);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if (local->need_selfheal) {
+            dht_rmdir_unlock(frame, this);
+            local->layout = dht_layout_get(this, local->loc.inode);
 
-        return 0;
-err:
-        DHT_STACK_UNWIND (mkdir, frame, -1, local->op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
+            /* TODO: neater interface needed below */
+            local->stbuf.ia_type = local->loc.inode->ia_type;
 
-        return 0;
-}
+            gf_uuid_copy(local->gfid, local->loc.inode->gfid);
 
-int
-dht_mkdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
-{
-        dht_local_t  *local                   = NULL;
-        dht_conf_t   *conf                    = NULL;
-        int           op_errno                = -1, ret = -1;
-        xlator_t     *hashed_subvol           = NULL;
-        char          pgfid[GF_UUID_BUF_SIZE] = {0};
-        call_stub_t  *stub                    = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        gf_uuid_unparse (loc->parent->gfid, pgfid);
-
-        conf = this->private;
-
-        if (!params || !dict_get (params, "gfid-req")) {
-                op_errno = EPERM;
-                gf_msg_callingfn (this->name, GF_LOG_WARNING, op_errno,
-                                  DHT_MSG_GFID_NULL, "mkdir: %s is received "
-                                  "without gfid-req %p", loc->path, params);
-                goto err;
-        }
+            /* Use a different frame or else the rmdir op_ret is
+             * overwritten by that of the selfheal */
 
-        dht_get_du_info (frame, this, loc);
+            heal_frame = copy_frame(frame);
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKDIR);
-        if (!local) {
-                op_errno = ENOMEM;
+            if (heal_frame == NULL) {
                 goto err;
-        }
+            }
 
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        if (hashed_subvol == NULL) {
-                gf_msg_debug (this->name, 0,
-                              "hashed subvol not found for %s",
-                              loc->path);
-                local->op_errno = EIO;
+            heal_local = dht_local_init(heal_frame, &local->loc, NULL, 0);
+            if (!heal_local) {
+                DHT_STACK_DESTROY(heal_frame);
                 goto err;
-        }
-
-
-        local->hashed_subvol = hashed_subvol;
-        local->mode = mode;
-        local->umask = umask;
-        if (params)
-                local->params = dict_ref (params);
-
-        local->inode  = inode_ref (loc->inode);
+            }
 
-        local->layout = dht_layout_new (this, conf->subvolume_cnt);
-        if (!local->layout) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+            heal_local->inode = inode_ref(local->loc.inode);
+            heal_local->main_frame = frame;
+            gf_uuid_copy(heal_local->gfid, local->loc.inode->gfid);
 
-        /* set the newly created directory hash to the commit hash
-         * if the configuration option is set. If configuration option
-         * is not set, the older clients may still be connecting to the
-         * volume and hence we need to preserve the 1 in disk[0] part of the
-         * layout xattr */
-        if (conf->lookup_optimize)
-                local->layout->commit_hash = conf->vol_commit_hash;
-        else
-                local->layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
+            dht_selfheal_restore(heal_frame, dht_rmdir_selfheal_cbk,
+                                 &heal_local->loc, heal_local->layout);
+            return 0;
+        } else {
+            if (local->loc.parent) {
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->preparent, 0);
 
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->postparent, 1);
+            }
 
-        stub = fop_mkdir_stub (frame, dht_mkdir_guard_parent_layout_cbk, loc,
-                               mode, umask, params);
-        if (stub == NULL) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): "
-                        "creating stub failed.", pgfid, loc->name, loc->path);
-                local->op_errno = ENOMEM;
-                goto err;
-        }
+            dht_set_fixed_dir_stat(&local->preparent);
+            dht_set_fixed_dir_stat(&local->postparent);
 
-        ret = dht_guard_parent_layout_and_namespace (this, stub);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s) cannot wind lock request to "
-                        "guard parent layout", pgfid, loc->name, loc->path);
-                goto err;
+            dht_rmdir_unlock(frame, this);
+            DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                             &local->preparent, &local->postparent, NULL);
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = local ? local->op_errno : op_errno;
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
-
-        return 0;
+    DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
-
-int
-dht_rmdir_selfheal_cbk (call_frame_t *heal_frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_rmdir_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        dht_local_t  *heal_local = NULL;
-        call_frame_t *main_frame = NULL;
-
-        heal_local = heal_frame->local;
-        main_frame = heal_local->main_frame;
-        local = main_frame->local;
-
-        DHT_STACK_DESTROY (heal_frame);
-        dht_set_fixed_dir_stat (&local->preparent);
-        dht_set_fixed_dir_stat (&local->postparent);
-
-        DHT_STACK_UNWIND (rmdir, main_frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
-
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
+static int
+dht_rmdir_unlock(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
 
-int
-dht_rmdir_hashed_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int op_ret, int op_errno, struct iatt *preparent,
-                             struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        dht_local_t  *heal_local = NULL;
-        call_frame_t *heal_frame = NULL;
-        dht_conf_t   *conf = NULL;
-        int           this_call_cnt = 0;
-        xlator_t     *prev = NULL;
-        char gfid[GF_UUID_BUF_SIZE] ={0};
-
-        local = frame->local;
-        prev  = cookie;
-        conf = this->private;
-
-        gf_uuid_unparse(local->loc.gfid, gfid);
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        local->op_ret   = -1;
-                        if (conf->subvolume_cnt != 1) {
-                                if (op_errno != ENOENT && op_errno != EACCES
-                                    && op_errno != ESTALE) {
-                                        local->need_selfheal = 1;
-                                }
-                        }
-
-                        gf_msg_debug (this->name, op_errno,
-                                      "rmdir on %s for %s failed "
-                                      "(gfid = %s)",
-                                      prev->name, local->loc.path,
-                                      gfid);
-                        goto unlock;
-                }
-
-                dht_iatt_merge (this, &local->preparent, preparent, prev);
-                dht_iatt_merge (this, &local->postparent, postparent, prev);
-
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-               if (local->need_selfheal) {
-                        dht_rmdir_unlock (frame, this);
-                        local->layout =
-                                dht_layout_get (this, local->loc.inode);
-
-                        /* TODO: neater interface needed below */
-                        local->stbuf.ia_type = local->loc.inode->ia_type;
-
-                        gf_uuid_copy (local->gfid, local->loc.inode->gfid);
+    local = frame->local;
 
-                        /* Use a different frame or else the rmdir op_ret is
-                         * overwritten by that of the selfheal */
+    /* Unlock entrylk */
+    dht_unlock_entrylk_wrapper(frame, &local->lock[0].ns.directory_ns);
 
-                        heal_frame = copy_frame (frame);
+    /* Unlock inodelk */
+    lock_count = dht_lock_count(local->lock[0].ns.parent_layout.locks,
+                                local->lock[0].ns.parent_layout.lk_count);
 
-                        if (heal_frame == NULL) {
-                                goto err;
-                        }
+    if (lock_count == 0)
+        goto done;
 
-                        heal_local = dht_local_init (heal_frame,
-                                                     &local->loc,
-                                                     NULL, 0);
-                        if (!heal_local) {
-                                DHT_STACK_DESTROY (heal_frame);
-                                goto err;
-                        }
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL)
+        goto done;
 
-                        heal_local->inode = inode_ref (local->loc.inode);
-                        heal_local->main_frame = frame;
-                        gf_uuid_copy (heal_local->gfid, local->loc.inode->gfid);
-
-                        dht_selfheal_restore (heal_frame,
-                                              dht_rmdir_selfheal_cbk,
-                                              &heal_local->loc,
-                                              heal_local->layout);
-                        return 0;
-               } else {
-
-                        if (local->loc.parent) {
-                                dht_inode_ctx_time_update (local->loc.parent,
-                                                           this,
-                                                           &local->preparent,
-                                                           0);
-
-                                dht_inode_ctx_time_update (local->loc.parent,
-                                                           this,
-                                                           &local->postparent,
-                                                           1);
-                        }
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL)
+        goto done;
 
-                        dht_set_fixed_dir_stat (&local->preparent);
-                        dht_set_fixed_dir_stat (&local->postparent);
+    lock_local->lock[0].ns.parent_layout.locks = local->lock[0]
+                                                     .ns.parent_layout.locks;
+    lock_local->lock[0]
+        .ns.parent_layout.lk_count = local->lock[0].ns.parent_layout.lk_count;
 
-                        dht_rmdir_unlock (frame, this);
-                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
-                                          local->op_errno, &local->preparent,
-                                          &local->postparent, NULL);
-               }
-        }
+    local->lock[0].ns.parent_layout.locks = NULL;
+    local->lock[0].ns.parent_layout.lk_count = 0;
+    dht_unlock_inodelk(lock_frame, lock_local->lock[0].ns.parent_layout.locks,
+                       lock_local->lock[0].ns.parent_layout.lk_count,
+                       dht_rmdir_unlock_cbk);
+    lock_frame = NULL;
 
-        return 0;
-
-err:
-        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
-                          local->op_errno, NULL, NULL, NULL);
-        return 0;
+done:
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
 
+    return 0;
 }
 
+static int
+dht_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    int done = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *heal_local = NULL;
+    call_frame_t *heal_frame = NULL;
+    int ret = -1;
+
+    local = frame->local;
+    prev = cookie;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if ((op_errno != ENOENT) && (op_errno != ESTALE)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
 
-int
-dht_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        xlator_t     *prev = NULL;
-        int           done = 0;
-        char gfid[GF_UUID_BUF_SIZE] ={0};
-        dht_local_t  *heal_local = NULL;
-        call_frame_t *heal_frame = NULL;
-        int           ret        = -1;
-
-        local = frame->local;
-        prev  = cookie;
-
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if ((op_errno != ENOENT) && (op_errno != ESTALE)) {
-                                local->op_errno = op_errno;
-                                local->op_ret = -1;
-
-                                if (op_errno != EACCES)
-                                        local->need_selfheal = 1;
-                        }
-
-                        gf_uuid_unparse(local->loc.gfid, gfid);
-
-                        gf_msg_debug (this->name, op_errno,
-                                      "rmdir on %s for %s failed."
-                                      "(gfid = %s)",
-                                      prev->name, local->loc.path,
-                                      gfid);
-                        goto unlock;
-                }
-
-                /* Track if rmdir succeeded on atleast one subvol*/
-                local->fop_succeeded = 1;
-                dht_iatt_merge (this, &local->preparent, preparent, prev);
-                dht_iatt_merge (this, &local->postparent, postparent, prev);
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
+                if (op_errno != EACCES)
+                    local->need_selfheal = 1;
+            }
 
-        this_call_cnt = dht_frame_return (frame);
+            gf_uuid_unparse(local->loc.gfid, gfid);
 
-        /* if local->hashed_subvol, we are yet to wind to hashed_subvol. */
-        if (local->hashed_subvol && (this_call_cnt == 1)) {
-                done = 1;
-        } else if (!local->hashed_subvol && !this_call_cnt) {
-                done = 1;
+            gf_msg_debug(this->name, op_errno,
+                         "rmdir on %s for %s failed."
+                         "(gfid = %s)",
+                         prev->name, local->loc.path, gfid);
+            goto unlock;
         }
 
+        /* Track if rmdir succeeded on at least one subvol*/
+        local->fop_succeeded = 1;
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (done) {
-                if (local->need_selfheal && local->fop_succeeded) {
-                        dht_rmdir_unlock (frame, this);
-                        local->layout =
-                                dht_layout_get (this, local->loc.inode);
+    this_call_cnt = dht_frame_return(frame);
 
-                        /* TODO: neater interface needed below */
-                        local->stbuf.ia_type = local->loc.inode->ia_type;
+    /* if local->hashed_subvol, we are yet to wind to hashed_subvol. */
+    if (local->hashed_subvol && (this_call_cnt == 1)) {
+        done = 1;
+    } else if (!local->hashed_subvol && !this_call_cnt) {
+        done = 1;
+    }
 
-                        gf_uuid_copy (local->gfid, local->loc.inode->gfid);
-                        heal_frame = copy_frame (frame);
-                        if (heal_frame == NULL) {
-                                goto err;
-                        }
+    if (done) {
+        if (local->need_selfheal && local->fop_succeeded) {
+            dht_rmdir_unlock(frame, this);
+            local->layout = dht_layout_get(this, local->loc.inode);
 
-                        heal_local = dht_local_init (heal_frame, &local->loc,
-                                                     NULL, 0);
-                        if (!heal_local) {
-                                DHT_STACK_DESTROY (heal_frame);
-                                goto err;
-                        }
+            /* TODO: neater interface needed below */
+            local->stbuf.ia_type = local->loc.inode->ia_type;
 
-                        heal_local->inode = inode_ref (local->loc.inode);
-                        heal_local->main_frame = frame;
-                        gf_uuid_copy (heal_local->gfid, local->loc.inode->gfid);
-                        ret = dht_selfheal_restore (heal_frame,
-                                                    dht_rmdir_selfheal_cbk,
-                                                    &heal_local->loc,
-                                                    heal_local->layout);
-                        if (ret) {
-                                DHT_STACK_DESTROY (heal_frame);
-                                goto err;
-                        }
+            gf_uuid_copy(local->gfid, local->loc.inode->gfid);
+            heal_frame = copy_frame(frame);
+            if (heal_frame == NULL) {
+                goto err;
+            }
 
-                } else if (this_call_cnt) {
-                        /* If non-hashed subvol's have responded, proceed */
-                        if (local->op_ret == 0) {
-                                /* Delete the dir from the hashed subvol if:
-                                 * The fop succeeded on at least one subvol
-                                 *  and did not fail on any
-                                 *  or
-                                 *  The fop failed with ENOENT/ESTALE on
-                                 *  all subvols */
-
-                                STACK_WIND_COOKIE (frame, dht_rmdir_hashed_subvol_cbk,
-                                                   local->hashed_subvol,
-                                                   local->hashed_subvol,
-                                                   local->hashed_subvol->fops->rmdir,
-                                                   &local->loc, local->flags, NULL);
-                        } else {
-                         /* hashed-subvol was non-NULL and rmdir failed on
-                          * all non hashed-subvols. Unwind rmdir with
-                          * local->op_ret and local->op_errno. */
-                                dht_rmdir_unlock (frame, this);
-                                DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
-                                          local->op_errno, &local->preparent,
-                                          &local->postparent, NULL);
-
-                                return 0;
+            heal_local = dht_local_init(heal_frame, &local->loc, NULL, 0);
+            if (!heal_local) {
+                DHT_STACK_DESTROY(heal_frame);
+                goto err;
+            }
 
-                        }
-                } else if (!this_call_cnt) {
-                        /* All subvol's have responded, proceed */
+            heal_local->inode = inode_ref(local->loc.inode);
+            heal_local->main_frame = frame;
+            gf_uuid_copy(heal_local->gfid, local->loc.inode->gfid);
+            ret = dht_selfheal_restore(heal_frame, dht_rmdir_selfheal_cbk,
+                                       &heal_local->loc, heal_local->layout);
+            if (ret) {
+                DHT_STACK_DESTROY(heal_frame);
+                goto err;
+            }
 
-                        if (local->loc.parent) {
+        } else if (this_call_cnt) {
+            /* If non-hashed subvol's have responded, proceed */
+            if (local->op_ret == 0) {
+                /* Delete the dir from the hashed subvol if:
+                 * The fop succeeded on at least one subvol
+                 *  and did not fail on any
+                 *  or
+                 *  The fop failed with ENOENT/ESTALE on
+                 *  all subvols */
+
+                STACK_WIND_COOKIE(frame, dht_rmdir_hashed_subvol_cbk,
+                                  local->hashed_subvol, local->hashed_subvol,
+                                  local->hashed_subvol->fops->rmdir,
+                                  &local->loc, local->flags, NULL);
+            } else {
+                /* hashed-subvol was non-NULL and rmdir failed on
+                 * all non hashed-subvols. Unwind rmdir with
+                 * local->op_ret and local->op_errno. */
+                dht_rmdir_unlock(frame, this);
+                DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                                 &local->preparent, &local->postparent, NULL);
 
-                                dht_inode_ctx_time_update (local->loc.parent,
-                                                           this,
-                                                           &local->preparent,
-                                                           0);
+                return 0;
+            }
+        } else if (!this_call_cnt) {
+            /* All subvol's have responded, proceed */
 
-                                dht_inode_ctx_time_update (local->loc.parent,
-                                                           this,
-                                                           &local->postparent,
-                                                           1);
+            if (local->loc.parent) {
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->preparent, 0);
 
-                        }
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->postparent, 1);
+            }
 
-                        dht_set_fixed_dir_stat (&local->preparent);
-                        dht_set_fixed_dir_stat (&local->postparent);
+            dht_set_fixed_dir_stat(&local->preparent);
+            dht_set_fixed_dir_stat(&local->postparent);
 
-                        dht_rmdir_unlock (frame, this);
-                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
-                                          local->op_errno, &local->preparent,
-                                          &local->postparent, NULL);
-                }
+            dht_rmdir_unlock(frame, this);
+            DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                             &local->preparent, &local->postparent, NULL);
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (rmdir, frame, -1, local->op_errno, NULL, NULL, NULL);
-        return 0;
-
-}
-
-
-int
-dht_rmdir_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    DHT_STACK_UNWIND(rmdir, frame, -1, local->op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
-int
-dht_rmdir_unlock (call_frame_t *frame, xlator_t *this)
+static int
+dht_rmdir_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local      = NULL, *lock_local = NULL;
-        call_frame_t *lock_frame = NULL;
-        int           lock_count = 0;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *hashed_subvol;
 
-        local = frame->local;
-
-        /* Unlock entrylk */
-        dht_unlock_entrylk_wrapper (frame, &local->lock[0].ns.directory_ns);
-
-        /* Unlock inodelk */
-        lock_count = dht_lock_count (local->lock[0].ns.parent_layout.locks,
-                                     local->lock[0].ns.parent_layout.lk_count);
+    conf = this->private;
+    local = frame->local;
 
-        if (lock_count == 0)
-                goto done;
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring entrylk after inodelk failed rmdir for %s)",
+               local->loc.path);
 
-        lock_frame = copy_frame (frame);
-        if (lock_frame == NULL)
-                goto done;
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
 
-        lock_local = dht_local_init (lock_frame, &local->loc, NULL,
-                                     lock_frame->root->op);
-        if (lock_local == NULL)
-                goto done;
+    hashed_subvol = local->hashed_subvol;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (hashed_subvol && (hashed_subvol == conf->subvolumes[i]))
+            continue;
 
-        lock_local->lock[0].ns.parent_layout.locks = local->lock[0].ns.parent_layout.locks;
-        lock_local->lock[0].ns.parent_layout.lk_count = local->lock[0].ns.parent_layout.lk_count;
+        STACK_WIND_COOKIE(frame, dht_rmdir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i], conf->subvolumes[i]->fops->rmdir,
+                          &local->loc, local->flags, NULL);
+    }
 
-        local->lock[0].ns.parent_layout.locks = NULL;
-        local->lock[0].ns.parent_layout.lk_count = 0;
-        dht_unlock_inodelk (lock_frame,
-                            lock_local->lock[0].ns.parent_layout.locks,
-                            lock_local->lock[0].ns.parent_layout.lk_count,
-                            dht_rmdir_unlock_cbk);
-        lock_frame = NULL;
+    return 0;
 
-done:
-        if (lock_frame != NULL) {
-                DHT_STACK_DESTROY (lock_frame);
-        }
+err:
+    DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_rmdir_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int
+dht_rmdir_do(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf  = NULL;
-        int           i     = 0;
-        xlator_t     *hashed_subvol;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    xlator_t *hashed_subvol = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        VALIDATE_OR_GOTO (this->private, err);
+    VALIDATE_OR_GOTO(frame->local, err);
+    local = frame->local;
+    VALIDATE_OR_GOTO(this->private, out);
+    conf = this->private;
 
-        conf = this->private;
-        local = frame->local;
+    if (local->op_ret == -1)
+        goto out;
 
-        if (op_ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_INODE_LK_ERROR,
-                        "acquiring entrylk after inodelk failed rmdir for %s)",
-                        local->loc.path);
+    local->call_cnt = conf->subvolume_cnt;
 
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-                goto err;
-        }
+    /* first remove from non-hashed_subvol */
+    hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
 
-        hashed_subvol = local->hashed_subvol;
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (hashed_subvol &&
-                    (hashed_subvol == conf->subvolumes[i]))
-                        continue;
+    if (!hashed_subvol) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
 
-                STACK_WIND_COOKIE (frame, dht_rmdir_cbk, conf->subvolumes[i],
-                                   conf->subvolumes[i],
-                                   conf->subvolumes[i]->fops->rmdir,
-                                   &local->loc, local->flags, NULL);
-        }
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "Failed to get hashed subvol for %s (gfid = %s)",
+               local->loc.path, gfid);
+    } else {
+        local->hashed_subvol = hashed_subvol;
+    }
 
+    /* When DHT has only 1 child */
+    if (conf->subvolume_cnt == 1) {
+        STACK_WIND_COOKIE(frame, dht_rmdir_hashed_subvol_cbk,
+                          conf->subvolumes[0], conf->subvolumes[0],
+                          conf->subvolumes[0]->fops->rmdir, &local->loc,
+                          local->flags, NULL);
         return 0;
+    }
 
-err:
-        DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
+    local->current = &local->lock[0];
+    ret = dht_protect_namespace(frame, &local->loc, local->hashed_subvol,
+                                &local->current->ns, dht_rmdir_lock_cbk);
+    if (ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = errno ? errno : EINVAL;
+        goto out;
+    }
 
-        return 0;
-}
+    return 0;
 
+out:
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-int
-dht_rmdir_do (call_frame_t *frame, xlator_t *this)
+    DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, NULL);
+    return 0;
+err:
+    DHT_STACK_UNWIND(rmdir, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
+}
+
+static void
+dht_rmdir_readdirp_done(call_frame_t *readdirp_frame, xlator_t *this)
 {
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret = -1;
-        xlator_t     *hashed_subvol = NULL;
-        char gfid[GF_UUID_BUF_SIZE] ={0};
+    call_frame_t *main_frame = NULL;
+    dht_local_t *main_local = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
 
-        VALIDATE_OR_GOTO (this->private, err);
+    local = readdirp_frame->local;
+    main_frame = local->main_frame;
+    main_local = main_frame->local;
 
-        conf = this->private;
-        local = frame->local;
+    /* At least one readdirp failed.
+     * This is a bit hit or miss - if readdirp failed on more than
+     * one subvol, we don't know which error is returned.
+     */
+    if (local->op_ret == -1) {
+        main_local->op_ret = local->op_ret;
+        main_local->op_errno = local->op_errno;
+    }
 
-        if (local->op_ret == -1)
-                goto err;
-
-        local->call_cnt = conf->subvolume_cnt;
+    this_call_cnt = dht_frame_return(main_frame);
 
-        /* first remove from non-hashed_subvol */
-        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
+    if (is_last_call(this_call_cnt))
+        dht_rmdir_do(main_frame, this);
 
-        if (!hashed_subvol) {
-                gf_uuid_unparse(local->loc.gfid, gfid);
+    DHT_STACK_DESTROY(readdirp_frame);
+}
 
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                        "Failed to get hashed subvol for %s (gfid = %s)",
-                        local->loc.path, gfid);
-        } else {
-                local->hashed_subvol = hashed_subvol;
-        }
+/* Keep sending readdirp on the subvol until it returns no more entries
+ * It is possible that not all entries will fit in a single readdirp in
+ * which case the rmdir will keep failing with ENOTEMPTY
+ */
 
-        /* When DHT has only 1 child */
-        if (conf->subvolume_cnt == 1) {
-                STACK_WIND_COOKIE (frame, dht_rmdir_hashed_subvol_cbk,
-                                   conf->subvolumes[0], conf->subvolumes[0],
-                                   conf->subvolumes[0]->fops->rmdir,
-                                   &local->loc, local->flags, NULL);
-                return 0;
-        }
+static int
+dht_rmdir_readdirp_do(call_frame_t *readdirp_frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
 
-        local->current = &local->lock[0];
-        ret = dht_protect_namespace (frame, &local->loc, local->hashed_subvol,
-                                     &local->current->ns, dht_rmdir_lock_cbk);
-        if (ret < 0) {
-                local->op_ret = -1;
-                local->op_errno = errno ? errno : EINVAL;
-                goto err;
-        }
+    local = readdirp_frame->local;
 
+    if (local->op_ret == -1) {
+        /* there is no point doing another readdirp on this
+         * subvol . */
+        dht_rmdir_readdirp_done(readdirp_frame, this);
         return 0;
+    }
 
-err:
-        dht_set_fixed_dir_stat (&local->preparent);
-        dht_set_fixed_dir_stat (&local->postparent);
+    STACK_WIND_COOKIE(readdirp_frame, dht_rmdir_readdirp_cbk,
+                      local->hashed_subvol, local->hashed_subvol,
+                      local->hashed_subvol->fops->readdirp, local->fd, 4096, 0,
+                      local->xattr);
 
-        DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
-        return 0;
+    return 0;
 }
 
-
-int
-dht_rmdir_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                               int op_ret, int op_errno, struct iatt *preparent,
-                               struct iatt *postparent, dict_t *xdata)
+static int
+dht_rmdir_linkfile_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int op_ret, int op_errno, struct iatt *preparent,
+                              struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t    *local = NULL;
-        xlator_t       *prev = NULL;
-        xlator_t       *src = NULL;
-        call_frame_t   *readdirp_frame = NULL;
-        dht_local_t    *readdirp_local = NULL;
-        int             this_call_cnt = 0;
-        char gfid[GF_UUID_BUF_SIZE] ={0};
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    dht_local_t *readdirp_local = NULL;
+    int this_call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
+    local = frame->local;
+    prev = cookie;
+    src = prev;
 
-        local  = frame->local;
-        prev   = cookie;
-        src    = prev;
+    readdirp_frame = local->main_frame;
+    readdirp_local = readdirp_frame->local;
 
+    gf_uuid_unparse(local->loc.gfid, gfid);
 
-        readdirp_frame = local->main_frame;
-        readdirp_local = readdirp_frame->local;
-
-        gf_uuid_unparse(local->loc.gfid, gfid);
-
-        if (op_ret == 0) {
-                gf_msg_trace (this->name, 0,
-                              "Unlinked linkfile %s on %s, gfid = %s",
-                              local->loc.path, src->name, gfid);
-        } else {
-                if (op_errno != ENOENT) {
-                        readdirp_local->op_ret   = -1;
-                        readdirp_local->op_errno = op_errno;
-                }
-                gf_msg_debug (this->name, op_errno,
-                              "Unlink of %s on %s failed. (gfid = %s)",
-                              local->loc.path, src->name, gfid);
+    if (op_ret == 0) {
+        gf_msg_trace(this->name, 0, "Unlinked linkfile %s on %s, gfid = %s",
+                     local->loc.path, src->name, gfid);
+    } else {
+        if (op_errno != ENOENT) {
+            readdirp_local->op_ret = -1;
+            readdirp_local->op_errno = op_errno;
         }
+        gf_msg_debug(this->name, op_errno,
+                     "Unlink of %s on %s failed. (gfid = %s)", local->loc.path,
+                     src->name, gfid);
+    }
 
-        this_call_cnt = dht_frame_return (readdirp_frame);
+    this_call_cnt = dht_frame_return(readdirp_frame);
 
-        if (is_last_call (this_call_cnt))
-                dht_rmdir_readdirp_do (readdirp_frame, this);
+    if (is_last_call(this_call_cnt))
+        dht_rmdir_readdirp_do(readdirp_frame, this);
 
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
+static int
+dht_rmdir_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, inode_t *inode,
+                     struct iatt *stbuf, dict_t *xattr, struct iatt *parent)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    dht_local_t *readdirp_local = NULL;
+    int this_call_cnt = 0;
+    dht_conf_t *conf = this->private;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    prev = cookie;
+    src = prev;
+
+    gf_msg_debug(this->name, 0, "dht_rmdir_lookup_cbk %s", local->loc.path);
+
+    readdirp_frame = local->main_frame;
+    readdirp_local = readdirp_frame->local;
+
+    if (op_ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_FILE_LOOKUP_FAILED,
+               "lookup failed for %s on %s", local->loc.path, src->name);
+        goto err;
+    }
+
+    if (!check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name)) {
+        readdirp_local->op_ret = -1;
+        readdirp_local->op_errno = ENOTEMPTY;
 
-int
-dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, inode_t *inode,
-                      struct iatt *stbuf, dict_t *xattr, struct iatt *parent)
-{
-        dht_local_t    *local = NULL;
-        xlator_t       *prev = NULL;
-        xlator_t       *src = NULL;
-        call_frame_t   *readdirp_frame = NULL;
-        dht_local_t    *readdirp_local = NULL;
-        int             this_call_cnt = 0;
-        dht_conf_t     *conf = this->private;
-        char               gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        prev  = cookie;
-        src   = prev;
-
-        gf_msg_debug (this->name, 0, "dht_rmdir_lookup_cbk %s",
-                      local->loc.path);
-
-        readdirp_frame = local->main_frame;
-        readdirp_local = readdirp_frame->local;
-
-        if (op_ret != 0) {
-
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_FILE_LOOKUP_FAILED,
-                        "lookup failed for %s on %s  (type=0%o)",
-                        local->loc.path, src->name, stbuf->ia_type);
-                goto err;
-        }
-
-        if (!check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
-                readdirp_local->op_ret  = -1;
-                readdirp_local->op_errno = ENOTEMPTY;
-
-                gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_uuid_unparse(local->loc.gfid, gfid);
 
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_NOT_LINK_FILE_ERROR,
-                        "%s on %s is not a linkfile (type=0%o, gfid = %s)",
-                        local->loc.path, src->name, stbuf->ia_type, gfid);
-                goto err;
-        }
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NOT_LINK_FILE_ERROR,
+               "%s on %s is not a linkfile (type=0%o, gfid = %s)",
+               local->loc.path, src->name, stbuf->ia_type, gfid);
+        goto err;
+    }
 
-        STACK_WIND_COOKIE (frame, dht_rmdir_linkfile_unlink_cbk, src,
-                           src, src->fops->unlink, &local->loc, 0, NULL);
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_rmdir_linkfile_unlink_cbk, src, src,
+                      src->fops->unlink, &local->loc, 0, NULL);
+    return 0;
 err:
 
-        this_call_cnt = dht_frame_return (readdirp_frame);
-        if (is_last_call (this_call_cnt)) {
-                dht_rmdir_readdirp_do (readdirp_frame, this);
-        }
+    this_call_cnt = dht_frame_return(readdirp_frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_rmdir_readdirp_do(readdirp_frame, this);
+    }
 
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
-
-int
-dht_rmdir_cached_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int op_ret, int op_errno, inode_t *inode,
-                             struct iatt *stbuf, dict_t *xattr,
-                             struct iatt *parent)
-{
-        dht_local_t    *local             = NULL;
-        xlator_t       *src               = NULL;
-        call_frame_t   *readdirp_frame    = NULL;
-        dht_local_t    *readdirp_local    = NULL;
-        int             this_call_cnt     = 0;
-        dht_conf_t     *conf              = this->private;
-        dict_t         *xattrs            = NULL;
-        int             ret               = 0;
-
-        local = frame->local;
-        src   = local->hashed_subvol;
-
-
-        /* main_frame here is the readdirp_frame */
-
-        readdirp_frame = local->main_frame;
-        readdirp_local = readdirp_frame->local;
-
-        gf_msg_debug (this->name, 0, "returning for %s ",
-                      local->loc.path);
-
-        if (op_ret == 0) {
-                readdirp_local->op_ret  = -1;
-                readdirp_local->op_errno = ENOTEMPTY;
-
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_SUBVOL_ERROR,
-                        "%s found on cached subvol %s",
-                        local->loc.path, src->name);
-                goto err;
-        } else if (op_errno != ENOENT) {
-                readdirp_local->op_ret  = -1;
-                readdirp_local->op_errno = op_errno;
-
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_SUBVOL_ERROR,
-                        "%s not found on cached subvol %s",
-                        local->loc.path, src->name);
-                goto err;
-        }
-
-        xattrs = dict_new ();
-        if (!xattrs) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        DHT_MSG_NO_MEMORY, "dict_new failed");
-                goto err;
-        }
-
-        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value: key = %s",
-                        conf->link_xattr_name);
-                if (xattrs)
-                        dict_unref (xattrs);
-                goto err;
-        }
-        STACK_WIND_COOKIE (frame, dht_rmdir_lookup_cbk, src, src,
-                           src->fops->lookup, &local->loc, xattrs);
+static int
+dht_rmdir_cached_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int op_ret, int op_errno, inode_t *inode,
+                            struct iatt *stbuf, dict_t *xattr,
+                            struct iatt *parent)
+{
+    dht_local_t *local = NULL;
+    xlator_t *src = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    dht_local_t *readdirp_local = NULL;
+    int this_call_cnt = 0;
+    dht_conf_t *conf = this->private;
+    dict_t *xattrs = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    src = local->hashed_subvol;
+
+    /* main_frame here is the readdirp_frame */
+
+    readdirp_frame = local->main_frame;
+    readdirp_local = readdirp_frame->local;
+
+    gf_msg_debug(this->name, 0, "returning for %s ", local->loc.path);
+
+    if (op_ret == 0) {
+        readdirp_local->op_ret = -1;
+        readdirp_local->op_errno = ENOTEMPTY;
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SUBVOL_ERROR,
+               "%s found on cached subvol %s", local->loc.path, src->name);
+        goto err;
+    } else if (op_errno != ENOENT) {
+        readdirp_local->op_ret = -1;
+        readdirp_local->op_errno = op_errno;
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_SUBVOL_ERROR,
+               "%s not found on cached subvol %s", local->loc.path, src->name);
+        goto err;
+    }
+
+    xattrs = dict_new();
+    if (!xattrs) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "dict_new failed");
+        goto err;
+    }
+
+    ret = dict_set_uint32(xattrs, conf->link_xattr_name, 256);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s",
+               conf->link_xattr_name);
         if (xattrs)
-                dict_unref (xattrs);
-
-        return 0;
+            dict_unref(xattrs);
+        goto err;
+    }
+    STACK_WIND_COOKIE(frame, dht_rmdir_lookup_cbk, src, src, src->fops->lookup,
+                      &local->loc, xattrs);
+    if (xattrs)
+        dict_unref(xattrs);
+
+    return 0;
 err:
 
-        this_call_cnt = dht_frame_return (readdirp_frame);
+    this_call_cnt = dht_frame_return(readdirp_frame);
 
-        /* Once all the lookups/unlinks etc have returned, proceed to wind
-         * readdirp on the subvol again until no entries are returned.
-         * This is required if there are more entries than can be returned
-         * in a single readdirp call.
-         */
+    /* Once all the lookups/unlinks etc have returned, proceed to wind
+     * readdirp on the subvol again until no entries are returned.
+     * This is required if there are more entries than can be returned
+     * in a single readdirp call.
+     */
 
-        if (is_last_call (this_call_cnt))
-                dht_rmdir_readdirp_do (readdirp_frame, this);
+    if (is_last_call(this_call_cnt))
+        dht_rmdir_readdirp_do(readdirp_frame, this);
 
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
+static int
+dht_rmdir_is_subvol_empty(call_frame_t *frame, xlator_t *this,
+                          gf_dirent_t *entries, xlator_t *src)
+{
+    int ret = 0;
+    int build_ret = 0;
+    gf_dirent_t *trav = NULL;
+    call_frame_t *lookup_frame = NULL;
+    dht_local_t *lookup_local = NULL;
+    dht_local_t *local = NULL;
+    dict_t *xattrs = NULL;
+    dht_conf_t *conf = this->private;
+    xlator_t *subvol = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int count = 0;
+    gf_boolean_t unwind = _gf_false;
+
+    local = frame->local;
+
+    list_for_each_entry(trav, &entries->list, list)
+    {
+        if (strcmp(trav->d_name, ".") == 0)
+            continue;
+        if (strcmp(trav->d_name, "..") == 0)
+            continue;
+        if (check_is_linkfile(NULL, (&trav->d_stat), trav->dict,
+                              conf->link_xattr_name)) {
+            count++;
+            continue;
+        }
+
+        /* this entry is either a directory which is neither "." nor "..",
+           or a non directory which is not a linkfile. the directory is to
+           be treated as non-empty
+        */
+        return 0;
+    }
 
-int
-dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
-                           gf_dirent_t *entries, xlator_t *src)
-{
-        int                 ret          = 0;
-        int                 build_ret    = 0;
-        gf_dirent_t        *trav         = NULL;
-        call_frame_t       *lookup_frame = NULL;
-        dht_local_t        *lookup_local = NULL;
-        dht_local_t        *local        = NULL;
-        dict_t             *xattrs       = NULL;
-        dht_conf_t         *conf         = this->private;
-        xlator_t           *subvol       = NULL;
-        char                gfid[GF_UUID_BUF_SIZE] = {0};
-        int                 count        = 0;
-        gf_boolean_t        unwind        = _gf_false;
-
+    xattrs = dict_new();
+    if (!xattrs) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "dict_new failed");
+        return -1;
+    }
 
-        local = frame->local;
+    ret = dict_set_uint32(xattrs, conf->link_xattr_name, 256);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s",
+               conf->link_xattr_name);
 
-        list_for_each_entry (trav, &entries->list, list) {
-                if (strcmp (trav->d_name, ".") == 0)
-                        continue;
-                if (strcmp (trav->d_name, "..") == 0)
-                        continue;
-                if (check_is_linkfile (NULL, (&trav->d_stat), trav->dict,
-                                              conf->link_xattr_name)) {
-                        count++;
-                        continue;
-                }
+        if (xattrs)
+            dict_unref(xattrs);
+        return -1;
+    }
 
-                /* this entry is either a directory which is neither "." nor "..",
-                   or a non directory which is not a linkfile. the directory is to
-                   be treated as non-empty
-                */
-                return 0;
-        }
+    local->call_cnt = count;
+    ret = 0;
 
-        xattrs = dict_new ();
-        if (!xattrs) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        DHT_MSG_NO_MEMORY, "dict_new failed");
-                return -1;
-        }
+    list_for_each_entry(trav, &entries->list, list)
+    {
+        if (strcmp(trav->d_name, ".") == 0)
+            continue;
+        if (strcmp(trav->d_name, "..") == 0)
+            continue;
 
-        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value: key = %s",
-                        conf->link_xattr_name);
+        lookup_frame = copy_frame(frame);
 
-                if (xattrs)
-                        dict_unref (xattrs);
-                return -1;
+        if (!lookup_frame) {
+            /* out of memory, let the rmdir fail
+               (as non-empty, unfortunately) */
+            goto err;
         }
 
-        local->call_cnt = count;
-        ret = 0;
-
-        list_for_each_entry (trav, &entries->list, list) {
-                if (strcmp (trav->d_name, ".") == 0)
-                        continue;
-                if (strcmp (trav->d_name, "..") == 0)
-                        continue;
-
-                lookup_frame = copy_frame (frame);
-
-                if (!lookup_frame) {
-                        /* out of memory, let the rmdir fail
-                           (as non-empty, unfortunately) */
-                        goto err;
-                }
-
-                lookup_local = dht_local_init (lookup_frame, NULL, NULL,
-                                               GF_FOP_LOOKUP);
-                if (!lookup_local) {
-                        goto err;
-                }
-
-                lookup_frame->local = lookup_local;
-                lookup_local->main_frame = frame;
-                lookup_local->hashed_subvol = src;
+        lookup_local = dht_local_init(lookup_frame, NULL, NULL, GF_FOP_LOOKUP);
+        if (!lookup_local) {
+            goto err;
+        }
 
-                build_ret = dht_build_child_loc (this, &lookup_local->loc,
-                                                 &local->loc, trav->d_name);
-                if (build_ret != 0)
-                        goto err;
+        lookup_frame->local = lookup_local;
+        lookup_local->main_frame = frame;
+        lookup_local->hashed_subvol = src;
 
-                gf_uuid_copy (lookup_local->loc.gfid, trav->d_stat.ia_gfid);
+        build_ret = dht_build_child_loc(this, &lookup_local->loc, &local->loc,
+                                        trav->d_name);
+        if (build_ret != 0)
+            goto err;
 
-                gf_uuid_unparse(lookup_local->loc.gfid, gfid);
+        gf_uuid_copy(lookup_local->loc.gfid, trav->d_stat.ia_gfid);
 
-                gf_msg_trace (this->name, 0,
-                              "looking up %s on subvolume %s, gfid = %s",
-                              lookup_local->loc.path, src->name, gfid);
+        gf_uuid_unparse(lookup_local->loc.gfid, gfid);
 
-                subvol = dht_linkfile_subvol (this, NULL, &trav->d_stat,
-                                              trav->dict);
-                if (!subvol) {
+        gf_msg_trace(this->name, 0, "looking up %s on subvolume %s, gfid = %s",
+                     lookup_local->loc.path, src->name, gfid);
 
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_INVALID_LINKFILE,
-                                "Linkfile does not have link subvolume. "
-                                "path = %s, gfid = %s",
-                                lookup_local->loc.path, gfid);
+        subvol = dht_linkfile_subvol(this, NULL, &trav->d_stat, trav->dict);
+        if (!subvol || (subvol == src)) {
+            /* we need to delete the linkto file if it does not have a
+             * valid subvol or it points to itself.
+             */
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_INVALID_LINKFILE,
+                   "Linkfile does not have link subvolume. "
+                   "path = %s, gfid = %s",
+                   lookup_local->loc.path, gfid);
 
-                        gf_msg_debug (this->name, 0,
-                                      "looking up %s on subvol %s, gfid = %s",
-                                      lookup_local->loc.path, src->name, gfid);
+            gf_msg_debug(this->name, 0, "looking up %s on subvol %s, gfid = %s",
+                         lookup_local->loc.path, src->name, gfid);
 
-                        STACK_WIND_COOKIE (lookup_frame, dht_rmdir_lookup_cbk,
-                                           src, src, src->fops->lookup,
-                                           &lookup_local->loc, xattrs);
-                } else {
-                        gf_msg_debug (this->name, 0,
-                                      "Looking up linkfile target %s on "
-                                      " subvol %s, gfid = %s",
-                                      lookup_local->loc.path, subvol->name,
-                                      gfid);
-
-                        STACK_WIND (lookup_frame, dht_rmdir_cached_lookup_cbk,
-                                    subvol, subvol->fops->lookup,
-                                    &lookup_local->loc, xattrs);
-                }
-                ret++;
+            STACK_WIND_COOKIE(lookup_frame, dht_rmdir_lookup_cbk, src, src,
+                              src->fops->lookup, &lookup_local->loc, xattrs);
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "Looking up linkfile target %s on "
+                         " subvol %s, gfid = %s",
+                         lookup_local->loc.path, subvol->name, gfid);
 
-                lookup_frame = NULL;
-                lookup_local = NULL;
+            STACK_WIND(lookup_frame, dht_rmdir_cached_lookup_cbk, subvol,
+                       subvol->fops->lookup, &lookup_local->loc, xattrs);
         }
+        ret++;
 
-        if (xattrs)
-                dict_unref (xattrs);
+        lookup_frame = NULL;
+        lookup_local = NULL;
+    }
 
-        return ret;
+    if (xattrs)
+        dict_unref(xattrs);
+
+    return ret;
 err:
-        if (xattrs)
-                dict_unref (xattrs);
+    if (xattrs)
+        dict_unref(xattrs);
 
-        if (lookup_frame)
-                DHT_STACK_DESTROY (lookup_frame);
+    if (lookup_frame)
+        DHT_STACK_DESTROY(lookup_frame);
 
-        /* Handle the case where the wound calls have unwound before the
-         * loop processing is done
-         */
+    /* Handle the case where the wound calls have unwound before the
+     * loop processing is done
+     */
 
-        LOCK (&frame->lock);
-        {
-                local->op_ret = -1;
-                local->op_errno = ENOTEMPTY;
+    LOCK(&frame->lock);
+    {
+        local->op_ret = -1;
+        local->op_errno = ENOTEMPTY;
 
-                local->call_cnt -= (count - ret);
-                if (!local->call_cnt)
-                        unwind = _gf_true;
-        }
-        UNLOCK (&frame->lock);
+        local->call_cnt -= (count - ret);
+        if (!local->call_cnt)
+            unwind = _gf_true;
+    }
+    UNLOCK(&frame->lock);
 
-        if (!unwind) {
-                return ret;
-        }
-        return 0;
+    if (!unwind) {
+        return ret;
+    }
+    return 0;
 }
 
-
-
 /*
  * No more entries on this subvol. Proceed to the actual rmdir operation.
  */
 
-void
-dht_rmdir_readdirp_done (call_frame_t *readdirp_frame, xlator_t *this)
+static int
+dht_rmdir_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, gf_dirent_t *entries,
+                       dict_t *xdata)
 {
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src = NULL;
+    int ret = 0;
+    char *path = NULL;
 
-        call_frame_t      *main_frame    = NULL;
-        dht_local_t       *main_local         = NULL;
-        dht_local_t       *local         = NULL;
-        int                this_call_cnt = 0;
-
+    local = frame->local;
+    prev = cookie;
+    src = prev;
 
-        local = readdirp_frame->local;
-        main_frame = local->main_frame;
-        main_local = main_frame->local;
-
-        /* At least one readdirp failed.
-         * This is a bit hit or miss - if readdirp failed on more than
-         * one subvol, we don't know which error is returned.
+    if (op_ret > 2) {
+        /* dht_rmdir_is_subvol_empty() may free the frame,
+         * copy path for logging.
          */
-        if (local->op_ret == -1) {
-                main_local->op_ret = local->op_ret;
-                main_local->op_errno = local->op_errno;
-        }
-
-        this_call_cnt = dht_frame_return (main_frame);
-
-        if (is_last_call (this_call_cnt))
-                dht_rmdir_do (main_frame, this);
-
-
-        DHT_STACK_DESTROY (readdirp_frame);
-}
-
+        path = gf_strdup(local->loc.path);
 
+        ret = dht_rmdir_is_subvol_empty(frame, this, entries, src);
 
-int
-dht_rmdir_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, gf_dirent_t *entries,
-                        dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        xlator_t     *prev = NULL;
-        xlator_t     *src = NULL;
-        int           ret = 0;
-
-
-        local = frame->local;
-        prev  = cookie;
-        src   = prev;
-
-        if (op_ret > 2) {
-                ret = dht_rmdir_is_subvol_empty (frame, this, entries, src);
-
-                switch (ret) {
-                case 0: /* non linkfiles exist */
-                        gf_msg_trace (this->name, 0,
-                                      "readdir on %s for %s returned %d "
-                                      "entries", prev->name,
-                                      local->loc.path, op_ret);
-                        local->op_ret = -1;
-                        local->op_errno = ENOTEMPTY;
-                        goto done;
-                default:
-                        /* @ret number of linkfiles are getting unlinked */
-                        gf_msg_trace (this->name, 0,
-                                      "readdir on %s for %s found %d "
-                                      "linkfiles", prev->name,
-                                      local->loc.path, ret);
-                        break;
-                }
-
-        }
-
-
-        if (ret) {
-                return 0;
+        switch (ret) {
+            case 0: /* non linkfiles exist */
+                gf_msg_trace(this->name, 0,
+                             "readdir on %s for %s returned %d "
+                             "entries",
+                             prev->name, local->loc.path, op_ret);
+                local->op_ret = -1;
+                local->op_errno = ENOTEMPTY;
+                break;
+            default:
+                /* @ret number of linkfiles are getting unlinked */
+                gf_msg_trace(this->name, 0,
+                             "readdir on %s for %s found %d "
+                             "linkfiles",
+                             prev->name, path, ret);
+                break;
         }
+    }
 
-done:
-        /* readdirp failed or no linkto files were found on this subvol */
+    /* readdirp failed or no linkto files were found on this subvol */
+    if (!ret)
+        dht_rmdir_readdirp_done(frame, this);
 
-        dht_rmdir_readdirp_done (frame, this);
-        return 0;
+    GF_FREE(path);
+    return 0;
 }
 
-/* Keep sending readdirp on the subvol until it returns no more entries
- * It is possible that not all entries will fit in a single readdirp in which
- * case the rmdir will keep failing with ENOTEMPTY
- */
-
-int
-dht_rmdir_readdirp_do (call_frame_t *readdirp_frame, xlator_t *this)
-{
-        dht_local_t *local  = NULL;
-
-        local = readdirp_frame->local;
+static int
+dht_rmdir_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
+    xlator_t *prev = NULL;
+    int ret = 0;
+    dht_conf_t *conf = this->private;
+    dict_t *dict = NULL;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *readdirp_local = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    int cnt = 0;
+
+    local = frame->local;
+    prev = cookie;
+
+    this_call_cnt = dht_frame_return(frame);
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
 
-        if (local->op_ret == -1) {
-                /* there is no point doing another readdirp on this
-                 * subvol . */
-                dht_rmdir_readdirp_done (readdirp_frame, this);
-                return 0;
+        gf_msg_debug(this->name, op_errno,
+                     "opendir on %s for %s failed, "
+                     "gfid = %s,",
+                     prev->name, local->loc.path, gfid);
+        if ((op_errno != ENOENT) && (op_errno != ESTALE)) {
+            local->op_ret = -1;
+            local->op_errno = op_errno;
         }
+        goto err;
+    }
 
-        STACK_WIND_COOKIE (readdirp_frame, dht_rmdir_readdirp_cbk,
-                           local->hashed_subvol,
-                           local->hashed_subvol,
-                           local->hashed_subvol->fops->readdirp,
-                           local->fd, 4096, 0, local->xattr);
-
-
+    if (!is_last_call(this_call_cnt))
         return 0;
 
-}
+    if (local->op_ret == -1)
+        goto err;
 
+    fd_bind(fd);
 
-int
-dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
-{
-        dht_local_t  *local         = NULL;
-        int           this_call_cnt = -1;
-        xlator_t     *prev          = NULL;
-        int           ret           = 0;
-        dht_conf_t   *conf          = this->private;
-        dict_t       *dict          = NULL;
-        int           i             = 0;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
-        dht_local_t  *readdirp_local = NULL;
-        call_frame_t *readdirp_frame = NULL;
-        int           cnt           = 0;
+    dict = dict_new();
+    if (!dict) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
 
-        local = frame->local;
-        prev  = cookie;
+    ret = dict_set_uint32(dict, conf->link_xattr_name, 256);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: Failed to set dictionary value:key = %s", local->loc.path,
+               conf->link_xattr_name);
 
+    cnt = local->call_cnt = conf->subvolume_cnt;
 
-        this_call_cnt = dht_frame_return (frame);
-        if (op_ret == -1) {
-                gf_uuid_unparse(local->loc.gfid, gfid);
-
-                gf_msg_debug (this->name, op_errno,
-                              "opendir on %s for %s failed, "
-                              "gfid = %s,",
-                              prev->name, local->loc.path, gfid);
-                if ((op_errno != ENOENT) && (op_errno != ESTALE)) {
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                }
-                goto err;
-        }
+    /* Create a separate frame per subvol as we might need
+     * to resend readdirp multiple times to get all the
+     * entries.
+     */
 
-        if (!is_last_call (this_call_cnt))
-                return 0;
-
-        if (local->op_ret == -1)
-                goto err;
-
-        fd_bind (fd);
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        readdirp_frame = copy_frame(frame);
 
-        dict = dict_new ();
-        if (!dict) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto err;
+        if (!readdirp_frame) {
+            cnt--;
+            /* Reduce the local->call_cnt as well */
+            (void)dht_frame_return(frame);
+            continue;
         }
 
-        ret = dict_set_uint32 (dict, conf->link_xattr_name, 256);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value:key = %s",
-                        local->loc.path, conf->link_xattr_name);
-
-        cnt = local->call_cnt = conf->subvolume_cnt;
-
-
-        /* Create a separate frame per subvol as we might need
-         * to resend readdirp multiple times to get all the
-         * entries.
-         */
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-
-                readdirp_frame = copy_frame (frame);
-
-                if (!readdirp_frame) {
-                        cnt--;
-                        /* Reduce the local->call_cnt as well */
-                        (void) dht_frame_return (frame);
-                        continue;
-                }
-
-                readdirp_local = dht_local_init (readdirp_frame, &local->loc,
-                                                 local->fd, 0);
-
-                if (!readdirp_local) {
-                        DHT_STACK_DESTROY (readdirp_frame);
-                        cnt--;
-                        /* Reduce the local->call_cnt as well */
-                        dht_frame_return (frame);
-                        continue;
-                }
-                readdirp_local->main_frame = frame;
-                readdirp_local->op_ret = 0;
-                readdirp_local->xattr = dict_ref (dict);
-                /* overload this field to save the subvol info */
-                readdirp_local->hashed_subvol = conf->subvolumes[i];
+        readdirp_local = dht_local_init(readdirp_frame, &local->loc, local->fd,
+                                        0);
 
-                STACK_WIND_COOKIE (readdirp_frame, dht_rmdir_readdirp_cbk,
-                                   conf->subvolumes[i], conf->subvolumes[i],
-                                   conf->subvolumes[i]->fops->readdirp,
-                                   readdirp_local->fd, 4096, 0,
-                                   readdirp_local->xattr);
+        if (!readdirp_local) {
+            DHT_STACK_DESTROY(readdirp_frame);
+            cnt--;
+            /* Reduce the local->call_cnt as well */
+            dht_frame_return(frame);
+            continue;
         }
+        readdirp_local->main_frame = frame;
+        readdirp_local->op_ret = 0;
+        readdirp_local->xattr = dict_ref(dict);
+        /* overload this field to save the subvol info */
+        readdirp_local->hashed_subvol = conf->subvolumes[i];
 
-        if (dict)
-                dict_unref (dict);
+        STACK_WIND_COOKIE(readdirp_frame, dht_rmdir_readdirp_cbk,
+                          conf->subvolumes[i], conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->readdirp,
+                          readdirp_local->fd, 4096, 0, readdirp_local->xattr);
+    }
 
-        /* Could not wind readdirp to any subvol */
+    if (dict)
+        dict_unref(dict);
 
-        if (!cnt)
-                goto err;
+    /* Could not wind readdirp to any subvol */
 
-        return 0;
+    if (!cnt)
+        goto err;
+
+    return 0;
 
 err:
-        if (is_last_call (this_call_cnt)) {
-                dht_rmdir_do (frame, this);
-        }
+    if (is_last_call(this_call_cnt)) {
+        dht_rmdir_do(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-           dict_t *xdata)
-{
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_RMDIR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->call_cnt = conf->subvolume_cnt;
-        local->op_ret   = 0;
-        local->fop_succeeded = 0;
-
-        local->flags = flags;
-
-        local->fd = fd_create (local->loc.inode, frame->root->pid);
-        if (!local->fd) {
-
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        if (flags) {
-                return dht_rmdir_do (frame, this);
-        }
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND_COOKIE (frame, dht_rmdir_opendir_cbk,
-                                   conf->subvolumes[i], conf->subvolumes[i],
-                                   conf->subvolumes[i]->fops->opendir,
-                                   loc, local->fd, NULL);
-        }
-
-        return 0;
+dht_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int ret = -1;
+    dict_t *xattr_req = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_RMDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->call_cnt = conf->subvolume_cnt;
+    local->op_ret = 0;
+    local->fop_succeeded = 0;
+
+    local->flags = flags;
+
+    local->fd = fd_create(local->loc.inode, frame->root->pid);
+    if (!local->fd) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (flags) {
+        return dht_rmdir_do(frame, this);
+    }
+    if (xdata) {
+        xattr_req = dict_ref(xdata);
+    } else {
+        xattr_req = dict_new();
+    }
+    if (xattr_req) {
+        ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+        /* If parallel-readdir is enabled, this is required
+         * to handle stale linkto files in the directory
+         * being deleted. If this fails, log an error but
+         * do not prevent the operation.
+         */
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "%s: failed to set key %s",
+                   loc->path, conf->link_xattr_name);
+        }
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "%s: failed to set key %s",
+               loc->path, conf->link_xattr_name);
+    }
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_rmdir_opendir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->opendir, loc, local->fd,
+                          xattr_req);
+    }
+
+    if (xattr_req) {
+        dict_unref(xattr_req);
+    }
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rmdir, frame, -1, op_errno,
-                          NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_entrylk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int
+dht_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    DHT_STACK_UNWIND(entrylk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 /* TODO
@@ -10547,769 +10603,789 @@ dht_entrylk_cbk (call_frame_t *frame, void *cookie,
  * as described in the bug 1311002.
  */
 int
-dht_entrylk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, const char *basename,
-             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
-{
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_ENTRYLK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+dht_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_uuid_unparse(loc->gfid, gfid);
+    local = dht_local_init(frame, loc, NULL, GF_FOP_ENTRYLK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s, "
-                              "gfid = %s", loc->path, gfid);
-                op_errno = EINVAL;
-                goto err;
-        }
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_uuid_unparse(loc->gfid, gfid);
 
-        local->call_cnt = 1;
+        gf_msg_debug(this->name, 0,
+                     "no cached subvolume for path=%s, "
+                     "gfid = %s",
+                     loc->path, gfid);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_entrylk_cbk,
-                    subvol, subvol->fops->entrylk,
-                    volume, loc, basename, cmd, type, xdata);
+    local->call_cnt = 1;
 
-        return 0;
+    STACK_WIND(frame, dht_entrylk_cbk, subvol, subvol->fops->entrylk, volume,
+               loc, basename, cmd, type, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(entrylk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_fentrylk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int
+dht_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, NULL);
-        return 0;
+    DHT_STACK_UNWIND(fentrylk, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
-
 int
-dht_fentrylk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, const char *basename,
-              entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+dht_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        char               gfid[GF_UUID_BUF_SIZE] = {0};
-
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO(fd->inode, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(fd->inode, err);
 
-        gf_uuid_unparse(fd->inode->gfid, gfid);
+    gf_uuid_unparse(fd->inode->gfid, gfid);
 
-        subvol = dht_subvol_get_cached (this, fd->inode);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "No cached subvolume for fd=%p,"
-                              " gfid = %s", fd, gfid);
-                op_errno = EINVAL;
-                goto err;
-        }
+    subvol = dht_subvol_get_cached(this, fd->inode);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0,
+                     "No cached subvolume for fd=%p,"
+                     " gfid = %s",
+                     fd, gfid);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_fentrylk_cbk,
-                    subvol, subvol->fops->fentrylk,
-                    volume, fd, basename, cmd, type, xdata);
+    STACK_WIND(frame, dht_fentrylk_cbk, subvol, subvol->fops->fentrylk, volume,
+               fd, basename, cmd, type, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fentrylk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-dht_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int32_t
+dht_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local                   = NULL;
-        int           this_call_cnt           = 0;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret < 0 && op_errno != ENOTCONN) {
-                        local->op_errno = op_errno;
-                        goto unlock;
-                }
-                local->op_ret = 0;
+    LOCK(&frame->lock);
+    {
+        if (op_ret < 0 && op_errno != ENOTCONN) {
+            local->op_errno = op_errno;
+            goto unlock;
         }
+        local->op_ret = 0;
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (ipc, frame, local->op_ret, local->op_errno,
-                                  NULL);
-        }
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(ipc, frame, local->op_ret, local->op_errno, NULL);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-
 int32_t
-dht_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+dht_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        dht_local_t  *local    = NULL;
-        int           op_errno = EINVAL;
-        dht_conf_t   *conf     = NULL;
-        int           call_cnt = 0;
-        int           i        = 0;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+    dht_conf_t *conf = NULL;
+    int call_cnt = 0;
+    int i = 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
 
-        if (op != GF_IPC_TARGET_UPCALL)
-                goto wind_default;
+    if (op != GF_IPC_TARGET_UPCALL)
+        goto wind_default;
 
-        VALIDATE_OR_GOTO (this->private, err);
-        conf = this->private;
+    VALIDATE_OR_GOTO(this->private, err);
+    conf = this->private;
 
-        local = dht_local_init (frame, NULL, NULL, GF_FOP_IPC);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = dht_local_init(frame, NULL, NULL, GF_FOP_IPC);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        call_cnt        = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
 
-        if (xdata) {
-                if (dict_set_int8 (xdata, conf->xattr_name, 0) < 0)
-                        goto err;
-        }
+    if (xdata) {
+        if (dict_set_int8(xdata, conf->xattr_name, 0) < 0)
+            goto err;
+    }
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_ipc_cbk, conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->ipc, op, xdata);
-        }
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND(frame, dht_ipc_cbk, conf->subvolumes[i],
+                   conf->subvolumes[i]->fops->ipc, op, xdata);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (ipc, frame, -1, op_errno, NULL);
+    DHT_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 
 wind_default:
-        STACK_WIND (frame, default_ipc_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ipc, op, xdata);
-        return 0;
+    STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
 }
 
-
 int
-dht_forget (xlator_t *this, inode_t *inode)
+dht_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t        ctx_int = 0;
-        dht_inode_ctx_t *ctx    = NULL;
-        dht_layout_t *layout = NULL;
+    uint64_t ctx_int = 0;
+    dht_inode_ctx_t *ctx = NULL;
+    dht_layout_t *layout = NULL;
 
-        inode_ctx_del (inode, this, &ctx_int);
+    inode_ctx_del(inode, this, &ctx_int);
 
-        if (!ctx_int)
-                return 0;
+    if (!ctx_int)
+        return 0;
 
-        ctx = (dht_inode_ctx_t *) (long) ctx_int;
+    ctx = (dht_inode_ctx_t *)(long)ctx_int;
 
-        layout = ctx->layout;
-        ctx->layout = NULL;
-        dht_layout_unref (this, layout);
-        GF_FREE (ctx);
+    layout = ctx->layout;
+    ctx->layout = NULL;
+    dht_layout_unref(this, layout);
+    GF_FREE(ctx);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_notify (xlator_t *this, int event, void *data, ...)
-{
-        xlator_t                *subvol = NULL;
-        int                      cnt    = -1;
-        int                      i      = -1;
-        dht_conf_t              *conf   = NULL;
-        int                      ret    = -1;
-        int                      propagate = 0;
-
-        int                      had_heard_from_all = 0;
-        int                      have_heard_from_all = 0;
-        struct timeval           time = {0,};
-        gf_defrag_info_t        *defrag = NULL;
-        dict_t                  *dict   = NULL;
-        gf_defrag_type           cmd    = 0;
-        dict_t                  *output = NULL;
-        va_list                  ap;
-        dht_methods_t           *methods = NULL;
-        struct gf_upcall        *up_data = NULL;
-        struct gf_upcall_cache_invalidation *up_ci = NULL;
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-
-        methods = &(conf->methods);
-
-        /* had all subvolumes reported status once till now? */
-        had_heard_from_all = 1;
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->last_event[i]) {
-                        had_heard_from_all = 0;
-                }
-        }
-
-        switch (event) {
+dht_notify(xlator_t *this, int event, void *data, ...)
+{
+    xlator_t *subvol = NULL;
+    int cnt = -1;
+    int i = -1;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int propagate = 0;
+
+    int had_heard_from_all = 0;
+    int have_heard_from_all = 0;
+    gf_defrag_info_t *defrag = NULL;
+    dict_t *dict = NULL;
+    gf_defrag_type cmd = 0;
+    dict_t *output = NULL;
+    va_list ap;
+    struct gf_upcall *up_data = NULL;
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    /* had all subvolumes reported status once till now? */
+    had_heard_from_all = 1;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->last_event[i]) {
+            had_heard_from_all = 0;
+        }
+    }
+
+    switch (event) {
         case GF_EVENT_CHILD_UP:
-                subvol = data;
+            subvol = data;
 
-                conf->gen++;
+            conf->gen++;
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (subvol == conf->subvolumes[i]) {
-                                cnt = i;
-                                break;
-                        }
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (subvol == conf->subvolumes[i]) {
+                    cnt = i;
+                    break;
                 }
+            }
 
-                if (cnt == -1) {
-                        gf_msg_debug (this->name, 0,
-                                      "got GF_EVENT_CHILD_UP bad "
-                                      "subvolume %s",
-                                      subvol->name);
-                        break;
-                }
+            if (cnt == -1) {
+                gf_msg_debug(this->name, 0,
+                             "got GF_EVENT_CHILD_UP bad "
+                             "subvolume %s",
+                             subvol->name);
+                break;
+            }
 
-                gettimeofday (&time, NULL);
-                LOCK (&conf->subvolume_lock);
-                {
-                        conf->subvolume_status[cnt] = 1;
-                        conf->last_event[cnt] = event;
-                        conf->subvol_up_time[cnt] = time.tv_sec;
-                }
-                UNLOCK (&conf->subvolume_lock);
+            LOCK(&conf->subvolume_lock);
+            {
+                conf->subvolume_status[cnt] = 1;
+                conf->last_event[cnt] = event;
+                conf->subvol_up_time[cnt] = gf_time();
+            }
+            UNLOCK(&conf->subvolume_lock);
 
-                /* one of the node came back up, do a stat update */
-                dht_get_du_info_for_subvol (this, cnt);
+            /* one of the node came back up, do a stat update */
+            dht_get_du_info_for_subvol(this, cnt);
 
-                break;
+            break;
 
         case GF_EVENT_SOME_DESCENDENT_UP:
-                subvol = data;
-                conf->gen++;
-                propagate = 1;
+            subvol = data;
+            conf->gen++;
+            propagate = 1;
 
-                break;
+            break;
 
         case GF_EVENT_SOME_DESCENDENT_DOWN:
-                subvol = data;
-                propagate = 1;
+            subvol = data;
+            propagate = 1;
 
-                break;
+            break;
 
         case GF_EVENT_CHILD_DOWN:
-                subvol = data;
-
-                if (conf->assert_no_child_down) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_CHILD_DOWN,
-                                "Received CHILD_DOWN. Exiting");
-                        if (conf->defrag) {
-                                gf_defrag_stop (conf,
-                                                GF_DEFRAG_STATUS_FAILED, NULL);
-                        } else {
-                                kill (getpid(), SIGTERM);
-                        }
-                }
+            subvol = data;
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (subvol == conf->subvolumes[i]) {
-                                cnt = i;
-                                break;
-                        }
-                }
-
-                if (cnt == -1) {
-                        gf_msg_debug (this->name, 0,
-                                      "got GF_EVENT_CHILD_DOWN bad "
-                                      "subvolume %s", subvol->name);
-                        break;
+            if (conf->assert_no_child_down) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_CHILD_DOWN,
+                       "Received CHILD_DOWN. Exiting");
+                if (conf->defrag) {
+                    gf_defrag_stop(conf, GF_DEFRAG_STATUS_FAILED, NULL);
+                } else {
+                    kill(getpid(), SIGTERM);
                 }
+            }
 
-                LOCK (&conf->subvolume_lock);
-                {
-                        conf->subvolume_status[cnt] = 0;
-                        conf->last_event[cnt] = event;
-                        conf->subvol_up_time[cnt] = 0;
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (subvol == conf->subvolumes[i]) {
+                    cnt = i;
+                    break;
                 }
-                UNLOCK (&conf->subvolume_lock);
+            }
 
-                for (i = 0; i < conf->subvolume_cnt; i++)
-                        if (conf->last_event[i] != event)
-                                event = GF_EVENT_SOME_DESCENDENT_DOWN;
+            if (cnt == -1) {
+                gf_msg_debug(this->name, 0,
+                             "got GF_EVENT_CHILD_DOWN bad "
+                             "subvolume %s",
+                             subvol->name);
                 break;
+            }
 
-        case GF_EVENT_CHILD_CONNECTING:
-                subvol = data;
+            LOCK(&conf->subvolume_lock);
+            {
+                conf->subvolume_status[cnt] = 0;
+                conf->last_event[cnt] = event;
+                conf->subvol_up_time[cnt] = 0;
+            }
+            UNLOCK(&conf->subvolume_lock);
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (subvol == conf->subvolumes[i]) {
-                                cnt = i;
-                                break;
-                        }
-                }
+            for (i = 0; i < conf->subvolume_cnt; i++)
+                if (conf->last_event[i] != event)
+                    event = GF_EVENT_SOME_DESCENDENT_DOWN;
+            break;
 
-                if (cnt == -1) {
-                        gf_msg_debug (this->name, 0,
-                                      "got GF_EVENT_CHILD_CONNECTING"
-                                      " bad subvolume %s",
-                                      subvol->name);
-                        break;
-                }
+        case GF_EVENT_CHILD_CONNECTING:
+            subvol = data;
 
-                LOCK (&conf->subvolume_lock);
-                {
-                        conf->last_event[cnt] = event;
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (subvol == conf->subvolumes[i]) {
+                    cnt = i;
+                    break;
                 }
-                UNLOCK (&conf->subvolume_lock);
+            }
 
+            if (cnt == -1) {
+                gf_msg_debug(this->name, 0,
+                             "got GF_EVENT_CHILD_CONNECTING"
+                             " bad subvolume %s",
+                             subvol->name);
                 break;
-        case GF_EVENT_VOLUME_DEFRAG:
-        {
-                if (!conf->defrag) {
-                        return ret;
-                }
-                defrag = conf->defrag;
+            }
 
-                dict = data;
-                va_start (ap, data);
-                output = va_arg (ap, dict_t*);
+            LOCK(&conf->subvolume_lock);
+            {
+                conf->last_event[cnt] = event;
+            }
+            UNLOCK(&conf->subvolume_lock);
 
-                ret = dict_get_int32 (dict, "rebalance-command",
-                                      (int32_t*)&cmd);
-                if (ret) {
-                        va_end (ap);
-                        return ret;
-                }
-                LOCK (&defrag->lock);
-                {
-                        if (defrag->is_exiting)
-                                goto unlock;
-                        if ((cmd == GF_DEFRAG_CMD_STATUS) ||
-                            (cmd == GF_DEFRAG_CMD_STATUS_TIER) ||
-                            (cmd == GF_DEFRAG_CMD_DETACH_STATUS))
-                                gf_defrag_status_get (conf, output);
-                        else if (cmd == GF_DEFRAG_CMD_START_DETACH_TIER)
-                                gf_defrag_start_detach_tier(defrag);
-                        else if (cmd == GF_DEFRAG_CMD_DETACH_START)
-                                defrag->cmd = GF_DEFRAG_CMD_DETACH_START;
-                        else if (cmd == GF_DEFRAG_CMD_STOP ||
-                                 cmd == GF_DEFRAG_CMD_STOP_DETACH_TIER ||
-                                 cmd == GF_DEFRAG_CMD_DETACH_STOP)
-                                gf_defrag_stop (conf,
-                                                GF_DEFRAG_STATUS_STOPPED, output);
-                        else if (cmd == GF_DEFRAG_CMD_PAUSE_TIER)
-                                ret = gf_defrag_pause_tier (this, defrag);
-                        else if (cmd == GF_DEFRAG_CMD_RESUME_TIER)
-                                ret = gf_defrag_resume_tier (this, defrag);
-                }
-unlock:
-                UNLOCK (&defrag->lock);
-                va_end (ap);
+            break;
+        case GF_EVENT_VOLUME_DEFRAG: {
+            if (!conf->defrag) {
                 return ret;
-                break;
+            }
+            defrag = conf->defrag;
+
+            dict = data;
+            va_start(ap, data);
+            output = va_arg(ap, dict_t *);
+
+            ret = dict_get_int32(dict, "rebalance-command", (int32_t *)&cmd);
+            if (ret) {
+                va_end(ap);
+                return ret;
+            }
+            LOCK(&defrag->lock);
+            {
+                if (defrag->is_exiting)
+                    goto unlock;
+                if ((cmd == GF_DEFRAG_CMD_STATUS) ||
+                    (cmd == GF_DEFRAG_CMD_DETACH_STATUS))
+                    gf_defrag_status_get(conf, output);
+                else if (cmd == GF_DEFRAG_CMD_DETACH_START)
+                    defrag->cmd = GF_DEFRAG_CMD_DETACH_START;
+                else if (cmd == GF_DEFRAG_CMD_STOP ||
+                         cmd == GF_DEFRAG_CMD_DETACH_STOP)
+                    gf_defrag_stop(conf, GF_DEFRAG_STATUS_STOPPED, output);
+            }
+        unlock:
+            UNLOCK(&defrag->lock);
+            va_end(ap);
+            return ret;
+            break;
         }
         case GF_EVENT_UPCALL:
-                up_data = (struct gf_upcall *)data;
-                if (up_data->event_type != GF_UPCALL_CACHE_INVALIDATION)
-                        break;
-                up_ci = (struct gf_upcall_cache_invalidation *)up_data->data;
-
-                /* Since md-cache will be aggressively filtering lookups,
-                 * the stale layout issue will be more pronounced. Hence
-                 * when a layout xattr is changed by the rebalance process
-                 * notify all the md-cache clients to invalidate the existing
-                 * stat cache and send the lookup next time*/
-                if (up_ci->dict && dict_get (up_ci->dict, conf->xattr_name))
-                        up_ci->flags |= UP_EXPLICIT_LOOKUP;
-
-                /* TODO: Instead of invalidating iatt, update the new
-                 * hashed/cached subvolume in dht inode_ctx */
-                if (IS_DHT_LINKFILE_MODE (&up_ci->stat))
-                        up_ci->flags |= UP_EXPLICIT_LOOKUP;
-
-                propagate = 1;
+            up_data = (struct gf_upcall *)data;
+            if (up_data->event_type != GF_UPCALL_CACHE_INVALIDATION)
                 break;
+            up_ci = (struct gf_upcall_cache_invalidation *)up_data->data;
+
+            /* Since md-cache will be aggressively filtering lookups,
+             * the stale layout issue will be more pronounced. Hence
+             * when a layout xattr is changed by the rebalance process
+             * notify all the md-cache clients to invalidate the existing
+             * stat cache and send the lookup next time*/
+            if (up_ci->dict && dict_get(up_ci->dict, conf->xattr_name))
+                up_ci->flags |= UP_EXPLICIT_LOOKUP;
+
+            /* TODO: Instead of invalidating iatt, update the new
+             * hashed/cached subvolume in dht inode_ctx */
+            if (IS_DHT_LINKFILE_MODE(&up_ci->stat))
+                up_ci->flags |= UP_EXPLICIT_LOOKUP;
+
+            propagate = 1;
+            break;
         default:
-                propagate = 1;
-                break;
-        }
-
+            propagate = 1;
+            break;
+    }
+
+    /* have all subvolumes reported status once by now? */
+    have_heard_from_all = 1;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->last_event[i])
+            have_heard_from_all = 0;
+    }
+
+    /* if all subvols have reported status, no need to hide anything
+       or wait for anything else. Just propagate blindly */
+    if (have_heard_from_all) {
+        propagate = 1;
+    }
+
+    if (!had_heard_from_all && have_heard_from_all) {
+        static int run_defrag = 0;
+        /* This is the first event which completes aggregation
+           of events from all subvolumes. If at least one subvol
+           had come up, propagate CHILD_UP, but only this time
+        */
+        event = GF_EVENT_CHILD_DOWN;
 
-        /* have all subvolumes reported status once by now? */
-        have_heard_from_all = 1;
         for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->last_event[i])
-                        have_heard_from_all = 0;
-        }
-
-        /* if all subvols have reported status, no need to hide anything
-           or wait for anything else. Just propagate blindly */
-        if (have_heard_from_all) {
-                propagate = 1;
+            if (conf->last_event[i] == GF_EVENT_CHILD_UP) {
+                event = GF_EVENT_CHILD_UP;
+                break;
+            }
 
+            if (conf->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
+                event = GF_EVENT_CHILD_CONNECTING;
+                /* continue to check other events for CHILD_UP */
+            }
         }
 
-
-        if (!had_heard_from_all && have_heard_from_all) {
-                /* This is the first event which completes aggregation
-                   of events from all subvolumes. If at least one subvol
-                   had come up, propagate CHILD_UP, but only this time
-                */
-                event = GF_EVENT_CHILD_DOWN;
-
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->last_event[i] == GF_EVENT_CHILD_UP) {
-                                event = GF_EVENT_CHILD_UP;
-                                break;
-                        }
-
-                        if (conf->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
-                                event = GF_EVENT_CHILD_CONNECTING;
-                                /* continue to check other events for CHILD_UP */
-                        }
-                }
-
-                /* Rebalance is started with assert_no_child_down. So we do
-                 * not need to handle CHILD_DOWN event here.
-                 *
-                 * If there is a graph switch, we should not restart the
-                 * rebalance daemon. Use 'run_defrag' to indicate if the
-                 * thread has already started.
-                 */
-                 if (conf->defrag && !run_defrag) {
-                        if (methods->migration_needed(this)) {
-                                run_defrag = 1;
-                                ret = gf_thread_create(&conf->defrag->th,
-                                                       NULL,
-                                                       gf_defrag_start, this,
-                                                       "dhtdg");
-                                if (ret) {
-                                        GF_FREE (conf->defrag);
-                                        conf->defrag = NULL;
-                                        kill (getpid(), SIGTERM);
-                                }
-                        }
-                }
+        /* Rebalance is started with assert_no_child_down. So we do
+         * not need to handle CHILD_DOWN event here.
+         *
+         * If there is a graph switch, we should not restart the
+         * rebalance daemon. Use 'run_defrag' to indicate if the
+         * thread has already started.
+         */
+        if (conf->defrag && !run_defrag) {
+            run_defrag = 1;
+            ret = gf_thread_create(&conf->defrag->th, NULL, gf_defrag_start,
+                                   this, "dhtdg");
+            if (ret) {
+                GF_FREE(conf->defrag);
+                conf->defrag = NULL;
+                kill(getpid(), SIGTERM);
+            }
         }
+    }
 
-        ret = 0;
-        if (propagate)
-                ret = default_notify (this, event, data);
+    ret = 0;
+    if (propagate)
+        ret = default_notify(this, event, data);
 out:
-        return ret;
+    return ret;
 }
 
 int
-dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+dht_inode_ctx_layout_get(inode_t *inode, xlator_t *this, dht_layout_t **layout)
 {
-        dht_inode_ctx_t         *ctx            = NULL;
-        int                      ret            = -1;
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        ret = dht_inode_ctx_get (inode, this, &ctx);
+    ret = dht_inode_ctx_get(inode, this, &ctx);
 
-        if (!ret && ctx) {
-                if (ctx->layout) {
-                        if (layout)
-                                *layout = ctx->layout;
-                        ret = 0;
-                } else {
-                        ret = -1;
-                }
+    if (!ret && ctx) {
+        if (ctx->layout) {
+            if (layout)
+                *layout = ctx->layout;
+            ret = 0;
+        } else {
+            ret = -1;
         }
+    }
 
-        return ret;
+    return ret;
 }
 
 void
-dht_log_new_layout_for_dir_selfheal (xlator_t *this, loc_t *loc,
-                                 dht_layout_t *layout)
-{
-
-        char                    string[2048] = {0};
-        char                  *output_string = NULL;
-        int                              len = 0;
-        int                              off = 0;
-        int                                i = 0;
-        gf_loglevel_t             log_level = gf_log_get_loglevel();
-        int                              ret = 0;
-        int                   max_string_len = 0;
+dht_log_new_layout_for_dir_selfheal(xlator_t *this, loc_t *loc,
+                                    dht_layout_t *layout)
+{
+    char string[2048] = {0};
+    char *output_string = NULL;
+    int len = 0;
+    int off = 0;
+    int i = 0;
+    gf_loglevel_t log_level = gf_log_get_loglevel();
+    int ret = 0;
+
+    if (log_level < GF_LOG_INFO)
+        return;
 
-        if (log_level < GF_LOG_INFO)
-                return;
+    if (!layout)
+        return;
 
-        if (!layout)
-                return;
+    if (!layout->cnt)
+        return;
 
-        if (!layout->cnt)
-                return;
+    if (!loc)
+        return;
 
-        if (!loc)
-                return;
+    if (!loc->path)
+        return;
 
-        if (!loc->path)
-                return;
+    ret = snprintf(string, sizeof(string), "Setting layout of %s with ",
+                   loc->path);
 
-        max_string_len = sizeof (string);
+    if (ret < 0)
+        return;
 
-        ret = snprintf (string, max_string_len, "Setting layout of %s with ",
-                        loc->path);
+    len += ret;
+
+    /* Calculation  of total length of the string required to calloc
+     * output_string. Log includes subvolume-name, start-range, end-range
+     * and err value.
+     *
+     * This log will help to debug cases where:
+     * a) Different processes set different layout of a directory.
+     * b) Error captured in lookup, which will be filled in layout->err
+     * (like ENOENT, ESTALE etc)
+     */
+
+    for (i = 0; i < layout->cnt; i++) {
+        ret = snprintf(string, sizeof(string),
+                       "[Subvol_name: %s, Err: %d , Start: "
+                       "0x%x, Stop: 0x%x, Hash: 0x%x], ",
+                       layout->list[i].xlator->name, layout->list[i].err,
+                       layout->list[i].start, layout->list[i].stop,
+                       layout->list[i].commit_hash);
 
         if (ret < 0)
-                return;
+            return;
 
         len += ret;
+    }
 
-       /* Calculation  of total length of the string required to calloc
-        * output_string. Log includes subvolume-name, start-range, end-range and
-        * err value.
-        *
-        * This log will help to debug cases where:
-        * a) Different processes set different layout of a directory.
-        * b) Error captured in lookup, which will be filled in layout->err
-        * (like ENOENT, ESTALE etc)
-        */
+    len++;
 
-        for (i = 0; i < layout->cnt; i++) {
+    output_string = GF_MALLOC(len + 1, gf_common_mt_char);
 
-                ret  = snprintf (string, max_string_len,
-                                 "[Subvol_name: %s, Err: %d , Start: "
-                                 "%"PRIu32 " , Stop: %"PRIu32 " , Hash: %"
-                                 PRIu32 " ], ",
-                                 layout->list[i].xlator->name,
-                                 layout->list[i].err, layout->list[i].start,
-                                 layout->list[i].stop,
-                                 layout->list[i].commit_hash);
-
-                if (ret < 0)
-                        return;
-
-                len += ret;
-
-        }
+    if (!output_string)
+        return;
 
-        len++;
+    ret = snprintf(output_string, len + 1, "Setting layout of %s with ",
+                   loc->path);
 
-        output_string = GF_CALLOC (len, sizeof (char), gf_common_mt_char);
+    if (ret < 0)
+        goto err;
 
-        if (!output_string)
-                return;
+    off += ret;
 
-        ret = snprintf (output_string, len, "Setting layout of %s with ",
-                        loc->path);
+    for (i = 0; i < layout->cnt; i++) {
+        ret = snprintf(output_string + off, len - off,
+                       "[Subvol_name: %s, Err: %d , Start: "
+                       "0x%x, Stop: 0x%x, Hash: 0x%x], ",
+                       layout->list[i].xlator->name, layout->list[i].err,
+                       layout->list[i].start, layout->list[i].stop,
+                       layout->list[i].commit_hash);
 
         if (ret < 0)
-                goto err;
+            goto err;
 
         off += ret;
+    }
 
-
-        for (i = 0; i < layout->cnt; i++) {
-
-                ret  =  snprintf (output_string + off, len - off,
-                                  "[Subvol_name: %s, Err: %d , Start: "
-                                  "%"PRIu32 " , Stop: %"PRIu32 " , Hash: %"
-                                  PRIu32  " ], ",
-                                  layout->list[i].xlator->name,
-                                  layout->list[i].err, layout->list[i].start,
-                                  layout->list[i].stop,
-                                  layout->list[i].commit_hash);
-
-                if (ret < 0)
-                        goto err;
-
-                off += ret;
-
-        }
-
-        gf_msg (this->name, GF_LOG_DEBUG, 0, DHT_MSG_LOG_FIXED_LAYOUT,
-                "%s", output_string);
+    gf_msg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_LOG_FIXED_LAYOUT, "%s",
+           output_string);
 
 err:
-        GF_FREE (output_string);
+    GF_FREE(output_string);
 }
 
-int32_t dht_migration_get_dst_subvol(xlator_t *this, dht_local_t  *local)
+int32_t
+dht_migration_get_dst_subvol(xlator_t *this, dht_local_t *local)
 {
-        int ret = -1;
+    int ret = -1;
 
-        if (!local)
-                goto out;
+    if (!local)
+        goto out;
 
-        local->rebalance.target_node =
-                dht_subvol_get_hashed (this, &local->loc);
+    local->rebalance.target_node = dht_subvol_get_hashed(this, &local->loc);
 
-        if (local->rebalance.target_node)
-                ret = 0;
+    if (local->rebalance.target_node)
+        ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-int32_t dht_migration_needed(xlator_t *this)
+/*
+This function should not be called more then once during a FOP
+handling path. It is valid only for for ops on files
+*/
+int32_t
+dht_set_local_rebalance(xlator_t *this, dht_local_t *local, struct iatt *stbuf,
+                        struct iatt *prebuf, struct iatt *postbuf,
+                        dict_t *xdata)
 {
-        gf_defrag_info_t        *defrag = NULL;
-        dht_conf_t              *conf   = NULL;
-        int                      ret = 0;
+    if (!local)
+        return -1;
 
-        conf = this->private;
+    if (local->rebalance.set) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_REBAL_STRUCT_SET,
+               "local->rebalance already set");
+    }
 
-        GF_VALIDATE_OR_GOTO ("dht", conf, out);
-        GF_VALIDATE_OR_GOTO ("dht", conf->defrag, out);
+    if (stbuf)
+        memcpy(&local->rebalance.stbuf, stbuf, sizeof(struct iatt));
 
-        defrag = conf->defrag;
+    if (prebuf)
+        memcpy(&local->rebalance.prebuf, prebuf, sizeof(struct iatt));
 
-        if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER) &&
-            (defrag->cmd != GF_DEFRAG_CMD_START_DETACH_TIER))
-                ret = 1;
+    if (postbuf)
+        memcpy(&local->rebalance.postbuf, postbuf, sizeof(struct iatt));
 
-out:
-        return ret;
-}
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
 
+    local->rebalance.set = 1;
 
+    return 0;
+}
 
-/*
-This function should not be called more then once during a FOP
-handling path. It is valid only for for ops on files
-*/
-int32_t dht_set_local_rebalance (xlator_t *this, dht_local_t *local,
-                                 struct iatt *stbuf,
-                                 struct iatt *prebuf, struct iatt *postbuf,
-                                 dict_t *xdata)
+int32_t
+dht_release(xlator_t *this, fd_t *fd)
 {
+    return dht_fd_ctx_destroy(this, fd);
+}
 
-        if (!local)
-                return -1;
-
-        if (local->rebalance.set) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_REBAL_STRUCT_SET,
-                        "local->rebalance already set");
-        }
+static int
+dht_pt_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, inode_t *inode, struct iatt *stbuf,
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
 
+    local = frame->local;
 
-        if (stbuf)
-                memcpy (&local->rebalance.stbuf, stbuf, sizeof (struct iatt));
+    if (!op_ret) {
+        dht_layout_set(this, inode, local->layout);
+    }
 
-        if (prebuf)
-                memcpy (&local->rebalance.prebuf, prebuf, sizeof (struct iatt));
+    DHT_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, NULL);
 
-        if (postbuf)
-                memcpy (&local->rebalance.postbuf, postbuf,
-                        sizeof (struct iatt));
+    return 0;
+}
 
-        if (xdata)
-                local->rebalance.xdata = dict_ref (xdata);
+int32_t
+dht_pt_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
+{
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    bool free_xdata = false;
+    int ret = 0;
+    int op_errno = 0;
+    int32_t *disk_layout_p = NULL;
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = dht_layout_new(this, conf->subvolume_cnt);
+    if (!layout)
+        goto wind;
+
+    local->layout = layout;
+
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata)
+            goto wind;
+        free_xdata = true;
+    }
+
+    /*Set the xlator or the following will crash*/
+    layout->list[0].xlator = conf->subvolumes[0];
+
+    dht_selfheal_layout_new_directory(frame, loc, layout);
+
+    dht_disk_layout_extract(this, layout, 0, &disk_layout_p);
+
+    ret = dict_set_bin(xdata, conf->xattr_name, disk_layout_p, 4 * 4);
+    if (ret) {
+        gf_msg("dht", GF_LOG_DEBUG, EINVAL, DHT_MSG_DICT_SET_FAILED,
+               "dht layout dict set failed");
+    }
+wind:
+    STACK_WIND(frame, dht_pt_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    if (free_xdata)
+        dict_unref(xdata);
+    return 0;
 
-        local->rebalance.set = 1;
+err:
+    op_errno = local ? local->op_errno : op_errno;
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-gf_boolean_t
-dht_is_tier_xlator (xlator_t *this)
+static int
+dht_pt_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
+    dht_conf_t *conf = NULL;
 
-        if (strcmp (this->type, "cluster/tier") == 0)
-                return _gf_true;
-        return _gf_false;
-}
+    conf = this->private;
+    dict_del(xattr, conf->xattr_name);
+    dict_del(xattr, conf->mds_xattr_key);
+    dict_del(xattr, conf->commithash_xattr_name);
 
-int32_t
-dht_release (xlator_t *this, fd_t *fd)
-{
-        return dht_fd_ctx_destroy (this, fd);
+    if (frame->root->pid >= 0) {
+        GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+        GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
+    }
+
+    DHT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
+    return 0;
 }
 
 int
-dht_remove_stale_linkto (void *data)
+dht_pt_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *key, dict_t *xdata)
 {
-        call_frame_t    *frame = NULL;
-        dht_local_t     *local = NULL;
-        xlator_t        *this  = NULL;
-        dict_t          *xdata_in = NULL;
-        int             ret = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", data, out);
+    STACK_WIND(frame, dht_pt_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, key, xdata);
+    return 0;
+}
 
-        frame = data;
-        local = frame->local;
-        this = frame->this;
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", local, out);
-        GF_VALIDATE_OR_GOTO ("dht", local->link_subvol, out);
+static int
+dht_pt_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    dht_conf_t *conf = NULL;
 
-        xdata_in = dict_new ();
-        if (!xdata_in)
-                goto out;
+    conf = this->private;
+    dict_del(xattr, conf->xattr_name);
 
-        ret = dht_fill_dict_to_avoid_unlink_of_migrating_file (xdata_in);
-        if (ret) {
-                  gf_msg (this->name, GF_LOG_WARNING, -ret, 0,
-                                "Failed to set keys for stale linkto"
-                                "deletion on path %s", local->loc.path);
-                  goto out;
-        }
+    if (frame->root->pid >= 0) {
+        GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+        GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
+    }
 
-        ret = syncop_unlink (local->link_subvol, &local->loc, xdata_in, NULL);
-        if (ret) {
-                  gf_msg (this->name, GF_LOG_WARNING, -ret, 0,
-                                "Removal of linkto failed"
-                                " on path %s at subvol %s",
-                                local->loc.path, local->link_subvol->name);
+    DHT_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, xattr, xdata);
+    return 0;
+}
 
-        }
-out:
-        if (xdata_in)
-                dict_unref (xdata_in);
-        return ret;
+int
+dht_pt_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+                 dict_t *xdata)
+{
+    STACK_WIND(frame, dht_pt_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, key, xdata);
+    return 0;
 }
 
+/* The job of this function is to check if all the xlators have updated
+ * error in the layout. */
 int
-dht_remove_stale_linkto_cbk (int ret, call_frame_t *sync_frame, void *data)
+dht_dir_layout_error_check(xlator_t *this, inode_t *inode)
 {
-        DHT_STACK_DESTROY (sync_frame);
-        return 0;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+
+    layout = dht_layout_get(this, inode);
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err == 0) {
+            return 0;
+        }
+    }
+
+    /* Returning the first xlator error as all xlators have errors */
+    return layout->list[0].err;
 }
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index b82d3c4d267..fe0dc3db34a 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -9,156 +9,148 @@
 */
 
 #include <regex.h>
-#include <signal.h>
-#include <fnmatch.h>
 
 #include "dht-mem-types.h"
 #include "dht-messages.h"
-#include "call-stub.h"
+#include <glusterfs/call-stub.h>
 #include "libxlator.h"
-#include "syncop.h"
-#include "refcount.h"
-#include "timer.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/refcount.h>
+#include <glusterfs/timer.h>
 #include "protocol-common.h"
-#include "glusterfs-acl.h"
+#include <glusterfs/glusterfs-acl.h>
 
 #ifndef _DHT_H
 #define _DHT_H
 
-#define GF_XATTR_FIX_LAYOUT_KEY         "distribute.fix.layout"
-#define GF_XATTR_TIER_LAYOUT_FIXED_KEY  "trusted.tier.fix.layout.complete"
-#define GF_XATTR_FILE_MIGRATE_KEY       "trusted.distribute.migrate-data"
-#define DHT_MDS_STR                     "mds"
-#define GF_DHT_LOOKUP_UNHASHED_OFF      0
-#define GF_DHT_LOOKUP_UNHASHED_ON       1
-#define GF_DHT_LOOKUP_UNHASHED_AUTO     2
-#define DHT_PATHINFO_HEADER             "DISTRIBUTE:"
-#define DHT_FILE_MIGRATE_DOMAIN         "dht.file.migrate"
+#define GF_XATTR_FIX_LAYOUT_KEY "distribute.fix.layout"
+#define GF_XATTR_FILE_MIGRATE_KEY "trusted.distribute.migrate-data"
+#define DHT_MDS_STR "mds"
+#define GF_DHT_LOOKUP_UNHASHED_OFF 0
+#define GF_DHT_LOOKUP_UNHASHED_ON 1
+#define GF_DHT_LOOKUP_UNHASHED_AUTO 2
+#define DHT_PATHINFO_HEADER "DISTRIBUTE:"
+#define DHT_FILE_MIGRATE_DOMAIN "dht.file.migrate"
 /* Layout synchronization */
-#define DHT_LAYOUT_HEAL_DOMAIN          "dht.layout.heal"
+#define DHT_LAYOUT_HEAL_DOMAIN "dht.layout.heal"
 /* Namespace synchronization */
-#define DHT_ENTRY_SYNC_DOMAIN           "dht.entry.sync"
-#define TIERING_MIGRATION_KEY           "tiering.migration"
-#define DHT_LAYOUT_HASH_INVALID         1
-#define MAX_REBAL_THREADS               sysconf(_SC_NPROCESSORS_ONLN)
+#define DHT_ENTRY_SYNC_DOMAIN "dht.entry.sync"
+#define DHT_LAYOUT_HASH_INVALID 1
+#define MAX_REBAL_THREADS sysconf(_SC_NPROCESSORS_ONLN)
 
-#define DHT_DIR_STAT_BLOCKS          8
-#define DHT_DIR_STAT_SIZE            4096
+#define DHT_DIR_STAT_BLOCKS 8
+#define DHT_DIR_STAT_SIZE 4096
 
+/* Virtual xattr for subvols status */
 
-/* Virtual xattrs for debugging */
+#define DHT_SUBVOL_STATUS_KEY "dht.subvol.status"
 
-#define DHT_DBG_HASHED_SUBVOL_KEY         "dht.file.hashed-subvol"
+/* Virtual xattrs for debugging */
 
-
-/* Array to hold custom xattr keys
-*/
-extern char *xattrs_to_heal[];
+#define DHT_DBG_HASHED_SUBVOL_PATTERN "dht.file.hashed-subvol.*"
+#define DHT_DBG_HASHED_SUBVOL_KEY "dht.file.hashed-subvol."
 
 /* Rebalance nodeuuid flags */
-#define REBAL_NODEUUID_MINE   0x01
-
+#define REBAL_NODEUUID_MINE 0x01
 
-typedef int (*dht_selfheal_dir_cbk_t) (call_frame_t *frame, void *cookie,
-                                       xlator_t     *this,
-                                       int32_t       op_ret, int32_t op_errno,
-                                       dict_t       *xdata);
-typedef int (*dht_defrag_cbk_fn_t) (xlator_t        *this, xlator_t *dst_node,
-                                    call_frame_t    *frame, int ret);
+typedef int (*dht_selfheal_dir_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, dict_t *xdata);
+typedef int (*dht_defrag_cbk_fn_t)(xlator_t *this, xlator_t *dst_node,
+                                   call_frame_t *frame, int ret);
 
-typedef int (*dht_refresh_layout_unlock) (call_frame_t *frame, xlator_t *this,
+typedef int (*dht_refresh_layout_unlock)(call_frame_t *frame, xlator_t *this,
                                          int op_ret, int invoke_cbk);
 
-typedef int (*dht_refresh_layout_done_handle) (call_frame_t *frame);
+typedef int (*dht_refresh_layout_done_handle)(call_frame_t *frame);
 
 struct dht_layout {
-        int                spread_cnt;  /* layout spread count per directory,
-                                           is controlled by 'setxattr()' with
-                                           special key */
-        int                cnt;
-        int                preset;
-        /*
-         * The last *configuration* state for which this directory was known
-         * to be in balance.  The corresponding vol_commit_hash changes
-         * whenever bricks are added or removed.  This value changes when a
-         * (full) rebalance is complete.  If they match, it's safe to assume
-         * that every file is where it should be and there's no need to do
-         * lookups for files elsewhere.  If they don't, then we have to do a
-         * global lookup to be sure.
-         */
-        uint32_t           commit_hash;
-        /*
-         * The *runtime* state of the volume, changes when connections to
-         * bricks are made or lost.
-         */
-        int                gen;
-        int                type;
-        gf_atomic_t        ref; /* use with dht_conf_t->layout_lock */
-        uint32_t           search_unhashed;
-        struct {
-                int        err;   /* 0 = normal
-                                     -1 = dir exists and no xattr
-                                     >0 = dir lookup failed with errno
-                                  */
-                uint32_t   start;
-                uint32_t   stop;
-                uint32_t   commit_hash;
-                xlator_t  *xlator;
-        } list[];
+    int spread_cnt; /* layout spread count per directory,
+                       is controlled by 'setxattr()' with
+                       special key */
+    int cnt;
+    int preset;
+    /*
+     * The last *configuration* state for which this directory was known
+     * to be in balance.  The corresponding vol_commit_hash changes
+     * whenever bricks are added or removed.  This value changes when a
+     * (full) rebalance is complete.  If they match, it's safe to assume
+     * that every file is where it should be and there's no need to do
+     * lookups for files elsewhere.  If they don't, then we have to do a
+     * global lookup to be sure.
+     */
+    uint32_t commit_hash;
+    /*
+     * The *runtime* state of the volume, changes when connections to
+     * bricks are made or lost.
+     */
+    int gen;
+    int type;
+    gf_atomic_t ref; /* use with dht_conf_t->layout_lock */
+    uint32_t search_unhashed;
+    struct {
+        int err; /* 0 = normal
+                    -1 = dir exists and no xattr
+                    >0 = dir lookup failed with errno
+                 */
+        uint32_t start;
+        uint32_t stop;
+        uint32_t commit_hash;
+        xlator_t *xlator;
+    } list[];
 };
-typedef struct dht_layout  dht_layout_t;
+typedef struct dht_layout dht_layout_t;
 
 struct dht_stat_time {
-        uint32_t        atime;
-        uint32_t        atime_nsec;
-        uint32_t        ctime;
-        uint32_t        ctime_nsec;
-        uint32_t        mtime;
-        uint32_t        mtime_nsec;
+    uint32_t atime;
+    uint32_t atime_nsec;
+    uint32_t ctime;
+    uint32_t ctime_nsec;
+    uint32_t mtime;
+    uint32_t mtime_nsec;
 };
 
 typedef struct dht_stat_time dht_stat_time_t;
 
 struct dht_inode_ctx {
-        dht_layout_t    *layout;
-        dht_stat_time_t  time;
-        xlator_t        *lock_subvol;
-        xlator_t        *mds_subvol;     /* This is only used for directories */
+    dht_layout_t *layout;
+    dht_stat_time_t time;
+    xlator_t *lock_subvol;
+    xlator_t *mds_subvol; /* This is only used for directories */
 };
 
 typedef struct dht_inode_ctx dht_inode_ctx_t;
 
-
 typedef enum {
-        DHT_HASH_TYPE_DM,
-        DHT_HASH_TYPE_DM_USER,
+    DHT_HASH_TYPE_DM,
+    DHT_HASH_TYPE_DM_USER,
 } dht_hashfn_type_t;
 
 typedef enum {
-        DHT_INODELK,
-        DHT_ENTRYLK,
+    DHT_INODELK,
+    DHT_ENTRYLK,
 } dht_lock_type_t;
 
 /* rebalance related */
 struct dht_rebalance_ {
-        xlator_t            *from_subvol;
-        xlator_t            *target_node;
-        off_t                offset;
-        size_t               size;
-        int32_t              flags;
-        int                  count;
-        struct iobref       *iobref;
-        struct iovec        *vector;
-        struct iatt          stbuf;
-        struct iatt          prebuf;
-        struct iatt          postbuf;
-        dht_defrag_cbk_fn_t  target_op_fn;
-        dict_t              *xdata;
-        dict_t              *xattr;
-        dict_t              *dict;
-        int32_t              set;
-        struct gf_flock      flock;
-        int                  lock_cmd;
+    xlator_t *from_subvol;
+    xlator_t *target_node;
+    off_t offset;
+    size_t size;
+    int32_t flags;
+    int count;
+    struct iobref *iobref;
+    struct iovec *vector;
+    struct iatt stbuf;
+    struct iatt prebuf;
+    struct iatt postbuf;
+    dht_defrag_cbk_fn_t target_op_fn;
+    dict_t *xdata;
+    dict_t *xattr;
+    dict_t *dict;
+    struct gf_flock flock;
+    int32_t set;
+    int lock_cmd;
 };
 
 /**
@@ -166,63 +158,62 @@ struct dht_rebalance_ {
  * events
  **/
 typedef enum {
-        qdstatfs_action_OFF = 0,
-        qdstatfs_action_REPLACE,
-        qdstatfs_action_NEGLECT,
-        qdstatfs_action_COMPARE,
+    qdstatfs_action_OFF = 0,
+    qdstatfs_action_REPLACE,
+    qdstatfs_action_NEGLECT,
+    qdstatfs_action_COMPARE,
 } qdstatfs_action_t;
 
 typedef enum {
-        REACTION_INVALID,
-        FAIL_ON_ANY_ERROR,
-        IGNORE_ENOENT_ESTALE,
-        IGNORE_ENOENT_ESTALE_EIO,
+    REACTION_INVALID,
+    FAIL_ON_ANY_ERROR,
+    IGNORE_ENOENT_ESTALE,
+    IGNORE_ENOENT_ESTALE_EIO,
 } dht_reaction_type_t;
 
 struct dht_skip_linkto_unlink {
-
-        gf_boolean_t    handle_valid_link;
-        int             opend_fd_count;
-        xlator_t        *hash_links_to;
-        uuid_t          cached_gfid;
-        uuid_t          hashed_gfid;
+    xlator_t *hash_links_to;
+    uuid_t cached_gfid;
+    uuid_t hashed_gfid;
+    int opend_fd_count;
+    gf_boolean_t handle_valid_link;
 };
 
 typedef struct {
-        xlator_t            *xl;
-        loc_t                loc; /* contains/points to inode to lock on. */
-        short                type; /* read/write lock.                     */
-        char                *domain;  /* Only locks within a single domain
-                                * contend with each other
-                                */
-        char                *basename; /* Required for entrylk */
-        gf_lkowner_t         lk_owner;
-        gf_boolean_t         locked;
-        dht_reaction_type_t  do_on_failure;
+    xlator_t *xl;
+    loc_t loc;      /* contains/points to inode to lock on. */
+    char *domain;   /* Only locks within a single domain
+                     * contend with each other
+                     */
+    char *basename; /* Required for entrylk */
+    gf_boolean_t locked;
+    dht_reaction_type_t do_on_failure;
+    short type; /* read/write lock.                     */
+    gf_lkowner_t lk_owner;
 } dht_lock_t;
 
 /* The lock structure represents inodelk. */
 typedef struct {
-        fop_inodelk_cbk_t   inodelk_cbk;
-        dht_lock_t        **locks;
-        int                 lk_count;
-        dht_reaction_type_t reaction;
-
-        /* whether locking failed on _any_ of the "locks" above */
-        int                 op_ret;
-        int                 op_errno;
+    fop_inodelk_cbk_t inodelk_cbk;
+    dht_lock_t **locks;
+    int lk_count;
+    dht_reaction_type_t reaction;
+
+    /* whether locking failed on _any_ of the "locks" above */
+    int op_ret;
+    int op_errno;
 } dht_ilock_wrap_t;
 
 /* The lock structure represents entrylk. */
 typedef struct {
-        fop_entrylk_cbk_t   entrylk_cbk;
-        dht_lock_t        **locks;
-        int                 lk_count;
-        dht_reaction_type_t reaction;
-
-        /* whether locking failed on _any_ of the "locks" above */
-        int                 op_ret;
-        int                 op_errno;
+    fop_entrylk_cbk_t entrylk_cbk;
+    dht_lock_t **locks;
+    int lk_count;
+    dht_reaction_type_t reaction;
+
+    /* whether locking failed on _any_ of the "locks" above */
+    int op_ret;
+    int op_errno;
 } dht_elock_wrap_t;
 
 /* The first member of dht_dir_transaction_t should be of type dht_ilock_wrap_t.
@@ -231,1267 +222,1163 @@ typedef struct {
  * lock[0].ns.parent_layout (like in dht_local_wipe).
  */
 typedef union {
-        union {
-                 dht_ilock_wrap_t my_layout;
-                 dht_ilock_wrap_t parent_layout;
-        } layout;
-        struct dht_namespace {
-                dht_ilock_wrap_t parent_layout;
-                dht_elock_wrap_t directory_ns;
-                fop_entrylk_cbk_t ns_cbk;
-        } ns;
+    union {
+        dht_ilock_wrap_t my_layout;
+        dht_ilock_wrap_t parent_layout;
+    } layout;
+    struct dht_namespace {
+        dht_ilock_wrap_t parent_layout;
+        dht_elock_wrap_t directory_ns;
+        fop_entrylk_cbk_t ns_cbk;
+    } ns;
 } dht_dir_transaction_t;
 
-typedef
-int (*dht_selfheal_layout_t)(call_frame_t *frame, loc_t *loc,
-                             dht_layout_t *layout);
-
-typedef
-gf_boolean_t (*dht_need_heal_t)(call_frame_t *frame, dht_layout_t **inmem,
-                                dht_layout_t **ondisk);
+typedef int (*dht_selfheal_layout_t)(call_frame_t *frame, loc_t *loc,
+                                     dht_layout_t *layout);
 
-typedef struct {
-        uint64_t                 blocks_used;
-        uint64_t                 pblocks_used;
-        uint64_t                 files_used;
-        uint64_t                 pfiles_used;
-        uint64_t                 unhashed_blocks_used;
-        uint64_t                 unhashed_pblocks_used;
-        uint64_t                 unhashed_files_used;
-        uint64_t                 unhashed_pfiles_used;
-        uint64_t                 unhashed_fsid;
-        uint64_t                 hashed_fsid;
-} tier_statvfs_t;
+typedef gf_boolean_t (*dht_need_heal_t)(call_frame_t *frame,
+                                        dht_layout_t **inmem,
+                                        dht_layout_t **ondisk);
 
 struct dht_local {
-        int                      call_cnt;
-        loc_t                    loc;
-        loc_t                    loc2;
-        int                      op_ret;
-        int                      op_errno;
-        int                      layout_mismatch;
-        /* Use stbuf as the postbuf, when we require both
-         * pre and post attrs */
-        struct iatt              stbuf;
-        struct iatt              mds_stbuf;
-        struct iatt              prebuf;
-        struct iatt              preoldparent;
-        struct iatt              postoldparent;
-        struct iatt              preparent;
-        struct iatt              postparent;
-        struct statvfs           statvfs;
-        tier_statvfs_t           tier_statvfs;
-        fd_t                    *fd;
-        inode_t                 *inode;
-        dict_t                  *params;
-        dict_t                  *xattr;
-        dict_t                  *mds_xattr;
-        dict_t                  *xdata;      /* dict used to save xdata response by xattr fop */
-        dict_t                  *xattr_req;
-        dht_layout_t            *layout;
-        size_t                   size;
-        ino_t                    ia_ino;
-        xlator_t                *src_hashed, *src_cached;
-        xlator_t                *dst_hashed, *dst_cached;
-        xlator_t                *cached_subvol;
-        xlator_t                *hashed_subvol;
-        xlator_t                *mds_subvol; /* This is use for dir only */
-        char                     need_selfheal;
-        char                     need_xattr_heal;
-        int                      file_count;
-        int                      dir_count;
-        call_frame_t            *main_frame;
-        int                      fop_succeeded;
-        struct {
-                fop_mknod_cbk_t  linkfile_cbk;
-                struct iatt      stbuf;
-                loc_t            loc;
-                inode_t         *inode;
-                dict_t          *xattr;
-                xlator_t        *srcvol;
-        } linkfile;
-        struct {
-                uint32_t                hole_cnt;
-                uint32_t                overlaps_cnt;
-                uint32_t                down;
-                uint32_t                misc;
-                uint32_t                missing_cnt;
-                dht_selfheal_dir_cbk_t  dir_cbk;
-                dht_selfheal_layout_t   healer;
-                dht_need_heal_t         should_heal;
-                gf_boolean_t            force_mkdir;
-                dht_layout_t           *layout, *refreshed_layout;
-        } selfheal;
-
-        dht_refresh_layout_unlock              refresh_layout_unlock;
-        dht_refresh_layout_done_handle         refresh_layout_done;
-
-        uint32_t                 uid;
-        uint32_t                 gid;
-
-        /* needed by nufa */
-        int32_t flags;
-        mode_t  mode;
-        dev_t   rdev;
-        mode_t  umask;
-
-        /* need for file-info */
-        char   *xattr_val;
-        char   *key;
-
-        /* which xattr request? */
-        char xsel[256];
-        int32_t alloc_len;
-
-        /* gfid related */
-        uuid_t  gfid;
-
-        /* flag used to make sure we need to return estale in
-           {lookup,revalidate}_cbk */
-        char return_estale;
-        char need_lookup_everywhere;
-
-        glusterfs_fop_t      fop;
-
-        gf_boolean_t     linked;
-        xlator_t        *link_subvol;
-
-        struct dht_rebalance_ rebalance;
-        xlator_t        *first_up_subvol;
-
-        gf_boolean_t     quota_deem_statfs;
-
-        gf_boolean_t     added_link;
-        gf_boolean_t     is_linkfile;
-
-        struct dht_skip_linkto_unlink  skip_unlink;
-
-        dht_dir_transaction_t lock[2], *current;
-
-        /* inodelks during filerename for backward compatibility */
-        dht_lock_t           **rename_inodelk_backward_compatible;
-        int                    rename_inodelk_bc_count;
-
-        short           lock_type;
-
-        call_stub_t *stub;
-        int32_t      parent_disk_layout[4];
-
-        /* To hold dentries of readdir spawning across subvols */
-        gf_dirent_t   entries;
-        size_t        filled;
-
-        /* rename rollback */
-        int    *ret_cache ;
-
-        /* fd open check */
-        gf_boolean_t fd_checked;
-        /* This is use only for directory operation */
-        int32_t valid;
-        gf_boolean_t heal_layout;
-        int32_t mds_heal_fresh_lookup;
-        loc_t        loc2_copy;
-        gf_boolean_t locked;
-        gf_boolean_t dont_create_linkto;
+    loc_t loc;
+    loc_t loc2;
+    int call_cnt;
+    int op_ret;
+    int op_errno;
+    int layout_mismatch;
+    /* Use stbuf as the postbuf, when we require both
+     * pre and post attrs */
+    struct iatt stbuf;
+    struct iatt mds_stbuf;
+    struct iatt prebuf;
+    struct iatt preoldparent;
+    struct iatt postoldparent;
+    struct iatt preparent;
+    struct iatt postparent;
+    struct statvfs statvfs;
+    fd_t *fd;
+    inode_t *inode;
+    dict_t *params;
+    dict_t *xattr;
+    dict_t *mds_xattr;
+    dict_t *xdata; /* dict used to save xdata response by xattr fop */
+    dict_t *xattr_req;
+    dht_layout_t *layout;
+    size_t size;
+    ino_t ia_ino;
+    xlator_t *src_hashed, *src_cached;
+    xlator_t *dst_hashed, *dst_cached;
+    xlator_t *cached_subvol;
+    xlator_t *hashed_subvol;
+    xlator_t *mds_subvol; /* This is use for dir only */
+    int file_count;
+    int dir_count;
+    call_frame_t *main_frame;
+    int fop_succeeded;
+    struct {
+        fop_mknod_cbk_t linkfile_cbk;
+        struct iatt stbuf;
+        loc_t loc;
+        inode_t *inode;
+        dict_t *xattr;
+        xlator_t *srcvol;
+    } linkfile;
+    struct {
+        uint32_t hole_cnt;
+        uint32_t overlaps_cnt;
+        uint32_t down;
+        uint32_t misc;
+        dht_selfheal_dir_cbk_t dir_cbk;
+        dht_selfheal_layout_t healer;
+        dht_need_heal_t should_heal;
+        dht_layout_t *layout, *refreshed_layout;
+        uint32_t missing_cnt;
+        gf_boolean_t force_mkdir;
+    } selfheal;
+
+    dht_refresh_layout_unlock refresh_layout_unlock;
+    dht_refresh_layout_done_handle refresh_layout_done;
+
+    uint32_t uid;
+    uint32_t gid;
+    pid_t pid;
+
+    glusterfs_fop_t fop;
+
+    /* need for file-info */
+    char *xattr_val;
+    char *key;
+
+    /* needed by nufa */
+    int32_t flags;
+    mode_t mode;
+    dev_t rdev;
+    mode_t umask;
+
+    /* which xattr request? */
+    char xsel[256];
+    int32_t alloc_len;
+
+    /* gfid related */
+    uuid_t gfid;
+    uuid_t gfid_req;
+
+    xlator_t *link_subvol;
+
+    struct dht_rebalance_ rebalance;
+    xlator_t *first_up_subvol;
+
+    struct dht_skip_linkto_unlink skip_unlink;
+
+    dht_dir_transaction_t lock[2], *current;
+
+    /* inodelks during filerename for backward compatibility */
+    dht_lock_t **rename_inodelk_backward_compatible;
+
+    call_stub_t *stub;
+    int32_t parent_disk_layout[4];
+
+    /* rename rollback */
+    int *ret_cache;
+
+    loc_t loc2_copy;
+
+    int rename_inodelk_bc_count;
+    /* This is use only for directory operation */
+    int32_t valid;
+    int32_t mds_heal_fresh_lookup;
+    short lock_type;
+    char need_selfheal;
+    char need_xattr_heal;
+    char need_attrheal;
+    /* flag used to make sure we need to return estale in
+       {lookup,revalidate}_cbk */
+    char return_estale;
+    char need_lookup_everywhere;
+    /* fd open check */
+    gf_boolean_t fd_checked;
+    gf_boolean_t linked;
+    gf_boolean_t added_link;
+    gf_boolean_t is_linkfile;
+    gf_boolean_t quota_deem_statfs;
+    gf_boolean_t heal_layout;
+    gf_boolean_t locked;
+    gf_boolean_t dont_create_linkto;
+    gf_boolean_t gfid_missing;
 };
 typedef struct dht_local dht_local_t;
 
 /* du - disk-usage */
 struct dht_du {
-        double   avail_percent;
-        double   avail_inodes;
-        uint64_t avail_space;
-        uint32_t log;
-        uint32_t chunks;
-        uint32_t total_blocks;
-        uint32_t avail_blocks;
-        uint32_t frsize; /*fragment size*/
+    double avail_percent;
+    double avail_inodes;
+    uint64_t avail_space;
+    uint32_t log;
+    uint32_t chunks;
+    uint32_t total_blocks;
+    uint32_t avail_blocks;
+    uint32_t frsize; /*fragment size*/
 };
 typedef struct dht_du dht_du_t;
 
 enum gf_defrag_type {
-        GF_DEFRAG_CMD_NONE = 0,
-        GF_DEFRAG_CMD_START = 1,
-        GF_DEFRAG_CMD_STOP = 1 + 1,
-        GF_DEFRAG_CMD_STATUS = 1 + 2,
-        GF_DEFRAG_CMD_START_LAYOUT_FIX = 1 + 3,
-        GF_DEFRAG_CMD_START_FORCE = 1 + 4,
-        GF_DEFRAG_CMD_START_TIER = 1 + 5,
-        GF_DEFRAG_CMD_STATUS_TIER = 1 + 6,
-        GF_DEFRAG_CMD_START_DETACH_TIER = 1 + 7,
-        GF_DEFRAG_CMD_STOP_DETACH_TIER = 1 + 8,
-        GF_DEFRAG_CMD_PAUSE_TIER = 1 + 9,
-        GF_DEFRAG_CMD_RESUME_TIER = 1 + 10,
-        GF_DEFRAG_CMD_DETACH_STATUS = 1 + 11,
-        GF_DEFRAG_CMD_STOP_TIER = 1 + 12,
-        GF_DEFRAG_CMD_DETACH_START = 1 + 13,
-        GF_DEFRAG_CMD_DETACH_COMMIT = 1 + 14,
-        GF_DEFRAG_CMD_DETACH_COMMIT_FORCE = 1 + 15,
-        GF_DEFRAG_CMD_DETACH_STOP = 1 + 16,
-        /* new labels are used so it will help
-         * while removing old labels by easily differentiating.
-         * A few labels are added so that the count remains same
-         * between this enum and the ones on the xdr file.
-         * different values for the same enum cause errors and
-         * confusion.
-         */
+    GF_DEFRAG_CMD_NONE = 0,
+    GF_DEFRAG_CMD_START = 1,
+    GF_DEFRAG_CMD_STOP = 1 + 1,
+    GF_DEFRAG_CMD_STATUS = 1 + 2,
+    GF_DEFRAG_CMD_START_LAYOUT_FIX = 1 + 3,
+    GF_DEFRAG_CMD_START_FORCE = 1 + 4,
+    GF_DEFRAG_CMD_DETACH_STATUS = 1 + 11,
+    GF_DEFRAG_CMD_DETACH_START = 1 + 13,
+    GF_DEFRAG_CMD_DETACH_COMMIT = 1 + 14,
+    GF_DEFRAG_CMD_DETACH_COMMIT_FORCE = 1 + 15,
+    GF_DEFRAG_CMD_DETACH_STOP = 1 + 16,
+    /* new labels are used so it will help
+     * while removing old labels by easily differentiating.
+     * A few labels are added so that the count remains same
+     * between this enum and the ones on the xdr file.
+     * different values for the same enum cause errors and
+     * confusion.
+     */
 };
 typedef enum gf_defrag_type gf_defrag_type;
 
 enum gf_defrag_status_t {
-        GF_DEFRAG_STATUS_NOT_STARTED,
-        GF_DEFRAG_STATUS_STARTED,
-        GF_DEFRAG_STATUS_STOPPED,
-        GF_DEFRAG_STATUS_COMPLETE,
-        GF_DEFRAG_STATUS_FAILED,
-        GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
-        GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
-        GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
-        GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED,
+    GF_DEFRAG_STATUS_NOT_STARTED,
+    GF_DEFRAG_STATUS_STARTED,
+    GF_DEFRAG_STATUS_STOPPED,
+    GF_DEFRAG_STATUS_COMPLETE,
+    GF_DEFRAG_STATUS_FAILED,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED,
 };
 typedef enum gf_defrag_status_t gf_defrag_status_t;
 
 typedef struct gf_defrag_pattern_list gf_defrag_pattern_list_t;
 
 struct gf_defrag_pattern_list {
-        char                       path_pattern[256];
-        uint64_t                   size;
-        gf_defrag_pattern_list_t  *next;
+    char path_pattern[256];
+    uint64_t size;
+    gf_defrag_pattern_list_t *next;
 };
 
 struct dht_container {
-        union {
-                struct list_head             list;
-                struct {
-                        struct _gf_dirent_t *next;
-                        struct _gf_dirent_t *prev;
-                };
+    union {
+        struct list_head list;
+        struct {
+            struct _gf_dirent_t *next;
+            struct _gf_dirent_t *prev;
         };
-        gf_dirent_t     *df_entry;
-        xlator_t        *this;
-        loc_t           *parent_loc;
-        dict_t          *migrate_data;
-        int             local_subvol_index;
+    };
+    gf_dirent_t *df_entry;
+    xlator_t *this;
+    loc_t *parent_loc;
+    dict_t *migrate_data;
+    int local_subvol_index;
 };
 
-typedef enum tier_mode_ {
-        TIER_MODE_NONE = 0,
-        TIER_MODE_TEST,
-        TIER_MODE_WM
-} tier_mode_t;
-
-typedef enum tier_pause_state_ {
-        TIER_RUNNING = 0,
-        TIER_REQUEST_PAUSE,
-        TIER_PAUSED
-} tier_pause_state_t;
-
-/* This Structure is only used in tiering fixlayout */
-typedef struct gf_tier_fix_layout_arg {
-        xlator_t                *this;
-        dict_t                  *fix_layout;
-        pthread_t               thread_id;
-} gf_tier_fix_layout_arg_t;
-
-typedef struct gf_tier_conf {
-        int                          is_tier;
-        int                          watermark_hi;
-        int                          watermark_low;
-        int                          watermark_last;
-        unsigned long                block_size;
-        fsblkcnt_t                   blocks_total;
-        fsblkcnt_t                   blocks_used;
-        int                          percent_full;
-        uint64_t                     max_migrate_bytes;
-        int                          max_migrate_files;
-        int                          query_limit;
-        tier_mode_t                  mode;
-        /* These flags are only used for tier-compact */
-        gf_boolean_t                 compact_active;
-        /* These 3 flags are set to true when the client changes the */
-        /* compaction mode on the command line. */
-        /* When they are set, the daemon will trigger compaction as */
-        /* soon as possible to activate or deactivate compaction. */
-        /* If in the middle of a compaction, then the switches take */
-        /* effect on the next compaction, not the current one. */
-        /* If the user switches it off, we want to avoid needless */
-        /* compactions. */
-        /* If the user switches it on, they want to compact as soon */
-        /* as possible. */
-        gf_boolean_t                 compact_mode_switched;
-        gf_boolean_t                 compact_mode_switched_hot;
-        gf_boolean_t                 compact_mode_switched_cold;
-        int                          tier_max_promote_size;
-        int                          tier_promote_frequency;
-        int                          tier_demote_frequency;
-        int                          tier_compact_hot_frequency;
-        int                          tier_compact_cold_frequency;
-        uint64_t                     st_last_promoted_size;
-        uint64_t                     st_last_demoted_size;
-        tier_pause_state_t           pause_state;
-        struct synctask             *pause_synctask;
-        gf_timer_t                  *pause_timer;
-        pthread_mutex_t              pause_mutex;
-        int                          promote_in_progress;
-        int                          demote_in_progress;
-        /* This Structure is only used in tiering fixlayout */
-        gf_tier_fix_layout_arg_t     tier_fix_layout_arg;
-        /* Indicates the index of the first queryfile picked
-         * in the last cycle of promote or demote */
-        int32_t last_promote_qfile_index;
-        int32_t last_demote_qfile_index;
-        char                         volname[GD_VOLUME_NAME_MAX + 1];
-} gf_tier_conf_t;
-
 typedef struct nodeuuid_info {
-        char info; /* Set to 1 is this is my node's uuid*/
-        uuid_t uuid; /* Store the nodeuuid as well for debugging*/
+    char info;   /* Set to 1 is this is my node's uuid*/
+    uuid_t uuid; /* Store the nodeuuid as well for debugging*/
 } nodeuuid_info_t;
 
 typedef struct subvol_nodeuuids_info {
-        nodeuuid_info_t *elements;
-        int count;
+    nodeuuid_info_t *elements;
+    int count;
 } subvol_nodeuuids_info_t;
 
-
 struct gf_defrag_info_ {
-        uint64_t                     total_files;
-        uint64_t                     total_data;
-        uint64_t                     num_files_lookedup;
-        uint64_t                     total_failures;
-        uint64_t                     skipped;
-        uint64_t                     num_dirs_processed;
-        uint64_t                     size_processed;
-        gf_lock_t                    lock;
-        int                          cmd;
-        pthread_t                    th;
-        gf_defrag_status_t           defrag_status;
-        struct rpc_clnt             *rpc;
-        uint32_t                     connected;
-        uint32_t                     is_exiting;
-        pid_t                        pid;
-        inode_t                     *root_inode;
-        uuid_t                       node_uuid;
-        struct timeval               start_time;
-        gf_boolean_t                 stats;
-        uint32_t                     new_commit_hash;
-        gf_defrag_pattern_list_t    *defrag_pattern;
-        gf_tier_conf_t               tier_conf;
-
-        /*Data Tiering params for scanner*/
-        uint64_t                     total_files_promoted;
-        uint64_t                     total_files_demoted;
-        int                          write_freq_threshold;
-        int                          read_freq_threshold;
-
-        pthread_cond_t               parallel_migration_cond;
-        pthread_mutex_t              dfq_mutex;
-        pthread_cond_t               rebalance_crawler_alarm;
-        int32_t                      q_entry_count;
-        int32_t                      global_error;
-        struct  dht_container       *queue;
-        int32_t                      crawl_done;
-        int32_t                      abort;
-        int32_t                      wakeup_crawler;
-
-        /*Throttle params*/
-        /*stands for reconfigured thread count*/
-        int32_t                      recon_thread_count;
-        /*stands for current running thread count*/
-        int32_t                      current_thread_count;
-        pthread_cond_t               df_wakeup_thread;
-
-        /* lock migration flag */
-        gf_boolean_t                 lock_migration_enabled;
-
-        /* backpointer to make it easier to write functions for rebalance */
-        xlator_t                     *this;
-
-        pthread_cond_t               fc_wakeup_cond;
-        pthread_mutex_t              fc_mutex;
+    uint64_t total_files;
+    uint64_t total_data;
+    uint64_t num_files_lookedup;
+    uint64_t total_failures;
+    uint64_t skipped;
+    uint64_t num_dirs_processed;
+    uint64_t size_processed;
+    gf_lock_t lock;
+    pthread_t th;
+    struct rpc_clnt *rpc;
+    uint32_t connected;
+    uint32_t is_exiting;
+    pid_t pid;
+    int cmd;
+    inode_t *root_inode;
+    uuid_t node_uuid;
+    time_t start_time;
+    uint32_t new_commit_hash;
+    gf_defrag_status_t defrag_status;
+    gf_defrag_pattern_list_t *defrag_pattern;
+
+    pthread_cond_t parallel_migration_cond;
+    pthread_mutex_t dfq_mutex;
+    pthread_cond_t rebalance_crawler_alarm;
+    int32_t q_entry_count;
+    int32_t global_error;
+    struct dht_container *queue;
+    int32_t crawl_done;
+    int32_t abort;
+    int32_t wakeup_crawler;
+
+    /*Throttle params*/
+    /*stands for reconfigured thread count*/
+    int32_t recon_thread_count;
+    pthread_cond_t df_wakeup_thread;
+
+    /* backpointer to make it easier to write functions for rebalance */
+    xlator_t *this;
+
+    pthread_cond_t fc_wakeup_cond;
+    pthread_mutex_t fc_mutex;
+
+    /*stands for current running thread count*/
+    int32_t current_thread_count;
+
+    gf_boolean_t stats;
+    /* lock migration flag */
+    gf_boolean_t lock_migration_enabled;
 };
 
 typedef struct gf_defrag_info_ gf_defrag_info_t;
 
 struct dht_methods_s {
-        int32_t      (*migration_get_dst_subvol)(xlator_t *this,
-                                                 dht_local_t *local);
-        int32_t      (*migration_other)(xlator_t *this,
-                                        gf_defrag_info_t *defrag);
-        int32_t      (*migration_needed)(xlator_t *this);
-        xlator_t*    (*layout_search)(xlator_t *this,
-                                      dht_layout_t *layout,
-                                         const char *name);
+    int32_t (*migration_get_dst_subvol)(xlator_t *this, dht_local_t *local);
+    int32_t (*migration_other)(xlator_t *this, gf_defrag_info_t *defrag);
+    xlator_t *(*layout_search)(xlator_t *this, dht_layout_t *layout,
+                               const char *name);
 };
 
 typedef struct dht_methods_s dht_methods_t;
 
 struct dht_conf {
-        gf_lock_t      subvolume_lock;
-        int            subvolume_cnt;
-        xlator_t     **subvolumes;
-        char          *subvolume_status;
-        int           *last_event;
-        dht_layout_t **file_layouts;
-        dht_layout_t **dir_layouts;
-        unsigned int   search_unhashed;
-        gf_boolean_t   lookup_optimize;
-        int            gen;
-        dht_du_t      *du_stats;
-        double         min_free_disk;
-        double         min_free_inodes;
-        char           disk_unit;
-        int32_t        refresh_interval;
-        gf_boolean_t   unhashed_sticky_bit;
-        struct timeval last_stat_fetch;
-        gf_lock_t      layout_lock;
-        dict_t        *leaf_to_subvol;
-        void          *private;     /* Can be used by wrapper xlators over
-                                       dht */
-        gf_boolean_t   use_readdirp;
-        char           vol_uuid[UUID_SIZE + 1];
-        gf_boolean_t   assert_no_child_down;
-        time_t        *subvol_up_time;
-
-        /* This is the count used as the distribute layout for a directory */
-        /* Will be a global flag to control the layout spread count */
-        uint32_t       dir_spread_cnt;
-
-        /* to keep track of nodes which are decommissioned */
-        xlator_t     **decommissioned_bricks;
-        int            decommission_in_progress;
-        int            decommission_subvols_cnt;
-
-        /* defrag related */
-        gf_defrag_info_t *defrag;
-
-        /* Request to filter directory entries in readdir request */
-
-        gf_boolean_t    readdir_optimize;
-
-        /* Support regex-based name reinterpretation. */
-        regex_t         rsync_regex;
-        gf_boolean_t    rsync_regex_valid;
-        regex_t         extra_regex;
-        gf_boolean_t    extra_regex_valid;
-
-        /* Support variable xattr names. */
-        char            *xattr_name;
-        char            *mds_xattr_key;
-        char            *link_xattr_name;
-        char            *commithash_xattr_name;
-        char            *wild_xattr_name;
-
-        /* Support size-weighted rebalancing (heterogeneous bricks). */
-        gf_boolean_t    do_weighting;
-        gf_boolean_t    randomize_by_gfid;
-        int             dthrottle;
-
-        dht_methods_t   methods;
-
-        struct mem_pool *lock_pool;
-
-        /*local subvol storage for rebalance*/
-        xlator_t       **local_subvols;
-        subvol_nodeuuids_info_t       *local_nodeuuids;
-        int32_t          local_subvols_cnt;
-
-        /*
-         * "Commit hash" for this volume topology.  Changed whenever bricks
-         * are added or removed.
-         */
-        uint32_t        vol_commit_hash;
-        gf_boolean_t    vch_forced;
-
-        /* lock migration */
-
-        gf_boolean_t    lock_migration_enabled;
-        gf_lock_t       lock;
-
-        /* Hard link handle requirement for migration triggered from client*/
-        synclock_t      link_lock;
-
-        gf_boolean_t    use_fallocate;
-
-        gf_boolean_t    force_migration;
+    xlator_t **subvolumes;
+    char *subvolume_status;
+    int *last_event;
+    dht_layout_t **file_layouts;
+    dht_layout_t **dir_layouts;
+    unsigned int search_unhashed;
+    int gen;
+    dht_du_t *du_stats;
+    double min_free_disk;
+    double min_free_inodes;
+    int subvolume_cnt;
+    int32_t refresh_interval;
+    gf_lock_t subvolume_lock;
+    time_t last_stat_fetch;
+    gf_lock_t layout_lock;
+    dict_t *leaf_to_subvol;
+    void *private; /* Can be used by wrapper xlators over
+                      dht */
+    time_t *subvol_up_time;
+
+    /* to keep track of nodes which are decommissioned */
+    xlator_t **decommissioned_bricks;
+    int decommission_in_progress;
+    int decommission_subvols_cnt;
+
+    /* defrag related */
+    gf_defrag_info_t *defrag;
+
+    /* Support regex-based name reinterpretation. */
+    regex_t rsync_regex;
+    regex_t extra_regex;
+
+    /* Support variable xattr names. */
+    char *xattr_name;
+    char *mds_xattr_key;
+    char *link_xattr_name;
+    char *commithash_xattr_name;
+    char *wild_xattr_name;
+
+    dht_methods_t methods;
+
+    struct mem_pool *lock_pool;
+
+    /*local subvol storage for rebalance*/
+    xlator_t **local_subvols;
+    subvol_nodeuuids_info_t *local_nodeuuids;
+    int32_t local_subvols_cnt;
+
+    int dthrottle;
+
+    /* Hard link handle requirement for migration triggered from client*/
+    synclock_t link_lock;
+
+    /* lock migration */
+    gf_lock_t lock;
+
+    /* This is the count used as the distribute layout for a directory */
+    /* Will be a global flag to control the layout spread count */
+    uint32_t dir_spread_cnt;
+
+    /*
+     * "Commit hash" for this volume topology.  Changed whenever bricks
+     * are added or removed.
+     */
+    uint32_t vol_commit_hash;
+
+    char vol_uuid[UUID_SIZE + 1];
+
+    char disk_unit;
+
+    gf_boolean_t lock_migration_enabled;
+
+    gf_boolean_t vch_forced;
+
+    gf_boolean_t use_fallocate;
+
+    gf_boolean_t force_migration;
+
+    gf_boolean_t lookup_optimize;
+
+    gf_boolean_t unhashed_sticky_bit;
+
+    gf_boolean_t assert_no_child_down;
+
+    gf_boolean_t use_readdirp;
+
+    /* Request to filter directory entries in readdir request */
+    gf_boolean_t readdir_optimize;
+
+    gf_boolean_t rsync_regex_valid;
+
+    gf_boolean_t extra_regex_valid;
+
+    /* Support size-weighted rebalancing (heterogeneous bricks). */
+    gf_boolean_t do_weighting;
+
+    gf_boolean_t randomize_by_gfid;
 };
 typedef struct dht_conf dht_conf_t;
 
 struct dht_dfoffset_ctx {
-        xlator_t       *this;
-        off_t           offset;
-        int32_t         readdir_done;
+    xlator_t *this;
+    off_t offset;
+    int32_t readdir_done;
 };
 typedef struct dht_dfoffset_ctx dht_dfoffset_ctx_t;
 
 struct dht_disk_layout {
-        uint32_t           cnt;
-        uint32_t           type;
-        struct {
-                uint32_t   start;
-                uint32_t   stop;
-        } list[1];
+    uint32_t cnt;
+    uint32_t type;
+    struct {
+        uint32_t start;
+        uint32_t stop;
+    } list[1];
 };
 typedef struct dht_disk_layout dht_disk_layout_t;
 
 typedef enum {
-        GF_DHT_MIGRATE_DATA,
-        GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS,
-        GF_DHT_MIGRATE_HARDLINK,
-        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS
+    GF_DHT_MIGRATE_DATA,
+    GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS,
+    GF_DHT_MIGRATE_HARDLINK,
+    GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS
 } gf_dht_migrate_data_type_t;
 
 typedef enum {
-        GF_DHT_EQUAL_DISTRIBUTION,
-        GF_DHT_WEIGHTED_DISTRIBUTION
+    GF_DHT_EQUAL_DISTRIBUTION,
+    GF_DHT_WEIGHTED_DISTRIBUTION
 } dht_distribution_type_t;
 
 struct dir_dfmeta {
-        gf_dirent_t             *equeue;
-        dht_dfoffset_ctx_t      *offset_var;
-        struct list_head        **head;
-        struct list_head        **iterator;
-        int                     *fetch_entries;
+    gf_dirent_t *equeue;
+    dht_dfoffset_ctx_t *offset_var;
+    struct list_head **head;
+    struct list_head **iterator;
+    int *fetch_entries;
+    /* fds corresponding to local subvols only */
+    fd_t **lfd;
 };
 
 typedef struct dht_migrate_info {
-        xlator_t *src_subvol;
-        xlator_t *dst_subvol;
-        GF_REF_DECL;
+    xlator_t *src_subvol;
+    xlator_t *dst_subvol;
+    GF_REF_DECL;
 } dht_migrate_info_t;
 
-
-
 typedef struct dht_fd_ctx {
-        uint64_t opened_on_dst;
-        GF_REF_DECL;
+    uint64_t opened_on_dst;
+    GF_REF_DECL;
 } dht_fd_ctx_t;
 
-
 #define ENTRY_MISSING(op_ret, op_errno) (op_ret == -1 && op_errno == ENOENT)
 
-#define is_revalidate(loc) (dht_inode_ctx_layout_get ((loc)->inode, this, NULL) == 0)
+#define is_revalidate(loc)                                                     \
+    (dht_inode_ctx_layout_get((loc)->inode, this, NULL) == 0)
 
 #define is_last_call(cnt) (cnt == 0)
 
 #define DHT_MIGRATION_IN_PROGRESS 1
-#define DHT_MIGRATION_COMPLETED   2
+#define DHT_MIGRATION_COMPLETED 2
 
-#define check_is_linkfile(i,s,x,n) (IS_DHT_LINKFILE_MODE (s) && dict_get (x, n))
+#define check_is_linkfile(i, s, x, n)                                          \
+    (IS_DHT_LINKFILE_MODE(s) && dict_get(x, n))
 
-#define IS_DHT_MIGRATION_PHASE2(buf)  (                                 \
-                IA_ISREG ((buf)->ia_type) &&                            \
-                ((st_mode_from_ia ((buf)->ia_prot, (buf)->ia_type) &    \
-                  ~S_IFMT) == DHT_LINKFILE_MODE))
+#define IS_DHT_MIGRATION_PHASE2(buf)                                           \
+    (IA_ISREG((buf)->ia_type) &&                                               \
+     ((st_mode_from_ia((buf)->ia_prot, (buf)->ia_type) & ~S_IFMT) ==           \
+      DHT_LINKFILE_MODE))
 
-#define IS_DHT_MIGRATION_PHASE1(buf)  (                                 \
-                IA_ISREG ((buf)->ia_type) &&                            \
-                ((buf)->ia_prot.sticky == 1) &&                         \
-                ((buf)->ia_prot.sgid == 1))
+#define IS_DHT_MIGRATION_PHASE1(buf)                                           \
+    (IA_ISREG((buf)->ia_type) && ((buf)->ia_prot.sticky == 1) &&               \
+     ((buf)->ia_prot.sgid == 1))
 
-#define DHT_STRIP_PHASE1_FLAGS(buf)  do {                       \
-                if ((buf) && IS_DHT_MIGRATION_PHASE1(buf)) {    \
-                        (buf)->ia_prot.sticky = 0;              \
-                        (buf)->ia_prot.sgid = 0;                \
-                }                                               \
-        } while (0)
+#define DHT_STRIP_PHASE1_FLAGS(buf)                                            \
+    do {                                                                       \
+        if ((buf) && IS_DHT_MIGRATION_PHASE1(buf)) {                           \
+            (buf)->ia_prot.sticky = 0;                                         \
+            (buf)->ia_prot.sgid = 0;                                           \
+        }                                                                      \
+    } while (0)
 
-#define dht_inode_missing(op_errno) (op_errno == ENOENT || op_errno == ESTALE \
-                                     || op_errno == EIO) \
-/*Bad fix. Please revert the commit after fixing the bug 1329505*/
+#define dht_inode_missing(op_errno) (op_errno == ENOENT || op_errno == ESTALE)
 
-#define check_is_dir(i,s,x) (IA_ISDIR(s->ia_type))
+#define check_is_dir(i, s, x) (IA_ISDIR(s->ia_type))
 
 #define layout_is_sane(layout) ((layout) && (layout->cnt > 0))
 
-#define we_are_not_migrating(x)   ((x) == 1)
-
-#define DHT_STACK_UNWIND(fop, frame, params ...) do {           \
-                dht_local_t *__local = NULL;                    \
-                xlator_t    *__xl    = NULL;                    \
-                if (frame) {                                    \
-                        __xl         = frame->this;             \
-                        __local      = frame->local;            \
-                        frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
-                dht_local_wipe (__xl, __local);                 \
-        } while (0)
-
-#define DHT_STACK_DESTROY(frame) do {                   \
-                dht_local_t *__local = NULL;            \
-                xlator_t    *__xl    = NULL;            \
-                __xl                 = frame->this;     \
-                __local              = frame->local;    \
-                frame->local         = NULL;            \
-                STACK_DESTROY (frame->root);            \
-                dht_local_wipe (__xl, __local);         \
-        } while (0)
-
-#define DHT_UPDATE_TIME(ctx_sec, ctx_nsec, new_sec, new_nsec, inode, post) do {\
-                LOCK (&inode->lock);                                    \
-                {                                                       \
-                        if (ctx_sec == new_sec)                         \
-                                new_nsec = max (new_nsec, ctx_nsec);    \
-                        else if (ctx_sec > new_sec) {                   \
-                                new_sec = ctx_sec;                      \
-                                new_nsec = ctx_nsec;                    \
-                        }                                               \
-                        if (post) {                                     \
-                                ctx_sec = new_sec;                      \
-                                ctx_nsec = new_nsec;                    \
-                        }                                               \
-                }                                                       \
-                UNLOCK (&inode->lock);                                  \
-        } while (0)
-
-#define is_greater_time(a, an, b, bn) (((a) < (b)) || (((a) == (b)) && ((an) < (bn))))
-
-#define DHT_MARK_FOP_INTERNAL(xattr) do {                                      \
-                int tmp = -1;                                                  \
-                if (!xattr) {                                                  \
-                        xattr = dict_new ();                                   \
-                        if (!xattr)                                            \
-                                break;                                         \
-                }                                                              \
-                tmp = dict_set_str (xattr, GLUSTERFS_INTERNAL_FOP_KEY, "yes"); \
-                if (tmp) {                                                     \
-                        gf_msg (this->name, GF_LOG_ERROR, 0,                   \
-                                DHT_MSG_DICT_SET_FAILED,                       \
-                                "Failed to set dictionary value: key = %s,"    \
-                                " path = %s", GLUSTERFS_INTERNAL_FOP_KEY,      \
-                                 local->loc.path);                             \
-                }                                                              \
-        } while (0)
-
-dht_layout_t                            *dht_layout_new (xlator_t *this, int cnt);
-dht_layout_t                            *dht_layout_get (xlator_t *this, inode_t *inode);
-dht_layout_t                            *dht_layout_for_subvol (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_layout_search (xlator_t   *this, dht_layout_t *layout,
-                             const char *name);
+#define we_are_not_migrating(x) ((x) == 1)
+
+#define DHT_STACK_UNWIND(fop, frame, params...)                                \
+    do {                                                                       \
+        dht_local_t *__local = NULL;                                           \
+        xlator_t *__xl = NULL;                                                 \
+        if (frame) {                                                           \
+            __xl = frame->this;                                                \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        dht_local_wipe(__xl, __local);                                         \
+    } while (0)
+
+#define DHT_STACK_DESTROY(frame)                                               \
+    do {                                                                       \
+        dht_local_t *__local = NULL;                                           \
+        xlator_t *__xl = NULL;                                                 \
+        __xl = frame->this;                                                    \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        dht_local_wipe(__xl, __local);                                         \
+    } while (0)
+
+#define DHT_UPDATE_TIME(ctx_sec, ctx_nsec, new_sec, new_nsec, post)            \
+    do {                                                                       \
+        if (ctx_sec == new_sec)                                                \
+            new_nsec = max(new_nsec, ctx_nsec);                                \
+        else if (ctx_sec > new_sec) {                                          \
+            new_sec = ctx_sec;                                                 \
+            new_nsec = ctx_nsec;                                               \
+        }                                                                      \
+        if (post) {                                                            \
+            ctx_sec = new_sec;                                                 \
+            ctx_nsec = new_nsec;                                               \
+        }                                                                      \
+    } while (0)
+
+#define is_greater_time(a, an, b, bn)                                          \
+    (((a) < (b)) || (((a) == (b)) && ((an) < (bn))))
+
+#define DHT_MARK_FOP_INTERNAL(xattr)                                           \
+    do {                                                                       \
+        int tmp = -1;                                                          \
+        if (!xattr) {                                                          \
+            xattr = dict_new();                                                \
+            if (!xattr)                                                        \
+                break;                                                         \
+        }                                                                      \
+        tmp = dict_set_str(xattr, GLUSTERFS_INTERNAL_FOP_KEY, "yes");          \
+        if (tmp) {                                                             \
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,       \
+                   "Failed to set dictionary value: key = %s,"                 \
+                   " path = %s",                                               \
+                   GLUSTERFS_INTERNAL_FOP_KEY, local->loc.path);               \
+        }                                                                      \
+    } while (0)
+
+dht_layout_t *
+dht_layout_new(xlator_t *this, int cnt);
+dht_layout_t *
+dht_layout_get(xlator_t *this, inode_t *inode);
+dht_layout_t *
+dht_layout_for_subvol(xlator_t *this, xlator_t *subvol);
+xlator_t *
+dht_layout_search(xlator_t *this, dht_layout_t *layout, const char *name);
 int32_t
-dht_migration_get_dst_subvol(xlator_t *this, dht_local_t  *local);
+dht_migration_get_dst_subvol(xlator_t *this, dht_local_t *local);
 int32_t
 dht_migration_needed(xlator_t *this);
-int                                      dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout);
-int dht_layout_anomalies (xlator_t      *this, loc_t *loc, dht_layout_t *layout,
-                          uint32_t      *holes_p, uint32_t *overlaps_p,
-                          uint32_t      *missing_p, uint32_t *down_p,
-                          uint32_t      *misc_p, uint32_t *no_space_p);
-int dht_layout_dir_mismatch (xlator_t   *this, dht_layout_t *layout,
-                             xlator_t   *subvol, loc_t *loc, dict_t *xattr);
-
-xlator_t *dht_linkfile_subvol (xlator_t *this, inode_t *inode,
-                               struct iatt *buf, dict_t *xattr);
-int dht_linkfile_unlink (call_frame_t      *frame, xlator_t *this,
-                         xlator_t          *subvol, loc_t *loc);
-
-int dht_layouts_init (xlator_t *this, dht_conf_t *conf);
-int dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-                      int       op_ret, int op_errno, dict_t *xattr);
-
-int     dht_disk_layout_extract (xlator_t *this, dht_layout_t *layout,
-                             int       pos, int32_t **disk_layout_p);
-int dht_disk_layout_merge (xlator_t   *this, dht_layout_t *layout,
-                           int         pos, void *disk_layout_raw, int disk_layout_len);
 int
-dht_disk_layout_extract_for_subvol (xlator_t *this, dht_layout_t *layout,
-                                    xlator_t *subvol, int32_t **disk_layout_p);
+dht_layout_normalize(xlator_t *this, loc_t *loc, dht_layout_t *layout);
+void
+dht_layout_anomalies(xlator_t *this, loc_t *loc, dht_layout_t *layout,
+                     uint32_t *holes_p, uint32_t *overlaps_p,
+                     uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
+                     uint32_t *no_space_p);
+int
+dht_layout_dir_mismatch(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                        loc_t *loc, dict_t *xattr);
+xlator_t *
+dht_linkfile_subvol(xlator_t *this, inode_t *inode, struct iatt *buf,
+                    dict_t *xattr);
+int
+dht_linkfile_unlink(call_frame_t *frame, xlator_t *this, xlator_t *subvol,
+                    loc_t *loc);
 
-int dht_frame_return (call_frame_t *frame);
+int
+dht_layouts_init(xlator_t *this, dht_conf_t *conf);
+int
+dht_layout_merge(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                 int op_ret, int op_errno, dict_t *xattr);
 
-int dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol);
+int
+dht_disk_layout_extract(xlator_t *this, dht_layout_t *layout, int pos,
+                        int32_t **disk_layout_p);
+int
+dht_disk_layout_extract_for_subvol(xlator_t *this, dht_layout_t *layout,
+                                   xlator_t *subvol, int32_t **disk_layout_p);
 
-void dht_local_wipe (xlator_t *this, dht_local_t *local);
-dht_local_t *dht_local_init (call_frame_t    *frame, loc_t *loc, fd_t *fd,
-                             glusterfs_fop_t  fop);
-int dht_iatt_merge (xlator_t                 *this, struct iatt *to, struct iatt *from,
-                    xlator_t                 *subvol);
+int
+dht_frame_return(call_frame_t *frame);
 
-xlator_t *dht_subvol_get_hashed (xlator_t *this, loc_t *loc);
-xlator_t *dht_subvol_get_cached (xlator_t *this, inode_t *inode);
-xlator_t *dht_subvol_next (xlator_t *this, xlator_t *prev);
-xlator_t *dht_subvol_next_available (xlator_t *this, xlator_t *prev);
-int       dht_subvol_cnt (xlator_t *this, xlator_t *subvol);
+int
+dht_deitransform(xlator_t *this, uint64_t y, xlator_t **subvol);
 
-int dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p);
+void
+dht_local_wipe(xlator_t *this, dht_local_t *local);
+dht_local_t *
+dht_local_init(call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop);
+int
+dht_iatt_merge(xlator_t *this, struct iatt *to, struct iatt *from);
 
-int dht_linkfile_create (call_frame_t    *frame, fop_mknod_cbk_t linkfile_cbk,
-                         xlator_t        *this, xlator_t *tovol,
-                         xlator_t        *fromvol, loc_t *loc);
-int                                       dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc);
-int                                       dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc);
+xlator_t *
+dht_subvol_get_hashed(xlator_t *this, loc_t *loc);
+xlator_t *
+dht_subvol_get_cached(xlator_t *this, inode_t *inode);
+xlator_t *
+dht_subvol_next(xlator_t *this, xlator_t *prev);
+xlator_t *
+dht_subvol_next_available(xlator_t *this, xlator_t *prev);
 int
-dht_selfheal_directory (call_frame_t     *frame, dht_selfheal_dir_cbk_t cbk,
-                        loc_t            *loc, dht_layout_t *layout);
+dht_subvol_cnt(xlator_t *this, xlator_t *subvol);
 
 int
-dht_selfheal_directory_for_nameless_lookup (call_frame_t  *frame,
-                                            dht_selfheal_dir_cbk_t cbk,
-                                            loc_t  *loc, dht_layout_t *layout);
+dht_hash_compute(xlator_t *this, int type, const char *name, uint32_t *hash_p);
 
 int
-dht_selfheal_new_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
-                            dht_layout_t *layout);
+dht_linkfile_create(call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
+                    xlator_t *this, xlator_t *tovol, xlator_t *fromvol,
+                    loc_t *loc);
+int
+dht_lookup_everywhere(call_frame_t *frame, xlator_t *this, loc_t *loc);
 int
-dht_selfheal_restore (call_frame_t       *frame, dht_selfheal_dir_cbk_t cbk,
-                      loc_t              *loc, dht_layout_t *layout);
+dht_selfheal_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
+                       loc_t *loc, dht_layout_t *layout);
 int
-dht_layout_sort_volname (dht_layout_t *layout);
+dht_selfheal_new_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
+                           dht_layout_t *layout);
+int
+dht_selfheal_restore(call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
+                     loc_t *loc, dht_layout_t *layout);
+void
+dht_layout_sort_volname(dht_layout_t *layout);
 
-int dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc);
+int
+dht_get_du_info(call_frame_t *frame, xlator_t *this, loc_t *loc);
 
-gf_boolean_t dht_is_subvol_filled (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
-                                          dht_local_t *layout);
-int       dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx);
+gf_boolean_t
+dht_is_subvol_filled(xlator_t *this, xlator_t *subvol);
+xlator_t *
+dht_free_disk_available_subvol(xlator_t *this, xlator_t *subvol,
+                               dht_local_t *layout);
+int
+dht_get_du_info_for_subvol(xlator_t *this, int subvol_idx);
 
-int dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode);
-int dht_layout_index_for_subvol (dht_layout_t *layout, xlator_t *subvol);
-int           dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout);;
-void          dht_layout_unref (xlator_t *this, dht_layout_t *layout);
-dht_layout_t *dht_layout_ref (xlator_t *this, dht_layout_t *layout);
-xlator_t     *dht_first_up_subvol (xlator_t *this);
-xlator_t     *dht_last_up_subvol (xlator_t *this);
+int
+dht_layout_preset(xlator_t *this, xlator_t *subvol, inode_t *inode);
+int
+dht_layout_set(xlator_t *this, inode_t *inode, dht_layout_t *layout);
+;
+void
+dht_layout_unref(xlator_t *this, dht_layout_t *layout);
+dht_layout_t *
+dht_layout_ref(xlator_t *this, dht_layout_t *layout);
+int
+dht_layout_index_for_subvol(dht_layout_t *layout, xlator_t *subvol);
+xlator_t *
+dht_first_up_subvol(xlator_t *this);
+xlator_t *
+dht_last_up_subvol(xlator_t *this);
 
-int dht_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name);
+int
+dht_build_child_loc(xlator_t *this, loc_t *child, loc_t *parent, char *name);
 
-int dht_filter_loc_subvol_key (xlator_t  *this, loc_t *loc, loc_t *new_loc,
-                               xlator_t **subvol);
+int
+dht_filter_loc_subvol_key(xlator_t *this, loc_t *loc, loc_t *new_loc,
+                          xlator_t **subvol);
 
-int                                     dht_rename_cleanup (call_frame_t *frame);
-int dht_rename_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t           op_ret, int32_t op_errno,
-                         inode_t          *inode, struct iatt *stbuf,
-                         struct iatt      *preparent, struct iatt *postparent,
-                         dict_t *xdata);
+int
+dht_rename_cleanup(call_frame_t *frame);
+int
+dht_rename_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *stbuf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata);
 
-int dht_update_commit_hash_for_layout (call_frame_t *frame);
-int dht_fix_directory_layout (call_frame_t *frame,
-                              dht_selfheal_dir_cbk_t  dir_cbk,
-                              dht_layout_t           *layout);
+int
+dht_update_commit_hash_for_layout(call_frame_t *frame);
+int
+dht_fix_directory_layout(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                         dht_layout_t *layout);
 
-int dht_init_subvolumes (xlator_t *this, dht_conf_t *conf);
+int
+dht_init_subvolumes(xlator_t *this, dht_conf_t *conf);
 
 /* migration/rebalance */
-int dht_start_rebalance_task (xlator_t *this, call_frame_t *frame);
+int
+dht_start_rebalance_task(xlator_t *this, call_frame_t *frame);
 
-int dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame);
-int dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame);
+int
+dht_rebalance_in_progress_check(xlator_t *this, call_frame_t *frame);
+int
+dht_rebalance_complete_check(xlator_t *this, call_frame_t *frame);
 
 int
-dht_init_local_subvolumes (xlator_t *this, dht_conf_t *conf);
+dht_init_local_subvolumes(xlator_t *this, dht_conf_t *conf);
 
 /* FOPS */
-int32_t dht_lookup (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc,
-                    dict_t   *xattr_req);
-
-int32_t dht_stat (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t    *loc, dict_t *xdata);
-
-int32_t dht_fstat (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd, dict_t *xdata);
-
-int32_t dht_truncate (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t    *loc,
-                      off_t     offset, dict_t *xdata);
-
-int32_t dht_ftruncate (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t     *fd,
-                       off_t     offset, dict_t *xdata);
-
-int32_t dht_access (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc,
-                    int32_t   mask, dict_t *xdata);
-
-int32_t dht_readlink (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t    *loc,
-                      size_t    size, dict_t *xdata);
-
-int32_t dht_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
-
-int32_t dht_mkdir (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
-
-int32_t dht_unlink (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc, int xflag, dict_t *xdata);
-
-int32_t dht_rmdir (call_frame_t *frame, xlator_t *this,
-                   loc_t    *loc, int flags, dict_t *xdata);
-
-int32_t dht_symlink (call_frame_t   *frame, xlator_t *this,
-                     const char *linkpath, loc_t *loc, mode_t umask,
-                     dict_t *xdata);
-
-int32_t dht_rename (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *oldloc,
-                    loc_t    *newloc, dict_t *xdata);
-
-int32_t dht_link (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t    *oldloc,
-                  loc_t    *newloc, dict_t *xdata);
-
-int32_t dht_create (call_frame_t *frame, xlator_t *this,
-                    loc_t    *loc, int32_t flags, mode_t mode,
-                    mode_t umask, fd_t     *fd, dict_t *params);
-
-int32_t dht_open (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t    *loc,
-                  int32_t   flags, fd_t *fd, dict_t *xdata);
-
-int32_t dht_readv (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd,
-                   size_t    size,
-                   off_t     offset, uint32_t flags, dict_t *xdata);
-
-int32_t dht_writev (call_frame_t      *frame,
-                    xlator_t      *this,
-                    fd_t          *fd,
-                    struct iovec  *vector,
-                    int32_t        count,
-                    off_t          offset,
-                    uint32_t       flags,
-                    struct iobref *iobref, dict_t *xdata);
-
-int32_t dht_flush (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd, dict_t *xdata);
-
-int32_t dht_fsync (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd,
-                   int32_t   datasync, dict_t *xdata);
-
-int32_t dht_opendir (call_frame_t *frame,
-                     xlator_t *this,
-                     loc_t    *loc, fd_t *fd, dict_t *xdata);
-
-int32_t dht_fsyncdir (call_frame_t *frame,
-                      xlator_t *this,
-                      fd_t     *fd,
-                      int32_t   datasync, dict_t *xdata);
-
-int32_t dht_statfs (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc, dict_t *xdata);
-
-int32_t dht_setxattr (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t    *loc,
-                      dict_t   *dict,
-                      int32_t   flags, dict_t *xdata);
-
-int32_t dht_getxattr (call_frame_t   *frame,
-                      xlator_t   *this,
-                      loc_t      *loc,
-                      const char *name, dict_t *xdata);
-
-int32_t dht_fsetxattr (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t     *fd,
-                       dict_t   *dict,
-                       int32_t   flags, dict_t *xdata);
-
-int32_t dht_fgetxattr (call_frame_t   *frame,
-                       xlator_t   *this,
-                       fd_t       *fd,
-                       const char *name, dict_t *xdata);
-
-int32_t dht_removexattr (call_frame_t   *frame,
-                         xlator_t   *this,
-                         loc_t      *loc,
-                         const char *name, dict_t *xdata);
-int32_t dht_fremovexattr (call_frame_t   *frame,
-                          xlator_t   *this,
-                          fd_t      *fd,
-                          const char *name, dict_t *xdata);
-
-int32_t dht_lk (call_frame_t        *frame,
-                xlator_t        *this,
-                fd_t            *fd,
-                int32_t          cmd,
-                struct gf_flock *flock, dict_t *xdata);
-
-int32_t dht_lease (call_frame_t    *frame,
-                   xlator_t        *this,
-                   loc_t           *loc,
-                   struct gf_lease *lease, dict_t *xdata);
-
-int32_t dht_inodelk (call_frame_t *frame, xlator_t *this,
-                     const char      *volume, loc_t *loc, int32_t cmd,
-                     struct gf_flock *flock, dict_t *xdata);
-
-int32_t dht_finodelk (call_frame_t        *frame, xlator_t *this,
-                      const char      *volume, fd_t *fd, int32_t cmd,
-                      struct gf_flock *flock, dict_t *xdata);
-
-int32_t dht_entrylk (call_frame_t    *frame, xlator_t *this,
-                     const char  *volume, loc_t *loc, const char *basename,
-                     entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
-
-int32_t dht_fentrylk (call_frame_t    *frame, xlator_t *this,
-                      const char  *volume, fd_t *fd, const char *basename,
-                      entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
-
-int32_t dht_readdir (call_frame_t  *frame,
-                     xlator_t *this,
-                     fd_t     *fd,
-                     size_t    size, off_t off, dict_t *xdata);
-
-int32_t dht_readdirp (call_frame_t *frame,
-                      xlator_t *this,
-                      fd_t     *fd,
-                      size_t    size, off_t off, dict_t *dict);
-
-int32_t dht_xattrop (call_frame_t           *frame,
-                     xlator_t           *this,
-                     loc_t              *loc,
-                     gf_xattrop_flags_t  flags,
-                     dict_t             *dict, dict_t *xdata);
-
-int32_t dht_fxattrop (call_frame_t *frame,
-                      xlator_t           *this,
-                      fd_t               *fd,
-                      gf_xattrop_flags_t  flags,
-                      dict_t             *dict, dict_t *xdata);
-
-int32_t dht_forget (xlator_t *this, inode_t *inode);
-int32_t dht_setattr (call_frame_t  *frame, xlator_t *this, loc_t *loc,
-                     struct iatt   *stbuf, int32_t valid, dict_t *xdata);
-int32_t dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      struct iatt  *stbuf, int32_t valid, dict_t *xdata);
-int32_t dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      int32_t mode, off_t offset, size_t len, dict_t *xdata);
-int32_t dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    off_t offset, size_t len, dict_t *xdata);
-int32_t dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    off_t offset, off_t len, dict_t *xdata);
-int32_t dht_ipc (call_frame_t *frame, xlator_t *this, int32_t op,
-                 dict_t *xdata);
+int32_t
+dht_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req);
+
+int32_t
+dht_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int32_t
+dht_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
+
+int32_t
+dht_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata);
+
+int32_t
+dht_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata);
+
+int32_t
+dht_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata);
+
+int32_t
+dht_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata);
+
+int32_t
+dht_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata);
+
+int32_t
+dht_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata);
+
+int32_t
+dht_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata);
+
+int32_t
+dht_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata);
+
+int32_t
+dht_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *loc, mode_t umask, dict_t *xdata);
+
+int32_t
+dht_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata);
+
+int32_t
+dht_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata);
+
+int32_t
+dht_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *params);
+
+int32_t
+dht_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata);
+
+int32_t
+dht_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata);
+
+int32_t
+dht_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata);
+
+int32_t
+dht_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
+
+int32_t
+dht_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata);
+
+int32_t
+dht_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata);
+
+int32_t
+dht_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+             dict_t *xdata);
+
+int32_t
+dht_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int32_t
+dht_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata);
+
+int32_t
+dht_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata);
+
+int32_t
+dht_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata);
+
+int32_t
+dht_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata);
+
+int32_t
+dht_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata);
+int32_t
+dht_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata);
+
+int32_t
+dht_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+       struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+dht_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata);
+
+int32_t
+dht_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+dht_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             int32_t cmd, struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+dht_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata);
+
+int32_t
+dht_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata);
+
+int32_t
+dht_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *xdata);
+
+int32_t
+dht_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t off, dict_t *dict);
+
+int32_t
+dht_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+dht_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+dht_forget(xlator_t *this, inode_t *inode);
+int32_t
+dht_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata);
+int32_t
+dht_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+             int32_t valid, dict_t *xdata);
+int32_t
+dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata);
+int32_t
+dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata);
+int32_t
+dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata);
+int32_t
+dht_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata);
 
 int
 dht_set_subvol_range(xlator_t *this);
-int32_t dht_init (xlator_t *this);
-void    dht_fini (xlator_t *this);
-int     dht_reconfigure (xlator_t *this, dict_t *options);
-int32_t dht_notify (xlator_t *this, int32_t event, void *data, ...);
+int32_t
+dht_init(xlator_t *this);
+void
+dht_fini(xlator_t *this);
+int
+dht_reconfigure(xlator_t *this, dict_t *options);
+int32_t
+dht_notify(xlator_t *this, int32_t event, void *data, ...);
 
 /* definitions for nufa/switch */
-int dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, inode_t *inode,
-                        struct iatt *stbuf, dict_t *xattr,
-                        struct iatt *postparent);
-int dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+int
+dht_revalidate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent);
+int
+dht_lookup_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent);
+int
+dht_lookup_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                         int op_ret, int op_errno, inode_t *inode,
                         struct iatt *stbuf, dict_t *xattr,
                         struct iatt *postparent);
-int dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int op_ret, int op_errno,
-                             inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                             struct iatt *postparent);
-int dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                    struct iatt *postparent);
-int dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                    struct iatt *preparent, struct iatt *postparent,
-                    dict_t *xdata);
-int dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno,
-                     inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata);
-
-int
-gf_defrag_status_get (dht_conf_t *conf, dict_t *dict);
-
-void
-gf_defrag_set_pause_state (gf_tier_conf_t *tier_conf, tier_pause_state_t state);
-
-tier_pause_state_t
-gf_defrag_get_pause_state (gf_tier_conf_t *tier_conf);
-
 int
-gf_defrag_pause_tier (xlator_t *this, gf_defrag_info_t *defrag);
+dht_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+               struct iatt *postparent);
+int
+dht_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, fd_t *fd, inode_t *inode, struct iatt *stbuf,
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata);
+int
+dht_newfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, inode_t *inode, struct iatt *stbuf,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata);
 
-tier_pause_state_t
-gf_defrag_check_pause_tier (gf_tier_conf_t *defrag);
+int
+dht_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 int
-gf_defrag_resume_tier (xlator_t *this, gf_defrag_info_t *defrag);
+dht_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, dict_t *xattr, dict_t *xdata);
 
 int
-gf_defrag_start_detach_tier (gf_defrag_info_t *defrag);
+dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata);
+int
+gf_defrag_status_get(dht_conf_t *conf, dict_t *dict);
 
 int
-gf_defrag_stop (dht_conf_t *conf, gf_defrag_status_t status,
-                dict_t *output);
+gf_defrag_stop(dht_conf_t *conf, gf_defrag_status_t status, dict_t *output);
 
-void*
-gf_defrag_start (void *this);
+void *
+gf_defrag_start(void *this);
 
 int32_t
-gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, int *fop_errno);
+gf_defrag_handle_hardlink(xlator_t *this, loc_t *loc, int *fop_errno);
 int
-dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
+dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                  int flag, int *fop_errno);
 int
-dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this,
-                          dht_layout_t **layout_int);
+dht_inode_ctx_layout_get(inode_t *inode, xlator_t *this,
+                         dht_layout_t **layout_int);
 int
-dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
-                          dht_layout_t* layout_int);
+dht_inode_ctx_layout_set(inode_t *inode, xlator_t *this,
+                         dht_layout_t *layout_int);
 int
-dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
-                           int32_t update_ctx);
-void dht_inode_ctx_time_set (inode_t *inode, xlator_t *this, struct iatt *stat);
+dht_inode_ctx_time_update(inode_t *inode, xlator_t *this, struct iatt *stat,
+                          int32_t update_ctx);
+void
+dht_inode_ctx_time_set(inode_t *inode, xlator_t *this, struct iatt *stat);
 
-int dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx);
-int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx);
 int
-dht_dir_attr_heal (void *data);
+dht_inode_ctx_get(inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx);
 int
-dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data);
+dht_inode_ctx_set(inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx);
 int
-dht_dir_has_layout (dict_t *xattr, char *name);
-gf_boolean_t
-dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator);
+dht_dir_attr_heal(void *data);
+int
+dht_dir_attr_heal_done(int ret, call_frame_t *sync_frame, void *data);
 xlator_t *
-dht_subvol_with_free_space_inodes (xlator_t *this, xlator_t *subvol, xlator_t *ignore,
-                                   dht_layout_t *layout, uint64_t filesize);
+dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol,
+                                  xlator_t *ignore, dht_layout_t *layout,
+                                  uint64_t filesize);
 xlator_t *
-dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
-                                  dht_layout_t *layout);
+dht_subvol_maxspace_nonzeroinode(xlator_t *this, xlator_t *subvol,
+                                 dht_layout_t *layout);
 int
-dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this);
+dht_dir_has_layout(dict_t *xattr, char *name);
+int
+dht_linkfile_attr_heal(call_frame_t *frame, xlator_t *this);
 
-void
-dht_layout_dump (dht_layout_t  *layout, const char *prefix);
 int32_t
-dht_priv_dump (xlator_t *this);
+dht_priv_dump(xlator_t *this);
 int32_t
-dht_inodectx_dump (xlator_t *this, inode_t *inode);
+dht_inodectx_dump(xlator_t *this, inode_t *inode);
 
-int
-dht_inode_ctx_get_mig_info (xlator_t *this, inode_t *inode,
-                            xlator_t **src_subvol, xlator_t **dst_subvol);
 gf_boolean_t
-dht_mig_info_is_invalid (xlator_t *current, xlator_t *src_subvol,
-                         xlator_t *dst_subvol);
+dht_is_subvol_in_layout(dht_layout_t *layout, xlator_t *xlator);
 
 int
-dht_subvol_status (dht_conf_t *conf, xlator_t *subvol);
+dht_inode_ctx_get_mig_info(xlator_t *this, inode_t *inode,
+                           xlator_t **src_subvol, xlator_t **dst_subvol);
+gf_boolean_t
+dht_mig_info_is_invalid(xlator_t *current, xlator_t *src_subvol,
+                        xlator_t *dst_subvol);
 
-void
-dht_log_new_layout_for_dir_selfheal (xlator_t *this, loc_t *loc,
-                                     dht_layout_t *layout);
 int
-dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this);
+dht_subvol_status(dht_conf_t *conf, xlator_t *subvol);
 
-int
-dht_fill_dict_to_avoid_unlink_of_migrating_file (dict_t *dict);
+void
+dht_log_new_layout_for_dir_selfheal(xlator_t *this, loc_t *loc,
+                                    dht_layout_t *layout);
 
 int
-dht_layout_sort (dht_layout_t *layout);
+dht_layout_sort(dht_layout_t *layout);
 
 int
-dht_heal_full_path (void *data);
+dht_heal_full_path(void *data);
 
 int
-dht_heal_full_path_done (int op_ret, call_frame_t *frame, void *data);
+dht_heal_full_path_done(int op_ret, call_frame_t *frame, void *data);
 
 int
-dht_layout_missing_dirs (dht_layout_t *layout);
+dht_layout_missing_dirs(dht_layout_t *layout);
 
 int
-dht_refresh_layout (call_frame_t *frame);
-
-gf_boolean_t
-dht_is_tier_xlator (xlator_t *this);
+dht_refresh_layout(call_frame_t *frame);
 
 int
-dht_build_parent_loc (xlator_t *this, loc_t *parent, loc_t *child,
-                                                 int32_t *op_errno);
+dht_build_parent_loc(xlator_t *this, loc_t *parent, loc_t *child,
+                     int32_t *op_errno);
 
 int32_t
-dht_set_local_rebalance (xlator_t *this, dht_local_t *local,
-                         struct iatt *stbuf,
-                         struct iatt *prebuf,
-                         struct iatt *postbuf, dict_t *xdata);
+dht_set_local_rebalance(xlator_t *this, dht_local_t *local, struct iatt *stbuf,
+                        struct iatt *prebuf, struct iatt *postbuf,
+                        dict_t *xdata);
 void
-dht_build_root_loc (inode_t *inode, loc_t *loc);
+dht_build_root_loc(inode_t *inode, loc_t *loc);
 
 gf_boolean_t
-dht_fd_open_on_dst (xlator_t *this, fd_t *fd, xlator_t *dst);
+dht_fd_open_on_dst(xlator_t *this, fd_t *fd, xlator_t *dst);
 
 int32_t
-dht_fd_ctx_destroy (xlator_t *this, fd_t *fd);
+dht_fd_ctx_destroy(xlator_t *this, fd_t *fd);
 
 int32_t
-dht_release (xlator_t *this, fd_t *fd);
-
+dht_release(xlator_t *this, fd_t *fd);
 
 int32_t
-dht_set_fixed_dir_stat (struct iatt *stat);
+dht_set_fixed_dir_stat(struct iatt *stat);
 
-xlator_t*
-dht_get_lock_subvolume (xlator_t *this, struct gf_flock *lock,
-                        dht_local_t *local);
+xlator_t *
+dht_get_lock_subvolume(xlator_t *this, struct gf_flock *lock,
+                       dht_local_t *local);
 
 int
-dht_lk_inode_unref (call_frame_t *frame, int32_t op_ret);
+dht_lk_inode_unref(call_frame_t *frame, int32_t op_ret);
 
-void
-dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
-                     unsigned long frsize);
+int
+dht_fd_ctx_set(xlator_t *this, fd_t *fd, xlator_t *subvol);
 
 int
-add_opt(char **optsp, const char *opt);
+dht_check_and_open_fd_on_subvol(xlator_t *this, call_frame_t *frame);
+
+/* FD fop callbacks */
 
 int
-dht_aggregate_split_brain_xattr (dict_t *dst, char *key, data_t *value);
+dht_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+               dict_t *xdata);
 
 int
-dht_remove_stale_linkto (void *data);
+dht_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, dict_t *xdata);
 
 int
-dht_remove_stale_linkto_cbk (int ret, call_frame_t *sync_frame, void *data);
+dht_file_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata);
 
 int
-dht_fd_ctx_set (xlator_t *this, fd_t *fd, xlator_t *subvol);
+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata);
 
 int
-dht_check_and_open_fd_on_subvol (xlator_t *this, call_frame_t *frame);
+dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                dict_t *xdata);
 
-/* FD fop callbacks */
+int
+dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                  dict_t *xdata);
 
 int
-dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata);
+dht_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata);
 
 int
-dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, dict_t *xdata);
+dht_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata);
 
 int
-dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata);
+dht_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iovec *vector, int count, struct iatt *stbuf,
+              struct iobref *iobref, dict_t *xdata);
 
 int
-dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata);
+dht_file_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *stbuf, dict_t *xdata);
 
 int
-dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata);
+dht_file_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata);
 
 int
-dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata);
+dht_file_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata);
 
+/* All custom xattr heal functions */
 int
-dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata);
+dht_dir_heal_xattrs(void *data);
 
 int
-dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
-               dict_t *xdata);
+dht_dir_heal_xattrs_done(int ret, call_frame_t *sync_frame, void *data);
 
+int32_t
+dht_dict_set_array(dict_t *dict, char *key, int32_t value[], int32_t size);
 
 int
-dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno,
-               struct iovec *vector, int count, struct iatt *stbuf,
-               struct iobref *iobref, dict_t *xdata);
+dht_set_user_xattr(dict_t *dict, char *k, data_t *v, void *data);
 
-int
-dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata);
+void
+dht_dir_set_heal_xattr(xlator_t *this, dht_local_t *local, dict_t *dst,
+                       dict_t *src, int *uret, int *uflag);
 
 int
-dht_file_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int op_ret, int op_errno, dict_t *xdata);
+dht_dir_xattr_heal(xlator_t *this, dht_local_t *local, int *op_errno);
 
 int
-dht_file_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, dict_t *xdata);
+dht_common_mark_mdsxattr(call_frame_t *frame, int *errst, int flag);
 
-/* All custom xattr heal functions */
 int
-dht_dir_heal_xattrs (void *data);
+dht_inode_ctx_mdsvol_get(inode_t *inode, xlator_t *this, xlator_t **mdsvol);
 
 int
-dht_dir_heal_xattrs_done (int ret, call_frame_t *sync_frame, void *data);
+dht_selfheal_dir_setattr(call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
+                         int32_t valid, dht_layout_t *layout);
+
+/* Abstract out the DHT-IATT-IN-DICT */
 
 void
-dht_aggregate_xattr (dict_t *dst, dict_t *src);
+dht_selfheal_layout_new_directory(call_frame_t *frame, loc_t *loc,
+                                  dht_layout_t *new_layout);
 
-int32_t
-dht_dict_set_array(dict_t *dict, char *key, int32_t value[], int32_t size);
+int
+dht_pt_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+                 dict_t *xdata);
 
 int
-dht_set_user_xattr (dict_t *dict, char *k, data_t *v, void *data);
+dht_pt_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *key, dict_t *xdata);
 
-void
-dht_dir_set_heal_xattr (xlator_t *this, dht_local_t *local, dict_t *dst,
-                        dict_t *src, int *uret, int *uflag);
+int32_t
+dht_pt_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata);
 
 int
-dht_dir_xattr_heal (xlator_t *this, dht_local_t *local);
+dht_pt_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata);
 
 int32_t
-dht_dict_get_array (dict_t *dict, char *key, int32_t value[], int32_t size, int *errst);
-
-xlator_t *
-dht_inode_get_hashed_subvol (inode_t *inode, xlator_t *this, loc_t *loc);
+dht_check_remote_fd_failed_error(dht_local_t *local, int op_ret, int op_errno);
 
 int
-dht_common_mark_mdsxattr (call_frame_t *frame, int *errst, int flag);
+dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata);
 
-int
-dht_common_mark_mdsxattr_cbk (call_frame_t *frame, void *cookie,
-                              xlator_t *this, int op_ret, int op_errno,
-                              dict_t *xdata);
+int32_t
+dht_create_lock(call_frame_t *frame, xlator_t *subvol);
 
 int
-dht_inode_ctx_mdsvol_set (inode_t *inode, xlator_t *this,
-                          xlator_t *mds_subvol);
-int
-dht_inode_ctx_mdsvol_get (inode_t *inode, xlator_t *this,
-                          xlator_t **mdsvol);
+dht_set_parent_layout_in_dict(loc_t *loc, xlator_t *this, dht_local_t *local);
 
 int
-dht_selfheal_dir_setattr (call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
-                          int32_t valid, dht_layout_t *layout);
-
-/* Abstract out the DHT-IATT-IN-DICT */
+dht_dir_layout_error_check(xlator_t *this, inode_t *inode);
 
-
-int dht_request_iatt_in_xdata (xlator_t *this, dict_t *xattr_req);
-
-int dht_read_iatt_from_xdata (xlator_t *this, dict_t *xdata,
-                              struct iatt *stbuf);
-
-#endif/* _DHT_H */
+int
+dht_inode_ctx_mdsvol_set(inode_t *inode, xlator_t *this, xlator_t *mds_subvol);
+#endif /* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-diskusage.c b/xlators/cluster/dht/src/dht-diskusage.c
index c2b9b2e37c4..c0588828fdb 100644
--- a/xlators/cluster/dht/src/dht-diskusage.c
+++ b/xlators/cluster/dht/src/dht-diskusage.c
@@ -8,505 +8,480 @@
   cases as published by the Free Software Foundation.
 */
 
-
 /* TODO: add NS locking */
 
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "dht-messages.h"
-#include "defaults.h"
 
 #include <sys/time.h>
-#include "events.h"
-
+#include <glusterfs/events.h>
 
 int
-dht_du_info_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno, struct statvfs *statvfs,
-                 dict_t *xdata)
+dht_du_info_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
-	dht_conf_t    *conf         = NULL;
-	xlator_t      *prev          = NULL;
-	int            this_call_cnt = 0;
-	int            i = 0;
-	double         percent = 0;
-	double         percent_inodes = 0;
-	uint64_t       bytes = 0;
-        uint32_t       bpc;     /* blocks per chunk */
-        uint32_t       chunks   = 0;
-
-	conf = this->private;
-	prev = cookie;
-
-	if (op_ret == -1 || !statvfs) {
-		gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_GET_DISK_INFO_ERROR,
-			"failed to get disk info from %s", prev->name);
-		goto out;
-	}
-
-	if (statvfs->f_blocks) {
-		percent = (statvfs->f_bavail * 100) / statvfs->f_blocks;
-		bytes = (statvfs->f_bavail * statvfs->f_frsize);
-                /*
-                 * A 32-bit count of 1MB chunks allows a maximum brick size of
-                 * ~4PB.  It's possible that we could see a single local FS
-                 * bigger than that some day, but this code is likely to be
-                 * irrelevant by then.  Meanwhile, it's more important to keep
-                 * the chunk size small so the layout-calculation code that
-                 * uses this value can be tested on normal machines.
-                 */
-                bpc = (1 << 20) / statvfs->f_bsize;
-                chunks = (statvfs->f_blocks + bpc - 1) / bpc;
-	}
-
-	if (statvfs->f_files) {
-		percent_inodes = (statvfs->f_ffree * 100) / statvfs->f_files;
-	} else {
-                /*
-                 * Set percent inodes to 100 for dynamically allocated inode
-                 * filesystems. The rationale is that distribute need not
-                 * worry about total inodes; rather, let the 'create()' be
-                 * scheduled on the hashed subvol regardless of the total
-                 * inodes.
-		 */
-		percent_inodes = 100;
-	}
-
-	LOCK (&conf->subvolume_lock);
-	{
-		for (i = 0; i < conf->subvolume_cnt; i++)
-			if (prev == conf->subvolumes[i]) {
-				conf->du_stats[i].avail_percent = percent;
-				conf->du_stats[i].avail_space   = bytes;
-				conf->du_stats[i].avail_inodes  = percent_inodes;
-                                conf->du_stats[i].chunks        = chunks;
-                                conf->du_stats[i].total_blocks  = statvfs->f_blocks;
-                                conf->du_stats[i].avail_blocks  = statvfs->f_bavail;
-                                conf->du_stats[i].frsize        = statvfs->f_frsize;
-
-                                gf_msg_debug (this->name, 0,
-				              "subvolume '%s': avail_percent "
-					      "is: %.2f and avail_space "
-                                              "is: %" PRIu64" and avail_inodes"
-                                              " is: %.2f",
-					      prev->name,
-					      conf->du_stats[i].avail_percent,
-					      conf->du_stats[i].avail_space,
-					      conf->du_stats[i].avail_inodes);
-                                break;  /* no point in looping further */
-			}
-	}
-	UNLOCK (&conf->subvolume_lock);
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
+    int i = 0;
+    double percent = 0;
+    double percent_inodes = 0;
+    uint64_t bytes = 0;
+    uint32_t bpc; /* blocks per chunk */
+    uint32_t chunks = 0;
+
+    conf = this->private;
+    prev = cookie;
+
+    if (op_ret == -1 || !statvfs) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno,
+               DHT_MSG_GET_DISK_INFO_ERROR, "failed to get disk info from %s",
+               prev->name);
+        goto out;
+    }
+
+    if (statvfs->f_blocks) {
+        percent = (statvfs->f_bavail * 100) / statvfs->f_blocks;
+        bytes = (statvfs->f_bavail * statvfs->f_frsize);
+        /*
+         * A 32-bit count of 1MB chunks allows a maximum brick size of
+         * ~4PB.  It's possible that we could see a single local FS
+         * bigger than that some day, but this code is likely to be
+         * irrelevant by then.  Meanwhile, it's more important to keep
+         * the chunk size small so the layout-calculation code that
+         * uses this value can be tested on normal machines.
+         */
+        bpc = (1 << 20) / statvfs->f_bsize;
+        chunks = (statvfs->f_blocks + bpc - 1) / bpc;
+    }
+
+    if (statvfs->f_files) {
+        percent_inodes = (statvfs->f_ffree * 100) / statvfs->f_files;
+    } else {
+        /*
+         * Set percent inodes to 100 for dynamically allocated inode
+         * filesystems. The rationale is that distribute need not
+         * worry about total inodes; rather, let the 'create()' be
+         * scheduled on the hashed subvol regardless of the total
+         * inodes.
+         */
+        percent_inodes = 100;
+    }
+
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = 0; i < conf->subvolume_cnt; i++)
+            if (prev == conf->subvolumes[i]) {
+                conf->du_stats[i].avail_percent = percent;
+                conf->du_stats[i].avail_space = bytes;
+                conf->du_stats[i].avail_inodes = percent_inodes;
+                conf->du_stats[i].chunks = chunks;
+                conf->du_stats[i].total_blocks = statvfs->f_blocks;
+                conf->du_stats[i].avail_blocks = statvfs->f_bavail;
+                conf->du_stats[i].frsize = statvfs->f_frsize;
+
+                gf_msg_debug(this->name, 0,
+                             "subvolume '%s': avail_percent "
+                             "is: %.2f and avail_space "
+                             "is: %" PRIu64
+                             " and avail_inodes"
+                             " is: %.2f",
+                             prev->name, conf->du_stats[i].avail_percent,
+                             conf->du_stats[i].avail_space,
+                             conf->du_stats[i].avail_inodes);
+                break; /* no point in looping further */
+            }
+    }
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_DESTROY (frame);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_DESTROY(frame);
 
-	return 0;
+    return 0;
 }
 
 int
-dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx)
+dht_get_du_info_for_subvol(xlator_t *this, int subvol_idx)
 {
-	dht_conf_t    *conf         = NULL;
-	call_frame_t  *statfs_frame = NULL;
-	dht_local_t   *statfs_local = NULL;
-	call_pool_t   *pool         = NULL;
-	loc_t          tmp_loc      = {0,};
-
-	conf = this->private;
-	pool = this->ctx->pool;
-
-	statfs_frame = create_frame (this, pool);
-	if (!statfs_frame) {
-		goto err;
-	}
-
-	/* local->fop value is not used in this case */
-	statfs_local = dht_local_init (statfs_frame, NULL, NULL,
-				       GF_FOP_MAXVALUE);
-	if (!statfs_local) {
-		goto err;
-	}
-
-        /* make it root gfid, should be enough to get the proper info back */
-        tmp_loc.gfid[15] = 1;
-
-	statfs_local->call_cnt = 1;
-	STACK_WIND_COOKIE (statfs_frame, dht_du_info_cbk,
-		           conf->subvolumes[subvol_idx],
-                           conf->subvolumes[subvol_idx],
-		           conf->subvolumes[subvol_idx]->fops->statfs,
-		           &tmp_loc, NULL);
-
-	return 0;
+    dht_conf_t *conf = NULL;
+    call_frame_t *statfs_frame = NULL;
+    dht_local_t *statfs_local = NULL;
+    call_pool_t *pool = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+
+    conf = this->private;
+    pool = this->ctx->pool;
+
+    statfs_frame = create_frame(this, pool);
+    if (!statfs_frame) {
+        goto err;
+    }
+
+    /* local->fop value is not used in this case */
+    statfs_local = dht_local_init(statfs_frame, NULL, NULL, GF_FOP_MAXVALUE);
+    if (!statfs_local) {
+        goto err;
+    }
+
+    /* make it root gfid, should be enough to get the proper info back */
+    tmp_loc.gfid[15] = 1;
+
+    statfs_local->call_cnt = 1;
+    STACK_WIND_COOKIE(
+        statfs_frame, dht_du_info_cbk, conf->subvolumes[subvol_idx],
+        conf->subvolumes[subvol_idx],
+        conf->subvolumes[subvol_idx]->fops->statfs, &tmp_loc, NULL);
+
+    return 0;
 err:
-	if (statfs_frame)
-		DHT_STACK_DESTROY (statfs_frame);
+    if (statfs_frame)
+        DHT_STACK_DESTROY(statfs_frame);
 
-	return -1;
+    return -1;
 }
 
 int
-dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_get_du_info(call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-	int            i            = 0;
-        int            ret          = -1;
-	dht_conf_t    *conf         = NULL;
-	call_frame_t  *statfs_frame = NULL;
-	dht_local_t   *statfs_local = NULL;
-	struct timeval tv           = {0,};
-        loc_t          tmp_loc      = {0,};
-
-	conf  = this->private;
-
-	gettimeofday (&tv, NULL);
-
-        /* make it root gfid, should be enough to get the proper
-           info back */
-        tmp_loc.gfid[15] = 1;
-
-	if (tv.tv_sec > (conf->refresh_interval
-			 + conf->last_stat_fetch.tv_sec)) {
-
-		statfs_frame = copy_frame (frame);
-		if (!statfs_frame) {
-			goto err;
-		}
-
-		/* In this case, 'local->fop' is not used */
-		statfs_local = dht_local_init (statfs_frame, loc, NULL,
-					       GF_FOP_MAXVALUE);
-		if (!statfs_local) {
-			goto err;
-		}
-
-                statfs_local->params = dict_new ();
-                if (!statfs_local->params)
-                        goto err;
-
-                ret = dict_set_int8 (statfs_local->params,
-                                     GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set "
-                                GF_INTERNAL_IGNORE_DEEM_STATFS" in dict");
-                        goto err;
-                }
+    int i = 0;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    call_frame_t *statfs_frame = NULL;
+    dht_local_t *statfs_local = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    time_t now;
+
+    conf = this->private;
+    now = gf_time();
+    /* make it root gfid, should be enough to get the proper
+       info back */
+    tmp_loc.gfid[15] = 1;
+
+    if (now > (conf->refresh_interval + conf->last_stat_fetch)) {
+        statfs_frame = copy_frame(frame);
+        if (!statfs_frame) {
+            goto err;
+        }
+
+        /* In this case, 'local->fop' is not used */
+        statfs_local = dht_local_init(statfs_frame, loc, NULL, GF_FOP_MAXVALUE);
+        if (!statfs_local) {
+            goto err;
+        }
+
+        statfs_local->params = dict_new();
+        if (!statfs_local->params)
+            goto err;
+
+        ret = dict_set_int8(statfs_local->params,
+                            GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set " GF_INTERNAL_IGNORE_DEEM_STATFS " in dict");
+            goto err;
+        }
+
+        statfs_local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            STACK_WIND_COOKIE(statfs_frame, dht_du_info_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->statfs, &tmp_loc,
+                              statfs_local->params);
+        }
 
-		statfs_local->call_cnt = conf->subvolume_cnt;
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			STACK_WIND_COOKIE (statfs_frame, dht_du_info_cbk,
-				           conf->subvolumes[i],
-                                           conf->subvolumes[i],
-				           conf->subvolumes[i]->fops->statfs,
-				           &tmp_loc, statfs_local->params);
-		}
-
-		conf->last_stat_fetch.tv_sec = tv.tv_sec;
-	}
-	return 0;
+        conf->last_stat_fetch = now;
+    }
+    return 0;
 err:
-	if (statfs_frame)
-		DHT_STACK_DESTROY (statfs_frame);
+    if (statfs_frame)
+        DHT_STACK_DESTROY(statfs_frame);
 
-	return -1;
+    return -1;
 }
 
-
 gf_boolean_t
-dht_is_subvol_filled (xlator_t *this, xlator_t *subvol)
+dht_is_subvol_filled(xlator_t *this, xlator_t *subvol)
 {
-	int         i = 0;
-        char      vol_name[256];
-	dht_conf_t *conf = NULL;
-	gf_boolean_t subvol_filled_inodes = _gf_false;
-	gf_boolean_t subvol_filled_space = _gf_false;
-	gf_boolean_t is_subvol_filled = _gf_false;
-        double usage = 0;
-
-	conf = this->private;
-
-	/* Check for values above specified percent or free disk */
-	LOCK (&conf->subvolume_lock);
-	{
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			if (subvol == conf->subvolumes[i]) {
-				if (conf->disk_unit == 'p') {
-					if (conf->du_stats[i].avail_percent <
-					    conf->min_free_disk) {
-						subvol_filled_space = _gf_true;
-						break;
-					}
-
-				} else {
-					if (conf->du_stats[i].avail_space <
-					    conf->min_free_disk) {
-						subvol_filled_space = _gf_true;
-						break;
-					}
-				}
-				if (conf->du_stats[i].avail_inodes <
-				    conf->min_free_inodes) {
-					subvol_filled_inodes = _gf_true;
-					break;
-				}
-			}
-		}
-	}
-	UNLOCK (&conf->subvolume_lock);
-
-	if (subvol_filled_space && conf->subvolume_status[i]) {
-		if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
-                        usage = 100 - conf->du_stats[i].avail_percent;
-
-			gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_SUBVOL_INSUFF_SPACE,
-				"disk space on subvolume '%s' is getting "
-				"full (%.2f %%), consider adding more bricks",
-				subvol->name, usage);
-
-                        strncpy(vol_name, this->name, sizeof(vol_name));
-                        vol_name[(strlen(this->name)-4)] = '\0';
-
-                        gf_event(EVENT_DHT_DISK_USAGE,
-                                 "volume=%s;subvol=%s;usage=%.2f %%",
-                                 vol_name, subvol->name, usage);
-		}
-	}
-
-	if (subvol_filled_inodes && conf->subvolume_status[i]) {
-		if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
-                        usage = 100 - conf->du_stats[i].avail_inodes;
-			gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                                DHT_MSG_SUBVOL_INSUFF_INODES,
-				"inodes on subvolume '%s' are at "
-				"(%.2f %%), consider adding more bricks",
-				subvol->name, usage);
-
-                        strncpy(vol_name, this->name, sizeof(vol_name));
-                        vol_name[(strlen(this->name)-4)] = '\0';
-
-                        gf_event(EVENT_DHT_INODES_USAGE,
-                                 "volume=%s;subvol=%s;usage=%.2f %%",
-                                  vol_name, subvol->name, usage);
-		}
-	}
-
-	is_subvol_filled = (subvol_filled_space || subvol_filled_inodes);
-
-	return is_subvol_filled;
-}
+    int i = 0;
+    char vol_name[256];
+    dht_conf_t *conf = NULL;
+    gf_boolean_t subvol_filled_inodes = _gf_false;
+    gf_boolean_t subvol_filled_space = _gf_false;
+    gf_boolean_t is_subvol_filled = _gf_false;
+    double usage = 0;
+
+    conf = this->private;
+
+    /* Check for values above specified percent or free disk */
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (subvol == conf->subvolumes[i]) {
+                if (conf->disk_unit == 'p') {
+                    if (conf->du_stats[i].avail_percent < conf->min_free_disk) {
+                        subvol_filled_space = _gf_true;
+                        break;
+                    }
+
+                } else {
+                    if (conf->du_stats[i].avail_space < conf->min_free_disk) {
+                        subvol_filled_space = _gf_true;
+                        break;
+                    }
+                }
+                if (conf->du_stats[i].avail_inodes < conf->min_free_inodes) {
+                    subvol_filled_inodes = _gf_true;
+                    break;
+                }
+            }
+        }
+    }
+    UNLOCK(&conf->subvolume_lock);
 
+    if (subvol_filled_space && conf->subvolume_status[i]) {
+        if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
+            usage = 100 - conf->du_stats[i].avail_percent;
 
-/*Get the best subvolume to create the file in*/
-xlator_t *
-dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
-                                dht_local_t *local)
-{
-	xlator_t   *avail_subvol = NULL;
-	dht_conf_t *conf = NULL;
-        dht_layout_t *layout = NULL;
-        loc_t      *loc = NULL;
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SUBVOL_INSUFF_SPACE,
+                   "disk space on subvolume '%s' is getting "
+                   "full (%.2f %%), consider adding more bricks",
+                   subvol->name, usage);
 
-	conf = this->private;
-        if (!local)
-                goto out;
-        loc = &local->loc;
-        if (!local->layout) {
-                layout = dht_layout_get (this, loc->parent);
-
-                if (!layout) {
-                        gf_msg_debug (this->name, 0,
-                                      "Missing layout. path=%s,"
-                                      " parent gfid = %s", loc->path,
-                                      uuid_utoa (loc->parent->gfid));
-                        goto out;
-                }
-        } else {
-                layout = dht_layout_ref (this, local->layout);
-        }
+            (void)snprintf(vol_name, sizeof(vol_name), "%s", this->name);
+            vol_name[(strlen(this->name) - 4)] = '\0';
 
-        LOCK (&conf->subvolume_lock);
-	{
-                avail_subvol = dht_subvol_with_free_space_inodes(this, subvol, NULL,
-                                                                 layout, 0);
-                if(!avail_subvol)
-                {
-                        avail_subvol = dht_subvol_maxspace_nonzeroinode(this,
-                                                                        subvol,
-                                                                        layout);
-                }
+            gf_event(EVENT_DHT_DISK_USAGE, "volume=%s;subvol=%s;usage=%.2f %%",
+                     vol_name, subvol->name, usage);
+        }
+    }
+
+    if (subvol_filled_inodes && conf->subvolume_status[i]) {
+        if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
+            usage = 100 - conf->du_stats[i].avail_inodes;
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_SUBVOL_INSUFF_INODES,
+                   "inodes on subvolume '%s' are at "
+                   "(%.2f %%), consider adding more bricks",
+                   subvol->name, usage);
+
+            (void)snprintf(vol_name, sizeof(vol_name), "%s", this->name);
+            vol_name[(strlen(this->name) - 4)] = '\0';
+
+            gf_event(EVENT_DHT_INODES_USAGE,
+                     "volume=%s;subvol=%s;usage=%.2f %%", vol_name,
+                     subvol->name, usage);
+        }
+    }
 
-	}
-	UNLOCK (&conf->subvolume_lock);
-out:
-	if (!avail_subvol) {
-		gf_msg_debug (this->name, 0,
-		              "No subvolume has enough free space \
-                              and/or inodes to create");
-                avail_subvol = subvol;
-	}
+    is_subvol_filled = (subvol_filled_space || subvol_filled_inodes);
 
-        if (layout)
-                dht_layout_unref (this, layout);
-	return avail_subvol;
+    return is_subvol_filled;
 }
 
-static inline
-int32_t dht_subvol_has_err (dht_conf_t *conf, xlator_t *this, xlator_t *ignore,
-                            dht_layout_t *layout)
+/*Get the best subvolume to create the file in*/
+xlator_t *
+dht_free_disk_available_subvol(xlator_t *this, xlator_t *subvol,
+                               dht_local_t *local)
 {
-        int ret = -1;
-        int i   = 0;
-
-        if (!this || !layout)
-                goto out;
-
-        /* this check is meant for rebalance process. The source of the file
-         * should be ignored for space check */
-        if (this == ignore) {
-                goto out;
+    xlator_t *avail_subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    loc_t *loc = NULL;
+
+    conf = this->private;
+    if (!local)
+        goto out;
+    loc = &local->loc;
+    if (!local->layout) {
+        layout = dht_layout_get(this, loc->parent);
+
+        if (!layout) {
+            gf_msg_debug(this->name, 0,
+                         "Missing layout. path=%s,"
+                         " parent gfid = %s",
+                         loc->path, uuid_utoa(loc->parent->gfid));
+            goto out;
+        }
+    } else {
+        layout = dht_layout_ref(this, local->layout);
+    }
+
+    LOCK(&conf->subvolume_lock);
+    {
+        avail_subvol = dht_subvol_with_free_space_inodes(this, subvol, NULL,
+                                                         layout, 0);
+        if (!avail_subvol) {
+            avail_subvol = dht_subvol_maxspace_nonzeroinode(this, subvol,
+                                                            layout);
         }
+    }
+    UNLOCK(&conf->subvolume_lock);
+out:
+    if (!avail_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "No subvolume has enough free space \
+                              and/or inodes to create");
+        avail_subvol = subvol;
+    }
 
+    if (layout)
+        dht_layout_unref(this, layout);
+    return avail_subvol;
+}
 
-        /* check if subvol has layout errors, before selecting it */
-        for (i = 0; i < layout->cnt; i++) {
-                if (!strcmp (layout->list[i].xlator->name, this->name) &&
-                     (layout->list[i].err != 0)) {
-                        ret = -1;
-                        goto out;
-                }
+static inline int32_t
+dht_subvol_has_err(dht_conf_t *conf, xlator_t *this, xlator_t *ignore,
+                   dht_layout_t *layout)
+{
+    int ret = -1;
+    int i = 0;
+
+    if (!this || !layout)
+        goto out;
+
+    /* this check is meant for rebalance process. The source of the file
+     * should be ignored for space check */
+    if (this == ignore) {
+        goto out;
+    }
+
+    /* check if subvol has layout errors, before selecting it */
+    for (i = 0; i < layout->cnt; i++) {
+        if (!strcmp(layout->list[i].xlator->name, this->name) &&
+            (layout->list[i].err != 0)) {
+            ret = -1;
+            goto out;
         }
+    }
 
-        /* discard decommissioned subvol */
-        if (conf->decommission_subvols_cnt) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->decommissioned_bricks[i] &&
-                            conf->decommissioned_bricks[i] == this) {
-                                ret = -1;
-                                goto out;
-                        }
-                }
+    /* discard decommissioned subvol */
+    if (conf->decommission_subvols_cnt) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->decommissioned_bricks[i] &&
+                conf->decommissioned_bricks[i] == this) {
+                ret = -1;
+                goto out;
+            }
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /*Get subvolume which has both space and inodes more than the min criteria*/
 xlator_t *
-dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol, xlator_t *ignore,
-                                  dht_layout_t *layout, uint64_t filesize)
+dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol,
+                                  xlator_t *ignore, dht_layout_t *layout,
+                                  uint64_t filesize)
 {
-        int i = 0;
-        double max = 0;
-        double max_inodes = 0;
-        int    ignore_subvol = 0;
-        uint64_t total_blocks = 0;
-        uint64_t avail_blocks = 0;
-        uint64_t frsize = 0;
-        double   post_availspace = 0;
-        double   post_percent = 0;
-
-        xlator_t *avail_subvol = NULL;
-        dht_conf_t *conf = NULL;
-
-        conf = this->private;
-
-        for(i=0; i < conf->subvolume_cnt; i++) {
-                /* check if subvol has layout errors and also it is not a
-                 * decommissioned brick, before selecting it */
-                ignore_subvol = dht_subvol_has_err (conf, conf->subvolumes[i],
-                                                    ignore, layout);
-                if (ignore_subvol)
-                        continue;
-
-                if ((conf->disk_unit == 'p') &&
-                    (conf->du_stats[i].avail_percent > conf->min_free_disk) &&
-                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
-                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
-                            (conf->du_stats[i].avail_percent > max)) {
-                                max = conf->du_stats[i].avail_percent;
-                                max_inodes = conf->du_stats[i].avail_inodes;
-                                avail_subvol = conf->subvolumes[i];
-                                total_blocks = conf->du_stats[i].total_blocks;
-                                avail_blocks = conf->du_stats[i].avail_blocks;
-                                frsize       = conf->du_stats[i].frsize;
-                        }
-                }
-
-                if ((conf->disk_unit != 'p') &&
-                    (conf->du_stats[i].avail_space > conf->min_free_disk) &&
-                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
-                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
-                            (conf->du_stats[i].avail_space > max)) {
-                                max = conf->du_stats[i].avail_space;
-                                max_inodes = conf->du_stats[i].avail_inodes;
-                                avail_subvol = conf->subvolumes[i];
-                        }
-                }
+    int i = 0;
+    double max = 0;
+    double max_inodes = 0;
+    int ignore_subvol = 0;
+    uint64_t total_blocks = 0;
+    uint64_t avail_blocks = 0;
+    uint64_t frsize = 0;
+    double post_availspace = 0;
+    double post_percent = 0;
+
+    xlator_t *avail_subvol = NULL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        /* check if subvol has layout errors and also it is not a
+         * decommissioned brick, before selecting it */
+        ignore_subvol = dht_subvol_has_err(conf, conf->subvolumes[i], ignore,
+                                           layout);
+        if (ignore_subvol)
+            continue;
+
+        if ((conf->disk_unit == 'p') &&
+            (conf->du_stats[i].avail_percent > conf->min_free_disk) &&
+            (conf->du_stats[i].avail_inodes > conf->min_free_inodes)) {
+            if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                (conf->du_stats[i].avail_percent > max)) {
+                max = conf->du_stats[i].avail_percent;
+                max_inodes = conf->du_stats[i].avail_inodes;
+                avail_subvol = conf->subvolumes[i];
+                total_blocks = conf->du_stats[i].total_blocks;
+                avail_blocks = conf->du_stats[i].avail_blocks;
+                frsize = conf->du_stats[i].frsize;
+            }
         }
 
-        if (avail_subvol) {
-                if (conf->disk_unit == 'p') {
-                        post_availspace = (avail_blocks * frsize) - filesize;
-                        post_percent = (post_availspace * 100) / (total_blocks * frsize);
-                        if (post_percent < conf->min_free_disk)
-                                avail_subvol = NULL;
-                }
-                if (conf->disk_unit != 'p') {
-                        if ((max - filesize) < conf->min_free_disk)
-                                avail_subvol = NULL;
-                }
+        if ((conf->disk_unit != 'p') &&
+            (conf->du_stats[i].avail_space > conf->min_free_disk) &&
+            (conf->du_stats[i].avail_inodes > conf->min_free_inodes)) {
+            if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                (conf->du_stats[i].avail_space > max)) {
+                max = conf->du_stats[i].avail_space;
+                max_inodes = conf->du_stats[i].avail_inodes;
+                avail_subvol = conf->subvolumes[i];
+            }
         }
+    }
+
+    if (avail_subvol) {
+        if (conf->disk_unit == 'p') {
+            post_availspace = (avail_blocks * frsize) - filesize;
+            post_percent = (post_availspace * 100) / (total_blocks * frsize);
+            if (post_percent < conf->min_free_disk)
+                avail_subvol = NULL;
+        }
+        if (conf->disk_unit != 'p') {
+            if ((max - filesize) < conf->min_free_disk)
+                avail_subvol = NULL;
+        }
+    }
 
-        return avail_subvol;
+    return avail_subvol;
 }
 
-
-/* Get subvol which has atleast one inode and maximum space */
+/* Get subvol which has at least one inode and maximum space */
 xlator_t *
-dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
-                                  dht_layout_t *layout)
+dht_subvol_maxspace_nonzeroinode(xlator_t *this, xlator_t *subvol,
+                                 dht_layout_t *layout)
 {
-        int         i = 0;
-        double      max = 0;
-        int         ignore_subvol = 0;
-
-        xlator_t   *avail_subvol = NULL;
-        dht_conf_t *conf = NULL;
-
-        conf = this->private;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                /* check if subvol has layout errors and also it is not a
-                 * decommissioned brick, before selecting it*/
-
-                ignore_subvol = dht_subvol_has_err (conf, conf->subvolumes[i], NULL,
-                                                    layout);
-                if (ignore_subvol)
-                        continue;
-
-                if (conf->disk_unit == 'p') {
-                        if ((conf->du_stats[i].avail_percent > max)
-                            && (conf->du_stats[i].avail_inodes > 0 )) {
-                                max = conf->du_stats[i].avail_percent;
-                                avail_subvol = conf->subvolumes[i];
-                        }
-               } else {
-                         if ((conf->du_stats[i].avail_space > max)
-                            && (conf->du_stats[i].avail_inodes > 0)) {
-                                 max = conf->du_stats[i].avail_space;
-                                 avail_subvol = conf->subvolumes[i];
-                         }
-               }
+    int i = 0;
+    double max = 0;
+    int ignore_subvol = 0;
+
+    xlator_t *avail_subvol = NULL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        /* check if subvol has layout errors and also it is not a
+         * decommissioned brick, before selecting it*/
+
+        ignore_subvol = dht_subvol_has_err(conf, conf->subvolumes[i], NULL,
+                                           layout);
+        if (ignore_subvol)
+            continue;
+
+        if (conf->disk_unit == 'p') {
+            if ((conf->du_stats[i].avail_percent > max) &&
+                (conf->du_stats[i].avail_inodes > 0)) {
+                max = conf->du_stats[i].avail_percent;
+                avail_subvol = conf->subvolumes[i];
+            }
+        } else {
+            if ((conf->du_stats[i].avail_space > max) &&
+                (conf->du_stats[i].avail_inodes > 0)) {
+                max = conf->du_stats[i].avail_space;
+                avail_subvol = conf->subvolumes[i];
+            }
         }
+    }
 
-        return avail_subvol;
+    return avail_subvol;
 }
diff --git a/xlators/cluster/dht/src/dht-hashfn.c b/xlators/cluster/dht/src/dht-hashfn.c
index f8e614a40aa..acda67c312a 100644
--- a/xlators/cluster/dht/src/dht-hashfn.c
+++ b/xlators/cluster/dht/src/dht-hashfn.c
@@ -8,104 +8,103 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "hashfn.h"
-
+#include <glusterfs/hashfn.h>
 
-int
-dht_hash_compute_internal (int type, const char *name, uint32_t *hash_p)
+static int
+dht_hash_compute_internal(int type, const char *name, const int len,
+                          uint32_t *hash_p)
 {
-        int      ret = 0;
-        uint32_t hash = 0;
+    int ret = 0;
+    uint32_t hash = 0;
 
-        switch (type) {
+    switch (type) {
         case DHT_HASH_TYPE_DM:
         case DHT_HASH_TYPE_DM_USER:
-                hash = gf_dm_hashfn (name, strlen (name));
-                break;
+            hash = gf_dm_hashfn(name, len);
+            break;
         default:
-                ret = -1;
-                break;
-        }
+            ret = -1;
+            break;
+    }
 
-        if (ret == 0) {
-                *hash_p = hash;
-        }
+    if (ret == 0) {
+        *hash_p = hash;
+    }
 
-        return ret;
+    return ret;
 }
 
-
-static
-gf_boolean_t
-dht_munge_name (const char *original, char *modified,
-                size_t len, regex_t *re)
+/* The function returns:
+ * 0  : in case no munge took place
+ * >0 : the length (inc. terminating NULL!) of the newly modified string,
+ *      if it was munged.
+ */
+static int
+dht_munge_name(const char *original, char *modified, size_t len, regex_t *re)
 {
-        regmatch_t  matches[2] = {{0}, };
-        size_t      new_len    = 0;
-        int         ret        = 0;
-
-        ret = regexec(re, original, 2, matches, 0);
-
-        if (ret != REG_NOMATCH) {
-                if (matches[1].rm_so != -1) {
-                        new_len = matches[1].rm_eo - matches[1].rm_so;
-                        /* Equal would fail due to the NUL at the end. */
-                        if (new_len < len) {
-                                memcpy (modified,original+matches[1].rm_so,
-                                        new_len);
-                                modified[new_len] = '\0';
-                                return _gf_true;
-                        }
-                }
+    regmatch_t matches[2] = {
+        {0},
+    };
+    size_t new_len = 0;
+    int ret = 0;
+
+    ret = regexec(re, original, 2, matches, 0);
+
+    if (ret != REG_NOMATCH) {
+        if (matches[1].rm_so != -1) {
+            new_len = matches[1].rm_eo - matches[1].rm_so;
+            /* Equal would fail due to the NUL at the end. */
+            if (new_len < len) {
+                memcpy(modified, original + matches[1].rm_so, new_len);
+                modified[new_len] = '\0';
+                return new_len + 1; /* +1 for the terminating NULL */
+            }
         }
+    }
 
-        /* This is guaranteed safe because of how the dest was allocated. */
-        strcpy(modified, original);
-        return _gf_false;
+    /* This is guaranteed safe because of how the dest was allocated. */
+    strcpy(modified, original);
+    return 0;
 }
 
 int
-dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
+dht_hash_compute(xlator_t *this, int type, const char *name, uint32_t *hash_p)
 {
-        char            *rsync_friendly_name    = NULL;
-        dht_conf_t      *priv                   = NULL;
-        size_t           len                    = 0;
-        gf_boolean_t     munged                 = _gf_false;
-
-        priv = this->private;
-
-        LOCK (&priv->lock);
-        {
-                if (priv->extra_regex_valid) {
-                        len = strlen(name) + 1;
-                        rsync_friendly_name = alloca(len);
-                        munged = dht_munge_name (name, rsync_friendly_name, len,
-                                                 &priv->extra_regex);
-                }
-
-                if (!munged && priv->rsync_regex_valid) {
-                        len = strlen(name) + 1;
-                        rsync_friendly_name = alloca(len);
-                        gf_msg_trace (this->name, 0, "trying regex for %s",
-                                      name);
-                        munged = dht_munge_name (name, rsync_friendly_name, len,
-                                                 &priv->rsync_regex);
-                        if (munged) {
-                                gf_msg_debug (this->name, 0,
-                                              "munged down to %s",
-                                              rsync_friendly_name);
-                        }
-                }
-        }
-        UNLOCK (&priv->lock);
+    char *rsync_friendly_name = NULL;
+    dht_conf_t *priv = NULL;
+    size_t len = 0;
+    int munged = 0;
+
+    priv = this->private;
+
+    if (name == NULL)
+        return -1;
 
-        if (!munged) {
-                rsync_friendly_name = (char *)name;
+    len = strlen(name) + 1;
+    rsync_friendly_name = alloca(len);
+
+    LOCK(&priv->lock);
+    {
+        if (priv->extra_regex_valid) {
+            munged = dht_munge_name(name, rsync_friendly_name, len,
+                                    &priv->extra_regex);
         }
 
-        return dht_hash_compute_internal (type, rsync_friendly_name, hash_p);
+        if (!munged && priv->rsync_regex_valid) {
+            gf_msg_trace(this->name, 0, "trying regex for %s", name);
+            munged = dht_munge_name(name, rsync_friendly_name, len,
+                                    &priv->rsync_regex);
+        }
+    }
+    UNLOCK(&priv->lock);
+    if (munged) {
+        gf_msg_debug(this->name, 0, "munged down to %s", rsync_friendly_name);
+        len = munged;
+    } else {
+        rsync_friendly_name = (char *)name;
+    }
+
+    return dht_hash_compute_internal(type, rsync_friendly_name, len - 1,
+                                     hash_p);
 }
diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c
index c301b1fd47e..3f2fe43d5f3 100644
--- a/xlators/cluster/dht/src/dht-helper.c
+++ b/xlators/cluster/dht/src/dht-helper.c
@@ -8,266 +8,249 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
 #include "dht-lock.h"
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
 
 static void
-dht_free_fd_ctx (dht_fd_ctx_t *fd_ctx)
+dht_free_fd_ctx(dht_fd_ctx_t *fd_ctx)
 {
-        GF_FREE (fd_ctx);
+    GF_FREE(fd_ctx);
 }
 
-
 int32_t
-dht_fd_ctx_destroy (xlator_t *this, fd_t *fd)
+dht_fd_ctx_destroy(xlator_t *this, fd_t *fd)
 {
-        dht_fd_ctx_t *fd_ctx  = NULL;
-        uint64_t      value   = 0;
-        int32_t       ret     = -1;
+    dht_fd_ctx_t *fd_ctx = NULL;
+    uint64_t value = 0;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        ret = fd_ctx_del (fd, this, &value);
-        if (ret) {
-                goto out;
-        }
+    ret = fd_ctx_del(fd, this, &value);
+    if (ret) {
+        goto out;
+    }
 
-        fd_ctx = (dht_fd_ctx_t *)value;
-        if (fd_ctx) {
-                GF_REF_PUT (fd_ctx);
-        }
+    fd_ctx = (dht_fd_ctx_t *)(uintptr_t)value;
+    if (fd_ctx) {
+        GF_REF_PUT(fd_ctx);
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 static int
-__dht_fd_ctx_set (xlator_t *this, fd_t *fd, xlator_t *dst)
+__dht_fd_ctx_set(xlator_t *this, fd_t *fd, xlator_t *dst)
 {
-        dht_fd_ctx_t *fd_ctx  = NULL;
-        uint64_t      value   = 0;
-        int           ret     = -1;
+    dht_fd_ctx_t *fd_ctx = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        fd_ctx = GF_CALLOC (1, sizeof (*fd_ctx), gf_dht_mt_fd_ctx_t);
+    fd_ctx = GF_CALLOC(1, sizeof(*fd_ctx), gf_dht_mt_fd_ctx_t);
 
-        if (!fd_ctx) {
-                goto out;
-        }
+    if (!fd_ctx) {
+        goto out;
+    }
 
-        fd_ctx->opened_on_dst = (uint64_t) dst;
-        GF_REF_INIT (fd_ctx, dht_free_fd_ctx);
+    fd_ctx->opened_on_dst = (uint64_t)(uintptr_t)dst;
+    GF_REF_INIT(fd_ctx, dht_free_fd_ctx);
 
-        value = (uint64_t) fd_ctx;
+    value = (uint64_t)(uintptr_t)fd_ctx;
 
-        ret = __fd_ctx_set (fd, this, value);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_FD_CTX_SET_FAILED,
-                        "Failed to set fd ctx in fd=0x%p", fd);
-                GF_REF_PUT (fd_ctx);
-        }
+    ret = __fd_ctx_set(fd, this, value);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FD_CTX_SET_FAILED,
+                "fd=0x%p", fd, NULL);
+        GF_REF_PUT(fd_ctx);
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
-
 int
-dht_fd_ctx_set (xlator_t *this, fd_t *fd, xlator_t *dst)
+dht_fd_ctx_set(xlator_t *this, fd_t *fd, xlator_t *dst)
 {
-        dht_fd_ctx_t *fd_ctx  = NULL;
-        uint64_t      value   = 0;
-        int           ret     = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_get (fd, this, &value);
-                if (ret && value) {
-
-                        fd_ctx = (dht_fd_ctx_t *) value;
-                        if (fd_ctx->opened_on_dst == (uint64_t) dst)  {
-                                /* This could happen due to racing
-                                 * check_progress tasks*/
-                                goto unlock;
-                        } else {
-                                /* This would be a big problem*/
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_INVALID_VALUE,
-                                        "Different dst found in the fd ctx");
-
-                                /* Overwrite and hope for the best*/
-                                fd_ctx->opened_on_dst = (uint64_t)dst;
-                                goto unlock;
-                        }
+    dht_fd_ctx_t *fd_ctx = NULL;
+    uint64_t value = 0;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_get(fd, this, &value);
+        if (ret && value) {
+            fd_ctx = (dht_fd_ctx_t *)(uintptr_t)value;
+            if (fd_ctx->opened_on_dst == (uint64_t)(uintptr_t)dst) {
+                /* This could happen due to racing
+                 * check_progress tasks*/
+                goto unlock;
+            } else {
+                /* This would be a big problem*/
+                /* Overwrite and hope for the best*/
+                fd_ctx->opened_on_dst = (uint64_t)(uintptr_t)dst;
+                UNLOCK(&fd->lock);
+                gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_VALUE,
+                        NULL);
 
-                }
-                ret = __dht_fd_ctx_set (this, fd, dst);
+                goto out;
+            }
         }
+        ret = __dht_fd_ctx_set(this, fd, dst);
+    }
 unlock:
-        UNLOCK (&fd->lock);
+    UNLOCK(&fd->lock);
 out:
-        return ret;
+    return ret;
 }
 
-
-
-static
-dht_fd_ctx_t *
-dht_fd_ctx_get (xlator_t *this, fd_t *fd)
+static dht_fd_ctx_t *
+dht_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        dht_fd_ctx_t *fd_ctx  = NULL;
-        int           ret     = -1;
-        uint64_t      tmp_val = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_get (fd, this, &tmp_val);
-                if ((ret < 0) || (tmp_val == 0)) {
-                        UNLOCK (&fd->lock);
-                        goto out;
-                }
+    dht_fd_ctx_t *fd_ctx = NULL;
+    int ret = -1;
+    uint64_t tmp_val = 0;
 
-                fd_ctx = (dht_fd_ctx_t *)tmp_val;
-                GF_REF_GET (fd_ctx);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_get(fd, this, &tmp_val);
+        if ((ret < 0) || (tmp_val == 0)) {
+            goto unlock;
         }
-        UNLOCK (&fd->lock);
+
+        fd_ctx = (dht_fd_ctx_t *)(uintptr_t)tmp_val;
+        GF_REF_GET(fd_ctx);
+    }
+unlock:
+    UNLOCK(&fd->lock);
 
 out:
-        return fd_ctx;
+    return fd_ctx;
 }
 
 gf_boolean_t
-dht_fd_open_on_dst (xlator_t *this, fd_t *fd, xlator_t *dst)
+dht_fd_open_on_dst(xlator_t *this, fd_t *fd, xlator_t *dst)
 {
-        dht_fd_ctx_t  *fd_ctx  = NULL;
-        gf_boolean_t   opened  = _gf_false;
+    dht_fd_ctx_t *fd_ctx = NULL;
+    gf_boolean_t opened = _gf_false;
 
-        fd_ctx = dht_fd_ctx_get (this, fd);
+    fd_ctx = dht_fd_ctx_get(this, fd);
 
-        if (fd_ctx) {
-                if (fd_ctx->opened_on_dst == (uint64_t) dst) {
-                        opened = _gf_true;
-                }
-                GF_REF_PUT (fd_ctx);
+    if (fd_ctx) {
+        if (fd_ctx->opened_on_dst == (uint64_t)(uintptr_t)dst) {
+            opened = _gf_true;
         }
+        GF_REF_PUT(fd_ctx);
+    }
 
-        return opened;
+    return opened;
 }
 
-
 void
-dht_free_mig_info (void *data)
+dht_free_mig_info(void *data)
 {
-        dht_migrate_info_t *miginfo = NULL;
+    dht_migrate_info_t *miginfo = NULL;
 
-        miginfo = data;
-        GF_FREE (miginfo);
+    miginfo = data;
+    GF_FREE(miginfo);
 
-        return;
+    return;
 }
 
 static int
-dht_inode_ctx_set_mig_info (xlator_t *this, inode_t *inode,
-                            xlator_t *src_subvol, xlator_t *dst_subvol)
+dht_inode_ctx_set_mig_info(xlator_t *this, inode_t *inode, xlator_t *src_subvol,
+                           xlator_t *dst_subvol)
 {
-        dht_migrate_info_t *miginfo = NULL;
-        uint64_t            value   = 0;
-        int                 ret     = -1;
+    dht_migrate_info_t *miginfo = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        miginfo = GF_CALLOC (1, sizeof (*miginfo), gf_dht_mt_miginfo_t);
-        if (miginfo == NULL)
-                goto out;
+    miginfo = GF_CALLOC(1, sizeof(*miginfo), gf_dht_mt_miginfo_t);
+    if (miginfo == NULL)
+        goto out;
 
-        miginfo->src_subvol = src_subvol;
-        miginfo->dst_subvol = dst_subvol;
-        GF_REF_INIT (miginfo, dht_free_mig_info);
+    miginfo->src_subvol = src_subvol;
+    miginfo->dst_subvol = dst_subvol;
+    GF_REF_INIT(miginfo, dht_free_mig_info);
 
-        value = (uint64_t) miginfo;
+    value = (uint64_t)(uintptr_t)miginfo;
 
-        ret = inode_ctx_set1 (inode, this, &value);
-        if (ret < 0) {
-                GF_REF_PUT (miginfo);
-        }
+    ret = inode_ctx_set1(inode, this, &value);
+    if (ret < 0) {
+        GF_REF_PUT(miginfo);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-dht_inode_ctx_get_mig_info (xlator_t *this, inode_t *inode,
-                            xlator_t **src_subvol, xlator_t **dst_subvol)
+dht_inode_ctx_get_mig_info(xlator_t *this, inode_t *inode,
+                           xlator_t **src_subvol, xlator_t **dst_subvol)
 {
-        int                 ret         = -1;
-        uint64_t            tmp_miginfo = 0;
-        dht_migrate_info_t *miginfo     = NULL;
-
-        LOCK (&inode->lock);
-        {
-                ret =  __inode_ctx_get1 (inode, this, &tmp_miginfo);
-                if ((ret < 0) || (tmp_miginfo == 0)) {
-                        UNLOCK (&inode->lock);
-                        goto out;
-                }
+    int ret = -1;
+    uint64_t tmp_miginfo = 0;
+    dht_migrate_info_t *miginfo = NULL;
 
-                miginfo = (dht_migrate_info_t *)tmp_miginfo;
-                GF_REF_GET (miginfo);
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get1(inode, this, &tmp_miginfo);
+        if ((ret < 0) || (tmp_miginfo == 0)) {
+            UNLOCK(&inode->lock);
+            goto out;
         }
-        UNLOCK (&inode->lock);
 
-        if (src_subvol)
-                *src_subvol = miginfo->src_subvol;
+        miginfo = (dht_migrate_info_t *)(uintptr_t)tmp_miginfo;
+        GF_REF_GET(miginfo);
+    }
+    UNLOCK(&inode->lock);
+
+    if (src_subvol)
+        *src_subvol = miginfo->src_subvol;
 
-        if (dst_subvol)
-                *dst_subvol = miginfo->dst_subvol;
+    if (dst_subvol)
+        *dst_subvol = miginfo->dst_subvol;
 
-        GF_REF_PUT (miginfo);
+    GF_REF_PUT(miginfo);
 
 out:
-        return ret;
+    return ret;
 }
 
 gf_boolean_t
-dht_mig_info_is_invalid (xlator_t *current, xlator_t *src_subvol,
-                      xlator_t *dst_subvol)
+dht_mig_info_is_invalid(xlator_t *current, xlator_t *src_subvol,
+                        xlator_t *dst_subvol)
 {
-
-/* Not set
- */
-        if (!src_subvol || !dst_subvol)
-                return _gf_true;
-
-/* Invalid scenarios:
- * The src_subvol does not match the subvol on which the current op was sent
- * so the cached subvol has changed between the last mig_info_set and now.
- * src_subvol == dst_subvol. The file was migrated without any FOP detecting
- * a P2 so the old dst is now the current subvol.
- *
- * There is still one scenario where the info could be outdated - if
- * file has undergone multiple migrations and ends up on the same src_subvol
- * on which the mig_info was first set.
- */
-        if ((current == dst_subvol) || (current != src_subvol))
-                return _gf_true;
-
-        return _gf_false;
+    /* Not set
+     */
+    if (!src_subvol || !dst_subvol)
+        return _gf_true;
+
+    /* Invalid scenarios:
+     * The src_subvol does not match the subvol on which the current op was sent
+     * so the cached subvol has changed between the last mig_info_set and now.
+     * src_subvol == dst_subvol. The file was migrated without any FOP detecting
+     * a P2 so the old dst is now the current subvol.
+     *
+     * There is still one scenario where the info could be outdated - if
+     * file has undergone multiple migrations and ends up on the same src_subvol
+     * on which the mig_info was first set.
+     */
+    if ((current == dst_subvol) || (current != src_subvol))
+        return _gf_true;
+
+    return _gf_false;
 }
 
-
-
 /* Used to check if fd fops have the fd opened on the cached subvol
  * This is required when:
  * 1. an fd is opened on FILE1 on subvol1
@@ -279,985 +262,974 @@ dht_mig_info_is_invalid (xlator_t *current, xlator_t *src_subvol,
  *
  */
 
-
 int
-dht_check_and_open_fd_on_subvol_complete (int ret, call_frame_t *frame,
-                                          void *data)
+dht_check_and_open_fd_on_subvol_complete(int ret, call_frame_t *frame,
+                                         void *data)
 {
-        glusterfs_fop_t     fop     = 0;
-        dht_local_t        *local   = NULL;
-        xlator_t           *subvol  = NULL;
-        xlator_t           *this  = NULL;
-        fd_t               *fd      = NULL;
-        int                 op_errno = -1;
-
-        local = frame->local;
-        this = frame->this;
-        fop = local->fop;
-        subvol = local->cached_subvol;
-        fd = local->fd;
-
-        if (ret) {
-                op_errno = local->op_errno;
-                goto handle_err;
-        }
-
-        switch (fop) {
-
+    glusterfs_fop_t fop = 0;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    fd_t *fd = NULL;
+    int op_errno = -1;
+
+    local = frame->local;
+    this = frame->this;
+    fop = local->fop;
+    subvol = local->cached_subvol;
+    fd = local->fd;
+
+    if (ret) {
+        op_errno = local->op_errno;
+        goto handle_err;
+    }
+
+    switch (fop) {
         case GF_FOP_WRITE:
-                STACK_WIND_COOKIE (frame, dht_writev_cbk, subvol, subvol,
-                                   subvol->fops->writev, fd,
-                                   local->rebalance.vector,
-                                   local->rebalance.count,
-                                   local->rebalance.offset,
-                                   local->rebalance.flags,
-                                   local->rebalance.iobref, local->xattr_req);
-                break;
+            STACK_WIND_COOKIE(frame, dht_writev_cbk, subvol, subvol,
+                              subvol->fops->writev, fd, local->rebalance.vector,
+                              local->rebalance.count, local->rebalance.offset,
+                              local->rebalance.flags, local->rebalance.iobref,
+                              local->xattr_req);
+            break;
 
         case GF_FOP_FLUSH:
-                STACK_WIND (frame, dht_flush_cbk, subvol,
-                            subvol->fops->flush, fd, local->xattr_req);
-                break;
+            STACK_WIND(frame, dht_flush_cbk, subvol, subvol->fops->flush, fd,
+                       local->xattr_req);
+            break;
 
         case GF_FOP_FSETATTR:
-                STACK_WIND_COOKIE (frame, dht_file_setattr_cbk, subvol,
-                                   subvol, subvol->fops->fsetattr, fd,
-                                   &local->rebalance.stbuf,
-                                   local->rebalance.flags,
-                                   local->xattr_req);
-                break;
+            STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                              subvol->fops->fsetattr, fd,
+                              &local->rebalance.stbuf, local->rebalance.flags,
+                              local->xattr_req);
+            break;
 
         case GF_FOP_ZEROFILL:
-                STACK_WIND_COOKIE (frame, dht_zerofill_cbk, subvol, subvol,
-                                   subvol->fops->zerofill, fd,
-                                   local->rebalance.offset,
-                                   local->rebalance.size, local->xattr_req);
+            STACK_WIND_COOKIE(frame, dht_zerofill_cbk, subvol, subvol,
+                              subvol->fops->zerofill, fd,
+                              local->rebalance.offset, local->rebalance.size,
+                              local->xattr_req);
 
-                break;
+            break;
 
         case GF_FOP_DISCARD:
-                STACK_WIND_COOKIE (frame, dht_discard_cbk, subvol, subvol,
-                                   subvol->fops->discard, local->fd,
-                                   local->rebalance.offset,
-                                   local->rebalance.size,
-                                   local->xattr_req);
-                break;
+            STACK_WIND_COOKIE(frame, dht_discard_cbk, subvol, subvol,
+                              subvol->fops->discard, local->fd,
+                              local->rebalance.offset, local->rebalance.size,
+                              local->xattr_req);
+            break;
 
         case GF_FOP_FALLOCATE:
-                STACK_WIND_COOKIE (frame, dht_fallocate_cbk, subvol, subvol,
-                                   subvol->fops->fallocate, fd,
-                                   local->rebalance.flags,
-                                   local->rebalance.offset,
-                                   local->rebalance.size,
-                                   local->xattr_req);
-                break;
+            STACK_WIND_COOKIE(frame, dht_fallocate_cbk, subvol, subvol,
+                              subvol->fops->fallocate, fd,
+                              local->rebalance.flags, local->rebalance.offset,
+                              local->rebalance.size, local->xattr_req);
+            break;
 
         case GF_FOP_FTRUNCATE:
-                STACK_WIND_COOKIE (frame, dht_truncate_cbk, subvol, subvol,
-                                   subvol->fops->ftruncate, fd,
-                                   local->rebalance.offset, local->xattr_req);
-                break;
+            STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                              subvol->fops->ftruncate, fd,
+                              local->rebalance.offset, local->xattr_req);
+            break;
 
         case GF_FOP_FSYNC:
-                STACK_WIND_COOKIE (frame, dht_fsync_cbk, subvol, subvol,
-                                   subvol->fops->fsync, local->fd,
-                                   local->rebalance.flags, local->xattr_req);
-                break;
+            STACK_WIND_COOKIE(frame, dht_fsync_cbk, subvol, subvol,
+                              subvol->fops->fsync, local->fd,
+                              local->rebalance.flags, local->xattr_req);
+            break;
 
         case GF_FOP_READ:
-                STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,
-                            local->fd, local->rebalance.size,
-                            local->rebalance.offset,
-                            local->rebalance.flags, local->xattr_req);
-                break;
+            STACK_WIND(frame, dht_readv_cbk, subvol, subvol->fops->readv,
+                       local->fd, local->rebalance.size,
+                       local->rebalance.offset, local->rebalance.flags,
+                       local->xattr_req);
+            break;
 
         case GF_FOP_FSTAT:
-                STACK_WIND_COOKIE (frame, dht_file_attr_cbk, subvol,
-                                   subvol, subvol->fops->fstat, fd,
-                                   local->xattr_req);
-                break;
+            STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                              subvol->fops->fstat, fd, local->xattr_req);
+            break;
 
         case GF_FOP_FSETXATTR:
-                STACK_WIND_COOKIE (frame, dht_file_setxattr_cbk, subvol,
-                                   subvol, subvol->fops->fsetxattr, local->fd,
-                                   local->rebalance.xattr,
-                                   local->rebalance.flags, local->xattr_req);
-                break;
+            STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                              subvol->fops->fsetxattr, local->fd,
+                              local->rebalance.xattr, local->rebalance.flags,
+                              local->xattr_req);
+            break;
 
         case GF_FOP_FREMOVEXATTR:
-                STACK_WIND_COOKIE (frame, dht_file_removexattr_cbk, subvol,
-                                   subvol, subvol->fops->fremovexattr,
-                                   local->fd, local->key, local->xattr_req);
-
-                break;
-
+            STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                              subvol->fops->fremovexattr, local->fd, local->key,
+                              local->xattr_req);
+
+            break;
+
+        case GF_FOP_FXATTROP:
+            STACK_WIND(frame, dht_common_xattrop_cbk, subvol,
+                       subvol->fops->fxattrop, local->fd,
+                       local->rebalance.flags, local->rebalance.xattr,
+                       local->xattr_req);
+            break;
+
+        case GF_FOP_FGETXATTR:
+            STACK_WIND(frame, dht_getxattr_cbk, subvol, subvol->fops->fgetxattr,
+                       local->fd, local->key, NULL);
+            break;
+
+        case GF_FOP_FINODELK:
+            STACK_WIND(frame, dht_finodelk_cbk, subvol, subvol->fops->finodelk,
+                       local->key, local->fd, local->rebalance.lock_cmd,
+                       &local->rebalance.flock, local->xattr_req);
+            break;
         default:
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_UNKNOWN_FOP,
-                        "Unknown FOP on fd (%p) on file %s @ %s",
-                        fd, uuid_utoa (fd->inode->gfid),
-                        subvol->name);
-                break;
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UNKNOWN_FOP, "fd=%p",
+                    fd, "gfid=%s", uuid_utoa(fd->inode->gfid), "name=%s",
+                    subvol->name, NULL);
+            break;
+    }
 
-        }
+    goto out;
 
-        goto out;
-
-        /* Could not open the fd on the dst. Unwind */
+    /* Could not open the fd on the dst. Unwind */
 
 handle_err:
 
-        switch (fop) {
-
+    switch (fop) {
         case GF_FOP_WRITE:
-                DHT_STACK_UNWIND (writev, frame, -1,
-                                  op_errno, NULL, NULL, NULL);
-                break;
+            DHT_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
 
         case GF_FOP_FLUSH:
-                DHT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
-                break;
+            DHT_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+            break;
 
         case GF_FOP_FSETATTR:
-                DHT_STACK_UNWIND (fsetattr, frame, -1, op_errno,
-                                  NULL, NULL, NULL);
-                break;
+            DHT_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
 
         case GF_FOP_ZEROFILL:
-                DHT_STACK_UNWIND (zerofill, frame, -1, op_errno,
-                                  NULL, NULL, NULL);
-                break;
+            DHT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
 
         case GF_FOP_DISCARD:
-                DHT_STACK_UNWIND (discard, frame, -1, op_errno,
-                                  NULL, NULL, NULL);
-                break;
+            DHT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
 
         case GF_FOP_FALLOCATE:
-                DHT_STACK_UNWIND (fallocate, frame, -1, op_errno,
-                                  NULL, NULL, NULL);
-                break;
+            DHT_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
 
         case GF_FOP_FTRUNCATE:
-                DHT_STACK_UNWIND (ftruncate, frame, -1, op_errno,
-                                  NULL, NULL, NULL);
-                break;
+            DHT_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
 
         case GF_FOP_FSYNC:
-                DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
-                break;
+            DHT_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
 
         case GF_FOP_READ:
-                DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL,
-                                  0, NULL, NULL, NULL);
-                break;
+            DHT_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL,
+                             NULL);
+            break;
 
         case GF_FOP_FSTAT:
-                DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
-                break;
+            DHT_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
+            break;
 
         case GF_FOP_FSETXATTR:
-                DHT_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
-                break;
+            DHT_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
+            break;
 
         case GF_FOP_FREMOVEXATTR:
-                DHT_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
-                break;
+            DHT_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
+            break;
+
+        case GF_FOP_FXATTROP:
+            DHT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
+            break;
+
+        case GF_FOP_FGETXATTR:
+            DHT_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
+            break;
+
+        case GF_FOP_FINODELK:
+            DHT_STACK_UNWIND(finodelk, frame, -1, op_errno, NULL);
+            break;
 
         default:
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_UNKNOWN_FOP,
-                        "Unknown FOP on fd (%p) on file %s @ %s",
-                        fd, uuid_utoa (fd->inode->gfid),
-                        subvol->name);
-                break;
-        }
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UNKNOWN_FOP, "fd=%p",
+                    fd, "gfid=%s", uuid_utoa(fd->inode->gfid), "name=%s",
+                    subvol->name, NULL);
+            break;
+    }
 
 out:
 
-        return 0;
-
+    return 0;
 }
 
-
 /* Check once again if the fd has been opened on the cached subvol.
  * If not, open and update the fd_ctx.
  */
 
 int
-dht_check_and_open_fd_on_subvol_task (void *data)
+dht_check_and_open_fd_on_subvol_task(void *data)
 {
-        loc_t          loc        = {0,};
-        int            ret        = -1;
-        call_frame_t  *frame      = NULL;
-        dht_local_t   *local      = NULL;
-        fd_t          *fd         = NULL;
-        xlator_t      *this       = NULL;
-        xlator_t      *subvol     = NULL;
-
-
-        frame = data;
-        local = frame->local;
-        this = THIS;
-        fd = local->fd;
-        subvol = local->cached_subvol;
-
-        local->fd_checked = _gf_true;
-
-        if (fd_is_anonymous (fd) || dht_fd_open_on_dst (this, fd, subvol)) {
-                ret = 0;
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0,
-                      "Opening fd (%p, flags=0%o) on file %s @ %s",
-                      fd, fd->flags, uuid_utoa (fd->inode->gfid),
-                      subvol->name);
-
-
-        loc.inode = inode_ref (fd->inode);
-        gf_uuid_copy (loc.gfid, fd->inode->gfid);
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    fd_t *fd = NULL;
+    xlator_t *this = NULL;
+    xlator_t *subvol = NULL;
+
+    frame = data;
+    local = frame->local;
+    this = THIS;
+    fd = local->fd;
+    subvol = local->cached_subvol;
+
+    local->fd_checked = _gf_true;
+
+    if (fd_is_anonymous(fd) || dht_fd_open_on_dst(this, fd, subvol)) {
+        ret = 0;
+        goto out;
+    }
 
-        /* Open this on the dst subvol */
+    gf_msg_debug(this->name, 0, "Opening fd (%p, flags=0%o) on file %s @ %s",
+                 fd, fd->flags, uuid_utoa(fd->inode->gfid), subvol->name);
 
-        SYNCTASK_SETID(0, 0);
+    loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(loc.gfid, fd->inode->gfid);
 
-        ret = syncop_open (subvol, &loc,
-                           (fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)),
-                           fd, NULL, NULL);
+    /* Open this on the dst subvol */
 
-        if (ret < 0) {
+    SYNCTASK_SETID(0, 0);
 
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_OPEN_FD_ON_DST_FAILED,
-                        "Failed to open the fd"
-                        " (%p, flags=0%o) on file %s @ %s",
-                        fd, fd->flags, uuid_utoa (fd->inode->gfid),
-                        subvol->name);
-                /* This can happen if the cached subvol was updated in the
-                 * inode_ctx and the fd was opened on the new cached suvol
-                 * after this fop was wound on the old cached subvol.
-                 * As we do not close the fd on the old subvol (a leak)
-                 * don't treat ENOENT as an error and allow the phase1/phase2
-                 * checks to handle it.
-                 */
-
-                if ((-ret != ENOENT) && (-ret != ESTALE)) {
-                        local->op_errno = -ret;
-                        ret = -1;
-                } else {
-                        ret = 0;
-                }
+    ret = syncop_open(subvol, &loc, (fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)),
+                      fd, NULL, NULL);
 
-                local->op_errno = -ret;
-                ret = -1;
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_OPEN_FD_ON_DST_FAILED,
+                "fd=%p", fd, "flags=0%o", fd->flags, "gfid=%s",
+                uuid_utoa(fd->inode->gfid), "name=%s", subvol->name, NULL);
+        /* This can happen if the cached subvol was updated in the
+         * inode_ctx and the fd was opened on the new cached suvol
+         * after this fop was wound on the old cached subvol.
+         * As we do not close the fd on the old subvol (a leak)
+         * don't treat ENOENT as an error and allow the phase1/phase2
+         * checks to handle it.
+         */
 
+        if ((-ret != ENOENT) && (-ret != ESTALE)) {
+            local->op_errno = -ret;
+            ret = -1;
         } else {
-                dht_fd_ctx_set (this, fd, subvol);
+            ret = 0;
         }
 
-        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        local->op_errno = -ret;
+        ret = -1;
+
+    } else {
+        dht_fd_ctx_set(this, fd, subvol);
+    }
+
+    SYNCTASK_SETID(frame->root->uid, frame->root->gid);
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        return ret;
+    return ret;
 }
 
-
 int
-dht_check_and_open_fd_on_subvol (xlator_t *this, call_frame_t *frame)
+dht_check_and_open_fd_on_subvol(xlator_t *this, call_frame_t *frame)
 {
-        int ret            = -1;
-        dht_local_t *local = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
 
-/*
-        if (dht_fd_open_on_dst (this, fd, subvol))
-                goto out;
-*/
-        local = frame->local;
+    /*
+            if (dht_fd_open_on_dst (this, fd, subvol))
+                    goto out;
+    */
+    local = frame->local;
 
-        ret = synctask_new (this->ctx->env,
-                            dht_check_and_open_fd_on_subvol_task,
-                            dht_check_and_open_fd_on_subvol_complete,
-                            frame, frame);
+    ret = synctask_new(this->ctx->env, dht_check_and_open_fd_on_subvol_task,
+                       dht_check_and_open_fd_on_subvol_complete, frame, frame);
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                        "Failed to create synctask"
-                        " to check and open fd=%p", local->fd);
-        }
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SYNCTASK_CREATE_FAILED,
+                "to-check-and-open fd=%p", local->fd, NULL);
+    }
 
-        return ret;
+    return ret;
 }
 
-
-
 int
-dht_frame_return (call_frame_t *frame)
+dht_frame_return(call_frame_t *frame)
 {
-        dht_local_t *local = NULL;
-        int          this_call_cnt = -1;
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
 
-        if (!frame)
-                return -1;
+    if (!frame)
+        return -1;
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                this_call_cnt = --local->call_cnt;
-        }
-        UNLOCK (&frame->lock);
+    LOCK(&frame->lock);
+    {
+        this_call_cnt = --local->call_cnt;
+    }
+    UNLOCK(&frame->lock);
 
-        return this_call_cnt;
+    return this_call_cnt;
 }
 
+/*
+ * Use this function to specify which subvol you want the file created
+ * on - this need not be the hashed subvol.
+ * Format: <filename>@<this->name>:<subvol-name>
+ * Eg: file-1@vol1-dht:vol1-client-0
+ *     where vol1 is a pure distribute volume
+ *     will create file-1 on vol1-client-0
+ */
 
 int
-dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
-                           xlator_t **subvol)
+dht_filter_loc_subvol_key(xlator_t *this, loc_t *loc, loc_t *new_loc,
+                          xlator_t **subvol)
 {
-        char          *new_name  = NULL;
-        char          *new_path  = NULL;
-        xlator_list_t *trav      = NULL;
-        char           key[1024] = {0,};
-        int            ret       = 0; /* not found */
-
-        /* Why do other tasks if first required 'char' itself is not there */
-        if (!new_loc || !loc || !loc->name || !strchr (loc->name, '@')) {
-                /* Skip the GF_FREE checks here */
-                return ret;
-        }
-
-        trav = this->children;
-        while (trav) {
-                snprintf (key, 1024, "*@%s:%s", this->name, trav->xlator->name);
-                if (fnmatch (key, loc->name, FNM_NOESCAPE) == 0) {
-                        new_name = GF_CALLOC(strlen (loc->name),
-                                             sizeof (char),
-                                             gf_common_mt_char);
-                        if (!new_name)
-                                goto out;
-                        if (fnmatch (key, loc->path, FNM_NOESCAPE) == 0) {
-                                new_path = GF_CALLOC(strlen (loc->path),
-                                                     sizeof (char),
-                                                     gf_common_mt_char);
-                                if (!new_path)
-                                        goto out;
-                                strncpy (new_path, loc->path, (strlen (loc->path) -
-                                                               strlen (key) + 1));
-                        }
-                        strncpy (new_name, loc->name, (strlen (loc->name) -
-                                                       strlen (key) + 1));
-
-                        if (new_loc) {
-                                new_loc->path   = ((new_path) ? new_path:
-                                                   gf_strdup (loc->path));
-                                new_loc->name   = new_name;
-                                new_loc->inode  = inode_ref (loc->inode);
-                                new_loc->parent = inode_ref (loc->parent);
-                        }
-                        *subvol         = trav->xlator;
-                        ret = 1;  /* success */
-                        goto out;
-                }
-                trav = trav->next;
-        }
-out:
-        if (!ret) {
-                /* !success */
-                GF_FREE (new_path);
-                GF_FREE (new_name);
-        }
+    char *new_name = NULL;
+    char *new_path = NULL;
+    xlator_list_t *trav = NULL;
+    char key[1024] = {
+        0,
+    };
+    int ret = 0; /* not found */
+    int keylen = 0;
+    int name_len = 0;
+    int path_len = 0;
+
+    /* Why do other tasks if first required 'char' itself is not there */
+    if (!new_loc || !loc || !loc->name || !strchr(loc->name, '@')) {
+        /* Skip the GF_FREE checks here */
         return ret;
+    }
+
+    trav = this->children;
+    while (trav) {
+        keylen = snprintf(key, sizeof(key), "*@%s:%s", this->name,
+                          trav->xlator->name);
+        /* Ignore '*' */
+        keylen = keylen - 1;
+        if (fnmatch(key, loc->name, FNM_NOESCAPE) == 0) {
+            name_len = strlen(loc->name) - keylen;
+            new_name = GF_MALLOC(name_len + 1, gf_common_mt_char);
+            if (!new_name)
+                goto out;
+            if (fnmatch(key, loc->path, FNM_NOESCAPE) == 0) {
+                path_len = strlen(loc->path) - keylen;
+                new_path = GF_MALLOC(path_len + 1, gf_common_mt_char);
+                if (!new_path)
+                    goto out;
+                snprintf(new_path, path_len + 1, "%s", loc->path);
+            }
+            snprintf(new_name, name_len + 1, "%s", loc->name);
+
+            if (new_loc) {
+                new_loc->path = ((new_path) ? new_path : gf_strdup(loc->path));
+                new_loc->name = new_name;
+                new_loc->inode = inode_ref(loc->inode);
+                new_loc->parent = inode_ref(loc->parent);
+            }
+            *subvol = trav->xlator;
+            ret = 1; /* success */
+            goto out;
+        }
+        trav = trav->next;
+    }
+out:
+    if (!ret) {
+        /* !success */
+        GF_FREE(new_path);
+        GF_FREE(new_name);
+    }
+    return ret;
 }
 
 static xlator_t *
 dht_get_subvol_from_id(xlator_t *this, int client_id)
 {
-        xlator_t   *xl   = NULL;
-        dht_conf_t *conf = NULL;
-        char       *sid  = NULL;
-        int32_t     ret  = -1;
-
-        conf = this->private;
-
-        ret = gf_asprintf(&sid, "%d", client_id);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_ASPRINTF_FAILED, "asprintf failed while "
-                        "fetching subvol from the id");
-                goto out;
-        }
+    xlator_t *xl = NULL;
+    dht_conf_t *conf = NULL;
+    char *sid = NULL;
+    int32_t ret = -1;
 
-        if (dict_get_ptr(conf->leaf_to_subvol, sid, (void **) &xl))
-                xl = NULL;
+    conf = this->private;
 
-        GF_FREE (sid);
+    ret = gf_asprintf(&sid, "%d", client_id);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_ASPRINTF_FAILED, NULL);
+        goto out;
+    }
 
-out:
-        return xl;
+    if (dict_get_ptr(conf->leaf_to_subvol, sid, (void **)&xl))
+        xl = NULL;
+
+    GF_FREE(sid);
 
+out:
+    return xl;
 }
 
 int
-dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol_p)
+dht_deitransform(xlator_t *this, uint64_t y, xlator_t **subvol_p)
 {
-        int         client_id = 0;
-        xlator_t   *subvol = 0;
-        dht_conf_t *conf = NULL;
+    int client_id = 0;
+    xlator_t *subvol = 0;
+    dht_conf_t *conf = NULL;
 
-        if (!this->private)
-                return -1;
+    if (!this->private)
+        return -1;
 
-        conf = this->private;
+    conf = this->private;
 
-        client_id = gf_deitransform(this, y);
+    client_id = gf_deitransform(this, y);
 
-        subvol = dht_get_subvol_from_id(this, client_id);
+    subvol = dht_get_subvol_from_id(this, client_id);
 
-        if (!subvol)
-                subvol = conf->subvolumes[0];
+    if (!subvol)
+        subvol = conf->subvolumes[0];
 
-        if (subvol_p)
-                *subvol_p = subvol;
+    if (subvol_p)
+        *subvol_p = subvol;
 
-        return 0;
+    return 0;
 }
 
 void
-dht_local_wipe (xlator_t *this, dht_local_t *local)
+dht_local_wipe(xlator_t *this, dht_local_t *local)
 {
-        int i = 0;
+    int i = 0;
 
-        if (!local)
-                return;
+    if (!local)
+        return;
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->loc2);
-        loc_wipe (&local->loc2_copy);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->loc2);
+    loc_wipe(&local->loc2_copy);
 
-        if (local->xattr)
-                dict_unref (local->xattr);
+    if (local->xattr)
+        dict_unref(local->xattr);
 
-        if (local->inode)
-                inode_unref (local->inode);
+    if (local->inode)
+        inode_unref(local->inode);
 
-        if (local->layout) {
-                dht_layout_unref (this, local->layout);
-                local->layout = NULL;
-        }
+    if (local->layout) {
+        dht_layout_unref(this, local->layout);
+        local->layout = NULL;
+    }
 
-        loc_wipe (&local->linkfile.loc);
+    loc_wipe(&local->linkfile.loc);
 
-        if (local->linkfile.xattr)
-                dict_unref (local->linkfile.xattr);
+    if (local->linkfile.xattr)
+        dict_unref(local->linkfile.xattr);
 
-        if (local->linkfile.inode)
-                inode_unref (local->linkfile.inode);
+    if (local->linkfile.inode)
+        inode_unref(local->linkfile.inode);
 
-        if (local->fd) {
-                fd_unref (local->fd);
-                local->fd = NULL;
-        }
+    if (local->fd) {
+        fd_unref(local->fd);
+        local->fd = NULL;
+    }
 
-        if (local->params) {
-                dict_unref (local->params);
-                local->params = NULL;
-        }
+    if (local->params) {
+        dict_unref(local->params);
+        local->params = NULL;
+    }
 
-        if (local->xattr_req)
-                dict_unref (local->xattr_req);
-        if (local->mds_xattr)
-                dict_unref (local->mds_xattr);
-        if (local->xdata)
-                dict_unref (local->xdata);
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
+    if (local->mds_xattr)
+        dict_unref(local->mds_xattr);
+    if (local->xdata)
+        dict_unref(local->xdata);
 
-        if (local->selfheal.layout) {
-                dht_layout_unref (this, local->selfheal.layout);
-                local->selfheal.layout = NULL;
-        }
+    if (local->selfheal.layout) {
+        dht_layout_unref(this, local->selfheal.layout);
+        local->selfheal.layout = NULL;
+    }
 
-        if (local->selfheal.refreshed_layout) {
-                dht_layout_unref (this, local->selfheal.refreshed_layout);
-                local->selfheal.refreshed_layout = NULL;
-        }
-
-        for (i = 0; i < 2; i++) {
-                dht_lock_array_free (local->lock[i].ns.parent_layout.locks,
-                                     local->lock[i].ns.parent_layout.lk_count);
+    if (local->selfheal.refreshed_layout) {
+        dht_layout_unref(this, local->selfheal.refreshed_layout);
+        local->selfheal.refreshed_layout = NULL;
+    }
 
-                GF_FREE (local->lock[i].ns.parent_layout.locks);
+    for (i = 0; i < 2; i++) {
+        dht_lock_array_free(local->lock[i].ns.parent_layout.locks,
+                            local->lock[i].ns.parent_layout.lk_count);
 
-                dht_lock_array_free (local->lock[i].ns.directory_ns.locks,
-                                     local->lock[i].ns.directory_ns.lk_count);
-                GF_FREE (local->lock[i].ns.directory_ns.locks);
-        }
+        GF_FREE(local->lock[i].ns.parent_layout.locks);
 
-        GF_FREE (local->key);
+        dht_lock_array_free(local->lock[i].ns.directory_ns.locks,
+                            local->lock[i].ns.directory_ns.lk_count);
+        GF_FREE(local->lock[i].ns.directory_ns.locks);
+    }
 
-        if (local->rebalance.xdata)
-                dict_unref (local->rebalance.xdata);
+    GF_FREE(local->key);
 
-        if (local->rebalance.xattr)
-                dict_unref (local->rebalance.xattr);
+    if (local->rebalance.xdata)
+        dict_unref(local->rebalance.xdata);
 
-        if (local->rebalance.dict)
-                dict_unref (local->rebalance.dict);
+    if (local->rebalance.xattr)
+        dict_unref(local->rebalance.xattr);
 
-        GF_FREE (local->rebalance.vector);
+    if (local->rebalance.dict)
+        dict_unref(local->rebalance.dict);
 
-        if (local->rebalance.iobref)
-                iobref_unref (local->rebalance.iobref);
+    GF_FREE(local->rebalance.vector);
 
-        if (local->stub) {
-                call_stub_destroy (local->stub);
-                local->stub = NULL;
-        }
+    if (local->rebalance.iobref)
+        iobref_unref(local->rebalance.iobref);
 
-        if (local->ret_cache)
-                GF_FREE (local->ret_cache);
+    if (local->stub) {
+        call_stub_destroy(local->stub);
+        local->stub = NULL;
+    }
 
-        gf_dirent_free (&local->entries);
+    if (local->ret_cache)
+        GF_FREE(local->ret_cache);
 
-        mem_put (local);
+    mem_put(local);
 }
 
-
 dht_local_t *
-dht_local_init (call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop)
+dht_local_init(call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop)
 {
-        dht_local_t *local = NULL;
-        inode_t     *inode = NULL;
-        int          ret   = 0;
+    dht_local_t *local = NULL;
+    inode_t *inode = NULL;
+    int ret = 0;
 
-        local = mem_get0 (THIS->local_pool);
-        if (!local)
-                goto out;
+    local = mem_get0(THIS->local_pool);
+    if (!local)
+        goto out;
 
-        if (loc) {
-                ret = loc_copy (&local->loc, loc);
-                if (ret)
-                        goto out;
+    if (loc) {
+        ret = loc_copy(&local->loc, loc);
+        if (ret)
+            goto out;
 
-                inode = loc->inode;
-        }
+        inode = loc->inode;
+    }
 
-        if (fd) {
-                local->fd = fd_ref (fd);
-                if (!inode)
-                        inode = fd->inode;
-        }
+    if (fd) {
+        local->fd = fd_ref(fd);
+        if (!inode)
+            inode = fd->inode;
+    }
 
-        local->op_ret   = -1;
-        local->op_errno = EUCLEAN;
-        local->fop      = fop;
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+    local->fop = fop;
 
-        if (inode) {
-                local->layout   = dht_layout_get (frame->this, inode);
-                local->cached_subvol = dht_subvol_get_cached (frame->this,
-                                                              inode);
-        }
+    if (inode) {
+        local->layout = dht_layout_get(frame->this, inode);
+        local->cached_subvol = dht_subvol_get_cached(frame->this, inode);
+    }
 
-        INIT_LIST_HEAD (&local->entries.list);
-        frame->local = local;
+    frame->local = local;
 
 out:
-        if (ret) {
-                if (local)
-                        mem_put (local);
-                local = NULL;
-        }
-        return local;
+    if (ret) {
+        if (local)
+            mem_put(local);
+        local = NULL;
+    }
+    return local;
 }
 
 xlator_t *
-dht_first_up_subvol (xlator_t *this)
+dht_first_up_subvol(xlator_t *this)
 {
-        dht_conf_t *conf = NULL;
-        xlator_t   *child = NULL;
-        int         i = 0;
-        time_t      time = 0;
+    dht_conf_t *conf = NULL;
+    xlator_t *child = NULL;
+    int i = 0;
+    time_t time = 0;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        LOCK (&conf->subvolume_lock);
-        {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->subvol_up_time[i]) {
-                                if (!time) {
-                                        time = conf->subvol_up_time[i];
-                                        child = conf->subvolumes[i];
-                                } else if (time > conf->subvol_up_time[i]) {
-                                        time  = conf->subvol_up_time[i];
-                                        child = conf->subvolumes[i];
-                                }
-                        }
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvol_up_time[i]) {
+                if (!time) {
+                    time = conf->subvol_up_time[i];
+                    child = conf->subvolumes[i];
+                } else if (time > conf->subvol_up_time[i]) {
+                    time = conf->subvol_up_time[i];
+                    child = conf->subvolumes[i];
                 }
+            }
         }
-        UNLOCK (&conf->subvolume_lock);
+    }
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-        return child;
+    return child;
 }
 
 xlator_t *
-dht_last_up_subvol (xlator_t *this)
+dht_last_up_subvol(xlator_t *this)
 {
-        dht_conf_t *conf = NULL;
-        xlator_t   *child = NULL;
-        int         i = 0;
+    dht_conf_t *conf = NULL;
+    xlator_t *child = NULL;
+    int i = 0;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        LOCK (&conf->subvolume_lock);
-        {
-                for (i = conf->subvolume_cnt-1; i >= 0; i--) {
-                        if (conf->subvolume_status[i]) {
-                                child = conf->subvolumes[i];
-                                break;
-                        }
-                }
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = conf->subvolume_cnt - 1; i >= 0; i--) {
+            if (conf->subvolume_status[i]) {
+                child = conf->subvolumes[i];
+                break;
+            }
         }
-        UNLOCK (&conf->subvolume_lock);
+    }
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-        return child;
+    return child;
 }
 
 xlator_t *
-dht_subvol_get_hashed (xlator_t *this, loc_t *loc)
+dht_subvol_get_hashed(xlator_t *this, loc_t *loc)
 {
-        dht_layout_t *layout = NULL;
-        xlator_t     *subvol = NULL;
-        dht_conf_t *conf = NULL;
-        dht_methods_t *methods = NULL;
+    dht_layout_t *layout = NULL;
+    xlator_t *subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
 
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-        methods = &(conf->methods);
+    methods = &(conf->methods);
 
-        if (__is_root_gfid (loc->gfid)) {
-                subvol = dht_first_up_subvol (this);
-                goto out;
-        }
+    if (__is_root_gfid(loc->gfid)) {
+        subvol = dht_first_up_subvol(this);
+        goto out;
+    }
 
-        GF_VALIDATE_OR_GOTO (this->name, loc->parent, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->parent, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->name, out);
 
-        layout = dht_layout_get (this, loc->parent);
+    layout = dht_layout_get(this, loc->parent);
 
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "Missing layout. path=%s, parent gfid =%s",
-                              loc->path, uuid_utoa (loc->parent->gfid));
-                goto out;
-        }
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "Missing layout. path=%s, parent gfid =%s",
+                     loc->path, uuid_utoa(loc->parent->gfid));
+        goto out;
+    }
 
-        subvol = methods->layout_search (this, layout, loc->name);
+    subvol = methods->layout_search(this, layout, loc->name);
 
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "No hashed subvolume for path=%s",
-                              loc->path);
-                goto out;
-        }
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "No hashed subvolume for path=%s",
+                     loc->path);
+        goto out;
+    }
 
 out:
-        if (layout) {
-                dht_layout_unref (this, layout);
-        }
+    if (layout) {
+        dht_layout_unref(this, layout);
+    }
 
-        return subvol;
+    return subvol;
 }
 
-
 xlator_t *
-dht_subvol_get_cached (xlator_t *this, inode_t *inode)
+dht_subvol_get_cached(xlator_t *this, inode_t *inode)
 {
-        dht_layout_t *layout = NULL;
-        xlator_t     *subvol = NULL;
+    dht_layout_t *layout = NULL;
+    xlator_t *subvol = NULL;
 
-        GF_VALIDATE_OR_GOTO (this->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        layout = dht_layout_get (this, inode);
+    layout = dht_layout_get(this, inode);
 
-        if (!layout) {
-                goto out;
-        }
+    if (!layout) {
+        goto out;
+    }
 
-        subvol = layout->list[0].xlator;
+    subvol = layout->list[0].xlator;
 
 out:
-        if (layout) {
-                dht_layout_unref (this, layout);
-        }
+    if (layout) {
+        dht_layout_unref(this, layout);
+    }
 
-        return subvol;
+    return subvol;
 }
 
-
 xlator_t *
-dht_subvol_next (xlator_t *this, xlator_t *prev)
+dht_subvol_next(xlator_t *this, xlator_t *prev)
 {
-        dht_conf_t *conf = NULL;
-        int         i = 0;
-        xlator_t   *next = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *next = NULL;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == prev) {
-                        if ((i + 1) < conf->subvolume_cnt)
-                                next = conf->subvolumes[i + 1];
-                        break;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == prev) {
+            if ((i + 1) < conf->subvolume_cnt)
+                next = conf->subvolumes[i + 1];
+            break;
         }
+    }
 
 out:
-        return next;
+    return next;
 }
 
 /* This func wraps around, if prev is actually the last subvol.
  */
 xlator_t *
-dht_subvol_next_available (xlator_t *this, xlator_t *prev)
+dht_subvol_next_available(xlator_t *this, xlator_t *prev)
 {
-        dht_conf_t *conf = NULL;
-        int         i = 0;
-        xlator_t   *next = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *next = NULL;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == prev) {
-                        /* if prev is last in conf->subvolumes, then wrap
-                         * around.
-                         */
-                        if ((i + 1) < conf->subvolume_cnt) {
-                                next = conf->subvolumes[i + 1];
-                        } else {
-                                next = conf->subvolumes[0];
-                        }
-                        break;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == prev) {
+            /* if prev is last in conf->subvolumes, then wrap
+             * around.
+             */
+            if ((i + 1) < conf->subvolume_cnt) {
+                next = conf->subvolumes[i + 1];
+            } else {
+                next = conf->subvolumes[0];
+            }
+            break;
         }
+    }
 
 out:
-        return next;
+    return next;
 }
 int
-dht_subvol_cnt (xlator_t *this, xlator_t *subvol)
+dht_subvol_cnt(xlator_t *this, xlator_t *subvol)
 {
-        int i = 0;
-        int ret = -1;
-        dht_conf_t *conf = NULL;
+    int i = 0;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (subvol == conf->subvolumes[i]) {
-                        ret = i;
-                        break;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (subvol == conf->subvolumes[i]) {
+            ret = i;
+            break;
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
+#define set_if_greater(a, b)                                                   \
+    do {                                                                       \
+        if ((a) < (b))                                                         \
+            (a) = (b);                                                         \
+    } while (0)
 
-#define set_if_greater(a, b) do {               \
-                if ((a) < (b))                  \
-                        (a) = (b);              \
-        } while (0)
-
-
-#define set_if_greater_time(a, an, b, bn) do {                          \
-                if (((a) < (b)) || (((a) == (b)) && ((an) < (bn)))){    \
-                        (a) = (b);                                      \
-                        (an) = (bn);                                    \
-                }                                                       \
-        } while (0)                                                     \
-
+#define set_if_greater_time(a, an, b, bn)                                      \
+    do {                                                                       \
+        if (((a) < (b)) || (((a) == (b)) && ((an) < (bn)))) {                  \
+            (a) = (b);                                                         \
+            (an) = (bn);                                                       \
+        }                                                                      \
+    } while (0)
 
 int
-dht_iatt_merge (xlator_t *this, struct iatt *to,
-                struct iatt *from, xlator_t *subvol)
+dht_iatt_merge(xlator_t *this, struct iatt *to, struct iatt *from)
 {
-        if (!from || !to)
-                return 0;
-
-        to->ia_dev      = from->ia_dev;
-
-        gf_uuid_copy (to->ia_gfid, from->ia_gfid);
-
-        to->ia_ino      = from->ia_ino;
-        to->ia_prot     = from->ia_prot;
-        to->ia_type     = from->ia_type;
-        to->ia_nlink    = from->ia_nlink;
-        to->ia_rdev     = from->ia_rdev;
-        to->ia_size    += from->ia_size;
-        to->ia_blksize  = from->ia_blksize;
-        to->ia_blocks  += from->ia_blocks;
-
-        if (IA_ISDIR (from->ia_type)) {
-                to->ia_blocks = DHT_DIR_STAT_BLOCKS;
-                to->ia_size = DHT_DIR_STAT_SIZE;
-        }
-        set_if_greater (to->ia_uid, from->ia_uid);
-        set_if_greater (to->ia_gid, from->ia_gid);
-
-        set_if_greater_time(to->ia_atime, to->ia_atime_nsec,
-                            from->ia_atime, from->ia_atime_nsec);
-        set_if_greater_time (to->ia_mtime, to->ia_mtime_nsec,
-                             from->ia_mtime, from->ia_mtime_nsec);
-        set_if_greater_time (to->ia_ctime, to->ia_ctime_nsec,
-                             from->ia_ctime, from->ia_ctime_nsec);
-
+    if (!from || !to)
         return 0;
+
+    to->ia_dev = from->ia_dev;
+
+    gf_uuid_copy(to->ia_gfid, from->ia_gfid);
+
+    to->ia_ino = from->ia_ino;
+    to->ia_prot = from->ia_prot;
+    to->ia_type = from->ia_type;
+    to->ia_nlink = from->ia_nlink;
+    to->ia_rdev = from->ia_rdev;
+    to->ia_size += from->ia_size;
+    to->ia_blksize = from->ia_blksize;
+    to->ia_blocks += from->ia_blocks;
+
+    if (IA_ISDIR(from->ia_type)) {
+        to->ia_blocks = DHT_DIR_STAT_BLOCKS;
+        to->ia_size = DHT_DIR_STAT_SIZE;
+    }
+    set_if_greater(to->ia_uid, from->ia_uid);
+    set_if_greater(to->ia_gid, from->ia_gid);
+
+    set_if_greater_time(to->ia_atime, to->ia_atime_nsec, from->ia_atime,
+                        from->ia_atime_nsec);
+    set_if_greater_time(to->ia_mtime, to->ia_mtime_nsec, from->ia_mtime,
+                        from->ia_mtime_nsec);
+    set_if_greater_time(to->ia_ctime, to->ia_ctime_nsec, from->ia_ctime,
+                        from->ia_ctime_nsec);
+
+    return 0;
 }
 
 int
-dht_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
+dht_build_child_loc(xlator_t *this, loc_t *child, loc_t *parent, char *name)
 {
-        if (!child) {
-                goto err;
-        }
+    if (!child) {
+        goto err;
+    }
 
-        if (strcmp (parent->path, "/") == 0)
-                gf_asprintf ((char **)&child->path, "/%s", name);
-        else
-                gf_asprintf ((char **)&child->path, "%s/%s", parent->path, name);
+    if (strcmp(parent->path, "/") == 0)
+        gf_asprintf((char **)&child->path, "/%s", name);
+    else
+        gf_asprintf((char **)&child->path, "%s/%s", parent->path, name);
 
-        if (!child->path) {
-                goto err;
-        }
+    if (!child->path) {
+        goto err;
+    }
 
-        child->name = strrchr (child->path, '/');
-        if (child->name)
-                child->name++;
+    child->name = strrchr(child->path, '/');
+    if (child->name)
+        child->name++;
 
-        child->parent = inode_ref (parent->inode);
-        child->inode = inode_new (parent->inode->table);
+    child->parent = inode_ref(parent->inode);
+    child->inode = inode_new(parent->inode->table);
 
-        if (!child->inode) {
-                goto err;
-        }
+    if (!child->inode) {
+        goto err;
+    }
 
-        return 0;
+    return 0;
 err:
-        if (child) {
-            loc_wipe (child);
-        }
-        return -1;
+    if (child) {
+        loc_wipe(child);
+    }
+    return -1;
 }
 
 int
-dht_init_local_subvolumes (xlator_t *this, dht_conf_t *conf)
+dht_init_local_subvolumes(xlator_t *this, dht_conf_t *conf)
 {
-        xlator_list_t *subvols = NULL;
-        int            cnt = 0;
+    xlator_list_t *subvols = NULL;
+    int cnt = 0;
 
-        if (!conf)
-                return -1;
+    if (!conf)
+        return -1;
 
-        for (subvols = this->children; subvols; subvols = subvols->next)
-                cnt++;
+    for (subvols = this->children; subvols; subvols = subvols->next)
+        cnt++;
 
-        conf->local_subvols = GF_CALLOC (cnt, sizeof (xlator_t *),
-                                        gf_dht_mt_xlator_t);
+    conf->local_subvols = GF_CALLOC(cnt, sizeof(xlator_t *),
+                                    gf_dht_mt_xlator_t);
 
-        /* FIX FIX : do this dynamically*/
-        conf->local_nodeuuids = GF_CALLOC (cnt,
-                                           sizeof (subvol_nodeuuids_info_t),
-                                           gf_dht_nodeuuids_t);
+    /* FIX FIX : do this dynamically*/
+    conf->local_nodeuuids = GF_CALLOC(cnt, sizeof(subvol_nodeuuids_info_t),
+                                      gf_dht_nodeuuids_t);
 
-        if (!conf->local_subvols || !conf->local_nodeuuids) {
-                return -1;
-        }
+    if (!conf->local_subvols || !conf->local_nodeuuids) {
+        return -1;
+    }
 
-        conf->local_subvols_cnt = 0;
+    conf->local_subvols_cnt = 0;
 
-        return 0;
+    return 0;
 }
 
 int
-dht_init_subvolumes (xlator_t *this, dht_conf_t *conf)
+dht_init_subvolumes(xlator_t *this, dht_conf_t *conf)
 {
-        xlator_list_t *subvols = NULL;
-        int            cnt = 0;
+    xlator_list_t *subvols = NULL;
+    int cnt = 0;
 
-        if (!conf)
-                return -1;
+    if (!conf)
+        return -1;
 
-        for (subvols = this->children; subvols; subvols = subvols->next)
-                cnt++;
+    for (subvols = this->children; subvols; subvols = subvols->next)
+        cnt++;
 
-        conf->subvolumes = GF_CALLOC (cnt, sizeof (xlator_t *),
-                                      gf_dht_mt_xlator_t);
-        if (!conf->subvolumes) {
-                return -1;
-        }
-        conf->subvolume_cnt = cnt;
+    conf->subvolumes = GF_CALLOC(cnt, sizeof(xlator_t *), gf_dht_mt_xlator_t);
+    if (!conf->subvolumes) {
+        return -1;
+    }
+    conf->subvolume_cnt = cnt;
+    /* Doesn't make sense to do any dht layer tasks
+       if the subvol count is 1. Set it as pass_through */
+    if (cnt == 1)
+        this->pass_through = _gf_true;
 
-        conf->local_subvols_cnt = 0;
+    conf->local_subvols_cnt = 0;
 
-        dht_set_subvol_range(this);
+    dht_set_subvol_range(this);
 
-        cnt = 0;
-        for (subvols = this->children; subvols; subvols = subvols->next)
-                conf->subvolumes[cnt++] = subvols->xlator;
+    cnt = 0;
+    for (subvols = this->children; subvols; subvols = subvols->next)
+        conf->subvolumes[cnt++] = subvols->xlator;
 
-        conf->subvolume_status = GF_CALLOC (cnt, sizeof (char),
-                                            gf_dht_mt_char);
-        if (!conf->subvolume_status) {
-                return -1;
-        }
+    conf->subvolume_status = GF_CALLOC(cnt, sizeof(char), gf_dht_mt_char);
+    if (!conf->subvolume_status) {
+        return -1;
+    }
 
-        conf->last_event = GF_CALLOC (cnt, sizeof (int),
-                                      gf_dht_mt_char);
-        if (!conf->last_event) {
-                return -1;
-        }
+    conf->last_event = GF_CALLOC(cnt, sizeof(int), gf_dht_mt_char);
+    if (!conf->last_event) {
+        return -1;
+    }
 
-        conf->subvol_up_time = GF_CALLOC (cnt, sizeof (time_t),
-                                          gf_dht_mt_subvol_time);
-        if (!conf->subvol_up_time) {
-                return -1;
-        }
+    conf->subvol_up_time = GF_CALLOC(cnt, sizeof(time_t),
+                                     gf_dht_mt_subvol_time);
+    if (!conf->subvol_up_time) {
+        return -1;
+    }
 
-        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
-                                    gf_dht_mt_dht_du_t);
-        if (!conf->du_stats) {
-                return -1;
-        }
+    conf->du_stats = GF_CALLOC(conf->subvolume_cnt, sizeof(dht_du_t),
+                               gf_dht_mt_dht_du_t);
+    if (!conf->du_stats) {
+        return -1;
+    }
 
-        conf->decommissioned_bricks = GF_CALLOC (cnt, sizeof (xlator_t *),
-                                                 gf_dht_mt_xlator_t);
-        if (!conf->decommissioned_bricks) {
-                return -1;
-        }
+    conf->decommissioned_bricks = GF_CALLOC(cnt, sizeof(xlator_t *),
+                                            gf_dht_mt_xlator_t);
+    if (!conf->decommissioned_bricks) {
+        return -1;
+    }
 
-        return 0;
+    return 0;
 }
 
-
 /*
  op_ret values :
   0 : Success.
@@ -1266,270 +1238,274 @@ dht_init_subvolumes (xlator_t *this, dht_conf_t *conf)
 */
 
 static int
-dht_migration_complete_check_done (int op_ret, call_frame_t *frame, void *data)
+dht_migration_complete_check_done(int op_ret, call_frame_t *frame, void *data)
 {
-        dht_local_t *local  = NULL;
-        xlator_t    *subvol = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret != 0)
-                goto out;
+    if (op_ret != 0)
+        goto out;
 
-        if (local->cached_subvol == NULL) {
-                local->op_errno = EINVAL;
-                goto out;
-        }
+    if (local->cached_subvol == NULL) {
+        local->op_errno = EINVAL;
+        goto out;
+    }
 
-        subvol = local->cached_subvol;
+    subvol = local->cached_subvol;
 
 out:
-        local->rebalance.target_op_fn (THIS, subvol, frame, op_ret);
+    local->rebalance.target_op_fn(THIS, subvol, frame, op_ret);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_migration_complete_check_task (void *data)
+dht_migration_complete_check_task(void *data)
 {
-        int                 ret         = -1;
-        xlator_t           *src_node    = NULL;
-        xlator_t           *dst_node    = NULL, *linkto_target = NULL;
-        dht_local_t        *local       = NULL;
-        dict_t             *dict        = NULL;
-        struct iatt         stbuf       = {0,};
-        xlator_t           *this        = NULL;
-        call_frame_t       *frame       = NULL;
-        loc_t               tmp_loc     = {0,};
-        char               *path        = NULL;
-        dht_conf_t         *conf        = NULL;
-        inode_t            *inode       = NULL;
-        fd_t               *iter_fd     = NULL;
-        fd_t               *tmp         = NULL;
-        uint64_t            tmp_miginfo = 0;
-        dht_migrate_info_t *miginfo     = NULL;
-        int                 open_failed = 0;
-
-        this  = THIS;
-        frame = data;
-        local = frame->local;
-        conf = this->private;
-
-        src_node = local->cached_subvol;
-
-        if (!local->loc.inode && !local->fd) {
-                local->op_errno = EINVAL;
-                goto out;
-        }
-
-        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
-
-        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
-         * as root:root. If a fd is already open, access check wont be done*/
-
-        if (!local->loc.inode) {
-                ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        conf->link_xattr_name, NULL, NULL);
-        } else {
-                SYNCTASK_SETID (0, 0);
-                ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       conf->link_xattr_name, NULL, NULL);
-                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
-        }
-
-
-        /*
-         * Each DHT xlator layer has its own name for the linkto xattr.
-         * If the file mode bits indicate the the file is being migrated but
-         * this layer's linkto xattr is not set, it means that another
-         * DHT layer is migrating the file. In this case, return 1 so
-         * the mode bits can be passed on to the higher layer for appropriate
-         * action.
-         */
-        if (-ret == ENODATA) {
-                /* This DHT translator is not migrating this file */
-
-                ret = inode_ctx_reset1 (inode, this, &tmp_miginfo);
-                if (tmp_miginfo) {
-
-                        /* This can be a problem if the file was
-                         * migrated by two different layers. Raise
-                         * a warning here.
-                         */
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_HAS_MIGINFO,
-                                "%s: Found miginfo in the inode ctx",
-                                tmp_loc.path ? tmp_loc.path :
-                                uuid_utoa (tmp_loc.gfid));
-
-                        miginfo = (void *)tmp_miginfo;
-                        GF_REF_PUT (miginfo);
-                }
-                ret = 1;
-                goto out;
-        }
+    int ret = -1;
+    xlator_t *src_node = NULL;
+    xlator_t *dst_node = NULL, *linkto_target = NULL;
+    dht_local_t *local = NULL;
+    dict_t *dict = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    call_frame_t *frame = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    char *path = NULL;
+    dht_conf_t *conf = NULL;
+    inode_t *inode = NULL;
+    fd_t *iter_fd = NULL;
+    fd_t *tmp = NULL;
+    uint64_t tmp_miginfo = 0;
+    dht_migrate_info_t *miginfo = NULL;
+    gf_boolean_t skip_open = _gf_false;
+    int open_failed = 0;
+
+    this = THIS;
+    frame = data;
+    local = frame->local;
+    conf = this->private;
+
+    src_node = local->cached_subvol;
+
+    if (!local->loc.inode && !local->fd) {
+        local->op_errno = EINVAL;
+        goto out;
+    }
 
-        if (!ret)
-                linkto_target = dht_linkfile_subvol (this, NULL, NULL, dict);
+    inode = (!local->fd) ? local->loc.inode : local->fd->inode;
 
-        if (local->loc.inode) {
-                loc_copy (&tmp_loc, &local->loc);
-        } else {
-                tmp_loc.inode = inode_ref (inode);
-                gf_uuid_copy (tmp_loc.gfid, inode->gfid);
-        }
+    /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+     * as root:root. If a fd is already open, access check won't be done*/
 
-        ret = syncop_lookup (this, &tmp_loc, &stbuf, 0, 0, 0);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_FILE_LOOKUP_FAILED,
-                        "%s: failed to lookup the file on %s",
-                        tmp_loc.path ? tmp_loc.path : uuid_utoa (tmp_loc.gfid),
-                        this->name);
-                local->op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
+    if (!local->loc.inode) {
+        ret = syncop_fgetxattr(src_node, local->fd, &dict,
+                               conf->link_xattr_name, NULL, NULL);
+    } else {
+        SYNCTASK_SETID(0, 0);
+        ret = syncop_getxattr(src_node, &local->loc, &dict,
+                              conf->link_xattr_name, NULL, NULL);
+        SYNCTASK_SETID(frame->root->uid, frame->root->gid);
+    }
+
+    /*
+     * Each DHT xlator layer has its own name for the linkto xattr.
+     * If the file mode bits indicate the the file is being migrated but
+     * this layer's linkto xattr is not set, it means that another
+     * DHT layer is migrating the file. In this case, return 1 so
+     * the mode bits can be passed on to the higher layer for appropriate
+     * action.
+     */
+    if (-ret == ENODATA) {
+        /* This DHT translator is not migrating this file */
+
+        ret = inode_ctx_reset1(inode, this, &tmp_miginfo);
+        if (tmp_miginfo) {
+            /* This can be a problem if the file was
+             * migrated by two different layers. Raise
+             * a warning here.
+             */
+            gf_smsg(
+                this->name, GF_LOG_WARNING, 0, DHT_MSG_HAS_MIGINFO, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), NULL);
+
+            miginfo = (void *)(uintptr_t)tmp_miginfo;
+            GF_REF_PUT(miginfo);
+        }
+        ret = 1;
+        goto out;
+    }
+
+    if (!ret)
+        linkto_target = dht_linkfile_subvol(this, NULL, NULL, dict);
+
+    if (local->loc.inode) {
+        loc_copy(&tmp_loc, &local->loc);
+    } else {
+        tmp_loc.inode = inode_ref(inode);
+        gf_uuid_copy(tmp_loc.gfid, inode->gfid);
+    }
+
+    ret = syncop_lookup(this, &tmp_loc, &stbuf, 0, 0, 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_FILE_LOOKUP_FAILED,
+                "tmp=%s", tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "name=%s", this->name, NULL);
+        local->op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    dst_node = dht_subvol_get_cached(this, tmp_loc.inode);
+    if (linkto_target && dst_node != linkto_target) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_LINKFILE,
+                "linkto_target_name=%s", linkto_target->name, "dst_name=%s",
+                dst_node->name, NULL);
+    }
+
+    if (gf_uuid_compare(stbuf.ia_gfid, tmp_loc.inode->gfid)) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "dst_name=%s", dst_node->name, NULL);
+        ret = -1;
+        local->op_errno = EIO;
+        goto out;
+    }
 
-        dst_node = dht_subvol_get_cached (this, tmp_loc.inode);
-        if (linkto_target && dst_node != linkto_target) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_INVALID_LINKFILE,
-                        "linkto target (%s) is "
-                        "different from cached-subvol (%s). Treating %s as "
-                        "destination subvol", linkto_target->name,
-                        dst_node->name, dst_node->name);
-        }
+    /* update local. A layout is set in inode-ctx in lookup already */
 
-        if (gf_uuid_compare (stbuf.ia_gfid, tmp_loc.inode->gfid)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_GFID_MISMATCH,
-                                "%s: gfid different on the target file on %s",
-                                tmp_loc.path ? tmp_loc.path :
-                                uuid_utoa (tmp_loc.gfid), dst_node->name);
-                ret = -1;
-                local->op_errno = EIO;
-                goto out;
-        }
+    dht_layout_unref(this, local->layout);
 
-        /* update local. A layout is set in inode-ctx in lookup already */
+    local->layout = dht_layout_get(frame->this, inode);
+    local->cached_subvol = dst_node;
 
-        dht_layout_unref (this, local->layout);
+    ret = 0;
 
-        local->layout   = dht_layout_get (frame->this, inode);
-        local->cached_subvol = dst_node;
+    /* once we detect the migration complete, the inode-ctx2 is no more
+       required.. delete the ctx and also, it means, open() already
+       done on all the fd of inode */
+    ret = inode_ctx_reset1(inode, this, &tmp_miginfo);
+    if (tmp_miginfo) {
+        miginfo = (void *)(uintptr_t)tmp_miginfo;
+        GF_REF_PUT(miginfo);
+        goto out;
+    }
+
+    /* perform 'open()' on all the fd's present on the inode */
+    if (tmp_loc.path == NULL) {
+        inode_path(inode, NULL, &path);
+        if (path)
+            tmp_loc.path = path;
+    }
+
+    LOCK(&inode->lock);
+
+    if (list_empty(&inode->fd_list))
+        goto unlock;
+
+    /* perform open as root:root. There is window between linkfile
+     * creation(root:root) and setattr with the correct uid/gid
+     */
+    SYNCTASK_SETID(0, 0);
+
+    /* It's possible that we are the last user of iter_fd after each
+     * iteration. In this case the fd_unref() of iter_fd at the end of
+     * the loop will cause the destruction of the fd. So we need to
+     * iterate the list safely because iter_fd cannot be trusted.
+     */
+    iter_fd = list_entry((&inode->fd_list)->next, typeof(*iter_fd), inode_list);
+    while (&iter_fd->inode_list != (&inode->fd_list)) {
+        if (fd_is_anonymous(iter_fd) ||
+            (dht_fd_open_on_dst(this, iter_fd, dst_node))) {
+            if (!tmp) {
+                iter_fd = list_entry(iter_fd->inode_list.next, typeof(*iter_fd),
+                                     inode_list);
+                continue;
+            }
+            skip_open = _gf_true;
+        }
+        /* We need to release the inode->lock before calling
+         * syncop_open() to avoid possible deadlocks. However this
+         * can cause the iter_fd to be released by other threads.
+         * To avoid this, we take a reference before releasing the
+         * lock.
+         */
+        fd_ref(iter_fd);
 
-        ret = 0;
+        UNLOCK(&inode->lock);
 
-        /* once we detect the migration complete, the inode-ctx2 is no more
-           required.. delete the ctx and also, it means, open() already
-           done on all the fd of inode */
-        ret = inode_ctx_reset1 (inode, this, &tmp_miginfo);
-        if (tmp_miginfo) {
-                miginfo = (void *)tmp_miginfo;
-                GF_REF_PUT (miginfo);
-                goto out;
+        if (tmp) {
+            fd_unref(tmp);
+            tmp = NULL;
         }
+        if (skip_open)
+            goto next;
 
-        /* perform 'open()' on all the fd's present on the inode */
-        if (tmp_loc.path == NULL) {
-                inode_path (inode, NULL, &path);
-                if (path)
-                        tmp_loc.path = path;
+        /* flags for open are stripped down to allow following the
+         * new location of the file, otherwise we can get EEXIST or
+         * truncate the file again as rebalance is moving the data */
+        ret = syncop_open(dst_node, &tmp_loc,
+                          (iter_fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)),
+                          iter_fd, NULL, NULL);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_OPEN_FD_ON_DST_FAILED, "id=%p", iter_fd,
+                    "flags=0%o", iter_fd->flags, "path=%s", path, "name=%s",
+                    dst_node->name, NULL);
+
+            open_failed = 1;
+            local->op_errno = -ret;
+            ret = -1;
+        } else {
+            dht_fd_ctx_set(this, iter_fd, dst_node);
         }
 
+    next:
         LOCK(&inode->lock);
+        skip_open = _gf_false;
+        tmp = iter_fd;
+        iter_fd = list_entry(tmp->inode_list.next, typeof(*tmp), inode_list);
+    }
 
-        if (list_empty (&inode->fd_list))
-                goto unlock;
-
-        /* perform open as root:root. There is window between linkfile
-         * creation(root:root) and setattr with the correct uid/gid
-         */
-        SYNCTASK_SETID(0, 0);
-
-        /* It's possible that we are the last user of iter_fd after each
-         * iteration. In this case the fd_unref() of iter_fd at the end of
-         * the loop will cause the destruction of the fd. So we need to
-         * iterate the list safely because iter_fd cannot be trusted.
-         */
-        list_for_each_entry_safe (iter_fd, tmp, &inode->fd_list, inode_list) {
-
-                if (fd_is_anonymous (iter_fd))
-                        continue;
-
-                if (dht_fd_open_on_dst (this, iter_fd, dst_node))
-                        continue;
-
-                /* We need to release the inode->lock before calling
-                 * syncop_open() to avoid possible deadlocks. However this
-                 * can cause the iter_fd to be released by other threads.
-                 * To avoid this, we take a reference before releasing the
-                 * lock.
-                 */
-                __fd_ref(iter_fd);
-
-                UNLOCK(&inode->lock);
-
-                /* flags for open are stripped down to allow following the
-                 * new location of the file, otherwise we can get EEXIST or
-                 * truncate the file again as rebalance is moving the data */
-                ret = syncop_open (dst_node, &tmp_loc,
-                                   (iter_fd->flags &
-                                   ~(O_CREAT | O_EXCL | O_TRUNC)),
-                                   iter_fd, NULL, NULL);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_OPEN_FD_ON_DST_FAILED, "failed"
-                                " to open the fd"
-                                " (%p, flags=0%o) on file %s @ %s",
-                                iter_fd, iter_fd->flags, path,
-                                dst_node->name);
-
-                        open_failed = 1;
-                        local->op_errno = -ret;
-                        ret = -1;
-                } else {
-                        dht_fd_ctx_set (this, iter_fd, dst_node);
-                }
+    SYNCTASK_SETID(frame->root->uid, frame->root->gid);
 
-                fd_unref(iter_fd);
-
-                LOCK(&inode->lock);
-        }
-
-        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
-
-        if (open_failed) {
-                ret = -1;
-                goto unlock;
-        }
-        ret = 0;
+    if (open_failed) {
+        ret = -1;
+        goto unlock;
+    }
+    ret = 0;
 
 unlock:
-        UNLOCK(&inode->lock);
+    UNLOCK(&inode->lock);
+    if (tmp) {
+        fd_unref(tmp);
+        tmp = NULL;
+    }
 
 out:
-        if (dict) {
-                dict_unref (dict);
-        }
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        loc_wipe (&tmp_loc);
+    loc_wipe(&tmp_loc);
 
-        return ret;
+    return ret;
 }
 
 int
-dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame)
+dht_rebalance_complete_check(xlator_t *this, call_frame_t *frame)
 {
-        int         ret     = -1;
-
-        ret = synctask_new (this->ctx->env, dht_migration_complete_check_task,
-                            dht_migration_complete_check_done,
-                            frame, frame);
-        return ret;
+    int ret = -1;
 
+    ret = synctask_new(this->ctx->env, dht_migration_complete_check_task,
+                       dht_migration_complete_check_done, frame, frame);
+    return ret;
 }
 
 /* During 'in-progress' state, both nodes should have the file */
@@ -1540,785 +1516,789 @@ dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame)
   1 : File is being migrated but not by this DHT layer.
 */
 static int
-dht_inprogress_check_done (int op_ret, call_frame_t *frame, void *data)
+dht_inprogress_check_done(int op_ret, call_frame_t *frame, void *data)
 {
-        dht_local_t *local      = NULL;
-        xlator_t    *dst_subvol = NULL, *src_subvol = NULL;
-        inode_t     *inode      = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *dst_subvol = NULL, *src_subvol = NULL;
+    inode_t *inode = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret != 0)
-                goto out;
+    if (op_ret != 0)
+        goto out;
 
-        inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+    inode = local->loc.inode ? local->loc.inode : local->fd->inode;
 
-        dht_inode_ctx_get_mig_info (THIS, inode, &src_subvol, &dst_subvol);
-        if (dht_mig_info_is_invalid (local->cached_subvol,
-                                     src_subvol, dst_subvol)) {
-                dst_subvol = dht_subvol_get_cached (THIS, inode);
-                if (!dst_subvol) {
-                        local->op_errno = EINVAL;
-                        goto out;
-                }
+    dht_inode_ctx_get_mig_info(THIS, inode, &src_subvol, &dst_subvol);
+    if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol, dst_subvol)) {
+        dst_subvol = dht_subvol_get_cached(THIS, inode);
+        if (!dst_subvol) {
+            local->op_errno = EINVAL;
+            goto out;
         }
+    }
 
 out:
-        local->rebalance.target_op_fn (THIS, dst_subvol, frame, op_ret);
+    local->rebalance.target_op_fn(THIS, dst_subvol, frame, op_ret);
 
-        return 0;
+    return 0;
 }
 
 static int
-dht_rebalance_inprogress_task (void *data)
+dht_rebalance_inprogress_task(void *data)
 {
-        int           ret             = -1;
-        xlator_t     *src_node        = NULL;
-        xlator_t     *dst_node        = NULL;
-        dht_local_t  *local           = NULL;
-        dict_t       *dict            = NULL;
-        call_frame_t *frame           = NULL;
-        xlator_t     *this            = NULL;
-        char         *path            = NULL;
-        struct iatt   stbuf           = {0,};
-        loc_t         tmp_loc         = {0,};
-        dht_conf_t   *conf            = NULL;
-        inode_t      *inode           = NULL;
-        fd_t         *iter_fd         = NULL;
-        fd_t         *tmp             = NULL;
-        int           open_failed     = 0;
-        uint64_t      tmp_miginfo     = 0;
-        dht_migrate_info_t *miginfo   = NULL;
-
-
-        this  = THIS;
-        frame = data;
-        local = frame->local;
-        conf = this->private;
-
-        src_node = local->cached_subvol;
-
-        if (!local->loc.inode && !local->fd)
-                goto out;
+    int ret = -1;
+    xlator_t *src_node = NULL;
+    xlator_t *dst_node = NULL;
+    dht_local_t *local = NULL;
+    dict_t *dict = NULL;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    char *path = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    loc_t tmp_loc = {
+        0,
+    };
+    dht_conf_t *conf = NULL;
+    inode_t *inode = NULL;
+    fd_t *iter_fd = NULL;
+    fd_t *tmp = NULL;
+    int open_failed = 0;
+    uint64_t tmp_miginfo = 0;
+    dht_migrate_info_t *miginfo = NULL;
+    gf_boolean_t skip_open = _gf_false;
+
+    this = THIS;
+    frame = data;
+    local = frame->local;
+    conf = this->private;
+
+    src_node = local->cached_subvol;
+
+    if (!local->loc.inode && !local->fd)
+        goto out;
 
-        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+    inode = (!local->fd) ? local->loc.inode : local->fd->inode;
 
-        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
-         * as root:root. If a fd is already open, access check wont be done*/
-        if (local->loc.inode) {
-                SYNCTASK_SETID (0, 0);
-                ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       conf->link_xattr_name, NULL, NULL);
-                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
-        } else {
-                ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        conf->link_xattr_name, NULL, NULL);
-        }
+    /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+     * as root:root. If a fd is already open, access check won't be done*/
+    if (local->loc.inode) {
+        SYNCTASK_SETID(0, 0);
+        ret = syncop_getxattr(src_node, &local->loc, &dict,
+                              conf->link_xattr_name, NULL, NULL);
+        SYNCTASK_SETID(frame->root->uid, frame->root->gid);
+    } else {
+        ret = syncop_fgetxattr(src_node, local->fd, &dict,
+                               conf->link_xattr_name, NULL, NULL);
+    }
+
+    /*
+     * Each DHT xlator layer has its own name for the linkto xattr.
+     * If the file mode bits indicate the the file is being migrated but
+     * this layer's linkto xattr is not present, it means that another
+     * DHT layer is migrating the file. In this case, return 1 so
+     * the mode bits can be passed on to the higher layer for appropriate
+     * action.
+     */
+
+    if (-ret == ENODATA) {
+        /* This DHT layer is not migrating this file */
+        ret = inode_ctx_reset1(inode, this, &tmp_miginfo);
+        if (tmp_miginfo) {
+            /* This can be a problem if the file was
+             * migrated by two different layers. Raise
+             * a warning here.
+             */
+            gf_smsg(
+                this->name, GF_LOG_WARNING, 0, DHT_MSG_HAS_MIGINFO, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), NULL);
+            miginfo = (void *)(uintptr_t)tmp_miginfo;
+            GF_REF_PUT(miginfo);
+        }
+        ret = 1;
+        goto out;
+    }
 
-        /*
-         * Each DHT xlator layer has its own name for the linkto xattr.
-         * If the file mode bits indicate the the file is being migrated but
-         * this layer's linkto xattr is not present, it means that another
-         * DHT layer is migrating the file. In this case, return 1 so
-         * the mode bits can be passed on to the higher layer for appropriate
-         * action.
-         */
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_GET_XATTR_FAILED,
+                "path=%s", local->loc.path, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        if (-ret == ENODATA) {
-                /* This DHT layer is not migrating this file */
-                ret = inode_ctx_reset1 (inode, this, &tmp_miginfo);
-                if (tmp_miginfo) {
-                        /* This can be a problem if the file was
-                         * migrated by two different layers. Raise
-                         * a warning here.
-                         */
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_HAS_MIGINFO,
-                                "%s: Found miginfo in the inode ctx",
-                                tmp_loc.path ? tmp_loc.path :
-                                uuid_utoa (tmp_loc.gfid));
-                        miginfo = (void *)tmp_miginfo;
-                        GF_REF_PUT (miginfo);
-                }
-                ret = 1;
-                goto out;
-        }
+    dst_node = dht_linkfile_subvol(this, NULL, NULL, dict);
+    if (!dst_node) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GET_XATTR_FAILED,
+                "path=%s", local->loc.path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    local->rebalance.target_node = dst_node;
+
+    if (local->loc.inode) {
+        loc_copy(&tmp_loc, &local->loc);
+    } else {
+        tmp_loc.inode = inode_ref(inode);
+        gf_uuid_copy(tmp_loc.gfid, inode->gfid);
+    }
+
+    /* lookup on dst */
+    ret = syncop_lookup(dst_node, &tmp_loc, &stbuf, NULL, NULL, NULL);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_FILE_LOOKUP_FAILED,
+                "tmp=%s", tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "name=%s", dst_node->name, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_GET_XATTR_FAILED,
-                        "%s: failed to get the 'linkto' xattr",
-                        local->loc.path);
-                ret = -1;
-                goto out;
-        }
+    if (gf_uuid_compare(stbuf.ia_gfid, tmp_loc.inode->gfid)) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "name=%s", dst_node->name, NULL);
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+
+    if (tmp_loc.path == NULL) {
+        inode_path(inode, NULL, &path);
+        if (path)
+            tmp_loc.path = path;
+    }
+
+    LOCK(&inode->lock);
+
+    if (list_empty(&inode->fd_list))
+        goto unlock;
+
+    /* perform open as root:root. There is window between linkfile
+     * creation(root:root) and setattr with the correct uid/gid
+     */
+    SYNCTASK_SETID(0, 0);
+
+    /* It's possible that we are the last user of iter_fd after each
+     * iteration. In this case the fd_unref() of iter_fd at the end of
+     * the loop will cause the destruction of the fd. So we need to
+     * iterate the list safely because iter_fd cannot be trusted.
+     */
+    iter_fd = list_entry((&inode->fd_list)->next, typeof(*iter_fd), inode_list);
+    while (&iter_fd->inode_list != (&inode->fd_list)) {
+        /* We need to release the inode->lock before calling
+         * syncop_open() to avoid possible deadlocks. However this
+         * can cause the iter_fd to be released by other threads.
+         * To avoid this, we take a reference before releasing the
+         * lock.
+         */
 
-        dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
-        if (!dst_node) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_SUBVOL_NOT_FOUND,
-                        "%s: failed to get the 'linkto' xattr from dict",
-                        local->loc.path);
-                ret = -1;
-                goto out;
+        if (fd_is_anonymous(iter_fd) ||
+            (dht_fd_open_on_dst(this, iter_fd, dst_node))) {
+            if (!tmp) {
+                iter_fd = list_entry(iter_fd->inode_list.next, typeof(*iter_fd),
+                                     inode_list);
+                continue;
+            }
+            skip_open = _gf_true;
         }
 
-        local->rebalance.target_node = dst_node;
-
-        if (local->loc.inode) {
-                loc_copy (&tmp_loc, &local->loc);
-        } else {
-                tmp_loc.inode = inode_ref (inode);
-                gf_uuid_copy (tmp_loc.gfid, inode->gfid);
-        }
+        /* Yes, this is ugly but there isn't a cleaner way to do this
+         * the fd_ref is an atomic increment so not too bad. We want to
+         * reduce the number of inode locks and unlocks.
+         */
 
-        /* lookup on dst */
-        ret = syncop_lookup (dst_node, &tmp_loc, &stbuf, NULL,
-                             NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_FILE_LOOKUP_ON_DST_FAILED,
-                        "%s: failed to lookup the file on %s",
-                        tmp_loc.path ? tmp_loc.path : uuid_utoa (tmp_loc.gfid),
-                        dst_node->name);
-                ret = -1;
-                goto out;
-        }
+        fd_ref(iter_fd);
+        UNLOCK(&inode->lock);
 
-        if (gf_uuid_compare (stbuf.ia_gfid, tmp_loc.inode->gfid)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_GFID_MISMATCH,
-                        "%s: gfid different on the target file on %s",
-                        tmp_loc.path ? tmp_loc.path : uuid_utoa (tmp_loc.gfid),
-                        dst_node->name);
-                ret = -1;
-                goto out;
+        if (tmp) {
+            fd_unref(tmp);
+            tmp = NULL;
         }
-        ret = 0;
+        if (skip_open)
+            goto next;
 
-        if (tmp_loc.path == NULL) {
-                inode_path (inode, NULL, &path);
-                if (path)
-                        tmp_loc.path = path;
+        /* flags for open are stripped down to allow following the
+         * new location of the file, otherwise we can get EEXIST or
+         * truncate the file again as rebalance is moving the data */
+        ret = syncop_open(dst_node, &tmp_loc,
+                          (iter_fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)),
+                          iter_fd, NULL, NULL);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_OPEN_FD_ON_DST_FAILED, "fd=%p", iter_fd,
+                    "flags=0%o", iter_fd->flags, "path=%s", path, "name=%s",
+                    dst_node->name, NULL);
+            ret = -1;
+            open_failed = 1;
+        } else {
+            /* Potential fd leak if this fails here as it will be
+               reopened at the next Phase1/2 check */
+            dht_fd_ctx_set(this, iter_fd, dst_node);
         }
 
+    next:
         LOCK(&inode->lock);
+        skip_open = _gf_false;
+        tmp = iter_fd;
+        iter_fd = list_entry(tmp->inode_list.next, typeof(*tmp), inode_list);
+    }
 
-        if (list_empty (&inode->fd_list))
-                goto unlock;
-
-        /* perform open as root:root. There is window between linkfile
-         * creation(root:root) and setattr with the correct uid/gid
-         */
-        SYNCTASK_SETID (0, 0);
-
-        /* It's possible that we are the last user of iter_fd after each
-         * iteration. In this case the fd_unref() of iter_fd at the end of
-         * the loop will cause the destruction of the fd. So we need to
-         * iterate the list safely because iter_fd cannot be trusted.
-         */
-        list_for_each_entry_safe (iter_fd, tmp, &inode->fd_list, inode_list) {
-                if (fd_is_anonymous (iter_fd))
-                        continue;
-
-                if (dht_fd_open_on_dst (this, iter_fd, dst_node))
-                        continue;
-
-                /* We need to release the inode->lock before calling
-                 * syncop_open() to avoid possible deadlocks. However this
-                 * can cause the iter_fd to be released by other threads.
-                 * To avoid this, we take a reference before releasing the
-                 * lock.
-                 */
-                __fd_ref(iter_fd);
-
-                UNLOCK(&inode->lock);
-
-                /* flags for open are stripped down to allow following the
-                 * new location of the file, otherwise we can get EEXIST or
-                 * truncate the file again as rebalance is moving the data */
-                ret = syncop_open (dst_node, &tmp_loc,
-                                  (iter_fd->flags &
-                                   ~(O_CREAT | O_EXCL | O_TRUNC)),
-                                   iter_fd, NULL, NULL);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_OPEN_FD_ON_DST_FAILED,
-                                "failed to send open "
-                                "the fd (%p, flags=0%o) on file %s @ %s",
-                                iter_fd, iter_fd->flags, path,
-                                dst_node->name);
-                        ret = -1;
-                        open_failed = 1;
-                } else {
-                        /* Potential fd leak if this fails here as it will be
-                           reopened at the next Phase1/2 check */
-                        dht_fd_ctx_set (this, iter_fd, dst_node);
-                }
-
-                fd_unref(iter_fd);
-
-                LOCK(&inode->lock);
-        }
-
-        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+    SYNCTASK_SETID(frame->root->uid, frame->root->gid);
 
 unlock:
-        UNLOCK(&inode->lock);
-
-        if (open_failed) {
-                ret = -1;
-                goto out;
-        }
+    UNLOCK(&inode->lock);
+
+    if (tmp) {
+        fd_unref(tmp);
+        tmp = NULL;
+    }
+    if (open_failed) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dht_inode_ctx_set_mig_info (this, inode, src_node, dst_node);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_SET_INODE_CTX_FAILED,
-                        "%s: failed to set inode-ctx target file at %s",
-                        local->loc.path, dst_node->name);
-                goto out;
-        }
+    ret = dht_inode_ctx_set_mig_info(this, inode, src_node, dst_node);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                "path=%s", local->loc.path, "name=%s", dst_node->name, NULL);
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (dict) {
-                dict_unref (dict);
-        }
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        loc_wipe (&tmp_loc);
-        return ret;
+    loc_wipe(&tmp_loc);
+    return ret;
 }
 
 int
-dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame)
+dht_rebalance_in_progress_check(xlator_t *this, call_frame_t *frame)
 {
+    int ret = -1;
 
-        int         ret     = -1;
-
-        ret = synctask_new (this->ctx->env, dht_rebalance_inprogress_task,
-                            dht_inprogress_check_done,
-                            frame, frame);
-        return ret;
+    ret = synctask_new(this->ctx->env, dht_rebalance_inprogress_task,
+                       dht_inprogress_check_done, frame, frame);
+    return ret;
 }
 
 int
-dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
-                          dht_layout_t *layout_int)
+dht_inode_ctx_layout_set(inode_t *inode, xlator_t *this,
+                         dht_layout_t *layout_int)
 {
-        dht_inode_ctx_t         *ctx            = NULL;
-        int                      ret            = -1;
-
-        ret = dht_inode_ctx_get (inode, this, &ctx);
-        if (!ret && ctx) {
-                ctx->layout = layout_int;
-        } else {
-                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
-                if (!ctx)
-                        return ret;
-                ctx->layout = layout_int;
-        }
-
-        ret = dht_inode_ctx_set (inode, this, ctx);
-
-        return ret;
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+
+    ret = dht_inode_ctx_get(inode, this, &ctx);
+    if (!ret && ctx) {
+        ctx->layout = layout_int;
+    } else {
+        ctx = GF_CALLOC(1, sizeof(*ctx), gf_dht_mt_inode_ctx_t);
+        if (!ctx)
+            return ret;
+        ctx->layout = layout_int;
+    }
+
+    ret = dht_inode_ctx_set(inode, this, ctx);
+
+    return ret;
 }
 
-
 void
-dht_inode_ctx_time_set (inode_t *inode, xlator_t *this, struct iatt *stat)
+dht_inode_ctx_time_set(inode_t *inode, xlator_t *this, struct iatt *stat)
 {
-        dht_inode_ctx_t         *ctx            = NULL;
-        dht_stat_time_t         *time           = 0;
-        int                      ret            = -1;
+    dht_inode_ctx_t *ctx = NULL;
+    dht_stat_time_t *time = 0;
+    int ret = -1;
 
-        ret = dht_inode_ctx_get (inode, this, &ctx);
+    ret = dht_inode_ctx_get(inode, this, &ctx);
 
-        if (ret)
-		return;
+    if (ret)
+        return;
 
-        time = &ctx->time;
+    time = &ctx->time;
 
-	time->mtime      = stat->ia_mtime;
-	time->mtime_nsec = stat->ia_mtime_nsec;
+    time->mtime = stat->ia_mtime;
+    time->mtime_nsec = stat->ia_mtime_nsec;
 
-	time->ctime      = stat->ia_ctime;
-	time->ctime_nsec = stat->ia_ctime_nsec;
+    time->ctime = stat->ia_ctime;
+    time->ctime_nsec = stat->ia_ctime_nsec;
 
-	time->atime      = stat->ia_atime;
-	time->atime_nsec = stat->ia_atime_nsec;
+    time->atime = stat->ia_atime;
+    time->atime_nsec = stat->ia_atime_nsec;
 
-	return;
+    return;
 }
 
-
 int
-dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
-                           int32_t post)
+dht_inode_ctx_time_update(inode_t *inode, xlator_t *this, struct iatt *stat,
+                          int32_t post)
 {
-        dht_inode_ctx_t         *ctx            = NULL;
-        dht_stat_time_t         *time           = 0;
-        int                      ret            = -1;
-
-        GF_VALIDATE_OR_GOTO (this->name, stat, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        ret = dht_inode_ctx_get (inode, this, &ctx);
-
-        if (ret) {
-                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
-                if (!ctx)
-                        return -1;
-        }
-
-        time = &ctx->time;
-
-        DHT_UPDATE_TIME(time->mtime, time->mtime_nsec,
-                        stat->ia_mtime, stat->ia_mtime_nsec, inode, post);
-        DHT_UPDATE_TIME(time->ctime, time->ctime_nsec,
-                        stat->ia_ctime, stat->ia_ctime_nsec, inode, post);
-        DHT_UPDATE_TIME(time->atime, time->atime_nsec,
-                        stat->ia_atime, stat->ia_atime_nsec, inode, post);
-
-        ret = dht_inode_ctx_set (inode, this, ctx);
+    dht_inode_ctx_t *ctx = NULL;
+    dht_stat_time_t *time = 0;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(this->name, stat, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    ret = dht_inode_ctx_get(inode, this, &ctx);
+
+    if (ret) {
+        ctx = GF_CALLOC(1, sizeof(*ctx), gf_dht_mt_inode_ctx_t);
+        if (!ctx)
+            return -1;
+    }
+
+    time = &ctx->time;
+
+    LOCK(&inode->lock);
+    {
+        DHT_UPDATE_TIME(time->mtime, time->mtime_nsec, stat->ia_mtime,
+                        stat->ia_mtime_nsec, post);
+        DHT_UPDATE_TIME(time->ctime, time->ctime_nsec, stat->ia_ctime,
+                        stat->ia_ctime_nsec, post);
+        DHT_UPDATE_TIME(time->atime, time->atime_nsec, stat->ia_atime,
+                        stat->ia_atime_nsec, post);
+    }
+    UNLOCK(&inode->lock);
+
+    ret = dht_inode_ctx_set(inode, this, ctx);
 out:
-        return 0;
+    return 0;
 }
 
 int
-dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx)
+dht_inode_ctx_get(inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx)
 {
-        int             ret     = -1;
-        uint64_t        ctx_int = 0;
+    int ret = -1;
+    uint64_t ctx_int = 0;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        ret = inode_ctx_get (inode, this, &ctx_int);
+    ret = inode_ctx_get(inode, this, &ctx_int);
 
-        if (ret)
-                return ret;
+    if (ret)
+        return ret;
 
-        if (ctx)
-                *ctx = (dht_inode_ctx_t *) ctx_int;
+    if (ctx)
+        *ctx = (dht_inode_ctx_t *)(uintptr_t)ctx_int;
 out:
-        return ret;
+    return ret;
 }
 
-int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx)
+int
+dht_inode_ctx_set(inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx)
 {
-        int             ret = -1;
-        uint64_t        ctx_int = 0;
+    int ret = -1;
+    uint64_t ctx_int = 0;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, ctx, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, ctx, out);
 
-        ctx_int = (long)ctx;
-        ret = inode_ctx_set (inode, this, &ctx_int);
+    ctx_int = (long)ctx;
+    ret = inode_ctx_set(inode, this, &ctx_int);
 out:
-        return ret;
+    return ret;
 }
 
 int
-dht_subvol_status (dht_conf_t *conf, xlator_t *subvol)
+dht_subvol_status(dht_conf_t *conf, xlator_t *subvol)
 {
-        int i;
+    int i;
 
-        for (i=0 ; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == subvol) {
-                        return conf->subvolume_status[i];
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == subvol) {
+            return conf->subvolume_status[i];
         }
-        return 0;
+    }
+    return 0;
 }
 
-inode_t*
-dht_heal_path (xlator_t *this, char *path, inode_table_t *itable)
+inode_t *
+dht_heal_path(xlator_t *this, char *path, inode_table_t *itable)
 {
-        int             ret             = -1;
-        struct iatt     iatt            = {0, };
-        inode_t        *linked_inode    = NULL;
-        loc_t           loc             = {0, };
-        char           *bname           = NULL;
-        char           *save_ptr        = NULL;
-        uuid_t          gfid            = {0, };
-        char           *tmp_path        = NULL;
-
-
-        tmp_path = gf_strdup (path);
-        if (!tmp_path) {
-                goto out;
-        }
-
-        memset (gfid, 0, 16);
-        gfid[15] = 1;
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *linked_inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    char *bname = NULL;
+    char *save_ptr = NULL;
+    static uuid_t gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    char *tmp_path = NULL;
+
+    tmp_path = gf_strdup(path);
+    if (!tmp_path) {
+        goto out;
+    }
 
-        gf_uuid_copy (loc.pargfid, gfid);
-        loc.parent = inode_ref (itable->root);
+    gf_uuid_copy(loc.pargfid, gfid);
+    loc.parent = inode_ref(itable->root);
 
-        bname = strtok_r (tmp_path, "/",  &save_ptr);
+    bname = strtok_r(tmp_path, "/", &save_ptr);
 
-        /* sending a lookup on parent directory,
-         * Eg:  if  path is like /a/b/c/d/e/f/g/
-         * then we will send a lookup on a first and then b,c,d,etc
-         */
+    /* sending a lookup on parent directory,
+     * Eg:  if  path is like /a/b/c/d/e/f/g/
+     * then we will send a lookup on a first and then b,c,d,etc
+     */
 
-        while (bname) {
-                linked_inode = NULL;
-                loc.inode = inode_grep (itable, loc.parent, bname);
-                if (loc.inode == NULL) {
-                        loc.inode = inode_new (itable);
-                        if (loc.inode == NULL) {
-                                ret = -ENOMEM;
-                                goto out;
-                        }
-                } else {
-                        /*
-                         * Inode is already populated in the inode table.
-                         * Which means we already looked up the inde and
-                         * linked with a dentry. So that we will skip
-                         * lookup on this entry, and proceed to next.
-                         */
-                        bname = strtok_r (NULL, "/",  &save_ptr);
-                        inode_unref (loc.parent);
-                        loc.parent = loc.inode;
-                        gf_uuid_copy (loc.pargfid, loc.inode->gfid);
-                        loc.inode = NULL;
-                        continue;
-                }
+    while (bname) {
+        linked_inode = NULL;
+        loc.inode = inode_grep(itable, loc.parent, bname);
+        if (loc.inode == NULL) {
+            loc.inode = inode_new(itable);
+            if (loc.inode == NULL) {
+                ret = -ENOMEM;
+                goto out;
+            }
+        } else {
+            /*
+             * Inode is already populated in the inode table.
+             * Which means we already looked up the inode and
+             * linked with a dentry. So that we will skip
+             * lookup on this entry, and proceed to next.
+             */
+            linked_inode = loc.inode;
+            bname = strtok_r(NULL, "/", &save_ptr);
+            if (!bname) {
+                goto out;
+            }
+            inode_unref(loc.parent);
+            loc.parent = loc.inode;
+            gf_uuid_copy(loc.pargfid, loc.inode->gfid);
+            loc.inode = NULL;
+            continue;
+        }
 
-                loc.name = bname;
-                ret = loc_path (&loc, bname);
+        loc.name = bname;
+        ret = loc_path(&loc, bname);
 
-                ret = syncop_lookup (this, &loc, &iatt, NULL, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_INFO, -ret,
-                                DHT_MSG_DIR_SELFHEAL_FAILED,
-                                "Healing of path %s failed on subvolume %s for "
-                                "directory %s", path, this->name, bname);
-                        goto out;
-                }
+        ret = syncop_lookup(this, &loc, &iatt, NULL, NULL, NULL);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_SELFHEAL_FAILED,
+                    "path=%s", path, "subvolume=%s", this->name, "bname=%s",
+                    bname, NULL);
+            goto out;
+        }
 
-                linked_inode = inode_link (loc.inode, loc.parent, bname, &iatt);
-                if (!linked_inode)
-                        goto out;
+        linked_inode = inode_link(loc.inode, loc.parent, bname, &iatt);
+        if (!linked_inode)
+            goto out;
 
-                loc_wipe (&loc);
-                gf_uuid_copy (loc.pargfid, linked_inode->gfid);
-                loc.inode = NULL;
-                loc.parent = linked_inode;
+        loc_wipe(&loc);
+        gf_uuid_copy(loc.pargfid, linked_inode->gfid);
+        loc.inode = NULL;
 
-                bname = strtok_r (NULL, "/",  &save_ptr);
-        }
+        bname = strtok_r(NULL, "/", &save_ptr);
+        if (bname)
+            loc.parent = linked_inode;
+    }
 out:
-        inode_ref (linked_inode);
-        loc_wipe (&loc);
-        GF_FREE (tmp_path);
+    inode_ref(linked_inode);
+    loc_wipe(&loc);
+    GF_FREE(tmp_path);
 
-        return linked_inode;
+    return linked_inode;
 }
 
-
 int
-dht_heal_full_path (void *data)
+dht_heal_full_path(void *data)
 {
-        call_frame_t            *heal_frame     = data;
-        dht_local_t             *local          = NULL;
-        loc_t                    loc            = {0, };
-        dict_t                  *dict           = NULL;
-        char                    *path           = NULL;
-        int                      ret            = -1;
-        xlator_t                *source         = NULL;
-        xlator_t                *this           = NULL;
-        inode_table_t           *itable         = NULL;
-        inode_t                 *inode          = NULL;
-        inode_t                 *tmp_inode      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("DHT", heal_frame, out);
-
-        local = heal_frame->local;
-        this = heal_frame->this;
-        source = heal_frame->cookie;
-        heal_frame->cookie = NULL;
-        gf_uuid_copy (loc.gfid, local->gfid);
-
-        if (local->loc.inode)
-                loc.inode = inode_ref (local->loc.inode);
-        else
-                goto out;
-
-        itable = loc.inode->table;
-        ret = syncop_getxattr (source, &loc, &dict,
-                       GET_ANCESTRY_PATH_KEY, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_INFO, -ret,
-                        DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "Failed to get path from subvol %s. Aborting "
-                        "directory healing.", source->name);
-                goto out;
-        }
+    call_frame_t *heal_frame = data;
+    dht_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *path = NULL;
+    int ret = -1;
+    xlator_t *source = NULL;
+    xlator_t *this = NULL;
+    inode_table_t *itable = NULL;
+    inode_t *inode = NULL;
+    inode_t *tmp_inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("DHT", heal_frame, out);
+
+    local = heal_frame->local;
+    this = heal_frame->this;
+    source = heal_frame->cookie;
+    heal_frame->cookie = NULL;
+    gf_uuid_copy(loc.gfid, local->gfid);
+
+    if (local->loc.inode)
+        loc.inode = inode_ref(local->loc.inode);
+    else
+        goto out;
 
-        ret = dict_get_str (dict, GET_ANCESTRY_PATH_KEY, &path);
-        if (path) {
-                inode = dht_heal_path (this, path, itable);
-                if (inode && inode != local->inode) {
-                        /*
-                         * if inode returned by heal function is different
-                         * from what we passed, which means a racing thread
-                         * already linked a different inode for dentry.
-                         * So we will update our local->inode, so that we can
-                         * retrurn proper inode.
-                         */
-                        tmp_inode = local->inode;
-                        local->inode = inode;
-                        inode_unref (tmp_inode);
-                        tmp_inode = NULL;
-                } else {
-                        inode_unref (inode);
-                }
+    itable = loc.inode->table;
+    ret = syncop_getxattr(source, &loc, &dict, GET_ANCESTRY_PATH_KEY, NULL,
+                          NULL);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_HEAL_ABORT,
+                "subvol=%s", source->name, NULL);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, GET_ANCESTRY_PATH_KEY, &path);
+    if (path) {
+        inode = dht_heal_path(this, path, itable);
+        if (inode && inode != local->inode) {
+            /*
+             * if inode returned by heal function is different
+             * from what we passed, which means a racing thread
+             * already linked a different inode for dentry.
+             * So we will update our local->inode, so that we can
+             * retrurn proper inode.
+             */
+            tmp_inode = local->inode;
+            local->inode = inode;
+            inode_unref(tmp_inode);
+            tmp_inode = NULL;
+        } else {
+            inode_unref(inode);
         }
+    }
 
 out:
-        loc_wipe (&loc);
-        if (dict)
-                dict_unref (dict);
-        return 0;
+    loc_wipe(&loc);
+    if (dict)
+        dict_unref(dict);
+    return 0;
 }
 
 int
-dht_heal_full_path_done (int op_ret, call_frame_t *heal_frame, void *data)
+dht_heal_full_path_done(int op_ret, call_frame_t *heal_frame, void *data)
 {
-
-        call_frame_t            *main_frame       = NULL;
-        dht_local_t             *local            = NULL;
-        xlator_t                *this             = NULL;
-        int                     ret               = -1;
-
-        local = heal_frame->local;
-        main_frame = local->main_frame;
-        local->main_frame = NULL;
-        this = heal_frame->this;
-
-        dht_set_fixed_dir_stat (&local->postparent);
-        if (local->need_xattr_heal) {
-                local->need_xattr_heal = 0;
-                ret =  dht_dir_xattr_heal (this, local);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, ret,
-                                DHT_MSG_DIR_XATTR_HEAL_FAILED,
-                                "xattr heal failed for directory  %s ",
-                                local->loc.path);
+    call_frame_t *main_frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    int op_errno = 0;
+
+    local = heal_frame->local;
+    main_frame = local->main_frame;
+    local->main_frame = NULL;
+    this = heal_frame->this;
+
+    dht_set_fixed_dir_stat(&local->postparent);
+    if (local->need_xattr_heal) {
+        local->need_xattr_heal = 0;
+        ret = dht_dir_xattr_heal(this, local, &op_errno);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                    DHT_MSG_DIR_XATTR_HEAL_FAILED, "path=%s", local->loc.path,
+                    NULL);
         }
+    }
 
-        DHT_STACK_UNWIND (lookup, main_frame, 0, 0,
-                          local->inode, &local->stbuf, local->xattr,
-                          &local->postparent);
+    DHT_STACK_UNWIND(lookup, main_frame, 0, 0, local->inode, &local->stbuf,
+                     local->xattr, &local->postparent);
 
-        DHT_STACK_DESTROY (heal_frame);
-        return 0;
+    DHT_STACK_DESTROY(heal_frame);
+    return 0;
 }
 
 /* This function must be called inside an inode lock */
 int
-__dht_lock_subvol_set (inode_t *inode, xlator_t *this,
-                       xlator_t *lock_subvol)
+__dht_lock_subvol_set(inode_t *inode, xlator_t *this, xlator_t *lock_subvol)
 {
-        dht_inode_ctx_t         *ctx            = NULL;
-        int                      ret            = -1;
-        uint64_t                 value          = 0;
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+    uint64_t value = 0;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        ret = __inode_ctx_get0 (inode, this, &value);
-        if (ret || !value) {
-                return -1;
-        }
+    ret = __inode_ctx_get0(inode, this, &value);
+    if (ret || !value) {
+        return -1;
+    }
 
-        ctx = (dht_inode_ctx_t *) value;
-        ctx->lock_subvol = lock_subvol;
+    ctx = (dht_inode_ctx_t *)(uintptr_t)value;
+    ctx->lock_subvol = lock_subvol;
 out:
-        return ret;
+    return ret;
 }
 
-xlator_t*
-dht_get_lock_subvolume (xlator_t *this, struct gf_flock *lock,
-                        dht_local_t *local)
+xlator_t *
+dht_get_lock_subvolume(xlator_t *this, struct gf_flock *lock,
+                       dht_local_t *local)
 {
-        xlator_t                *subvol                  = NULL;
-        inode_t                 *inode                   = NULL;
-        int32_t                  ret                     = -1;
-        uint64_t                 value                   = 0;
-        xlator_t                *cached_subvol           = NULL;
-        dht_inode_ctx_t         *ctx                     = NULL;
-        char                     gfid[GF_UUID_BUF_SIZE]  = {0};
-
-        GF_VALIDATE_OR_GOTO (this->name, lock, out);
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    int32_t ret = -1;
+    uint64_t value = 0;
+    xlator_t *cached_subvol = NULL;
+    dht_inode_ctx_t *ctx = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        cached_subvol = local->cached_subvol;
+    GF_VALIDATE_OR_GOTO(this->name, lock, out);
+    GF_VALIDATE_OR_GOTO(this->name, local, out);
 
-        if (local->loc.inode || local->fd) {
-                inode = local->loc.inode ? local->loc.inode : local->fd->inode;
-        }
+    cached_subvol = local->cached_subvol;
 
-        if (!inode)
-                goto out;
+    if (local->loc.inode || local->fd) {
+        inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+    }
 
-        if (!(IA_ISDIR (inode->ia_type) || IA_ISINVAL (inode->ia_type))) {
-                /*
-                 * We may get non-linked inode for directories as part
-                 * of the selfheal code path. So checking  for IA_INVAL
-                 * type also. This will only happen for directory.
-                 */
-                subvol = local->cached_subvol;
-                goto out;
-        }
+    if (!inode)
+        goto out;
 
-        if (lock->l_type != F_UNLCK) {
-                /*
-                 * inode purging might happen on NFS between a lk
-                 * and unlk. Due to this lk and unlk might be sent
-                 * to different subvols.
-                 * So during a lock request, taking a ref on inode
-                 * to prevent inode purging. inode unref will happen
-                 * in unlock cbk code path.
-                 */
-                inode_ref (inode);
-        }
+    if (!(IA_ISDIR(inode->ia_type) || IA_ISINVAL(inode->ia_type))) {
+        /*
+         * We may get non-linked inode for directories as part
+         * of the selfheal code path. So checking  for IA_INVAL
+         * type also. This will only happen for directory.
+         */
+        subvol = local->cached_subvol;
+        goto out;
+    }
 
-        LOCK (&inode->lock);
-                ret = __inode_ctx_get0 (inode, this, &value);
-                if (!ret && value) {
-                        ctx = (dht_inode_ctx_t *) value;
-                        subvol = ctx->lock_subvol;
-                }
-                if (!subvol && lock->l_type != F_UNLCK && cached_subvol) {
-                        ret = __dht_lock_subvol_set (inode, this,
-                                                     cached_subvol);
-                        if (ret) {
-                                gf_uuid_unparse(inode->gfid, gfid);
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_SET_INODE_CTX_FAILED,
-                                        "Failed to set lock_subvol in "
-                                        "inode ctx for gfid %s",
-                                        gfid);
-                                goto unlock;
-                        }
-                        subvol = cached_subvol;
-                }
-unlock:
-        UNLOCK (&inode->lock);
-        if (!subvol && inode && lock->l_type != F_UNLCK) {
-                inode_unref (inode);
-        }
+    if (lock->l_type != F_UNLCK) {
+        /*
+         * inode purging might happen on NFS between a lk
+         * and unlk. Due to this lk and unlk might be sent
+         * to different subvols.
+         * So during a lock request, taking a ref on inode
+         * to prevent inode purging. inode unref will happen
+         * in unlock cbk code path.
+         */
+        inode_ref(inode);
+    }
+
+    LOCK(&inode->lock);
+    ret = __inode_ctx_get0(inode, this, &value);
+    if (!ret && value) {
+        ctx = (dht_inode_ctx_t *)(uintptr_t)value;
+        subvol = ctx->lock_subvol;
+    }
+    if (!subvol && lock->l_type != F_UNLCK && cached_subvol) {
+        ret = __dht_lock_subvol_set(inode, this, cached_subvol);
+        if (ret) {
+            gf_uuid_unparse(inode->gfid, gfid);
+            UNLOCK(&inode->lock);
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                    "lock_subvol gfid=%s", gfid, NULL);
+            goto post_unlock;
+        }
+        subvol = cached_subvol;
+    }
+    UNLOCK(&inode->lock);
+post_unlock:
+    if (!subvol && inode && lock->l_type != F_UNLCK) {
+        inode_unref(inode);
+    }
 out:
-        return subvol;
+    return subvol;
 }
 
 int
-dht_lk_inode_unref (call_frame_t *frame, int32_t op_ret)
+dht_lk_inode_unref(call_frame_t *frame, int32_t op_ret)
 {
-        int                     ret                     = -1;
-        dht_local_t            *local                   = NULL;
-        inode_t                *inode                   = NULL;
-        xlator_t               *this                    = NULL;
-        char                    gfid[GF_UUID_BUF_SIZE]  = {0};
+    int ret = -1;
+    dht_local_t *local = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        local = frame->local;
-        this = frame->this;
+    local = frame->local;
+    this = frame->this;
 
-        if (local->loc.inode || local->fd) {
-                inode = local->loc.inode ? local->loc.inode : local->fd->inode;
-        }
-        if (!inode) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LOCK_INODE_UNREF_FAILED,
-                        "Found a NULL inode. Failed to unref the inode");
-                goto out;
-        }
+    if (local->loc.inode || local->fd) {
+        inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+    }
+    if (!inode) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LOCK_INODE_UNREF_FAILED,
+                NULL);
+        goto out;
+    }
 
-        if (!(IA_ISDIR (inode->ia_type) || IA_ISINVAL (inode->ia_type))) {
-                ret = 0;
-                goto out;
-        }
+    if (!(IA_ISDIR(inode->ia_type) || IA_ISINVAL(inode->ia_type))) {
+        ret = 0;
+        goto out;
+    }
 
-        switch (local->lock_type) {
+    switch (local->lock_type) {
         case F_RDLCK:
         case F_WRLCK:
-                if (op_ret) {
-                        gf_uuid_unparse(inode->gfid, gfid);
-                        gf_msg_debug (this->name, 0,
-                                "lock request failed for gfid %s", gfid);
-                        inode_unref (inode);
-                        goto out;
-                }
-                break;
+            if (op_ret) {
+                gf_uuid_unparse(inode->gfid, gfid);
+                gf_msg_debug(this->name, 0, "lock request failed for gfid %s",
+                             gfid);
+                inode_unref(inode);
+                goto out;
+            }
+            break;
 
         case F_UNLCK:
-                if (!op_ret) {
-                        inode_unref (inode);
-                } else {
-                        gf_uuid_unparse(inode->gfid, gfid);
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_LOCK_INODE_UNREF_FAILED,
-                                "Unlock request failed for gfid %s."
-                                "Failed to unref the inode", gfid);
-                        goto out;
-                }
+            if (!op_ret) {
+                inode_unref(inode);
+            } else {
+                gf_uuid_unparse(inode->gfid, gfid);
+                gf_smsg(this->name, GF_LOG_WARNING, 0,
+                        DHT_MSG_LOCK_INODE_UNREF_FAILED, "gfid=%s", gfid, NULL);
+                goto out;
+            }
         default:
-                break;
-        }
-        ret = 0;
+            break;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* Code to update custom extended attributes from src dict to dst dict
-*/
+ */
 void
-dht_dir_set_heal_xattr (xlator_t *this, dht_local_t *local, dict_t *dst,
-                        dict_t *src, int *uret, int *uflag)
+dht_dir_set_heal_xattr(xlator_t *this, dht_local_t *local, dict_t *dst,
+                       dict_t *src, int *uret, int *uflag)
 {
-        int               ret                 = -1;
-        data_t           *keyval              = NULL;
-        int               luret               = -1;
-        int               luflag              = -1;
-        int               i                   = 0;
-
-        if (!src || !dst) {
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "src or dst is NULL. Failed to set "
-                        " dictionary value for path %s",
-                        local->loc.path);
-                return;
-        }
-        /* Check if any user xattr present in src dict and set
-           it to dst dict
-        */
-        luret = dict_foreach_fnmatch (src, "user.*",
-                                      dht_set_user_xattr, dst);
-        /* Check if any other custom xattr present in src dict
-           and set it to dst dict, here index start from 1 because
-           user xattr already checked in previous statement
-        */
-        for (i = 1; xattrs_to_heal[i]; i++) {
-                keyval = dict_get (src, xattrs_to_heal[i]);
-                if (keyval) {
-                        luflag = 1;
-                        ret = dict_set (dst, xattrs_to_heal[i], keyval);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "Failed to set dictionary value:key = %s for "
-                                        "path %s", xattrs_to_heal[i],
-                                        local->loc.path);
-                        keyval = NULL;
-                }
-        }
-        if (uret)
-                (*uret) = luret;
-        if (uflag)
-                (*uflag) = luflag;
+    int ret = -1;
+    data_t *keyval = NULL;
+    int luret = -1;
+    int luflag = -1;
+    int i = 0;
+    char **xattrs_to_heal;
+
+    if (!src || !dst) {
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_DST_NULL_SET_FAILED,
+                "path=%s", local->loc.path, NULL);
+        return;
+    }
+    /* Check if any user xattr present in src dict and set
+       it to dst dict
+    */
+    luret = dict_foreach_fnmatch(src, "user.*", dht_set_user_xattr, dst);
+    /* Check if any other custom xattr present in src dict
+       and set it to dst dict, here index start from 1 because
+       user xattr already checked in previous statement
+    */
+
+    xattrs_to_heal = get_xattrs_to_heal();
+
+    for (i = 1; xattrs_to_heal[i]; i++) {
+        keyval = dict_get(src, xattrs_to_heal[i]);
+        if (keyval) {
+            luflag = 1;
+            ret = dict_set(dst, xattrs_to_heal[i], keyval);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_WARNING, ENOMEM,
+                        DHT_MSG_DICT_SET_FAILED, "key=%s", xattrs_to_heal[i],
+                        "path=%s", local->loc.path, NULL);
+            keyval = NULL;
+        }
+    }
+    if (uret)
+        (*uret) = luret;
+    if (uflag)
+        (*uflag) = luflag;
 }
diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
index d1895eb2abb..dbb8070b0da 100644
--- a/xlators/cluster/dht/src/dht-inode-read.c
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -10,1661 +10,1649 @@
 
 #include "dht-common.h"
 
-int dht_access2 (xlator_t *this, xlator_t *dst_node,
-                 call_frame_t *frame, int ret);
-int dht_readv2 (xlator_t *this, xlator_t *dst_node,
-                call_frame_t *frame, int ret);
-int dht_attr2 (xlator_t *this, xlator_t *dst_node,
-               call_frame_t *frame, int ret);
-int dht_open2 (xlator_t *this, xlator_t *dst_node,
-               call_frame_t *frame, int ret);
-int dht_flush2 (xlator_t *this, xlator_t *dst_node,
-                call_frame_t *frame, int ret);
-int dht_lk2 (xlator_t *this, xlator_t *dst_node,
-             call_frame_t *frame, int ret);
-int dht_fsync2 (xlator_t *this, xlator_t *dst_node,
-                call_frame_t *frame, int ret);
-int
-dht_common_xattrop2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame,
-                     int ret);
-
-int
-dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+static int
+dht_access2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_readv2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_attr2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_open2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_flush2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_lk2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_fsync2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_common_xattrop2(xlator_t *this, xlator_t *subvol, call_frame_t *frame,
+                    int ret);
+
+static int
+dht_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, fd_t *fd, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *prev = NULL;
-        int           ret = 0;
-
-        local = frame->local;
-        prev = cookie;
-
-        local->op_errno = op_errno;
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->name);
-                goto out;
-        }
-
-        /* Update ctx if the fd has been opened on the target*/
-        if (!op_ret && (local->call_cnt == 1)) {
-                dht_fd_ctx_set (this, fd, prev);
-                goto out;
-        }
-
-        if (!op_ret || (local->call_cnt != 1))
-                goto out;
-
-        /* rebalance would have happened */
-        local->rebalance.target_op_fn = dht_open2;
-        ret = dht_rebalance_complete_check (this, frame);
-        if (!ret)
-                return 0;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    prev = cookie;
+
+    local->op_errno = op_errno;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    /* Update ctx if the fd has been opened on the target*/
+    if (!op_ret && (local->call_cnt == 1)) {
+        dht_fd_ctx_set(this, fd, prev);
+        goto out;
+    }
+
+    if (!op_ret || (local->call_cnt != 1))
+        goto out;
+
+    /* rebalance would have happened */
+    local->rebalance.target_op_fn = dht_open2;
+    ret = dht_rebalance_complete_check(this, frame);
+    if (!ret)
+        return 0;
 
 out:
-        DHT_STACK_UNWIND (open, frame, op_ret, op_errno, local->fd, xdata);
+    DHT_STACK_UNWIND(open, frame, op_ret, op_errno, local->fd, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_open2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_open2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int          op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        if (!frame || !frame->local)
-                goto out;
+    if (!frame || !frame->local)
+        goto out;
 
-        local = frame->local;
-        op_errno = ENOENT;
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        if (we_are_not_migrating (ret)) {
-                /* This DHT layer is not migrating the file */
-                DHT_STACK_UNWIND (open, frame, -1, local->op_errno,
-                                  NULL, local->rebalance.xdata);
-                return 0;
+    if (we_are_not_migrating(ret)) {
+        /* This DHT layer is not migrating the file */
+        DHT_STACK_UNWIND(open, frame, -1, local->op_errno, NULL,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        }
+    if (subvol == NULL)
+        goto out;
 
-        if (subvol == NULL)
-                goto out;
+    local->call_cnt = 2;
 
-        local->call_cnt = 2;
-
-        STACK_WIND_COOKIE (frame, dht_open_cbk, subvol, subvol,
-                           subvol->fops->open, &local->loc,
-                           local->rebalance.flags, local->fd, local->xattr_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_open_cbk, subvol, subvol, subvol->fops->open,
+                      &local->loc, local->rebalance.flags, local->fd,
+                      local->xattr_req);
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(open, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
-
 int
-dht_open (call_frame_t *frame, xlator_t *this,
-          loc_t *loc, int flags, fd_t *fd, dict_t *xdata)
+dht_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd,
+         dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, loc, fd, GF_FOP_OPEN);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local->rebalance.flags = flags;
-        local->call_cnt = 1;
+    local = dht_local_init(frame, loc, fd, GF_FOP_OPEN);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        STACK_WIND_COOKIE (frame, dht_open_cbk, subvol, subvol,
-                           subvol->fops->open, loc, flags, fd, xdata);
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
 
-        return 0;
+    local->rebalance.flags = flags;
+    local->call_cnt = 1;
+
+    STACK_WIND_COOKIE(frame, dht_open_cbk, subvol, subvol, subvol->fops->open,
+                      loc, flags, fd, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(open, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
+dht_file_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *stbuf, dict_t *xdata)
 {
-        xlator_t     *subvol1 = 0;
-        xlator_t     *subvol2 = 0;
-        dht_local_t  *local = NULL;
-        xlator_t     *prev = NULL;
-        int           ret = -1;
-        inode_t      *inode = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-        if ((local->fop == GF_FOP_FSTAT) && (op_ret == -1)
-             && (op_errno == EBADF) && !(local->fd_checked)) {
-
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto out;
-                return 0;
-        }
-
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->name);
-                goto out;
-        }
-
-        if (local->call_cnt != 1)
-                goto out;
+    xlator_t *subvol1 = 0;
+    xlator_t *subvol2 = 0;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    if ((local->fop == GF_FOP_FSTAT) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
         local->op_errno = op_errno;
-        local->op_ret = op_ret;
-
-        /* Check if the rebalance phase2 is true */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-
-                local->rebalance.target_op_fn = dht_attr2;
-                dht_set_local_rebalance (this, local, NULL, NULL,
-                                         stbuf, xdata);
-                inode = (local->fd) ? local->fd->inode : local->loc.inode;
-
-                dht_inode_ctx_get_mig_info (this, inode, &subvol1, &subvol2);
-                if (dht_mig_info_is_invalid (local->cached_subvol,
-                                             subvol1, subvol2)){
-                        /* Phase 2 of migration */
-                        ret = dht_rebalance_complete_check (this, frame);
-                        if (!ret)
-                                return 0;
-                } else {
-                        /* it is a non-fd op or it is an fd based Fop and
-                           opened on the dst.*/
-                       if (local->fd &&
-                           !dht_fd_open_on_dst (this, local->fd, subvol2)) {
-                                ret = dht_rebalance_complete_check (this, frame);
-                                if (!ret)
-                                        return 0;
-                        } else {
-                                dht_attr2 (this, subvol2, frame, 0);
-                                return 0;
-                        }
-                }
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1)
+        goto out;
+
+    local->op_errno = op_errno;
+    local->op_ret = op_ret;
+
+    /* Check if the rebalance phase2 is true */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        local->rebalance.target_op_fn = dht_attr2;
+        dht_set_local_rebalance(this, local, NULL, NULL, stbuf, xdata);
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get_mig_info(this, inode, &subvol1, &subvol2);
+        if (dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            /* Phase 2 of migration */
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            /* it is a non-fd op or it is an fd based Fop and
+               opened on the dst.*/
+            if (local->fd && !dht_fd_open_on_dst(this, local->fd, subvol2)) {
+                ret = dht_rebalance_complete_check(this, frame);
+                if (!ret)
+                    return 0;
+            } else {
+                dht_attr2(this, subvol2, frame, 0);
+                return 0;
+            }
         }
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (stat, frame, op_ret, op_errno, stbuf, xdata);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    DHT_STACK_UNWIND(stat, frame, op_ret, op_errno, stbuf, xdata);
 err:
-        return 0;
+    return 0;
 }
 
-int
-dht_attr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_attr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int          op_errno = EINVAL;
-
-        local = frame->local;
-        if (!local)
-                goto out;
-
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (stat, frame, local->op_ret, op_errno,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
-                return 0;
-        }
-
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(stat, frame, local->op_ret, op_errno,
+                         &local->rebalance.postbuf, local->rebalance.xdata);
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2;
+    local->call_cnt = 2;
 
-        if (local->fop == GF_FOP_FSTAT) {
-                STACK_WIND_COOKIE (frame, dht_file_attr_cbk, subvol, subvol,
-                                   subvol->fops->fstat, local->fd,
-                                   local->xattr_req);
-        } else {
-                STACK_WIND_COOKIE (frame, dht_file_attr_cbk, subvol, subvol,
-                                   subvol->fops->stat, &local->loc,
-                                   local->xattr_req);
-        }
+    if (local->fop == GF_FOP_FSTAT) {
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->fstat, local->fd, local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->stat, &local->loc, local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
-int
-dht_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
+static int
+dht_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, struct iatt *stbuf, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        xlator_t     *prev = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "subvolume %s returned -1",
-                                      prev->name);
-
-                        goto unlock;
-                }
-
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev);
-
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-out:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
-                                  &local->stbuf, xdata);
-        }
-err:
-        return 0;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    local = frame->local;
+    prev = cookie;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+
+            goto post_unlock;
+        }
+
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(stat, frame, local->op_ret, local->op_errno,
+                         &local->stbuf, xdata);
+    }
+
+    return 0;
 }
 
 int
-dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+dht_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_STAT);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
-
-        if (IA_ISREG (loc->inode->ia_type)) {
-                local->call_cnt = 1;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_STAT);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(loc->inode->ia_type)) {
+        local->call_cnt = 1;
 
-                subvol = local->cached_subvol;
+        subvol = local->cached_subvol;
 
-                STACK_WIND_COOKIE (frame, dht_file_attr_cbk, subvol, subvol,
-                                   subvol->fops->stat, loc, xdata);
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->stat, loc, xdata);
 
-                return 0;
-        }
+        return 0;
+    }
 
-        local->call_cnt = call_cnt = layout->cnt;
+    local->call_cnt = call_cnt = layout->cnt;
 
-        for (i = 0; i < call_cnt; i++) {
-                subvol = layout->list[i].xlator;
+    for (i = 0; i < call_cnt; i++) {
+        subvol = layout->list[i].xlator;
 
-                STACK_WIND_COOKIE (frame, dht_attr_cbk, subvol, subvol,
-                                   subvol->fops->stat, loc, xdata);
-        }
+        STACK_WIND_COOKIE(frame, dht_attr_cbk, subvol, subvol,
+                          subvol->fops->stat, loc, xdata);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+dht_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSTAT);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        layout = local->layout;
-        if (!layout) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                        "no layout for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
-
-        if (IA_ISREG (fd->inode->ia_type)) {
-                local->call_cnt = 1;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSTAT);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "no layout for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(fd->inode->ia_type)) {
+        local->call_cnt = 1;
 
-                subvol = local->cached_subvol;
+        subvol = local->cached_subvol;
 
-                STACK_WIND_COOKIE (frame, dht_file_attr_cbk, subvol,
-                                   subvol, subvol->fops->fstat, fd,
-                                   xdata);
-                return 0;
-        }
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->fstat, fd, xdata);
+        return 0;
+    }
 
-        local->call_cnt = call_cnt = layout->cnt;
+    local->call_cnt = call_cnt = layout->cnt;
 
-        for (i = 0; i < call_cnt; i++) {
-                subvol = layout->list[i].xlator;
-                STACK_WIND_COOKIE (frame, dht_attr_cbk, subvol, subvol,
-                                   subvol->fops->fstat, fd, xdata);
-        }
+    for (i = 0; i < call_cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND_COOKIE(frame, dht_attr_cbk, subvol, subvol,
+                          subvol->fops->fstat, fd, xdata);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno,
-               struct iovec *vector, int count, struct iatt *stbuf,
-               struct iobref *iobref, dict_t *xdata)
+dht_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iovec *vector, int count, struct iatt *stbuf,
+              struct iobref *iobref, dict_t *xdata)
 {
-        dht_local_t *local      = NULL;
-        int          ret        = 0;
-        xlator_t    *src_subvol = 0;
-        xlator_t    *dst_subvol = 0;
-
-        local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    dht_local_t *local = NULL;
+    int ret = 0;
+    xlator_t *src_subvol = 0;
+    xlator_t *dst_subvol = 0;
+
+    local = frame->local;
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /* This is already second try, no need for re-check */
+    if (local->call_cnt != 1)
+        goto out;
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        /* This is already second try, no need for re-check */
-        if (local->call_cnt != 1)
-                goto out;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno))
+        goto out;
 
-        if (op_ret == -1 && (op_errno == EBADF) && !(local->fd_checked)) {
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto out;
+    local->op_errno = op_errno;
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        local->op_ret = op_ret;
+        local->rebalance.target_op_fn = dht_readv2;
+        dht_set_local_rebalance(this, local, NULL, NULL, stbuf, xdata);
+        /* File would be migrated to other node */
+        ret = dht_inode_ctx_get_mig_info(this, local->fd->inode, &src_subvol,
+                                         &dst_subvol);
+
+        if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                    dst_subvol) ||
+            !dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret)
                 return 0;
+        } else {
+            /* value is already set in fd_ctx, that means no need
+               to check for whether its complete or not. */
+            dht_readv2(this, dst_subvol, frame, 0);
+            return 0;
         }
-
-        if ((op_ret == -1) && !dht_inode_missing(op_errno))
-                goto out;
-
-
-        local->op_errno = op_errno;
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-
-                local->op_ret = op_ret;
-                local->rebalance.target_op_fn = dht_readv2;
-                dht_set_local_rebalance (this, local, NULL, NULL,
-                                         stbuf, xdata);
-                /* File would be migrated to other node */
-                ret = dht_inode_ctx_get_mig_info (this, local->fd->inode,
-                                                  &src_subvol,
-                                                  &dst_subvol);
-
-                if (dht_mig_info_is_invalid (local->cached_subvol,
-                                             src_subvol, dst_subvol)
-                        || !dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
-
-                        ret = dht_rebalance_complete_check (this, frame);
-                        if (!ret)
-                                return 0;
-                } else {
-                        /* value is already set in fd_ctx, that means no need
-                           to check for whether its complete or not. */
-                        dht_readv2 (this, dst_subvol, frame, 0);
-                        return 0;
-                }
-        }
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
 
-        DHT_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count, stbuf,
-                          iobref, xdata);
+    DHT_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                     iobref, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_readv2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_readv2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local  = NULL;
-        int          op_errno = EINVAL;
-
-        local = frame->local;
-        if (!local)
-                goto out;
-
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (readv, frame, local->op_ret, op_errno,
-                                  NULL, 0, &local->rebalance.postbuf,
-                                  NULL, local->rebalance.xdata);
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(readv, frame, local->op_ret, op_errno, NULL, 0,
+                         &local->rebalance.postbuf, NULL,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2;
+    local->call_cnt = 2;
 
-        STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,
-                    local->fd, local->rebalance.size, local->rebalance.offset,
-                    local->rebalance.flags, local->xattr_req);
+    STACK_WIND(frame, dht_readv_cbk, subvol, subvol->fops->readv, local->fd,
+               local->rebalance.size, local->rebalance.offset,
+               local->rebalance.flags, local->xattr_req);
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int
-dht_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t off, uint32_t flags, dict_t *xdata)
+dht_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, off_t off,
+          uint32_t flags, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_READ);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
+    local = dht_local_init(frame, NULL, fd, GF_FOP_READ);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        local->rebalance.offset = off;
-        local->rebalance.size   = size;
-        local->rebalance.flags  = flags;
-        local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,
-                    local->fd, local->rebalance.size,
-                    local->rebalance.offset,
-                    local->rebalance.flags, local->xattr_req);
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
 
-        return 0;
+    local->rebalance.offset = off;
+    local->rebalance.size = size;
+    local->rebalance.flags = flags;
+    local->call_cnt = 1;
+
+    STACK_WIND(frame, dht_readv_cbk, subvol, subvol->fops->readv, local->fd,
+               local->rebalance.size, local->rebalance.offset,
+               local->rebalance.flags, local->xattr_req);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, dict_t *xdata)
 {
-        int          ret = -1;
-        dht_local_t *local = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *prev = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        if (!prev)
-                goto out;
-        if (local->call_cnt != 1)
-                goto out;
-        if ((op_ret == -1) && ((op_errno == ENOTCONN) ||
-                dht_inode_missing(op_errno)) &&
-                IA_ISDIR(local->loc.inode->ia_type)) {
-                subvol = dht_subvol_next_available (this, prev);
-                if (!subvol)
-                        goto out;
-
-                /* check if we are done with visiting every node */
-                if (subvol == local->cached_subvol) {
-                        goto out;
-                }
-
-                STACK_WIND_COOKIE (frame, dht_access_cbk, subvol, subvol,
-                                   subvol->fops->access, &local->loc,
-                                   local->rebalance.flags, NULL);
-                return 0;
-        }
-        if ((op_ret == -1) && dht_inode_missing(op_errno) &&
-                !(IA_ISDIR(local->loc.inode->ia_type))) {
-                /* File would be migrated to other node */
-                local->op_errno = op_errno;
-                local->rebalance.target_op_fn = dht_access2;
-                ret = dht_rebalance_complete_check (frame->this, frame);
-                if (!ret)
-                        return 0;
-        }
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *prev = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    if (!prev)
+        goto out;
+    if (local->call_cnt != 1)
+        goto out;
+    if ((op_ret == -1) &&
+        ((op_errno == ENOTCONN) || dht_inode_missing(op_errno)) &&
+        IA_ISDIR(local->loc.inode->ia_type)) {
+        subvol = dht_subvol_next_available(this, prev);
+        if (!subvol)
+            goto out;
+
+        /* check if we are done with visiting every node */
+        if (subvol == local->cached_subvol) {
+            goto out;
+        }
+
+        STACK_WIND_COOKIE(frame, dht_access_cbk, subvol, subvol,
+                          subvol->fops->access, &local->loc,
+                          local->rebalance.flags, NULL);
+        return 0;
+    }
+    if ((op_ret == -1) && dht_inode_missing(op_errno) &&
+        !(IA_ISDIR(local->loc.inode->ia_type))) {
+        /* File would be migrated to other node */
+        local->op_errno = op_errno;
+        local->rebalance.target_op_fn = dht_access2;
+        ret = dht_rebalance_complete_check(frame->this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
-        return 0;
+    DHT_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-int
-dht_access2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_access2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int          op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        local = frame->local;
-        if (!local)
-                goto out;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        op_errno = local->op_errno;
+    op_errno = local->op_errno;
 
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
 
-                DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
-                return 0;
-        }
+        DHT_STACK_UNWIND(access, frame, -1, op_errno, NULL);
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2;
+    local->call_cnt = 2;
 
-        STACK_WIND_COOKIE (frame, dht_access_cbk, subvol, subvol,
-                           subvol->fops->access, &local->loc,
-                           local->rebalance.flags, local->xattr_req);
+    STACK_WIND_COOKIE(frame, dht_access_cbk, subvol, subvol,
+                      subvol->fops->access, &local->loc, local->rebalance.flags,
+                      local->xattr_req);
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
-        return 0;
+    DHT_STACK_UNWIND(access, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-
 int
-dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
-            dict_t *xdata)
+dht_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_ACCESS);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->rebalance.flags = mask;
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
-
-        STACK_WIND_COOKIE (frame, dht_access_cbk, subvol, subvol,
-                           subvol->fops->access, loc, mask, xdata);
-
-        return 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_ACCESS);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.flags = mask;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_access_cbk, subvol, subvol,
+                      subvol->fops->access, loc, mask, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(access, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, dict_t *xdata)
+dht_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, dict_t *xdata)
 {
-        dht_local_t     *local  = NULL;
-        xlator_t        *subvol = 0;
-        int             ret = 0;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = 0;
+    int ret = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->op_errno = op_errno;
+    local->op_errno = op_errno;
 
-        if (local->call_cnt != 1)
-                goto out;
+    if (local->call_cnt != 1)
+        goto out;
 
-        if (op_ret == -1 && (op_errno == EBADF) && !(local->fd_checked)) {
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto out;
-                return 0;
-        }
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local->rebalance.target_op_fn = dht_flush2;
+    local->rebalance.target_op_fn = dht_flush2;
 
-        local->op_ret = op_ret;
-        local->op_errno = op_errno;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        /* If context is set, then send flush() it to the destination */
-        dht_inode_ctx_get_mig_info (this, local->fd->inode, NULL, &subvol);
-        if (subvol && dht_fd_open_on_dst (this, local->fd, subvol)) {
-                dht_flush2 (this, subvol, frame, 0);
-                return 0;
-        }
+    /* If context is set, then send flush() it to the destination */
+    dht_inode_ctx_get_mig_info(this, local->fd->inode, NULL, &subvol);
+    if (subvol && dht_fd_open_on_dst(this, local->fd, subvol)) {
+        dht_flush2(this, subvol, frame, 0);
+        return 0;
+    }
 
-        if (op_errno == EREMOTE) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret) {
-                        return 0;
-                }
+    if (op_errno == EREMOTE) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret) {
+            return 0;
         }
+    }
 
 out:
-        DHT_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
+    DHT_STACK_UNWIND(flush, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_flush2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_flush2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if ((frame == NULL) || (frame->local == NULL))
-                goto out;
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
 
-        local = frame->local;
+    local = frame->local;
 
-        op_errno = local->op_errno;
+    op_errno = local->op_errno;
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    local->call_cnt = 2; /* This is the second attempt */
 
-        STACK_WIND (frame, dht_flush_cbk,
-                    subvol, subvol->fops->flush, local->fd,
-                    local->xattr_req);
+    STACK_WIND(frame, dht_flush_cbk, subvol, subvol->fops->flush, local->fd,
+               local->xattr_req);
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
-        return 0;
+    DHT_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-
 int
-dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+dht_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FLUSH);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FLUSH);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_flush_cbk,
-                    subvol, subvol->fops->flush, fd, local->xattr_req);
-        return 0;
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    local->call_cnt = 1;
+
+    STACK_WIND(frame, dht_flush_cbk, subvol, subvol->fops->flush, fd,
+               local->xattr_req);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
-               dict_t *xdata)
+dht_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *prev = NULL;
-        int           ret = -1;
-        inode_t      *inode = NULL;
-        xlator_t     *src_subvol = 0;
-        xlator_t     *dst_subvol = 0;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
+    xlator_t *src_subvol = 0;
+    xlator_t *dst_subvol = 0;
 
-        local = frame->local;
-        prev = cookie;
+    local = frame->local;
+    prev = cookie;
 
-        local->op_errno = op_errno;
+    local->op_errno = op_errno;
 
-        if (op_ret == -1 && (op_errno == EBADF) && !(local->fd_checked)) {
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto out;
-                return 0;
-        }
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        if (op_ret == -1 && !dht_inode_missing(op_errno)) {
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->name);
-                goto out;
-        }
+    if (op_ret == -1 && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
 
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
+        goto out;
+    }
 
-        local->op_ret = op_ret;
-        inode = local->fd->inode;
-
-        local->rebalance.target_op_fn = dht_fsync2;
-        dht_set_local_rebalance (this, local, NULL, prebuf,
-                                 postbuf, xdata);
-
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+    local->op_ret = op_ret;
+    inode = local->fd->inode;
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+    local->rebalance.target_op_fn = dht_fsync2;
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-                dht_inode_ctx_get_mig_info (this, inode, &src_subvol, &dst_subvol);
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
 
-                if (dht_mig_info_is_invalid (local->cached_subvol, src_subvol,
-                                     dst_subvol) ||
-                      !dht_fd_open_on_dst (this, local->fd, dst_subvol)) {
+        dht_inode_ctx_get_mig_info(this, inode, &src_subvol, &dst_subvol);
 
-                        ret = dht_rebalance_in_progress_check (this, frame);
-                        if (!ret)
-                                return 0;
-                } else {
-                        dht_fsync2 (this, dst_subvol, frame, 0);
-                        return 0;
-                }
+        if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                    dst_subvol) ||
+            !dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+            ret = dht_rebalance_in_progress_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            dht_fsync2(this, dst_subvol, frame, 0);
+            return 0;
         }
-
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (fsync, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STACK_UNWIND(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_fsync2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_fsync2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
-
-        if ((frame == NULL) || (frame->local == NULL))
-                goto out;
-
-        local = frame->local;
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (fsync, frame, local->op_ret,
-                                  op_errno, &local->rebalance.prebuf,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(fsync, frame, local->op_ret, op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    local->call_cnt = 2; /* This is the second attempt */
 
-        STACK_WIND_COOKIE (frame, dht_fsync_cbk, subvol, subvol,
-                           subvol->fops->fsync, local->fd,
-                           local->rebalance.flags, local->xattr_req);
+    STACK_WIND_COOKIE(frame, dht_fsync_cbk, subvol, subvol, subvol->fops->fsync,
+                      local->fd, local->rebalance.flags, local->xattr_req);
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
-           dict_t *xdata)
+dht_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+          dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSYNC);
-        if (!local) {
-                op_errno = ENOMEM;
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSYNC);
+    if (!local) {
+        op_errno = ENOMEM;
 
-                goto err;
-        }
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
 
-        local->call_cnt = 1;
-        local->rebalance.flags = datasync;
+    local->call_cnt = 1;
+    local->rebalance.flags = datasync;
 
-        subvol = local->cached_subvol;
+    subvol = local->cached_subvol;
 
-        STACK_WIND_COOKIE (frame, dht_fsync_cbk, subvol, subvol,
-                           subvol->fops->fsync, local->fd,
-                           local->rebalance.flags, local->xattr_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_fsync_cbk, subvol, subvol, subvol->fops->fsync,
+                      local->fd, local->rebalance.flags, local->xattr_req);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* TODO: for 'lk()' call, we need some other special error, may be ESTALE to
    indicate that lock migration happened on the fd, so we can consider it as
    phase 2 of migration */
-int
-dht_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int op_ret, int op_errno, struct gf_flock *flock, dict_t *xdata)
+static int
+dht_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+           int op_errno, struct gf_flock *flock, dict_t *xdata)
 {
-        dht_local_t     *local  = NULL;
-        int             ret     = -1;
-        xlator_t        *subvol = NULL;
+    dht_local_t *local = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        if (local->call_cnt != 1)
-                goto out;
+    if (local->call_cnt != 1)
+        goto out;
 
-        local->rebalance.target_op_fn = dht_lk2;
+    local->rebalance.target_op_fn = dht_lk2;
 
-        local->op_ret = op_ret;
-        local->op_errno = op_errno;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        if (xdata)
-                local->rebalance.xdata = dict_ref (xdata);
-
-        if (op_errno == EREMOTE) {
-                dht_inode_ctx_get_mig_info (this, local->fd->inode,
-                                            NULL, &subvol);
-                if (subvol && dht_fd_open_on_dst (this, local->fd, subvol)) {
-                        dht_lk2 (this, subvol, frame, 0);
-                        return 0;
-                } else {
-                        ret = dht_rebalance_complete_check (this, frame);
-                        if (!ret) {
-                                return 0;
-                        }
-                }
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
+
+    if (op_errno == EREMOTE) {
+        dht_inode_ctx_get_mig_info(this, local->fd->inode, NULL, &subvol);
+        if (subvol && dht_fd_open_on_dst(this, local->fd, subvol)) {
+            dht_lk2(this, subvol, frame, 0);
+            return 0;
+        } else {
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret) {
+                return 0;
+            }
         }
+    }
 
 out:
-        dht_lk_inode_unref (frame, op_ret);
-        DHT_STACK_UNWIND (lk, frame, op_ret, op_errno, flock, xdata);
+    dht_lk_inode_unref(frame, op_ret);
+    DHT_STACK_UNWIND(lk, frame, op_ret, op_errno, flock, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_lk2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_lk2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if ((frame == NULL) || (frame->local == NULL))
-                goto out;
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
 
-        local = frame->local;
+    local = frame->local;
 
-        op_errno = local->op_errno;
+    op_errno = local->op_errno;
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    local->call_cnt = 2; /* This is the second attempt */
 
-        STACK_WIND (frame, dht_lk_cbk, subvol, subvol->fops->lk, local->fd,
-                    local->rebalance.lock_cmd, &local->rebalance.flock,
-                    local->xattr_req);
+    STACK_WIND(frame, dht_lk_cbk, subvol, subvol->fops->lk, local->fd,
+               local->rebalance.lock_cmd, &local->rebalance.flock,
+               local->xattr_req);
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL,  NULL);
-        return 0;
+    DHT_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
 int
-dht_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int cmd, struct gf_flock *flock, dict_t *xdata)
+dht_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
+       struct gf_flock *flock, dict_t *xdata)
 {
-        xlator_t        *lock_subvol    = NULL;
-        int             op_errno        = -1;
-        dht_local_t     *local          = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_LK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->lock_type = flock->l_type;
-        lock_subvol = dht_get_lock_subvolume (this, flock, local);
-        if (!lock_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no lock subvolume for path=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-/*
-        local->cached_subvol = lock_subvol;
-        ret = dht_check_and_open_fd_on_subvol (this, frame);
-        if (ret)
-                goto err;
-*/
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
-
-        local->rebalance.flock = *flock;
-        local->rebalance.lock_cmd = cmd;
-
-        local->call_cnt = 1;
-
-        STACK_WIND (frame, dht_lk_cbk, lock_subvol, lock_subvol->fops->lk, fd,
-                    cmd, flock, xdata);
-
-        return 0;
+    xlator_t *lock_subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_LK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->lock_type = flock->l_type;
+    lock_subvol = dht_get_lock_subvolume(this, flock, local);
+    if (!lock_subvol) {
+        gf_msg_debug(this->name, 0, "no lock subvolume for path=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /*
+            local->cached_subvol = lock_subvol;
+            ret = dht_check_and_open_fd_on_subvol (this, frame);
+            if (ret)
+                    goto err;
+    */
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    local->rebalance.flock = *flock;
+    local->rebalance.lock_cmd = cmd;
+
+    local->call_cnt = 1;
+
+    STACK_WIND(frame, dht_lk_cbk, lock_subvol, lock_subvol->fops->lk, fd, cmd,
+               flock, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_lease_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, struct gf_lease *lease, dict_t *xdata)
+static int
+dht_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct gf_lease *lease, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (lease, frame, op_ret, op_errno, lease, xdata);
+    DHT_STACK_UNWIND(lease, frame, op_ret, op_errno, lease, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_lease (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, struct gf_lease *lease, dict_t *xdata)
+dht_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        subvol = dht_subvol_get_cached (this, loc->inode);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
 
-        /* TODO: for rebalance, we need to preserve the fop arguments */
-        STACK_WIND (frame, dht_lease_cbk, subvol, subvol->fops->lease,
-                    loc, lease, xdata);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
 
-        return 0;
+    subvol = dht_subvol_get_cached(this, loc->inode);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* TODO: for rebalance, we need to preserve the fop arguments */
+    STACK_WIND(frame, dht_lease_cbk, subvol, subvol->fops->lease, loc, lease,
+               xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lease, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lease, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* Symlinks are currently not migrated, so no need for any check here */
-int
-dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, const char *path,
-                  struct iatt *stbuf, dict_t *xdata)
+static int
+dht_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, const char *path, struct iatt *stbuf,
+                 dict_t *xdata)
 {
-        dht_local_t *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
-        if (op_ret == -1)
-                goto err;
+    local = frame->local;
+    if (op_ret == -1)
+        goto err;
 
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-        }
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
 err:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf, xdata);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    DHT_STACK_UNWIND(readlink, frame, op_ret, op_errno, path, stbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-              dict_t *xdata)
+dht_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_READLINK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        STACK_WIND (frame, dht_readlink_cbk,
-                    subvol, subvol->fops->readlink,
-                    loc, size, xdata);
-
-        return 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_READLINK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    STACK_WIND(frame, dht_readlink_cbk, subvol, subvol->fops->readlink, loc,
+               size, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(readlink, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_common_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *dict,
-                        dict_t *xdata)
+/* Get both DHT_IATT_IN_XDATA_KEY and DHT_MODE_IN_XDATA_KEY
+ * Use DHT_MODE_IN_XDATA_KEY if available, else fall back to
+ * DHT_IATT_IN_XDATA_KEY
+ * This will return a dummy iatt with only the mode and type set
+ */
+static int
+dht_read_iatt_from_xdata(dict_t *xdata, struct iatt *stbuf)
 {
-        dht_local_t  *local          = NULL;
-        call_frame_t *call_frame     = NULL;
-        xlator_t     *prev           = NULL;
-        xlator_t     *src_subvol     = NULL;
-        xlator_t     *dst_subvol     = NULL;
-        struct iatt   stbuf          = {0,};
-        int           ret            = -1;
-        inode_t      *inode          = NULL;
-
-        local = frame->local;
-        call_frame = cookie;
-        prev = call_frame->this;
+    int ret = -1;
+    int32_t mode = 0;
 
-        local->op_errno = op_errno;
-
-        if ((op_ret == -1) && !dht_inode_missing (op_errno)) {
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1.",
-                              prev->name);
-                goto out;
-        }
+    ret = dict_get_int32(xdata, DHT_MODE_IN_XDATA_KEY, &mode);
 
-        if (local->call_cnt != 1)
-                goto out;
+    if (ret) {
+        ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
+    } else {
+        stbuf->ia_prot = ia_prot_from_st_mode(mode);
+        stbuf->ia_type = ia_type_from_st_mode(mode);
+    }
 
-        ret = dht_read_iatt_from_xdata (this, xdata, &stbuf);
-
-        if ((!op_ret) && (ret)) {
-                /* This is a potential problem and can cause corruption
-                 * with sharding.
-                 * Oh well. We tried.
-                 */
-                goto out;
-        }
-
-        local->op_ret = op_ret;
-        local->rebalance.target_op_fn = dht_common_xattrop2;
-        if (xdata)
-                local->rebalance.xdata = dict_ref (xdata);
+    return ret;
+}
 
-        if (dict)
-                local->rebalance.dict = dict_ref (dict);
+int
+dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    call_frame_t *call_frame = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int ret = -1;
+    inode_t *inode = NULL;
+
+    local = frame->local;
+    call_frame = cookie;
+    prev = call_frame->this;
+
+    local->op_errno = op_errno;
+
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1.",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1)
+        goto out;
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (&stbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+    ret = dht_read_iatt_from_xdata(xdata, &stbuf);
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (&stbuf)) {
+    if ((!op_ret) && (ret)) {
+        /* This is a potential problem and can cause corruption
+         * with sharding.
+         * Oh well. We tried.
+         */
+        goto out;
+    }
 
-                inode = local->loc.inode ? local->loc.inode : local->fd->inode;
-                dht_inode_ctx_get_mig_info (this, inode, &src_subvol,
-                                            &dst_subvol);
+    local->op_ret = op_ret;
+    local->rebalance.target_op_fn = dht_common_xattrop2;
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
 
-                if (dht_mig_info_is_invalid (local->cached_subvol, src_subvol,
-                                             dst_subvol) ||
-                      !dht_fd_open_on_dst (this, local->fd, dst_subvol)) {
+    if (dict)
+        local->rebalance.dict = dict_ref(dict);
 
-                        ret = dht_rebalance_in_progress_check (this, frame);
-                        if (!ret)
-                                return 0;
-                } else {
-                        dht_common_xattrop2 (this, dst_subvol, frame, 0);
-                        return 0;
-                }
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(&stbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(&stbuf)) {
+        inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+        dht_inode_ctx_get_mig_info(this, inode, &src_subvol, &dst_subvol);
+
+        if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                    dst_subvol) ||
+            !dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+            ret = dht_rebalance_in_progress_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            dht_common_xattrop2(this, dst_subvol, frame, 0);
+            return 0;
         }
-
+    }
 
 out:
-        if (local->fop == GF_FOP_XATTROP) {
-                DHT_STACK_UNWIND (xattrop, frame, op_ret, op_errno,
-                                  dict, xdata);
-        } else {
-                DHT_STACK_UNWIND (fxattrop, frame, op_ret, op_errno,
-                                  dict, xdata);
-        }
+    if (local->fop == GF_FOP_XATTROP) {
+        DHT_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    } else {
+        DHT_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    }
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_common_xattrop2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame,
-                     int ret)
+static int
+dht_common_xattrop2(xlator_t *this, xlator_t *subvol, call_frame_t *frame,
+                    int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
-
-        if ((frame == NULL) || (frame->local == NULL))
-                goto out;
-
-        local = frame->local;
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                if (local->fop == GF_FOP_XATTROP) {
-                        DHT_STACK_UNWIND (xattrop, frame, local->op_ret,
-                                          op_errno, local->rebalance.dict,
-                                          local->rebalance.xdata);
-                } else {
-                        DHT_STACK_UNWIND (fxattrop, frame, local->op_ret,
-                                          op_errno, local->rebalance.dict,
-                                          local->rebalance.xdata);
-                }
-
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if (subvol == NULL)
-                goto out;
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    local = frame->local;
+    op_errno = local->op_errno;
 
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
         if (local->fop == GF_FOP_XATTROP) {
-                STACK_WIND (frame, dht_common_xattrop_cbk, subvol,
-                            subvol->fops->xattrop, &local->loc,
-                            local->rebalance.flags, local->rebalance.xattr,
-                            local->xattr_req);
+            DHT_STACK_UNWIND(xattrop, frame, local->op_ret, op_errno,
+                             local->rebalance.dict, local->rebalance.xdata);
         } else {
-                STACK_WIND (frame, dht_common_xattrop_cbk, subvol,
-                            subvol->fops->fxattrop, local->fd,
-                            local->rebalance.flags, local->rebalance.xattr,
-                            local->xattr_req);
+            DHT_STACK_UNWIND(fxattrop, frame, local->op_ret, op_errno,
+                             local->rebalance.dict, local->rebalance.xdata);
         }
 
         return 0;
+    }
+
+    if (subvol == NULL)
+        goto out;
+
+    local->call_cnt = 2; /* This is the second attempt */
+
+    if (local->fop == GF_FOP_XATTROP) {
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol, subvol->fops->xattrop,
+                   &local->loc, local->rebalance.flags, local->rebalance.xattr,
+                   local->xattr_req);
+    } else {
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol,
+                   subvol->fops->fxattrop, local->fd, local->rebalance.flags,
+                   local->rebalance.xattr, local->xattr_req);
+    }
+
+    return 0;
 
 out:
 
-        /* If local is unavailable we could be unwinding the wrong
-         * function here */
+    /* If local is unavailable we could be unwinding the wrong
+     * function here */
 
-        if (local && (local->fop == GF_FOP_XATTROP)) {
-                DHT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
-        } else {
-                DHT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
-        }
-        return 0;
+    if (local && (local->fop == GF_FOP_XATTROP)) {
+        DHT_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
+    } else {
+        DHT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
+    }
+    return 0;
 }
 
-
-int
-dht_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+static int
+dht_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    DHT_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
-int
-dht_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+/* Set both DHT_IATT_IN_XDATA_KEY and DHT_MODE_IN_XDATA_KEY
+ * Use DHT_MODE_IN_XDATA_KEY if available. Else fall back to
+ * DHT_IATT_IN_XDATA_KEY
+ */
+static int
+dht_request_iatt_in_xdata(dict_t *xattr_req)
 {
-        xlator_t     *subvol   = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local    = NULL;
-        int           ret      = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_XATTROP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    int ret = -1;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for gfid=%s",
-                              uuid_utoa (loc->inode->gfid));
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        /* Todo : Handle dirs as well. At the moment the only xlator above dht
-         * that uses xattrop is sharding and that is only for files */
-
-        if (IA_ISDIR (loc->inode->ia_type)) {
-                STACK_WIND (frame, dht_xattrop_cbk, subvol,
-                            subvol->fops->xattrop, loc, flags, dict, xdata);
+    ret = dict_set_int8(xattr_req, DHT_MODE_IN_XDATA_KEY, 1);
+    ret = dict_set_int8(xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
 
-        } else {
-                local->xattr_req = xdata ? dict_ref(xdata) : dict_new ();
-                local->call_cnt = 1;
+    /* At least one call succeeded */
+    return ret;
+}
 
-                local->rebalance.xattr = dict_ref (dict);
-                local->rebalance.flags = flags;
+int
+dht_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_XATTROP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for gfid=%s",
+                     uuid_utoa(loc->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* Todo : Handle dirs as well. At the moment the only xlator above dht
+     * that uses xattrop is sharding and that is only for files */
+
+    if (IA_ISDIR(loc->inode->ia_type)) {
+        STACK_WIND(frame, dht_xattrop_cbk, subvol, subvol->fops->xattrop, loc,
+                   flags, dict, xdata);
+
+    } else {
+        local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+        local->call_cnt = 1;
 
-                ret = dht_request_iatt_in_xdata (this, local->xattr_req);
+        local->rebalance.xattr = dict_ref(dict);
+        local->rebalance.flags = flags;
 
-                if (ret) {
-                        gf_msg_debug (this->name, 0,
-                                      "Failed to set dictionary key %s file=%s",
-                                      DHT_IATT_IN_XDATA_KEY, loc->path);
-                }
+        ret = dht_request_iatt_in_xdata(local->xattr_req);
 
-                STACK_WIND (frame, dht_common_xattrop_cbk, subvol,
-                            subvol->fops->xattrop, loc,
-                            local->rebalance.flags, local->rebalance.xattr,
-                            local->xattr_req);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set dictionary key %s file=%s",
+                         DHT_IATT_IN_XDATA_KEY, loc->path);
         }
 
-        return 0;
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol, subvol->fops->xattrop,
+                   loc, local->rebalance.flags, local->rebalance.xattr,
+                   local->xattr_req);
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+static int
+dht_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    DHT_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int
-dht_fxattrop (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+dht_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local    = NULL;
-        int           ret      = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        subvol = dht_subvol_get_cached (this, fd->inode);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FXATTROP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        /* Todo : Handle dirs as well. At the moment the only xlator above dht
-         * that uses xattrop is sharding and that is only for files */
-
-        if (IA_ISDIR (fd->inode->ia_type)) {
-                STACK_WIND (frame, dht_fxattrop_cbk, subvol,
-                            subvol->fops->fxattrop, fd, flags, dict, xdata);
-
-        } else {
-                local->xattr_req = xdata ? dict_ref(xdata) : dict_new ();
-                local->call_cnt = 1;
-
-                local->rebalance.xattr = dict_ref (dict);
-                local->rebalance.flags = flags;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    subvol = dht_subvol_get_cached(this, fd->inode);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FXATTROP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /* Todo : Handle dirs as well. At the moment the only xlator above dht
+     * that uses xattrop is sharding and that is only for files */
+
+    if (IA_ISDIR(fd->inode->ia_type)) {
+        STACK_WIND(frame, dht_fxattrop_cbk, subvol, subvol->fops->fxattrop, fd,
+                   flags, dict, xdata);
+
+    } else {
+        local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+        local->call_cnt = 1;
 
-                ret = dht_request_iatt_in_xdata (this, local->xattr_req);
+        local->rebalance.xattr = dict_ref(dict);
+        local->rebalance.flags = flags;
 
-                if (ret) {
-                        gf_msg_debug (this->name, 0,
-                                      "Failed to set dictionary key %s fd=%p",
-                                      DHT_IATT_IN_XDATA_KEY, fd);
-                }
+        ret = dht_request_iatt_in_xdata(local->xattr_req);
 
-                STACK_WIND (frame, dht_common_xattrop_cbk, subvol,
-                            subvol->fops->fxattrop, fd,
-                            local->rebalance.flags, local->rebalance.xattr,
-                            local->xattr_req);
+        if (ret) {
+            gf_msg_debug(this->name, 0, "Failed to set dictionary key %s fd=%p",
+                         DHT_IATT_IN_XDATA_KEY, fd);
         }
 
-        return 0;
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol,
+                   subvol->fops->fxattrop, fd, local->rebalance.flags,
+                   local->rebalance.xattr, local->xattr_req);
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* Currently no translators on top of 'distribute' will be using
  * below fops, hence not implementing 'migration' related checks
  */
 
-int
-dht_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int
+dht_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        dht_lk_inode_unref (frame, op_ret);
-        DHT_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    dht_lk_inode_unref(frame, op_ret);
+    DHT_STACK_UNWIND(inodelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int32_t
-dht_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-             loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+dht_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        xlator_t        *lock_subvol            = NULL;
-        int              op_errno               = -1;
-        dht_local_t     *local                  = NULL;
+    xlator_t *lock_subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
+    local = dht_local_init(frame, loc, NULL, GF_FOP_INODELK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_INODELK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local->lock_type = lock->l_type;
+    lock_subvol = dht_get_lock_subvolume(this, lock, local);
+    if (!lock_subvol) {
+        gf_msg_debug(this->name, 0, "no lock subvolume for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        local->lock_type = lock->l_type;
-        lock_subvol = dht_get_lock_subvolume (this, lock, local);
-        if (!lock_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no lock subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local->call_cnt = 1;
 
-        local->call_cnt = 1;
+    STACK_WIND(frame, dht_inodelk_cbk, lock_subvol, lock_subvol->fops->inodelk,
+               volume, loc, cmd, lock, xdata);
 
-        STACK_WIND (frame, dht_inodelk_cbk,
-                    lock_subvol, lock_subvol->fops->inodelk,
-                    volume, loc, cmd, lock, xdata);
-
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(inodelk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        dht_lk_inode_unref (frame, op_ret);
-        DHT_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
-        return 0;
-}
+    dht_local_t *local = NULL;
+    int ret = 0;
 
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
 
-int
-dht_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-              fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
-{
-        xlator_t     *lock_subvol       = NULL;
-        dht_local_t  *local             = NULL;
-        int           op_errno          = -1;
-
+    local = frame->local;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_INODELK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local->call_cnt = 1;
-        local->lock_type = lock->l_type;
-
-        lock_subvol = dht_get_lock_subvolume (this, lock, local);
-        if (!lock_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no lock subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+out:
+    dht_lk_inode_unref(frame, op_ret);
+    DHT_STACK_UNWIND(finodelk, frame, op_ret, op_errno, xdata);
 
-/*
-        local->cached_subvol = lock_subvol;
-        ret = dht_check_and_open_fd_on_subvol (this, frame);
-        if (ret)
-                goto err;
-*/
-        STACK_WIND (frame, dht_finodelk_cbk, lock_subvol,
-                    lock_subvol->fops->finodelk,
-                    volume, fd, cmd, lock, xdata);
+    return 0;
+}
 
-        return 0;
+int
+dht_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+    xlator_t *lock_subvol = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_INODELK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->call_cnt = 1;
+    local->lock_type = lock->l_type;
+
+    lock_subvol = dht_get_lock_subvolume(this, lock, local);
+    if (!lock_subvol) {
+        gf_msg_debug(this->name, 0, "no lock subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /*
+            local->cached_subvol = lock_subvol;
+            ret = dht_check_and_open_fd_on_subvol (this, frame);
+            if (ret)
+                    goto err;
+    */
+    local->rebalance.flock = *lock;
+    local->rebalance.lock_cmd = cmd;
+    local->key = gf_strdup(volume);
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND(frame, dht_finodelk_cbk, lock_subvol,
+               lock_subvol->fops->finodelk, volume, fd, cmd, lock, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(finodelk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
index 226ee95e8b3..2f23ce90fbd 100644
--- a/xlators/cluster/dht/src/dht-inode-write.c
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -8,1487 +8,1397 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "dht-common.h"
 
-int dht_writev2 (xlator_t *this, xlator_t *subvol,
-                 call_frame_t *frame, int ret);
-int dht_truncate2 (xlator_t *this, xlator_t *subvol,
-                   call_frame_t *frame, int ret);
-int dht_setattr2 (xlator_t *this, xlator_t *subvol,
-                  call_frame_t *frame, int ret);
-int dht_fallocate2 (xlator_t *this, xlator_t *subvol,
-                    call_frame_t *frame, int ret);
-int dht_discard2 (xlator_t *this, xlator_t *subvol,
-                  call_frame_t *frame, int ret);
-int dht_zerofill2 (xlator_t *this, xlator_t *subvol,
-                   call_frame_t *frame, int ret);
+static int
+dht_writev2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_truncate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_setattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_fallocate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_discard2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_zerofill2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
 
 int
-dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+dht_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+               dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        xlator_t    *prev = NULL;
-        int          ret   = -1;
-        xlator_t    *subvol1 = NULL;
-        xlator_t    *subvol2 = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        /* writev fails with EBADF if dht has not yet opened the fd
-         * on the cached subvol. This could happen if the file was migrated
-         * and a lookup updated the cached subvol in the inode ctx.
-         * We only check once as this could be a valid bad fd error.
-         */
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *subvol1 = NULL;
+    xlator_t *subvol2 = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /* writev fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could be a valid bad fd error.
+     */
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        if (op_ret == -1 && (op_errno == EBADF) && !(local->fd_checked)) {
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto out;
-                return 0;
+    if (op_ret == -1 && !dht_inode_missing(op_errno)) {
+        local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, 0, "subvolume %s returned -1 (%s)", prev->name,
+                     strerror(op_errno));
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        /* preserve the modes of source */
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
-
-        if (op_ret == -1 && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
+        goto out;
+    }
+
+    local->rebalance.target_op_fn = dht_writev2;
+
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+
+    /* We might need to pass the stbuf information to the higher DHT
+     * layer for appropriate handling.
+     */
+
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        if (!local->xattr_req) {
+            local->xattr_req = dict_new();
+            if (!local->xattr_req) {
+                gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM,
+                       "insufficient memory");
+                local->op_errno = ENOMEM;
                 local->op_ret = -1;
-                gf_msg_debug (this->name, 0,
-                              "subvolume %s returned -1 (%s)",
-                              prev->name, strerror (op_errno));
                 goto out;
+            }
         }
 
-        if (local->call_cnt != 1) {
-                /* preserve the modes of source */
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
-        }
-
-        local->rebalance.target_op_fn = dht_writev2;
-
-        local->op_ret = op_ret;
-        local->op_errno = op_errno;
-
-        /* We might need to pass the stbuf information to the higher DHT
-         * layer for appropriate handling.
-         */
-
-        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
-
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
+        ret = dict_set_uint32(local->xattr_req, GF_PROTECT_FROM_EXTERNAL_WRITES,
+                              1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_DICT_SET_FAILED, 0,
+                   "Failed to set key %s in dictionary",
+                   GF_PROTECT_FROM_EXTERNAL_WRITES);
+            local->op_errno = ENOMEM;
+            local->op_ret = -1;
+            goto out;
         }
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                if (!dht_is_tier_xlator (this)) {
-                        if (!local->xattr_req) {
-                                local->xattr_req = dict_new ();
-                                if (!local->xattr_req) {
-                                        gf_msg (this->name, GF_LOG_ERROR,
-                                                DHT_MSG_NO_MEMORY,
-                                                ENOMEM, "insufficient memory");
-                                        local->op_errno = ENOMEM;
-                                        local->op_ret = -1;
-                                        goto out;
-                                }
-                        }
-
-                        ret = dict_set_uint32 (local->xattr_req,
-                                               GF_PROTECT_FROM_EXTERNAL_WRITES,
-                                               1);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR,
-                                        DHT_MSG_DICT_SET_FAILED, 0,
-                                        "Failed to set key %s in dictionary",
-                                        GF_PROTECT_FROM_EXTERNAL_WRITES);
-                                local->op_errno = ENOMEM;
-                                local->op_ret = -1;
-                                goto out;
-                        }
-                }
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
 
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-
-                ret = dht_inode_ctx_get_mig_info (this, local->fd->inode,
-                                                  &subvol1, &subvol2);
-                if (!dht_mig_info_is_invalid (local->cached_subvol,
-                                              subvol1, subvol2)) {
-                        if (dht_fd_open_on_dst (this, local->fd, subvol2)) {
-                                dht_writev2 (this, subvol2, frame, 0);
-                                return 0;
-                        }
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        ret = dht_inode_ctx_get_mig_info(this, local->fd->inode, &subvol1,
+                                         &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            if (dht_fd_open_on_dst(this, local->fd, subvol2)) {
+                dht_writev2(this, subvol2, frame, 0);
+                return 0;
+            }
         }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                          xdata);
+    DHT_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_writev2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_writev2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if ((frame == NULL) || (frame->local == NULL))
-                goto out;
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
 
-        local = frame->local;
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (writev, frame, local->op_ret,
-                                  local->op_errno, &local->rebalance.prebuf,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
-                return 0;
-        }
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        if (subvol == NULL)
-                goto out;
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(writev, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        local->call_cnt = 2; /* This is the second attempt */
+    if (subvol == NULL)
+        goto out;
 
-        STACK_WIND_COOKIE (frame, dht_writev_cbk, subvol,
-                           subvol, subvol->fops->writev,
-                           local->fd, local->rebalance.vector,
-                           local->rebalance.count,
-                           local->rebalance.offset, local->rebalance.flags,
-                           local->rebalance.iobref, local->xattr_req);
+    local->call_cnt = 2; /* This is the second attempt */
 
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_writev_cbk, subvol, subvol,
+                      subvol->fops->writev, local->fd, local->rebalance.vector,
+                      local->rebalance.count, local->rebalance.offset,
+                      local->rebalance.flags, local->rebalance.iobref,
+                      local->xattr_req);
+
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+    DHT_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int count, off_t off, uint32_t flags,
-            struct iobref *iobref, dict_t *xdata)
+dht_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int count, off_t off, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
-        xlator_t     *subvol   = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local    = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_WRITE);
-        if (!local) {
-
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
-
-        local->rebalance.vector = iov_dup (vector, count);
-        local->rebalance.offset = off;
-        local->rebalance.count = count;
-        local->rebalance.flags = flags;
-        local->rebalance.iobref = iobref_ref (iobref);
-        local->call_cnt = 1;
-
-        STACK_WIND_COOKIE (frame, dht_writev_cbk, subvol, subvol,
-                           subvol->fops->writev, fd,
-                           local->rebalance.vector,
-                           local->rebalance.count,
-                           local->rebalance.offset,
-                           local->rebalance.flags,
-                           local->rebalance.iobref, local->xattr_req);
-
-        return 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_WRITE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    local->rebalance.vector = iov_dup(vector, count);
+    local->rebalance.offset = off;
+    local->rebalance.count = count;
+    local->rebalance.flags = flags;
+    local->rebalance.iobref = iobref_ref(iobref);
+    local->call_cnt = 1;
+
+    STACK_WIND_COOKIE(frame, dht_writev_cbk, subvol, subvol,
+                      subvol->fops->writev, fd, local->rebalance.vector,
+                      local->rebalance.count, local->rebalance.offset,
+                      local->rebalance.flags, local->rebalance.iobref,
+                      local->xattr_req);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-
 int
-dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
+dht_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata)
 {
-        dht_local_t  *local      = NULL;
-        xlator_t     *prev       = NULL;
-        int           ret        = -1;
-        xlator_t     *src_subvol = NULL;
-        xlator_t     *dst_subvol = NULL;
-        inode_t      *inode      = NULL;
-
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-        /* Needs to be checked only for ftruncate.
-         * ftruncate fails with EBADF/EINVAL if dht has not yet opened the fd
-         * on the cached subvol. This could happen if the file was migrated
-         * and a lookup updated the cached subvol in the inode ctx.
-         * We only check once as this could actually be a valid error.
-         */
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* Needs to be checked only for ftruncate.
+     * ftruncate fails with EBADF/EINVAL if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+
+    if ((local->fop == GF_FOP_FTRUNCATE) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        if ((local->fop == GF_FOP_FTRUNCATE) && (op_ret == -1)
-             && ((op_errno == EBADF) || (op_errno == EINVAL))
-             && !(local->fd_checked)) {
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto out;
-                return 0;
-        }
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
 
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                local->op_ret = -1;
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->name);
+        goto out;
+    }
 
-                goto out;
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
+        goto out;
+    }
 
+    local->rebalance.target_op_fn = dht_truncate2;
 
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
-        }
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        local->rebalance.target_op_fn = dht_truncate2;
+    /* We might need to pass the stbuf information to the higher DHT
+     * layer for appropriate handling.
+     */
 
-        local->op_ret = op_ret;
-        local->op_errno = op_errno;
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-        /* We might need to pass the stbuf information to the higher DHT
-         * layer for appropriate handling.
-         */
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-
-                inode = (local->fd) ? local->fd->inode : local->loc.inode;
-
-                dht_inode_ctx_get_mig_info (this, inode, &src_subvol,
-                                            &dst_subvol);
-                if (!dht_mig_info_is_invalid (local->cached_subvol,
-                                              src_subvol, dst_subvol)) {
-                        if ((!local->fd) || ((local->fd) &&
-                            dht_fd_open_on_dst (this, local->fd, dst_subvol))) {
-                                dht_truncate2 (this, dst_subvol, frame, 0);
-                                return 0;
-                        }
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        dht_inode_ctx_get_mig_info(this, inode, &src_subvol, &dst_subvol);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                     dst_subvol)) {
+            if ((!local->fd) ||
+                ((local->fd) &&
+                 dht_fd_open_on_dst(this, local->fd, dst_subvol))) {
+                dht_truncate2(this, dst_subvol, frame, 0);
+                return 0;
+            }
         }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (truncate, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 err:
-        return 0;
+    return 0;
 }
 
-
-int
-dht_truncate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_truncate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if (!frame || !frame->local)
-                goto out;
-
-        local = frame->local;
-        op_errno = local->op_errno;
+    if (!frame || !frame->local)
+        goto out;
 
-        /* This dht xlator is not migrating the file  */
-        if (we_are_not_migrating (ret)) {
+    local = frame->local;
+    op_errno = local->op_errno;
 
-                DHT_STACK_UNWIND (truncate, frame, local->op_ret,
-                                  local->op_errno, &local->rebalance.prebuf,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
-                return 0;
-        }
+    /* This dht xlator is not migrating the file  */
+    if (we_are_not_migrating(ret)) {
+        DHT_STACK_UNWIND(truncate, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    local->call_cnt = 2; /* This is the second attempt */
 
-        if (local->fop == GF_FOP_TRUNCATE) {
-                STACK_WIND_COOKIE (frame, dht_truncate_cbk, subvol, subvol,
-                                   subvol->fops->truncate, &local->loc,
-                                   local->rebalance.offset, local->xattr_req);
-        } else {
-                STACK_WIND_COOKIE (frame, dht_truncate_cbk, subvol, subvol,
-                                   subvol->fops->ftruncate, local->fd,
-                                   local->rebalance.offset, local->xattr_req);
-        }
+    if (local->fop == GF_FOP_TRUNCATE) {
+        STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                          subvol->fops->truncate, &local->loc,
+                          local->rebalance.offset, local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                          subvol->fops->ftruncate, local->fd,
+                          local->rebalance.offset, local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-              dict_t *xdata)
+dht_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_TRUNCATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->rebalance.offset = offset;
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for gfid=%s",
-                              uuid_utoa (loc->inode->gfid));
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
-
-        STACK_WIND_COOKIE (frame, dht_truncate_cbk, subvol, subvol,
-                           subvol->fops->truncate, loc, offset, xdata);
-
-        return 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_TRUNCATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.offset = offset;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for gfid=%s",
+                     uuid_utoa(loc->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                      subvol->fops->truncate, loc, offset, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               dict_t *xdata)
+dht_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FTRUNCATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->rebalance.offset = offset;
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
-
-        STACK_WIND_COOKIE (frame, dht_truncate_cbk, subvol, subvol,
-                           subvol->fops->ftruncate, fd,
-                           local->rebalance.offset, local->xattr_req);
-        return 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FTRUNCATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.offset = offset;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                      subvol->fops->ftruncate, fd, local->rebalance.offset,
+                      local->xattr_req);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
+dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                  dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *prev = NULL;
-        int           ret = -1;
-        xlator_t    *src_subvol = NULL;
-        xlator_t    *dst_subvol = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-        /* fallocate fails with EBADF if dht has not yet opened the fd
-         * on the cached subvol. This could happen if the file was migrated
-         * and a lookup updated the cached subvol in the inode ctx.
-         * We only check once as this could actually be a valid error.
-         */
-
-        if ((op_ret == -1) && (op_errno == EBADF)
-             && !(local->fd_checked)) {
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto out;
-                return 0;
-        }
-
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                local->op_ret = -1;
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->name);
-
-                goto out;
-        }
-
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* fallocate fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local->op_ret = op_ret;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
         local->op_errno = op_errno;
-        local->rebalance.target_op_fn = dht_fallocate2;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
 
-        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
+        goto out;
+    }
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
-
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-
-                dht_inode_ctx_get_mig_info (this, local->fd->inode, &src_subvol,
-                                            &dst_subvol);
-                if (!dht_mig_info_is_invalid (local->cached_subvol,
-                                              src_subvol, dst_subvol)) {
-                        if (dht_fd_open_on_dst (this, local->fd, dst_subvol)) {
-                                dht_fallocate2 (this, dst_subvol, frame, 0);
-                                return 0;
-                        }
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        goto out;
+    }
+
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+    local->rebalance.target_op_fn = dht_fallocate2;
+
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        dht_inode_ctx_get_mig_info(this, local->fd->inode, &src_subvol,
+                                   &dst_subvol);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                     dst_subvol)) {
+            if (dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+                dht_fallocate2(this, dst_subvol, frame, 0);
+                return 0;
+            }
         }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (fallocate, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STACK_UNWIND(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                     xdata);
 err:
-        return 0;
+    return 0;
 }
 
-int
-dht_fallocate2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_fallocate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if (!frame || !frame->local)
-                goto out;
+    if (!frame || !frame->local)
+        goto out;
 
-        local = frame->local;
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (fallocate, frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->rebalance.prebuf,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
-                return 0;
-        }
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        if (subvol == NULL)
-                goto out;
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(fallocate, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        local->call_cnt = 2; /* This is the second attempt */
+    if (subvol == NULL)
+        goto out;
 
-	STACK_WIND_COOKIE (frame, dht_fallocate_cbk, subvol, subvol,
-                           subvol->fops->fallocate, local->fd,
-                           local->rebalance.flags, local->rebalance.offset,
-		           local->rebalance.size, local->xattr_req);
+    local->call_cnt = 2; /* This is the second attempt */
 
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_fallocate_cbk, subvol, subvol,
+                      subvol->fops->fallocate, local->fd,
+                      local->rebalance.flags, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
+
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-dht_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-	      off_t offset, size_t len, dict_t *xdata)
+dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FALLOCATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->rebalance.flags = mode;
-        local->rebalance.offset = offset;
-	local->rebalance.size = len;
-
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
-
-        STACK_WIND_COOKIE (frame, dht_fallocate_cbk, subvol, subvol,
-                           subvol->fops->fallocate, fd,
-                           local->rebalance.flags,
-                           local->rebalance.offset,
-                           local->rebalance.size,
-                           local->xattr_req);
-
-        return 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FALLOCATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.flags = mode;
+    local->rebalance.offset = offset;
+    local->rebalance.size = len;
+
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_fallocate_cbk, subvol, subvol,
+                      subvol->fops->fallocate, fd, local->rebalance.flags,
+                      local->rebalance.offset, local->rebalance.size,
+                      local->xattr_req);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *prev = NULL;
-        int           ret = -1;
-        xlator_t    *src_subvol = NULL;
-        xlator_t    *dst_subvol = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-
-        /* discard fails with EBADF if dht has not yet opened the fd
-         * on the cached subvol. This could happen if the file was migrated
-         * and a lookup updated the cached subvol in the inode ctx.
-         * We only check once as this could actually be a valid error.
-         */
-        if ((op_ret == -1) && (op_errno == EBADF)
-             && !(local->fd_checked)) {
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto out;
-                return 0;
-        }
-
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                local->op_ret = -1;
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->name);
-
-                goto out;
-        }
-
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* discard fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local->rebalance.target_op_fn = dht_discard2;
-        local->op_ret = op_ret;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
         local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
 
-        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
+        goto out;
+    }
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
-
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-
-                dht_inode_ctx_get_mig_info (this, local->fd->inode, &src_subvol,
-                                            &dst_subvol);
-                if (!dht_mig_info_is_invalid(local->cached_subvol,
-                                             src_subvol, dst_subvol)) {
-                        if (dht_fd_open_on_dst (this, local->fd, dst_subvol)) {
-                                dht_discard2 (this, dst_subvol, frame, 0);
-                                return 0;
-                        }
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        goto out;
+    }
+
+    local->rebalance.target_op_fn = dht_discard2;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        dht_inode_ctx_get_mig_info(this, local->fd->inode, &src_subvol,
+                                   &dst_subvol);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                     dst_subvol)) {
+            if (dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+                dht_discard2(this, dst_subvol, frame, 0);
+                return 0;
+            }
         }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (discard, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STACK_UNWIND(discard, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 err:
-        return 0;
+    return 0;
 }
 
-int
-dht_discard2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_discard2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if (!frame || !frame->local)
-                goto out;
+    if (!frame || !frame->local)
+        goto out;
 
-        local = frame->local;
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (discard, frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->rebalance.prebuf,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
-                return 0;
-        }
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        if (subvol == NULL)
-                goto out;
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(discard, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        local->call_cnt = 2; /* This is the second attempt */
+    if (subvol == NULL)
+        goto out;
 
-	STACK_WIND_COOKIE (frame, dht_discard_cbk, subvol, subvol,
-                           subvol->fops->discard, local->fd,
-                           local->rebalance.offset, local->rebalance.size,
-		           local->xattr_req);
+    local->call_cnt = 2; /* This is the second attempt */
 
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_discard_cbk, subvol, subvol,
+                      subvol->fops->discard, local->fd, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
+
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-dht_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	    size_t len, dict_t *xdata)
+dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_DISCARD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        local->rebalance.offset = offset;
-	local->rebalance.size = len;
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local = dht_local_init(frame, NULL, fd, GF_FOP_DISCARD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
+    local->rebalance.offset = offset;
+    local->rebalance.size = len;
 
-        STACK_WIND_COOKIE (frame, dht_discard_cbk, subvol, subvol,
-                           subvol->fops->discard, fd,
-                           local->rebalance.offset,
-                           local->rebalance.size,
-                           local->xattr_req);
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        return 0;
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_discard_cbk, subvol, subvol,
+                      subvol->fops->discard, fd, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata)
 {
-        dht_local_t  *local   = NULL;
-        xlator_t     *prev    = NULL;
-        int           ret     = -1;
-        xlator_t     *subvol1 = NULL, *subvol2 = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-        /* zerofill fails with EBADF if dht has not yet opened the fd
-         * on the cached subvol. This could happen if the file was migrated
-         * and a lookup updated the cached subvol in the inode ctx.
-         * We only check once as this could actually be a valid error.
-         */
-        if ((op_ret == -1) && (op_errno == EBADF)
-             && !(local->fd_checked)) {
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto out;
-                return 0;
-        }
-
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                local->op_ret = -1;
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->name);
-                goto out;
-        }
-
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *subvol1 = NULL, *subvol2 = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* zerofill fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local->rebalance.target_op_fn = dht_zerofill2;
-        local->op_ret = op_ret;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
         local->op_errno = op_errno;
-
-        dht_set_local_rebalance (this, local, NULL, prebuf, postbuf, xdata);
-
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
-
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-
-                ret = dht_inode_ctx_get_mig_info (this, local->fd->inode,
-                                                  &subvol1, &subvol2);
-                if (!dht_mig_info_is_invalid (local->cached_subvol,
-                                              subvol1, subvol2)) {
-                        if (dht_fd_open_on_dst (this, local->fd, subvol2)) {
-                                dht_zerofill2 (this, subvol2, frame, 0);
-                                return 0;
-                        }
-                }
-
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        goto out;
+    }
+
+    local->rebalance.target_op_fn = dht_zerofill2;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        ret = dht_inode_ctx_get_mig_info(this, local->fd->inode, &subvol1,
+                                         &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            if (dht_fd_open_on_dst(this, local->fd, subvol2)) {
+                dht_zerofill2(this, subvol2, frame, 0);
+                return 0;
+            }
         }
 
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (zerofill, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STACK_UNWIND(zerofill, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 err:
-        return 0;
+    return 0;
 }
 
-int
-dht_zerofill2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_zerofill2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if (!frame || !frame->local)
-                goto out;
+    if (!frame || !frame->local)
+        goto out;
 
-        local = frame->local;
+    local = frame->local;
 
-        op_errno = local->op_errno;
+    op_errno = local->op_errno;
 
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (zerofill, frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->rebalance.prebuf,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(zerofill, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
 
-                return 0;
-        }
+        return 0;
+    }
 
-        if (subvol == NULL)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    local->call_cnt = 2; /* This is the second attempt */
 
-        STACK_WIND_COOKIE (frame, dht_zerofill_cbk, subvol, subvol,
-                           subvol->fops->zerofill,
-                           local->fd, local->rebalance.offset,
-                           local->rebalance.size, local->xattr_req);
+    STACK_WIND_COOKIE(frame, dht_zerofill_cbk, subvol, subvol,
+                      subvol->fops->zerofill, local->fd,
+                      local->rebalance.offset, local->rebalance.size,
+                      local->xattr_req);
 
-        return 0;
+    return 0;
 
 out:
 
-        DHT_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-dht_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-            off_t len, dict_t *xdata)
+dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata)
 {
-        xlator_t     *subvol       = NULL;
-        int           op_errno     = -1;
-        dht_local_t  *local        = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_ZEROFILL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        local->rebalance.offset = offset;
-        local->rebalance.size = len;
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local = dht_local_init(frame, NULL, fd, GF_FOP_ZEROFILL);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
+    local->rebalance.offset = offset;
+    local->rebalance.size = len;
 
-        STACK_WIND_COOKIE (frame, dht_zerofill_cbk, subvol, subvol,
-                           subvol->fops->zerofill, fd,
-                           local->rebalance.offset,
-                           local->rebalance.size, local->xattr_req);
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        return 0;
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    STACK_WIND_COOKIE(frame, dht_zerofill_cbk, subvol, subvol,
+                      subvol->fops->zerofill, fd, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
 
-        return 0;
-}
+    return 0;
 
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
 
+    return 0;
+}
 
 /* handle cases of migration here for 'setattr()' calls */
 int
-dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+dht_file_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *prev = NULL;
-        int           ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
 
-        local = frame->local;
-        prev = cookie;
+    local = frame->local;
+    prev = cookie;
 
-        local->op_errno = op_errno;
+    local->op_errno = op_errno;
 
-        if ((local->fop == GF_FOP_FSETATTR) &&
-            (op_ret == -1) && (op_errno == EBADF) && !(local->fd_checked)) {
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto out;
-                return 0;
-        }
-
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->name);
-                goto out;
-        }
-
-        if (local->call_cnt != 1)
-                goto out;
+    if ((local->fop == GF_FOP_FSETATTR) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local->op_ret = op_ret;
-        local->op_errno = op_errno;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
 
-        local->rebalance.target_op_fn = dht_setattr2;
+    if (local->call_cnt != 1)
+        goto out;
 
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+    local->rebalance.target_op_fn = dht_setattr2;
 
-                dht_set_local_rebalance (this, local, NULL, prebuf,
-                                         postbuf, xdata);
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-        /* At the end of the migration process, whatever 'attr' we
-           have on source file will be migrated to destination file
-           in one shot, hence we don't need to check for in progress
-           state here (ie, PHASE1) */
+    /* At the end of the migration process, whatever 'attr' we
+       have on source file will be migrated to destination file
+       in one shot, hence we don't need to check for in progress
+       state here (ie, PHASE1) */
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (setattr, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STACK_UNWIND(setattr, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_setattr2 (xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+static int
+dht_setattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local    = NULL;
-        int32_t      op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if (!frame || !frame->local)
-                goto out;
+    if (!frame || !frame->local)
+        goto out;
 
-        local = frame->local;
-        op_errno = local->op_errno;
-
-        if (we_are_not_migrating (ret)) {
-                /* This dht xlator is not migrating the file. Unwind and
-                 * pass on the original mode bits so the higher DHT layer
-                 * can handle this.
-                 */
-                DHT_STACK_UNWIND (setattr, frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->rebalance.prebuf,
-                                  &local->rebalance.postbuf,
-                                  local->rebalance.xdata);
-                return 0;
-        }
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        if (subvol == NULL)
-                goto out;
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(setattr, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        local->call_cnt = 2; /* This is the second attempt */
-
-        if (local->fop == GF_FOP_SETATTR) {
-                STACK_WIND_COOKIE (frame, dht_file_setattr_cbk, subvol,
-                                   subvol, subvol->fops->setattr, &local->loc,
-                                   &local->rebalance.stbuf, local->rebalance.flags,
-                                   local->xattr_req);
-        } else {
-                STACK_WIND_COOKIE (frame, dht_file_setattr_cbk, subvol,
-                                   subvol, subvol->fops->fsetattr, local->fd,
-                                   &local->rebalance.stbuf, local->rebalance.flags,
-                                   local->xattr_req);
-        }
+    if (subvol == NULL)
+        goto out;
 
-        return 0;
+    local->call_cnt = 2; /* This is the second attempt */
+
+    if (local->fop == GF_FOP_SETATTR) {
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->setattr, &local->loc,
+                          &local->rebalance.stbuf, local->rebalance.flags,
+                          local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->fsetattr, local->fd,
+                          &local->rebalance.stbuf, local->rebalance.flags,
+                          local->xattr_req);
+    }
+
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /* Keep the existing code same for all the cases other than regular file */
 int
-dht_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, struct iatt *statpre,
-                 struct iatt *statpost, dict_t *xdata)
+dht_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *statpre, struct iatt *statpost,
+                dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        xlator_t     *prev = NULL;
-
-
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, op_errno,
-                                      "subvolume %s returned -1",
-                                      prev->name);
-                        goto unlock;
-                }
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
 
-                dht_iatt_merge (this, &local->prebuf, statpre, prev);
-                dht_iatt_merge (this, &local->stbuf, statpost, prev);
+    local = frame->local;
+    prev = cookie;
 
-                local->op_ret = 0;
-                local->op_errno = 0;
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+            goto post_unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-		if (local->op_ret == 0)
-			dht_inode_ctx_time_set (local->loc.inode, this,
-						&local->stbuf);
-                DHT_STACK_UNWIND (setattr, frame, local->op_ret, local->op_errno,
-                                  &local->prebuf, &local->stbuf, xdata);
-	}
 
-        return 0;
-}
+        dht_iatt_merge(this, &local->prebuf, statpre);
+        dht_iatt_merge(this, &local->stbuf, statpost);
 
+        local->op_ret = 0;
+        local->op_errno = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if (local->op_ret == 0)
+            dht_inode_ctx_time_set(local->loc.inode, this, &local->stbuf);
+        DHT_STACK_UNWIND(setattr, frame, local->op_ret, local->op_errno,
+                         &local->prebuf, &local->stbuf, xdata);
+    }
+
+    return 0;
+}
 
 /* Keep the existing code same for all the cases other than regular file */
 int
-dht_non_mds_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, struct iatt *statpre,
-                         struct iatt *statpost, dict_t *xdata)
+dht_non_mds_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, struct iatt *statpre,
+                        struct iatt *statpost, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        xlator_t     *prev = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
 
+    local = frame->local;
+    prev = cookie;
 
-        local = frame->local;
-        prev = cookie;
+    if (op_ret == -1) {
+        gf_msg(this->name, op_errno, 0, 0, "subvolume %s returned -1",
+               prev->name);
+        goto post_unlock;
+    }
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_msg (this->name, op_errno, 0,
-                                0, "subvolume %s returned -1",
-                                prev->name);
+    LOCK(&frame->lock);
+    {
+        dht_iatt_merge(this, &local->prebuf, statpre);
+        dht_iatt_merge(this, &local->stbuf, statpost);
 
-                        goto unlock;
-                }
-
-                dht_iatt_merge (this, &local->prebuf, statpre, prev);
-                dht_iatt_merge (this, &local->stbuf, statpost, prev);
-
-                local->op_ret = 0;
-                local->op_errno = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                dht_inode_ctx_time_set (local->loc.inode, this, &local->stbuf);
-                DHT_STACK_UNWIND (setattr, frame, 0, 0,
-                                  &local->prebuf, &local->stbuf, xdata);
-	}
-
-        return 0;
+        local->op_ret = 0;
+        local->op_errno = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_inode_ctx_time_set(local->loc.inode, this, &local->stbuf);
+        DHT_STACK_UNWIND(setattr, frame, 0, 0, &local->prebuf, &local->stbuf,
+                         xdata);
+    }
+
+    return 0;
 }
 
-
-
-
-
 int
-dht_mds_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, struct iatt *statpre,
-                     struct iatt *statpost, dict_t *xdata)
+dht_mds_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, struct iatt *statpre,
+                    struct iatt *statpost, dict_t *xdata)
 
 {
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf  = NULL;
-        xlator_t     *prev = NULL;
-        xlator_t     *mds_subvol = NULL;
-        struct iatt  loc_stbuf = {0,};
-        int i = 0;
-
-        local = frame->local;
-        prev = cookie;
-        conf = this->private;
-        mds_subvol = local->mds_subvol;
-
-        if (op_ret == -1) {
-                local->op_ret  = op_ret;
-                local->op_errno = op_errno;
-                gf_msg_debug (this->name, op_errno,
-                              "subvolume %s returned -1",
-                              prev->name);
-                goto out;
-        }
-
-        local->op_ret = 0;
-        loc_stbuf = local->stbuf;
-        dht_iatt_merge (this, &local->prebuf, statpre, prev);
-        dht_iatt_merge (this, &local->stbuf, statpost, prev);
-
-        local->call_cnt = conf->subvolume_cnt - 1;
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (mds_subvol == conf->subvolumes[i])
-                        continue;
-                STACK_WIND_COOKIE (frame, dht_non_mds_setattr_cbk,
-                                   conf->subvolumes[i], conf->subvolumes[i],
-                                   conf->subvolumes[i]->fops->setattr,
-                                   &local->loc, &loc_stbuf,
-                                   local->valid, local->xattr_req);
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *mds_subvol = NULL;
+    struct iatt loc_stbuf = {
+        0,
+    };
+    int i = 0;
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+    mds_subvol = local->mds_subvol;
+
+    if (op_ret == -1) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    local->op_ret = 0;
+    loc_stbuf = local->stbuf;
+    dht_iatt_merge(this, &local->prebuf, statpre);
+    dht_iatt_merge(this, &local->stbuf, statpost);
+
+    local->call_cnt = conf->subvolume_cnt - 1;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (mds_subvol == conf->subvolumes[i])
+            continue;
+        STACK_WIND_COOKIE(frame, dht_non_mds_setattr_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->setattr, &local->loc,
+                          &loc_stbuf, local->valid, local->xattr_req);
+    }
+
+    return 0;
 out:
-        DHT_STACK_UNWIND (setattr, frame, local->op_ret, local->op_errno,
-                          &local->prebuf, &local->stbuf, xdata);
+    DHT_STACK_UNWIND(setattr, frame, local->op_ret, local->op_errno,
+                     &local->prebuf, &local->stbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+dht_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
 {
-        xlator_t     *subvol     = NULL;
-        xlator_t     *mds_subvol = NULL;
-        dht_layout_t *layout     = NULL;
-        dht_local_t  *local      = NULL;
-        int           op_errno   = -1;
-        int           i          = -1;
-        int           ret        = -1;
-        int           call_cnt   = 0;
-        dht_conf_t   *conf       = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        conf = this->private;
-        local = dht_local_init (frame, loc, NULL, GF_FOP_SETATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (!layout_is_sane (layout)) {
-                gf_msg_debug (this->name, 0,
-                              "layout is not sane for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
-
-        if (IA_ISREG (loc->inode->ia_type)) {
-                /* in the regular file _cbk(), we need to check for
-                   migration possibilities */
-                local->rebalance.stbuf = *stbuf;
-                local->rebalance.flags = valid;
-                local->call_cnt = 1;
-                subvol = local->cached_subvol;
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    dht_layout_t *layout = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int ret = -1;
+    int call_cnt = 0;
+    dht_conf_t *conf = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    conf = this->private;
+    local = dht_local_init(frame, loc, NULL, GF_FOP_SETATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (!layout_is_sane(layout)) {
+        gf_msg_debug(this->name, 0, "layout is not sane for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(loc->inode->ia_type)) {
+        /* in the regular file _cbk(), we need to check for
+           migration possibilities */
+        local->rebalance.stbuf = *stbuf;
+        local->rebalance.flags = valid;
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
 
-                STACK_WIND_COOKIE (frame, dht_file_setattr_cbk, subvol,
-                                   subvol, subvol->fops->setattr, loc, stbuf,
-                                   valid, xdata);
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->setattr, loc, stbuf, valid, xdata);
 
-                return 0;
+        return 0;
+    }
+
+    local->call_cnt = call_cnt = layout->cnt;
+
+    if (IA_ISDIR(loc->inode->ia_type) && !__is_root_gfid(loc->inode->gfid) &&
+        call_cnt != 1) {
+        ret = dht_inode_ctx_mdsvol_get(loc->inode, this, &mds_subvol);
+        if (ret || !mds_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get mds subvol for path %s", local->loc.path);
+            op_errno = EINVAL;
+            goto err;
         }
 
-        local->call_cnt = call_cnt = layout->cnt;
-
-        if (IA_ISDIR (loc->inode->ia_type) &&
-            !__is_root_gfid (loc->inode->gfid) && call_cnt != 1) {
-                ret = dht_inode_ctx_mdsvol_get (loc->inode, this, &mds_subvol);
-                if (ret || !mds_subvol) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                                "Failed to get mds subvol for path %s",
-                                local->loc.path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                local->mds_subvol = mds_subvol;
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->subvolumes[i] ==  mds_subvol) {
-                                if (!conf->subvolume_status[i]) {
-                                        gf_msg (this->name, GF_LOG_WARNING,
-                                                layout->list[i].err,
-                                                DHT_MSG_HASHED_SUBVOL_DOWN,
-                                                "MDS subvol is down for path "
-                                                " %s Unable to set attr " ,
-                                                local->loc.path);
-                                        op_errno = ENOTCONN;
-                                        goto err;
-                                }
-                        }
-                }
-                local->valid = valid;
-                local->stbuf = *stbuf;
-
-                STACK_WIND_COOKIE (frame, dht_mds_setattr_cbk,
-                                   local->mds_subvol,
-                                   local->mds_subvol,
-                                   local->mds_subvol->fops->setattr,
-                                   loc, stbuf, valid, xdata);
-                return 0;
-        } else {
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND_COOKIE (frame, dht_setattr_cbk,
-                                           layout->list[i].xlator,
-                                           layout->list[i].xlator,
-                                           layout->list[i].xlator->fops->setattr,
-                                           loc, stbuf, valid, xdata);
+        local->mds_subvol = mds_subvol;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvolumes[i] == mds_subvol) {
+                if (!conf->subvolume_status[i]) {
+                    gf_msg(this->name, GF_LOG_WARNING, layout->list[i].err,
+                           DHT_MSG_HASHED_SUBVOL_DOWN,
+                           "MDS subvol is down for path "
+                           " %s Unable to set attr ",
+                           local->loc.path);
+                    op_errno = ENOTCONN;
+                    goto err;
                 }
+            }
         }
+        local->valid = valid;
+        local->stbuf = *stbuf;
 
+        STACK_WIND_COOKIE(frame, dht_mds_setattr_cbk, local->mds_subvol,
+                          local->mds_subvol, local->mds_subvol->fops->setattr,
+                          loc, stbuf, valid, xdata);
         return 0;
+    } else {
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_setattr_cbk, layout->list[i].xlator,
+                              layout->list[i].xlator,
+                              layout->list[i].xlator->fops->setattr, loc, stbuf,
+                              valid, xdata);
+        }
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
-              int32_t valid, dict_t *xdata)
+dht_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+             int32_t valid, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        dht_layout_t *layout = NULL;
-        dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-        int           i = -1;
-        int           call_cnt = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSETATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    dht_layout_t *layout = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSETATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (!layout_is_sane(layout)) {
+        gf_msg_debug(this->name, 0, "layout is not sane for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(fd->inode->ia_type)) {
+        /* in the regular file _cbk(), we need to check for
+           migration possibilities */
+        local->rebalance.stbuf = *stbuf;
+        local->rebalance.flags = valid;
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
 
-        if (!layout_is_sane (layout)) {
-                gf_msg_debug (this->name, 0,
-                              "layout is not sane for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
-
-        if (IA_ISREG (fd->inode->ia_type)) {
-                /* in the regular file _cbk(), we need to check for
-                   migration possibilities */
-                local->rebalance.stbuf = *stbuf;
-                local->rebalance.flags = valid;
-                local->call_cnt = 1;
-                subvol = local->cached_subvol;
-
-                STACK_WIND_COOKIE (frame, dht_file_setattr_cbk, subvol,
-                                   subvol, subvol->fops->fsetattr, fd,
-                                   &local->rebalance.stbuf,
-                                   local->rebalance.flags,
-                                   local->xattr_req);
-                return 0;
-        }
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->fsetattr, fd, &local->rebalance.stbuf,
+                          local->rebalance.flags, local->xattr_req);
+        return 0;
+    }
 
-        local->call_cnt = call_cnt = layout->cnt;
+    local->call_cnt = call_cnt = layout->cnt;
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND_COOKIE (frame, dht_setattr_cbk,
-                                   layout->list[i].xlator,
-                                   layout->list[i].xlator,
-                                   layout->list[i].xlator->fops->fsetattr,
-                                   fd, stbuf, valid, xdata);
-        }
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_setattr_cbk, layout->list[i].xlator,
+                          layout->list[i].xlator,
+                          layout->list[i].xlator->fops->fsetattr, fd, stbuf,
+                          valid, xdata);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c
index 97b98e01451..fda904c92c9 100644
--- a/xlators/cluster/dht/src/dht-layout.c
+++ b/xlators/cluster/dht/src/dht-layout.c
@@ -8,870 +8,801 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "byte-order.h"
-#include "dht-messages.h"
+#include <glusterfs/byte-order.h>
 #include "unittest/unittest.h"
 
+#define layout_base_size (sizeof(dht_layout_t))
 
-#define layout_base_size (sizeof (dht_layout_t))
-
-#define layout_entry_size (sizeof ((dht_layout_t *)NULL)->list[0])
+#define layout_entry_size (sizeof((dht_layout_t *)NULL)->list[0])
 
 #define layout_size(cnt) (layout_base_size + (cnt * layout_entry_size))
 
 dht_layout_t *
-dht_layout_new (xlator_t *this, int cnt)
+dht_layout_new(xlator_t *this, int cnt)
 {
-        dht_layout_t *layout = NULL;
-        dht_conf_t   *conf = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
 
-        REQUIRE(NULL != this);
-        REQUIRE(cnt >= 0);
+    REQUIRE(NULL != this);
+    REQUIRE(cnt >= 0);
 
-        conf = this->private;
+    conf = this->private;
 
-        layout = GF_CALLOC (1, layout_size (cnt),
-                            gf_dht_mt_dht_layout_t);
-        if (!layout) {
-                goto out;
-        }
+    layout = GF_CALLOC(1, layout_size(cnt), gf_dht_mt_dht_layout_t);
+    if (!layout) {
+        goto out;
+    }
 
-        layout->type = DHT_HASH_TYPE_DM;
-        layout->cnt = cnt;
+    layout->type = DHT_HASH_TYPE_DM;
+    layout->cnt = cnt;
 
-        if (conf) {
-                layout->spread_cnt = conf->dir_spread_cnt;
-                layout->gen = conf->gen;
-        }
+    if (conf) {
+        layout->spread_cnt = conf->dir_spread_cnt;
+        layout->gen = conf->gen;
+    }
 
-        GF_ATOMIC_INIT (layout->ref, 1);
+    GF_ATOMIC_INIT(layout->ref, 1);
 
-        ENSURE(NULL != layout);
-        ENSURE(layout->type == DHT_HASH_TYPE_DM);
-        ENSURE(layout->cnt == cnt);
-        ENSURE(GF_ATOMIC_GET (layout->ref) == 1);
+    ENSURE(NULL != layout);
+    ENSURE(layout->type == DHT_HASH_TYPE_DM);
+    ENSURE(layout->cnt == cnt);
+    ENSURE(GF_ATOMIC_GET(layout->ref) == 1);
 out:
-        return layout;
+    return layout;
 }
 
-
 dht_layout_t *
-dht_layout_get (xlator_t *this, inode_t *inode)
+dht_layout_get(xlator_t *this, inode_t *inode)
 {
-        dht_layout_t *layout = NULL;
-        int           ret = 0;
-
-        ret = dht_inode_ctx_layout_get (inode, this, &layout);
-        if ((!ret) && layout) {
-                GF_ATOMIC_INC (layout->ref);
-        }
-        return layout;
+    dht_layout_t *layout = NULL;
+    int ret = 0;
+
+    ret = dht_inode_ctx_layout_get(inode, this, &layout);
+    if ((!ret) && layout) {
+        GF_ATOMIC_INC(layout->ref);
+    }
+    return layout;
 }
 
-
 int
-dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
+dht_layout_set(xlator_t *this, inode_t *inode, dht_layout_t *layout)
 {
-        dht_conf_t   *conf = NULL;
-        int           oldret = -1;
-        int           ret = -1;
-        dht_layout_t *old_layout;
-
-        conf = this->private;
-        if (!conf || !layout)
-                goto out;
-
-        LOCK (&conf->layout_lock);
-        {
-                oldret = dht_inode_ctx_layout_get (inode, this, &old_layout);
-                if (layout)
-                        GF_ATOMIC_INC (layout->ref);
-                ret = dht_inode_ctx_layout_set (inode, this, layout);
-        }
-        UNLOCK (&conf->layout_lock);
-
-        if (!oldret) {
-                dht_layout_unref (this, old_layout);
-        }
-        if (ret)
-                GF_ATOMIC_DEC (layout->ref);
+    dht_conf_t *conf = NULL;
+    int oldret = -1;
+    int ret = -1;
+    dht_layout_t *old_layout;
+
+    conf = this->private;
+    if (!conf || !layout)
+        goto out;
+
+    LOCK(&conf->layout_lock);
+    {
+        oldret = dht_inode_ctx_layout_get(inode, this, &old_layout);
+        if (layout)
+            GF_ATOMIC_INC(layout->ref);
+        ret = dht_inode_ctx_layout_set(inode, this, layout);
+    }
+    UNLOCK(&conf->layout_lock);
+
+    if (!oldret) {
+        dht_layout_unref(this, old_layout);
+    }
+    if (ret)
+        GF_ATOMIC_DEC(layout->ref);
 
 out:
-        return ret;
+    return ret;
 }
 
-
 void
-dht_layout_unref (xlator_t *this, dht_layout_t *layout)
+dht_layout_unref(xlator_t *this, dht_layout_t *layout)
 {
-        int          ref = 0;
+    int ref = 0;
 
-        if (!layout || layout->preset || !this->private)
-                return;
+    if (!layout || layout->preset || !this->private)
+        return;
 
-        ref = GF_ATOMIC_DEC (layout->ref);
+    ref = GF_ATOMIC_DEC(layout->ref);
 
-        if (!ref)
-                GF_FREE (layout);
+    if (!ref)
+        GF_FREE(layout);
 }
 
-
 dht_layout_t *
-dht_layout_ref (xlator_t *this, dht_layout_t *layout)
+dht_layout_ref(xlator_t *this, dht_layout_t *layout)
 {
-        if (layout->preset || !this->private)
-                return layout;
+    if (layout->preset || !this->private)
+        return layout;
 
-        GF_ATOMIC_INC (layout->ref);
+    GF_ATOMIC_INC(layout->ref);
 
-        return layout;
+    return layout;
 }
 
-
 xlator_t *
-dht_layout_search (xlator_t *this, dht_layout_t *layout, const char *name)
+dht_layout_search(xlator_t *this, dht_layout_t *layout, const char *name)
 {
-        uint32_t   hash = 0;
-        xlator_t  *subvol = NULL;
-        int        i = 0;
-        int        ret = 0;
-
-        ret = dht_hash_compute (this, layout->type, name, &hash);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_COMPUTE_HASH_FAILED,
-                        "hash computation failed for type=%d name=%s",
-                        layout->type, name);
-                goto out;
-        }
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].start <= hash
-                    && layout->list[i].stop >= hash) {
-                        subvol = layout->list[i].xlator;
-                        break;
-                }
-        }
-
-        if (!subvol) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                        "no subvolume for hash (value) = %u", hash);
-        }
+    uint32_t hash = 0;
+    xlator_t *subvol = NULL;
+    int i = 0;
+    int ret = 0;
+
+    ret = dht_hash_compute(this, layout->type, name, &hash);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_COMPUTE_HASH_FAILED,
+                "type=%d", layout->type, "name=%s", name, NULL);
+        goto out;
+    }
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].start <= hash && layout->list[i].stop >= hash) {
+            subvol = layout->list[i].xlator;
+            break;
+        }
+    }
+
+    if (!subvol) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                "hash-value=0x%x", hash, NULL);
+    }
 
 out:
-        return subvol;
+    return subvol;
 }
 
-
 dht_layout_t *
-dht_layout_for_subvol (xlator_t *this, xlator_t *subvol)
+dht_layout_for_subvol(xlator_t *this, xlator_t *subvol)
 {
-        dht_conf_t   *conf = NULL;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == subvol) {
-                        layout = conf->file_layouts[i];
-                        break;
-                }
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == subvol) {
+            layout = conf->file_layouts[i];
+            break;
         }
+    }
 
 out:
-        return layout;
+    return layout;
 }
 
-
 int
-dht_layouts_init (xlator_t *this, dht_conf_t *conf)
+dht_layouts_init(xlator_t *this, dht_conf_t *conf)
 {
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           ret = -1;
-
-        if (!conf)
-                goto out;
-
-        conf->file_layouts = GF_CALLOC (conf->subvolume_cnt,
-                                        sizeof (dht_layout_t *),
-                                        gf_dht_mt_dht_layout_t);
-        if (!conf->file_layouts) {
-                goto out;
-        }
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int ret = -1;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                layout = dht_layout_new (this, 1);
+    if (!conf)
+        goto out;
 
-                if (!layout) {
-                        goto out;
-                }
+    conf->file_layouts = GF_CALLOC(conf->subvolume_cnt, sizeof(dht_layout_t *),
+                                   gf_dht_mt_dht_layout_t);
+    if (!conf->file_layouts) {
+        goto out;
+    }
 
-                layout->preset = 1;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        layout = dht_layout_new(this, 1);
 
-                layout->list[0].xlator = conf->subvolumes[i];
-
-                conf->file_layouts[i] = layout;
+        if (!layout) {
+            goto out;
         }
 
-        ret = 0;
+        layout->preset = 1;
+
+        layout->list[0].xlator = conf->subvolumes[i];
+
+        conf->file_layouts[i] = layout;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-dht_disk_layout_extract (xlator_t *this, dht_layout_t *layout,
-                         int pos, int32_t **disk_layout_p)
+dht_disk_layout_extract(xlator_t *this, dht_layout_t *layout, int pos,
+                        int32_t **disk_layout_p)
 {
-        int      ret = -1;
-        int32_t *disk_layout = NULL;
+    int ret = -1;
+    int32_t *disk_layout = NULL;
 
-        disk_layout = GF_CALLOC (5, sizeof (int),
-                                 gf_dht_mt_int32_t);
-        if (!disk_layout) {
-                goto out;
-        }
+    disk_layout = GF_CALLOC(5, sizeof(int), gf_dht_mt_int32_t);
+    if (!disk_layout) {
+        goto out;
+    }
 
-        disk_layout[0] = hton32 (layout->list[pos].commit_hash);
-        disk_layout[1] = hton32 (layout->type);
-        disk_layout[2] = hton32 (layout->list[pos].start);
-        disk_layout[3] = hton32 (layout->list[pos].stop);
+    disk_layout[0] = hton32(layout->list[pos].commit_hash);
+    disk_layout[1] = hton32(layout->type);
+    disk_layout[2] = hton32(layout->list[pos].start);
+    disk_layout[3] = hton32(layout->list[pos].stop);
 
-        if (disk_layout_p)
-                *disk_layout_p = disk_layout;
-        else
-                GF_FREE (disk_layout);
+    if (disk_layout_p)
+        *disk_layout_p = disk_layout;
+    else
+        GF_FREE(disk_layout);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-dht_disk_layout_extract_for_subvol (xlator_t *this, dht_layout_t *layout,
-                                    xlator_t *subvol, int32_t **disk_layout_p)
+dht_disk_layout_extract_for_subvol(xlator_t *this, dht_layout_t *layout,
+                                   xlator_t *subvol, int32_t **disk_layout_p)
 {
-        int i = 0;
+    int i = 0;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].xlator == subvol)
-                        break;
-        }
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == subvol)
+            break;
+    }
 
-        if (i == layout->cnt)
-                return -1;
+    if (i == layout->cnt)
+        return -1;
 
-        return dht_disk_layout_extract (this, layout, i, disk_layout_p);
+    return dht_disk_layout_extract(this, layout, i, disk_layout_p);
 }
 
-int
-dht_disk_layout_merge (xlator_t *this, dht_layout_t *layout,
-		       int pos, void *disk_layout_raw, int disk_layout_len)
+static int
+dht_disk_layout_merge(xlator_t *this, dht_layout_t *layout, int pos,
+                      void *disk_layout_raw, int disk_layout_len)
 {
-        int      type = 0;
-        int      start_off = 0;
-        int      stop_off = 0;
-        int      commit_hash = 0;
-        int      disk_layout[4];
+    int type = 0;
+    int start_off = 0;
+    int stop_off = 0;
+    int commit_hash = 0;
+    int disk_layout[4];
 
-	if (!disk_layout_raw) {
-		gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        DHT_MSG_LAYOUT_MERGE_FAILED,
-                        "error no layout on disk for merge");
-		return -1;
-	}
+    if (!disk_layout_raw) {
+        gf_smsg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                NULL);
+        return -1;
+    }
 
-	GF_ASSERT (disk_layout_len == sizeof (disk_layout));
+    GF_ASSERT(disk_layout_len == sizeof(disk_layout));
 
-        memcpy (disk_layout, disk_layout_raw, disk_layout_len);
+    memcpy(disk_layout, disk_layout_raw, disk_layout_len);
 
-        type = ntoh32 (disk_layout[1]);
-	switch (type) {
+    type = ntoh32(disk_layout[1]);
+    switch (type) {
         case DHT_HASH_TYPE_DM_USER:
-                gf_msg_debug (this->name, 0, "found user-set layout");
-                layout->type = type;
-                /* Fall through. */
-	case DHT_HASH_TYPE_DM:
-		break;
+            gf_msg_debug(this->name, 0, "found user-set layout");
+            layout->type = type;
+            /* Fall through. */
+        case DHT_HASH_TYPE_DM:
+            break;
         default:
-		gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        DHT_MSG_INVALID_DISK_LAYOUT,
-			"Invalid disk layout: "
-                        "Catastrophic error layout with unknown type found %d",
-			disk_layout[1]);
-		return -1;
-	}
-
-        commit_hash = ntoh32 (disk_layout[0]);
-        start_off = ntoh32 (disk_layout[2]);
-        stop_off  = ntoh32 (disk_layout[3]);
-
-        layout->list[pos].commit_hash = commit_hash;
-        layout->list[pos].start = start_off;
-        layout->list[pos].stop  = stop_off;
-
-        gf_msg_trace (this->name, 0,
-                      "merged to layout: %u - %u (type %d, hash %d) from %s",
-                      start_off, stop_off, commit_hash, type,
-                      layout->list[pos].xlator->name);
-
-        return 0;
+            gf_smsg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_INVALID_DISK_LAYOUT,
+                    "layout=%d", disk_layout[1], NULL);
+            return -1;
+    }
+
+    commit_hash = ntoh32(disk_layout[0]);
+    start_off = ntoh32(disk_layout[2]);
+    stop_off = ntoh32(disk_layout[3]);
+
+    layout->list[pos].commit_hash = commit_hash;
+    layout->list[pos].start = start_off;
+    layout->list[pos].stop = stop_off;
+
+    gf_msg_trace(this->name, 0,
+                 "merged to layout: 0x%x - 0x%x (hash 0x%x, type %d) from %s",
+                 start_off, stop_off, commit_hash, type,
+                 layout->list[pos].xlator->name);
+
+    return 0;
 }
 
 int
-dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-                  int op_ret, int op_errno, dict_t *xattr)
+dht_layout_merge(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                 int op_ret, int op_errno, dict_t *xattr)
 {
-        int         i     = 0;
-        int         ret   = -1;
-        int         err   = -1;
-        void       *disk_layout_raw = NULL;
-        int         disk_layout_len = 0;
-        dht_conf_t *conf = this->private;
-
-        if (op_ret != 0) {
-                err = op_errno;
-        }
+    int i = 0;
+    int ret = -1;
+    int err = -1;
+    void *disk_layout_raw = NULL;
+    int disk_layout_len = 0;
+    dht_conf_t *conf = this->private;
 
-        if (!layout)
-                goto out;
+    if (op_ret != 0) {
+        err = op_errno;
+    }
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].xlator == NULL) {
-                        layout->list[i].err    = err;
-                        layout->list[i].xlator = subvol;
-                        break;
-                }
-        }
+    if (!layout)
+        goto out;
 
-        if (op_ret != 0) {
-                ret = 0;
-                goto out;
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == NULL) {
+            layout->list[i].err = err;
+            layout->list[i].xlator = subvol;
+            break;
         }
+    }
 
-        if (xattr) {
-                /* during lookup and not mkdir */
-                ret = dict_get_ptr_and_len (xattr, conf->xattr_name,
-					    &disk_layout_raw, &disk_layout_len);
-        }
+    if (op_ret != 0) {
+        ret = 0;
+        goto out;
+    }
 
-        if (ret != 0) {
-                layout->list[i].err = 0;
-                gf_msg_trace (this->name, 0,
-                              "Missing disk layout on %s. err = %d",
-                              subvol->name, err);
-                ret = 0;
-                goto out;
-        }
+    if (xattr) {
+        /* during lookup and not mkdir */
+        ret = dict_get_ptr_and_len(xattr, conf->xattr_name, &disk_layout_raw,
+                                   &disk_layout_len);
+    }
 
-        ret = dht_disk_layout_merge (this, layout, i, disk_layout_raw,
-				     disk_layout_len);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LAYOUT_MERGE_FAILED,
-                        "layout merge from subvolume %s failed",
-                        subvol->name);
-                goto out;
-        }
+    if (ret != 0) {
+        layout->list[i].err = 0;
+        gf_msg_trace(this->name, 0, "Missing disk layout on %s. err = %d",
+                     subvol->name, err);
+        ret = 0;
+        goto out;
+    }
 
-        if (layout->commit_hash == 0) {
-                layout->commit_hash = layout->list[i].commit_hash;
-        } else if (layout->commit_hash != layout->list[i].commit_hash) {
-                layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
-        }
+    ret = dht_disk_layout_merge(this, layout, i, disk_layout_raw,
+                                disk_layout_len);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                "subvolume=%s", subvol->name, NULL);
+        goto out;
+    }
 
-        layout->list[i].err = 0;
+    if (layout->commit_hash == 0) {
+        layout->commit_hash = layout->list[i].commit_hash;
+    } else if (layout->commit_hash != layout->list[i].commit_hash) {
+        layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
+    }
+
+    layout->list[i].err = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 void
-dht_layout_entry_swap (dht_layout_t *layout, int i, int j)
+dht_layout_entry_swap(dht_layout_t *layout, int i, int j)
 {
-        uint32_t  start_swap = 0;
-        uint32_t  stop_swap = 0;
-        uint32_t  commit_hash_swap = 0;
-        xlator_t *xlator_swap = 0;
-        int       err_swap = 0;
-
-        start_swap  = layout->list[i].start;
-        stop_swap   = layout->list[i].stop;
-        xlator_swap = layout->list[i].xlator;
-        err_swap    = layout->list[i].err;
-        commit_hash_swap = layout->list[i].commit_hash;
-
-        layout->list[i].start  = layout->list[j].start;
-        layout->list[i].stop   = layout->list[j].stop;
-        layout->list[i].xlator = layout->list[j].xlator;
-        layout->list[i].err    = layout->list[j].err;
-        layout->list[i].commit_hash = layout->list[j].commit_hash;
-
-        layout->list[j].start  = start_swap;
-        layout->list[j].stop   = stop_swap;
-        layout->list[j].xlator = xlator_swap;
-        layout->list[j].err    = err_swap;
-        layout->list[j].commit_hash = commit_hash_swap;
+    uint32_t start_swap = 0;
+    uint32_t stop_swap = 0;
+    uint32_t commit_hash_swap = 0;
+    xlator_t *xlator_swap = 0;
+    int err_swap = 0;
+
+    start_swap = layout->list[i].start;
+    stop_swap = layout->list[i].stop;
+    xlator_swap = layout->list[i].xlator;
+    err_swap = layout->list[i].err;
+    commit_hash_swap = layout->list[i].commit_hash;
+
+    layout->list[i].start = layout->list[j].start;
+    layout->list[i].stop = layout->list[j].stop;
+    layout->list[i].xlator = layout->list[j].xlator;
+    layout->list[i].err = layout->list[j].err;
+    layout->list[i].commit_hash = layout->list[j].commit_hash;
+
+    layout->list[j].start = start_swap;
+    layout->list[j].stop = stop_swap;
+    layout->list[j].xlator = xlator_swap;
+    layout->list[j].err = err_swap;
+    layout->list[j].commit_hash = commit_hash_swap;
 }
 
 void
-dht_layout_range_swap (dht_layout_t *layout, int i, int j)
+dht_layout_range_swap(dht_layout_t *layout, int i, int j)
 {
-        uint32_t  start_swap = 0;
-        uint32_t  stop_swap = 0;
+    uint32_t start_swap = 0;
+    uint32_t stop_swap = 0;
 
-        start_swap  = layout->list[i].start;
-        stop_swap   = layout->list[i].stop;
+    start_swap = layout->list[i].start;
+    stop_swap = layout->list[i].stop;
 
-        layout->list[i].start  = layout->list[j].start;
-        layout->list[i].stop   = layout->list[j].stop;
+    layout->list[i].start = layout->list[j].start;
+    layout->list[i].stop = layout->list[j].stop;
 
-        layout->list[j].start  = start_swap;
-        layout->list[j].stop   = stop_swap;
+    layout->list[j].start = start_swap;
+    layout->list[j].stop = stop_swap;
 }
-
-int64_t
-dht_layout_entry_cmp_volname (dht_layout_t *layout, int i, int j)
+static int64_t
+dht_layout_entry_cmp_volname(dht_layout_t *layout, int i, int j)
 {
-        return (strcmp (layout->list[i].xlator->name,
-                        layout->list[j].xlator->name));
+    return (strcmp(layout->list[i].xlator->name, layout->list[j].xlator->name));
 }
 
 gf_boolean_t
-dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator)
+dht_is_subvol_in_layout(dht_layout_t *layout, xlator_t *xlator)
 {
-        int i = 0;
-
-        for (i = 0; i < layout->cnt; i++) {
-                /* Check if xlator is already part of layout, and layout is
-                 * non-zero. */
-                if (!strcmp (layout->list[i].xlator->name, xlator->name)) {
-                        if (layout->list[i].start != layout->list[i].stop)
-                                return _gf_true;
-                        break;
-                }
-        }
-        return _gf_false;
+    int i = 0;
+
+    for (i = 0; i < layout->cnt; i++) {
+        /* Check if xlator is already part of layout, and layout is
+         * non-zero. */
+        if (!strcmp(layout->list[i].xlator->name, xlator->name)) {
+            if (layout->list[i].start != layout->list[i].stop)
+                return _gf_true;
+            break;
+        }
+    }
+    return _gf_false;
 }
 
-int64_t
-dht_layout_entry_cmp (dht_layout_t *layout, int i, int j)
+static int64_t
+dht_layout_entry_cmp(dht_layout_t *layout, int i, int j)
 {
-        int64_t diff = 0;
+    int64_t diff = 0;
 
-        /* swap zero'ed out layouts to front, if needed */
-        if (!layout->list[j].start && !layout->list[j].stop) {
-                diff = (int64_t) layout->list[i].stop
-                       - (int64_t) layout->list[j].stop;
-                       goto out;
-        }
-        diff = (int64_t) layout->list[i].start
-                - (int64_t) layout->list[j].start;
+    /* swap zero'ed out layouts to front, if needed */
+    if (!layout->list[j].start && !layout->list[j].stop) {
+        diff = (int64_t)layout->list[i].stop - (int64_t)layout->list[j].stop;
+        goto out;
+    }
+    diff = (int64_t)layout->list[i].start - (int64_t)layout->list[j].start;
 
 out:
-        return diff;
+    return diff;
 }
 
-
 int
-dht_layout_sort (dht_layout_t *layout)
+dht_layout_sort(dht_layout_t *layout)
 {
-        int       i = 0;
-        int       j = 0;
-        int64_t   ret = 0;
+    int i = 0;
+    int j = 0;
+    int64_t ret = 0;
 
-        /* TODO: O(n^2) -- bad bad */
+    /* TODO: O(n^2) -- bad bad */
 
-        for (i = 0; i < layout->cnt - 1; i++) {
-                for (j = i + 1; j < layout->cnt; j++) {
-                        ret = dht_layout_entry_cmp (layout, i, j);
-                        if (ret > 0)
-                                dht_layout_entry_swap (layout, i, j);
-                }
+    for (i = 0; i < layout->cnt - 1; i++) {
+        for (j = i + 1; j < layout->cnt; j++) {
+            ret = dht_layout_entry_cmp(layout, i, j);
+            if (ret > 0)
+                dht_layout_entry_swap(layout, i, j);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_layout_sort_volname (dht_layout_t *layout)
+void
+dht_layout_sort_volname(dht_layout_t *layout)
 {
-        int       i = 0;
-        int       j = 0;
-        int64_t   ret = 0;
+    int i = 0;
+    int j = 0;
+    int64_t ret = 0;
 
-        /* TODO: O(n^2) -- bad bad */
+    /* TODO: O(n^2) -- bad bad */
 
-        for (i = 0; i < layout->cnt - 1; i++) {
-                for (j = i + 1; j < layout->cnt; j++) {
-                        ret = dht_layout_entry_cmp_volname (layout, i, j);
-                        if (ret > 0)
-                                dht_layout_entry_swap (layout, i, j);
-                }
+    for (i = 0; i < layout->cnt - 1; i++) {
+        for (j = i + 1; j < layout->cnt; j++) {
+            ret = dht_layout_entry_cmp_volname(layout, i, j);
+            if (ret > 0)
+                dht_layout_entry_swap(layout, i, j);
         }
-
-        return 0;
+    }
 }
 
-
-int
-dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
-                      uint32_t *holes_p, uint32_t *overlaps_p,
-                      uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
-                      uint32_t *no_space_p)
+void
+dht_layout_anomalies(xlator_t *this, loc_t *loc, dht_layout_t *layout,
+                     uint32_t *holes_p, uint32_t *overlaps_p,
+                     uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
+                     uint32_t *no_space_p)
 {
-        uint32_t    overlaps = 0;
-        uint32_t    missing  = 0;
-        uint32_t    down     = 0;
-        uint32_t    misc     = 0;
-        uint32_t    hole_cnt = 0;
-        uint32_t    overlap_cnt = 0;
-        int         i = 0;
-        int         ret = 0;
-        uint32_t    prev_stop = 0;
-        uint32_t    last_stop = 0;
-        char        is_virgin = 1;
-        uint32_t    no_space  = 0;
-
-        /* This function scans through the layout spread of a directory to
-           check if there are any anomalies. Prior to calling this function
-           the layout entries should be sorted in the ascending order.
-
-           If the layout entry has err != 0
-                then increment the corresponding anomaly.
-           else
-                if (start of the current layout entry > stop + 1 of previous
-                   non erroneous layout entry)
-                        then it indicates a hole in the layout
-                if (start of the current layout entry < stop + 1 of previous
-                    non erroneous layout entry)
-                         then it indicates an overlap in the layout
-        */
-        last_stop = layout->list[0].start - 1;
-        prev_stop = last_stop;
-
-        for (i = 0; i < layout->cnt; i++) {
-                switch (layout->list[i].err) {
-                case -1:
-                case ENOENT:
-                case ESTALE:
-                        missing++;
-                        continue;
-                case ENOTCONN:
-                        down++;
-                        continue;
-                case ENOSPC:
-                        no_space++;
-                        continue;
-                case 0:
-                        /* if err == 0 and start == stop, then it is a non misc++;
-                         * participating subvolume(spread-cnt). Then, do not
-                         * check for anomalies. If start != stop, then treat it
-                         * as misc err */
-                        if (layout->list[i].start == layout->list[i].stop) {
-                                continue;
-                        }
-                        break;
-                default:
-                        misc++;
-                        continue;
-                 }
-
-                is_virgin = 0;
-
-                if ((prev_stop + 1) < layout->list[i].start) {
-                        hole_cnt++;
+    uint32_t overlaps = 0;
+    uint32_t missing = 0;
+    uint32_t down = 0;
+    uint32_t misc = 0;
+    uint32_t hole_cnt = 0;
+    uint32_t overlap_cnt = 0;
+    int i = 0;
+    uint32_t prev_stop = 0;
+    uint32_t last_stop = 0;
+    char is_virgin = 1;
+    uint32_t no_space = 0;
+
+    /* This function scans through the layout spread of a directory to
+       check if there are any anomalies. Prior to calling this function
+       the layout entries should be sorted in the ascending order.
+
+       If the layout entry has err != 0
+            then increment the corresponding anomaly.
+       else
+            if (start of the current layout entry > stop + 1 of previous
+               non erroneous layout entry)
+                    then it indicates a hole in the layout
+            if (start of the current layout entry < stop + 1 of previous
+                non erroneous layout entry)
+                     then it indicates an overlap in the layout
+    */
+    last_stop = layout->list[0].start - 1;
+    prev_stop = last_stop;
+
+    for (i = 0; i < layout->cnt; i++) {
+        switch (layout->list[i].err) {
+            case -1:
+            case ENOENT:
+            case ESTALE:
+                missing++;
+                continue;
+            case ENOTCONN:
+                down++;
+                continue;
+            case ENOSPC:
+                no_space++;
+                continue;
+            case 0:
+                /* if err == 0 and start == stop, then it is a non misc++;
+                 * participating subvolume(spread-cnt). Then, do not
+                 * check for anomalies. If start != stop, then treat it
+                 * as misc err */
+                if (layout->list[i].start == layout->list[i].stop) {
+                    continue;
                 }
+                break;
+            default:
+                misc++;
+                continue;
+        }
 
-                if ((prev_stop + 1) > layout->list[i].start) {
-                        overlap_cnt++;
-                        overlaps += ((prev_stop + 1) - layout->list[i].start);
-                }
-                prev_stop = layout->list[i].stop;
+        is_virgin = 0;
+
+        if ((prev_stop + 1) < layout->list[i].start) {
+            hole_cnt++;
         }
 
-        if ((last_stop - prev_stop) || is_virgin)
-                hole_cnt++;
+        if ((prev_stop + 1) > layout->list[i].start) {
+            overlap_cnt++;
+            overlaps += ((prev_stop + 1) - layout->list[i].start);
+        }
+        prev_stop = layout->list[i].stop;
+    }
 
-        if (holes_p)
-                *holes_p = hole_cnt;
+    if ((last_stop - prev_stop) || is_virgin)
+        hole_cnt++;
 
-        if (overlaps_p)
-                *overlaps_p = overlap_cnt;
+    if (holes_p)
+        *holes_p = hole_cnt;
 
-        if (missing_p)
-                *missing_p = missing;
+    if (overlaps_p)
+        *overlaps_p = overlap_cnt;
 
-        if (down_p)
-                *down_p = down;
+    if (missing_p)
+        *missing_p = missing;
 
-        if (misc_p)
-                *misc_p = misc;
+    if (down_p)
+        *down_p = down;
 
-        if (no_space_p)
-                *no_space_p = no_space;
+    if (misc_p)
+        *misc_p = misc;
 
-        return ret;
+    if (no_space_p)
+        *no_space_p = no_space;
 }
 
-
 int
-dht_layout_missing_dirs (dht_layout_t *layout)
+dht_layout_missing_dirs(dht_layout_t *layout)
 {
-        int i = 0, missing = 0;
+    int i = 0, missing = 0;
 
-        if (layout == NULL)
-                goto out;
+    if (layout == NULL)
+        goto out;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if ((layout->list[i].err == ENOENT)
-                    || ((layout->list[i].err == -1)
-                        && (layout->list[i].start == 0)
-                        && (layout->list[i].stop == 0))) {
-                        missing++;
-                }
+    for (i = 0; i < layout->cnt; i++) {
+        if ((layout->list[i].err == ENOENT) ||
+            ((layout->list[i].err == -1) && (layout->list[i].start == 0) &&
+             (layout->list[i].stop == 0))) {
+            missing++;
         }
+    }
 
 out:
-        return missing;
+    return missing;
 }
 
-
 int
-dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout)
+dht_layout_normalize(xlator_t *this, loc_t *loc, dht_layout_t *layout)
 {
-        int          ret   = 0;
-        uint32_t     holes = 0;
-        uint32_t     overlaps = 0;
-        uint32_t     missing = 0;
-        uint32_t     down = 0;
-        uint32_t     misc = 0, missing_dirs = 0;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        ret = dht_layout_sort (layout);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LAYOUT_SORT_FAILED,
-                        "sort failed?! how the ....");
-                goto out;
-        }
-
-        gf_uuid_unparse(loc->gfid, gfid);
-
-        ret = dht_layout_anomalies (this, loc, layout,
-                                    &holes, &overlaps,
-                                    &missing, &down, &misc, NULL);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_FIND_LAYOUT_ANOMALIES_ERROR,
-                        "Error finding anomalies in %s, gfid = %s",
-                        loc->path, gfid);
-                goto out;
-        }
-
-        if (holes || overlaps) {
-                if (missing == layout->cnt) {
-                        gf_msg_debug (this->name, 0,
-                                      "Directory %s looked up first time"
-                                      " gfid = %s", loc->path, gfid);
-                } else {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_ANOMALIES_INFO,
-                                "Found anomalies in %s (gfid = %s). "
-                                "Holes=%d overlaps=%d",
-                                loc->path, gfid, holes, overlaps );
-                }
-                ret = -1;
-        }
-
-        if (ret >= 0) {
-                missing_dirs = dht_layout_missing_dirs (layout);
-                /* TODO During DHT selfheal rewrite (almost) find a better place
-                 * to detect this - probably in dht_layout_anomalies()
-                 */
-                if (missing_dirs > 0)
-                        ret += missing_dirs;
-        }
+    int ret = 0;
+    uint32_t holes = 0;
+    uint32_t overlaps = 0;
+    uint32_t missing = 0;
+    uint32_t down = 0;
+    uint32_t misc = 0, missing_dirs = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    ret = dht_layout_sort(layout);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_SORT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    gf_uuid_unparse(loc->gfid, gfid);
+
+    dht_layout_anomalies(this, loc, layout, &holes, &overlaps, &missing, &down,
+                         &misc, NULL);
+
+    if (holes || overlaps) {
+        if (missing == layout->cnt) {
+            gf_msg_debug(this->name, 0,
+                         "Directory %s looked up first time"
+                         " gfid = %s",
+                         loc->path, gfid);
+        } else {
+            gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_ANOMALIES_INFO,
+                    "path=%s", loc->path, "gfid=%s", gfid, "holes=%d", holes,
+                    "overlaps=%d", overlaps, NULL);
+        }
+        ret = -1;
+    }
+
+    if (ret >= 0) {
+        missing_dirs = dht_layout_missing_dirs(layout);
+        /* TODO During DHT selfheal rewrite (almost) find a better place
+         * to detect this - probably in dht_layout_anomalies()
+         */
+        if (missing_dirs > 0)
+            ret += missing_dirs;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-dht_dir_has_layout (dict_t *xattr, char *name)
+dht_dir_has_layout(dict_t *xattr, char *name)
 {
+    void *disk_layout_raw = NULL;
 
-        void     *disk_layout_raw = NULL;
-
-        return dict_get_ptr (xattr, name, &disk_layout_raw);
+    return dict_get_ptr(xattr, name, &disk_layout_raw);
 }
 
 int
-dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-                         loc_t *loc, dict_t *xattr)
+dht_layout_dir_mismatch(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                        loc_t *loc, dict_t *xattr)
 {
-        int         idx = 0;
-        int         pos = -1;
-        int         ret = 0;
-        int         err = 0;
-        int         dict_ret = 0;
-        int32_t     disk_layout[4];
-        void       *disk_layout_raw = NULL;
-        uint32_t    start_off = -1;
-        uint32_t    stop_off = -1;
-        uint32_t    commit_hash = -1;
-        dht_conf_t *conf = this->private;
-        char        gfid[GF_UUID_BUF_SIZE] = {0};
-
-        if(loc && loc->inode)
-                gf_uuid_unparse(loc->inode->gfid, gfid);
-
-        for (idx = 0; idx < layout->cnt; idx++) {
-                if (layout->list[idx].xlator == subvol) {
-                        pos = idx;
-                        break;
-                }
-        }
-
-        if (pos == -1) {
-                if (loc) {
-                        gf_msg_debug (this->name, 0,
-                                      "%s - no layout info for subvolume %s",
-                                      loc ? loc->path : "path not found",
-                                      subvol->name);
-                }
-                ret = 1;
-                goto out;
-        }
-
-        err = layout->list[pos].err;
-
-        if (!xattr) {
-                if (err == 0) {
-                        if (loc) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_DICT_GET_FAILED,
-                                        "%s: xattr dictionary is NULL",
-                                        loc->path);
-                        } else {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_DICT_GET_FAILED,
-                                        "path not found: "
-                                        "xattr dictionary is NULL");
-                        }
-                        ret = -1;
-                }
-                goto out;
-        }
-
-        dict_ret = dict_get_ptr (xattr, conf->xattr_name,
-                                 &disk_layout_raw);
-
-        if (dict_ret < 0) {
-                if (err == 0 && layout->list[pos].stop) {
-                        if (loc) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_DISK_LAYOUT_MISSING,
-                                    "%s: Disk layout missing, gfid = %s",
-                                    loc->path, gfid);
-                        } else {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_DISK_LAYOUT_MISSING,
-                                        "path not found: "
-                                        "Disk layout missing, gfid = %s",
-                                        gfid);
-                        }
-                        ret = -1;
-                }
-                goto out;
-        }
-
-        memcpy (disk_layout, disk_layout_raw, sizeof (disk_layout));
-
-        start_off = ntoh32 (disk_layout[2]);
-        stop_off  = ntoh32 (disk_layout[3]);
-        commit_hash = ntoh32 (disk_layout[0]);
-
-        if ((layout->list[pos].start != start_off)
-            || (layout->list[pos].stop != stop_off)
-            || (layout->list[pos].commit_hash != commit_hash)) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LAYOUT_INFO,
-                        "subvol: %s; inode layout - %"PRIu32" - %"PRIu32
-                        " - %"PRIu32"; "
-                        "disk layout - %"PRIu32" - %"PRIu32" - %"PRIu32,
-                        layout->list[pos].xlator->name,
-                        layout->list[pos].start, layout->list[pos].stop,
-                        layout->list[pos].commit_hash,
-                        start_off, stop_off, commit_hash);
-                ret = 1;
-        } else {
-                ret = 0;
-        }
+    int idx = 0;
+    int pos = -1;
+    int ret = 0;
+    int err = 0;
+    int dict_ret = 0;
+    int32_t disk_layout[4];
+    void *disk_layout_raw = NULL;
+    uint32_t start_off = -1;
+    uint32_t stop_off = -1;
+    uint32_t commit_hash = -1;
+    dht_conf_t *conf = this->private;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    if (loc && loc->inode)
+        gf_uuid_unparse(loc->inode->gfid, gfid);
+
+    for (idx = 0; idx < layout->cnt; idx++) {
+        if (layout->list[idx].xlator == subvol) {
+            pos = idx;
+            break;
+        }
+    }
+
+    if (pos == -1) {
+        if (loc) {
+            gf_msg_debug(this->name, 0, "%s - no layout info for subvolume %s",
+                         loc ? loc->path : "path not found", subvol->name);
+        }
+        ret = 1;
+        goto out;
+    }
+
+    err = layout->list[pos].err;
+
+    if (!xattr) {
+        if (err == 0) {
+            if (loc) {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_XATTR_DICT_NULL,
+                        "path=%s", loc->path, NULL);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_XATTR_DICT_NULL,
+                        "path not found", NULL);
+            }
+            ret = -1;
+        }
+        goto out;
+    }
+
+    dict_ret = dict_get_ptr(xattr, conf->xattr_name, &disk_layout_raw);
+
+    if (dict_ret < 0) {
+        if (err == 0 && layout->list[pos].stop) {
+            if (loc) {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_DISK_LAYOUT_MISSING,
+                        "path=%s", loc->path, "gfid=%s", gfid, NULL);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_DISK_LAYOUT_MISSING,
+                        "path not found"
+                        "gfid=%s",
+                        gfid, NULL);
+            }
+            ret = -1;
+        }
+        goto out;
+    }
+
+    memcpy(disk_layout, disk_layout_raw, sizeof(disk_layout));
+
+    start_off = ntoh32(disk_layout[2]);
+    stop_off = ntoh32(disk_layout[3]);
+    commit_hash = ntoh32(disk_layout[0]);
+
+    if ((layout->list[pos].start != start_off) ||
+        (layout->list[pos].stop != stop_off) ||
+        (layout->list[pos].commit_hash != commit_hash)) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_INFO, "subvol=%s",
+                layout->list[pos].xlator->name, "inode-layout:start=0x%x",
+                layout->list[pos].start, "inode-layout:stop=0x%x",
+                layout->list[pos].stop, "layout-commit-hash=0x%x; ",
+                layout->list[pos].commit_hash, "disk-layout:start-off=0x%x",
+                start_off, "disk-layout:top-off=0x%x", stop_off,
+                "commit-hash=0x%x", commit_hash, NULL);
+        ret = 1;
+    } else {
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode)
+dht_layout_preset(xlator_t *this, xlator_t *subvol, inode_t *inode)
 {
-        dht_layout_t *layout = NULL;
-        int           ret = -1;
-        dht_conf_t   *conf = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
+    layout = dht_layout_for_subvol(this, subvol);
+    if (!layout) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_NO_LAYOUT_INFO,
+                "subvolume=%s", subvol ? subvol->name : "<nil>", NULL);
+        ret = -1;
+        goto out;
+    }
 
-        layout = dht_layout_for_subvol (this, subvol);
-        if (!layout) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_SUBVOL_NO_LAYOUT_INFO,
-                        "no pre-set layout for subvolume %s",
-                        subvol ? subvol->name : "<nil>");
-                ret = -1;
-                goto out;
-        }
+    gf_msg_debug(this->name, 0, "file = %s, subvol = %s",
+                 uuid_utoa(inode->gfid), subvol ? subvol->name : "<nil>");
 
-        gf_msg_debug (this->name, 0, "file = %s, subvol = %s",
-                      uuid_utoa (inode->gfid), subvol ? subvol->name : "<nil>");
+    LOCK(&conf->layout_lock);
+    {
+        dht_inode_ctx_layout_set(inode, this, layout);
+    }
 
-        LOCK (&conf->layout_lock);
-        {
-                dht_inode_ctx_layout_set (inode, this, layout);
-        }
+    UNLOCK(&conf->layout_lock);
 
-        UNLOCK (&conf->layout_lock);
-
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-dht_layout_index_for_subvol (dht_layout_t *layout, xlator_t *subvol)
+dht_layout_index_for_subvol(dht_layout_t *layout, xlator_t *subvol)
 {
-        int i = 0, ret = -1;
+    int i = 0, ret = -1;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].xlator == subvol) {
-                        ret = i;
-                        break;
-                }
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == subvol) {
+            ret = i;
+            break;
         }
+    }
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-linkfile.c b/xlators/cluster/dht/src/dht-linkfile.c
index a6d994b4157..89ec6cca56e 100644
--- a/xlators/cluster/dht/src/dht-linkfile.c
+++ b/xlators/cluster/dht/src/dht-linkfile.c
@@ -8,351 +8,321 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "compat.h"
+#include <glusterfs/compat.h>
 #include "dht-common.h"
-#include "dht-messages.h"
 
-int
-dht_linkfile_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno,
-                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                         struct iatt *postparent)
+static int
+dht_linkfile_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent)
 {
-        char          is_linkfile   = 0;
-        dht_conf_t   *conf          = NULL;
-        dht_local_t  *local         = NULL;
-        xlator_t     *prev          = NULL;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        prev = cookie;
-        conf = this->private;
-
-        if (op_ret)
-                goto out;
-
-        gf_uuid_unparse(local->loc.gfid, gfid);
-
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                         conf->link_xattr_name);
-        if (!is_linkfile)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_NOT_LINK_FILE_ERROR,
-                        "got non-linkfile %s:%s, gfid = %s",
-                        prev->name, local->loc.path, gfid);
+    char is_linkfile = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    if (op_ret)
+        goto out;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
+    if (!is_linkfile)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NOT_LINK_FILE_ERROR,
+                "name=%s", prev->name, "path=%s", local->loc.path, "gfid=%s",
+                gfid, NULL);
 out:
-        local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
-                                      inode, stbuf, postparent, postparent,
-                                      xattr);
-        return 0;
+    local->linkfile.linkfile_cbk(frame, cookie, this, op_ret, op_errno, inode,
+                                 stbuf, postparent, postparent, xattr);
+    return 0;
 }
 
-#define is_equal(a, b) ((a) == (b))
-int
-dht_linkfile_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, inode_t *inode,
-                         struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
+static int
+dht_linkfile_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *subvol = NULL;
-        dict_t       *xattrs = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret = -1;
-
-        local = frame->local;
-
-        if (!op_ret)
-                local->linked = _gf_true;
-
-        FRAME_SU_UNDO (frame, dht_local_t);
-
-        if (op_ret && (op_errno == EEXIST)) {
-                conf = this->private;
-                subvol = cookie;
-                if (!subvol)
-                        goto out;
-                xattrs = dict_new ();
-                if (!xattrs)
-                        goto out;
-                ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value. key : %s",
-                                conf->link_xattr_name);
-                        goto out;
-                }
-
-                STACK_WIND_COOKIE (frame, dht_linkfile_lookup_cbk, subvol,
-                                   subvol, subvol->fops->lookup, &local->linkfile.loc,
-                                   xattrs);
-                if (xattrs)
-                        dict_unref (xattrs);
-                return 0;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    dict_t *xattrs = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+
+    local = frame->local;
+
+    if (!op_ret)
+        local->linked = _gf_true;
+
+    FRAME_SU_UNDO(frame, dht_local_t);
+
+    if (op_ret && (op_errno == EEXIST)) {
+        conf = this->private;
+        subvol = cookie;
+        if (!subvol)
+            goto out;
+        xattrs = dict_new();
+        if (!xattrs)
+            goto out;
+        ret = dict_set_uint32(xattrs, conf->link_xattr_name, 256);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                    "mame=%s", conf->link_xattr_name, NULL);
+            goto out;
         }
-out:
-        local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
-                                      inode, stbuf, preparent, postparent,
-                                      xdata);
+
+        STACK_WIND_COOKIE(frame, dht_linkfile_lookup_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->linkfile.loc, xattrs);
         if (xattrs)
-                dict_unref (xattrs);
+            dict_unref(xattrs);
         return 0;
+    }
+out:
+    local->linkfile.linkfile_cbk(frame, cookie, this, op_ret, op_errno, inode,
+                                 stbuf, preparent, postparent, xdata);
+    if (xattrs)
+        dict_unref(xattrs);
+    return 0;
 }
 
-
 int
-dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
-                     xlator_t *this,
-                     xlator_t *tovol, xlator_t *fromvol, loc_t *loc)
+dht_linkfile_create(call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
+                    xlator_t *this, xlator_t *tovol, xlator_t *fromvol,
+                    loc_t *loc)
 {
-        dht_local_t *local = NULL;
-        dict_t      *dict = NULL;
-        int          need_unref = 0;
-        int          ret = 0;
-        dht_conf_t  *conf = this->private;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        local->linkfile.linkfile_cbk = linkfile_cbk;
-        local->linkfile.srcvol = tovol;
-        loc_copy (&local->linkfile.loc, loc);
-
-        local->linked = _gf_false;
-
-        dict = local->params;
-        if (!dict) {
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-                need_unref = 1;
-        }
-
-
-        if (!gf_uuid_is_null (local->gfid)) {
-                gf_uuid_unparse(local->gfid, gfid);
-
-                ret = dict_set_gfuuid (dict, "gfid-req", local->gfid, true);
-                if (ret)
-                        gf_msg ("dht-linkfile", GF_LOG_INFO, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "%s: Failed to set dictionary value: "
-                                "key = gfid-req, gfid = %s ", loc->path, gfid);
-        } else {
-                gf_uuid_unparse(loc->gfid, gfid);
-        }
-
-        ret = dict_set_str (dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    dht_local_t *local = NULL;
+    dict_t *dict = NULL;
+    int need_unref = 0;
+    int ret = 0;
+    dht_conf_t *conf = this->private;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    local->linkfile.linkfile_cbk = linkfile_cbk;
+    local->linkfile.srcvol = tovol;
+    loc_copy(&local->linkfile.loc, loc);
+
+    local->linked = _gf_false;
+
+    dict = local->params;
+    if (!dict) {
+        dict = dict_new();
+        if (!dict)
+            goto out;
+        need_unref = 1;
+    }
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        gf_uuid_unparse(local->gfid, gfid);
+
+        ret = dict_set_gfuuid(dict, "gfid-req", local->gfid, true);
         if (ret)
-                gf_msg ("dht-linkfile", GF_LOG_INFO, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value: key = %s,"
-                        " gfid = %s", loc->path,
-                        GLUSTERFS_INTERNAL_FOP_KEY, gfid);
-
-        ret = dict_set_str (dict, conf->link_xattr_name, tovol->name);
-
-        if (ret < 0) {
-                gf_msg (frame->this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_CREATE_LINK_FAILED,
-                        "%s: failed to initialize linkfile data, gfid = %s",
-                        loc->path, gfid);
-                goto out;
-        }
-
-        local->link_subvol = fromvol;
-        /* Always create as root:root. dht_linkfile_attr_heal fixes the
-         * ownsership */
-        FRAME_SU_DO (frame, dht_local_t);
-        STACK_WIND_COOKIE (frame, dht_linkfile_create_cbk, fromvol, fromvol,
-                           fromvol->fops->mknod, loc,
-                           S_IFREG | DHT_LINKFILE_MODE, 0, 0, dict);
-
-        if (need_unref && dict)
-                dict_unref (dict);
-
-        return 0;
+            gf_smsg("dht-linkfile", GF_LOG_INFO, 0, DHT_MSG_DICT_SET_FAILED,
+                    "path=%s", loc->path, "gfid=%s", gfid, NULL);
+    } else {
+        gf_uuid_unparse(loc->gfid, gfid);
+    }
+
+    ret = dict_set_str(dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    if (ret)
+        gf_smsg("dht-linkfile", GF_LOG_INFO, 0, DHT_MSG_DICT_SET_FAILED,
+                "path=%s", loc->path, "key=%s", GLUSTERFS_INTERNAL_FOP_KEY,
+                "gfid=%s", gfid, NULL);
+
+    ret = dict_set_str(dict, conf->link_xattr_name, tovol->name);
+
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, DHT_MSG_CREATE_LINK_FAILED,
+                "path=%s", loc->path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    local->link_subvol = fromvol;
+    /* Always create as root:root. dht_linkfile_attr_heal fixes the
+     * ownsership */
+    FRAME_SU_DO(frame, dht_local_t);
+    STACK_WIND_COOKIE(frame, dht_linkfile_create_cbk, fromvol, fromvol,
+                      fromvol->fops->mknod, loc, S_IFREG | DHT_LINKFILE_MODE, 0,
+                      0, dict);
+
+    if (need_unref && dict)
+        dict_unref(dict);
+
+    return 0;
 out:
-        local->linkfile.linkfile_cbk (frame, frame->this, frame->this, -1, ENOMEM,
-                                      loc->inode, NULL, NULL, NULL, NULL);
+    local->linkfile.linkfile_cbk(frame, frame->this, frame->this, -1, ENOMEM,
+                                 loc->inode, NULL, NULL, NULL, NULL);
 
-        if (need_unref && dict)
-                dict_unref (dict);
+    if (need_unref && dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno,
-                         struct iatt *preparent, struct iatt *postparent,
-                         dict_t *xdata)
+dht_linkfile_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *preparent, struct iatt *postparent,
+                        dict_t *xdata)
 {
-        dht_local_t   *local = NULL;
-        xlator_t      *subvol = NULL;
-        char           gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        subvol = cookie;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
+    local = frame->local;
+    subvol = cookie;
 
-        if (op_ret == -1) {
-
-                gf_uuid_unparse(local->loc.gfid, gfid);
-                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                        DHT_MSG_UNLINK_FAILED,
-                        "Unlinking linkfile %s (gfid = %s)on "
-                        "subvolume %s failed ",
-                        local->loc.path, gfid, subvol->name);
-        }
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_UNLINK_FAILED,
+                "path=%s", local->loc.path, "gfid=%s", gfid, "subvolume=%s",
+                subvol->name, NULL);
+    }
 
-        DHT_STACK_DESTROY (frame);
+    DHT_STACK_DESTROY(frame);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_linkfile_unlink (call_frame_t *frame, xlator_t *this,
-                     xlator_t *subvol, loc_t *loc)
+dht_linkfile_unlink(call_frame_t *frame, xlator_t *this, xlator_t *subvol,
+                    loc_t *loc)
 {
-        call_frame_t *unlink_frame = NULL;
-        dht_local_t  *unlink_local = NULL;
+    call_frame_t *unlink_frame = NULL;
+    dht_local_t *unlink_local = NULL;
 
-        unlink_frame = copy_frame (frame);
-        if (!unlink_frame) {
-                goto err;
-        }
+    unlink_frame = copy_frame(frame);
+    if (!unlink_frame) {
+        goto err;
+    }
 
-        /* Using non-fop value here, as anyways, 'local->fop' is not used in
-           this particular case */
-        unlink_local = dht_local_init (unlink_frame, loc, NULL,
-                                       GF_FOP_MAXVALUE);
-        if (!unlink_local) {
-                goto err;
-        }
+    /* Using non-fop value here, as anyways, 'local->fop' is not used in
+       this particular case */
+    unlink_local = dht_local_init(unlink_frame, loc, NULL, GF_FOP_MAXVALUE);
+    if (!unlink_local) {
+        goto err;
+    }
 
-        STACK_WIND_COOKIE (unlink_frame, dht_linkfile_unlink_cbk, subvol,
-                           subvol, subvol->fops->unlink,
-                           &unlink_local->loc, 0, NULL);
+    STACK_WIND_COOKIE(unlink_frame, dht_linkfile_unlink_cbk, subvol, subvol,
+                      subvol->fops->unlink, &unlink_local->loc, 0, NULL);
 
-        return 0;
+    return 0;
 err:
-        if (unlink_frame)
-                DHT_STACK_DESTROY (unlink_frame);
+    if (unlink_frame)
+        DHT_STACK_DESTROY(unlink_frame);
 
-        return -1;
+    return -1;
 }
 
-
 xlator_t *
-dht_linkfile_subvol (xlator_t *this, inode_t *inode, struct iatt *stbuf,
-                     dict_t *xattr)
+dht_linkfile_subvol(xlator_t *this, inode_t *inode, struct iatt *stbuf,
+                    dict_t *xattr)
 {
-        dht_conf_t *conf = NULL;
-        xlator_t   *subvol = NULL;
-        void       *volname = NULL;
-        int         i = 0, ret = 0;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    void *volname = NULL;
+    int i = 0, ret = 0;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!xattr)
-                goto out;
+    if (!xattr)
+        goto out;
 
-        ret = dict_get_ptr (xattr, conf->link_xattr_name, &volname);
+    ret = dict_get_ptr(xattr, conf->link_xattr_name, &volname);
 
-        if ((-1 == ret) || !volname)
-                goto out;
+    if ((-1 == ret) || !volname)
+        goto out;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (strcmp (conf->subvolumes[i]->name, (char *)volname) == 0) {
-                        subvol = conf->subvolumes[i];
-                        break;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (strcmp(conf->subvolumes[i]->name, (char *)volname) == 0) {
+            subvol = conf->subvolumes[i];
+            break;
         }
+    }
 
 out:
-        return subvol;
+    return subvol;
 }
 
-int
-dht_linkfile_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int op_ret, int op_errno, struct iatt *statpre,
-                          struct iatt *statpost, dict_t *xdata)
+static int
+dht_linkfile_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, struct iatt *statpre,
+                         struct iatt *statpost, dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        loc_t *loc = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
 
-        local = frame->local;
-        loc = &local->loc;
+    local = frame->local;
+    loc = &local->loc;
 
-        if (op_ret)
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        DHT_MSG_SETATTR_FAILED,
-                        "Failed to set attr uid/gid on %s"
-                        " :<gfid:%s> ",
-                        (loc->path? loc->path: "NULL"),
-                        uuid_utoa(local->gfid));
+    if (op_ret)
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_SETATTR_FAILED,
+                "path=%s", (loc->path ? loc->path : "NULL"), "gfid=%s",
+                uuid_utoa(local->gfid), NULL);
 
-        DHT_STACK_DESTROY (frame);
+    DHT_STACK_DESTROY(frame);
 
-       return 0;
+    return 0;
 }
 
 int
-dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this)
+dht_linkfile_attr_heal(call_frame_t *frame, xlator_t *this)
 {
-        int           ret         = -1;
-        call_frame_t  *copy       = NULL;
-        dht_local_t   *local      = NULL;
-        dht_local_t   *copy_local = NULL;
-        xlator_t      *subvol     = NULL;
-        struct iatt   stbuf       = {0,};
-        dict_t        *xattr      = NULL;
-
-        local = frame->local;
-
-        GF_VALIDATE_OR_GOTO ("dht", local, out);
-        GF_VALIDATE_OR_GOTO ("dht", local->link_subvol, out);
-
-        if (local->stbuf.ia_type == IA_INVAL)
-                return 0;
+    int ret = -1;
+    call_frame_t *copy = NULL;
+    dht_local_t *local = NULL;
+    dht_local_t *copy_local = NULL;
+    xlator_t *subvol = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    dict_t *xattr = NULL;
+
+    local = frame->local;
+
+    GF_VALIDATE_OR_GOTO("dht", local, out);
+    GF_VALIDATE_OR_GOTO("dht", local->link_subvol, out);
+
+    if (local->stbuf.ia_type == IA_INVAL)
+        return 0;
 
-        DHT_MARK_FOP_INTERNAL (xattr);
+    DHT_MARK_FOP_INTERNAL(xattr);
 
-        gf_uuid_copy (local->loc.gfid, local->stbuf.ia_gfid);
+    gf_uuid_copy(local->loc.gfid, local->stbuf.ia_gfid);
 
-        copy = copy_frame (frame);
+    copy = copy_frame(frame);
 
-        if (!copy)
-                goto out;
+    if (!copy)
+        goto out;
 
-       copy_local = dht_local_init (copy, &local->loc, NULL, 0);
+    copy_local = dht_local_init(copy, &local->loc, NULL, 0);
 
-       if (!copy_local)
-               goto out;
+    if (!copy_local)
+        goto out;
 
-        stbuf = local->stbuf;
-        subvol = local->link_subvol;
+    stbuf = local->stbuf;
+    subvol = local->link_subvol;
 
-        copy->local = copy_local;
+    copy->local = copy_local;
 
-        FRAME_SU_DO (copy, dht_local_t);
+    FRAME_SU_DO(copy, dht_local_t);
 
-        STACK_WIND (copy, dht_linkfile_setattr_cbk, subvol,
-                    subvol->fops->setattr, &copy_local->loc,
-                    &stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), xattr);
-        ret = 0;
+    STACK_WIND(copy, dht_linkfile_setattr_cbk, subvol, subvol->fops->setattr,
+               &copy_local->loc, &stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+               xattr);
+    ret = 0;
 out:
-        if ((ret < 0) && (copy))
-                DHT_STACK_DESTROY (copy);
+    if ((ret < 0) && (copy))
+        DHT_STACK_DESTROY(copy);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-lock.c b/xlators/cluster/dht/src/dht-lock.c
index 3f389eafa75..638821ccee5 100644
--- a/xlators/cluster/dht/src/dht-lock.c
+++ b/xlators/cluster/dht/src/dht-lock.c
@@ -11,1252 +11,1250 @@
 #include "dht-lock.h"
 
 static char *
-dht_lock_asprintf (dht_lock_t *lock)
+dht_lock_asprintf(dht_lock_t *lock)
 {
-        char *lk_buf                = NULL;
-        char gfid[GF_UUID_BUF_SIZE] = {0, };
+    char *lk_buf = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
 
-        if (lock == NULL)
-                goto out;
+    if (lock == NULL)
+        goto out;
 
-        uuid_utoa_r (lock->loc.gfid, gfid);
+    uuid_utoa_r(lock->loc.gfid, gfid);
 
-        gf_asprintf (&lk_buf, "%s:%s", lock->xl->name, gfid);
+    gf_asprintf(&lk_buf, "%s:%s", lock->xl->name, gfid);
 
 out:
-        return lk_buf;
+    return lk_buf;
 }
 
 static void
-dht_log_lk_array (char *name, gf_loglevel_t log_level, dht_lock_t **lk_array,
-                  int count)
+dht_log_lk_array(char *name, gf_loglevel_t log_level, dht_lock_t **lk_array,
+                 int count)
 {
-        int   i      = 0;
-        char *lk_buf = NULL;
+    int i = 0;
+    char *lk_buf = NULL;
 
-        if ((lk_array == NULL) || (count == 0))
-                goto out;
+    if ((lk_array == NULL) || (count == 0))
+        goto out;
 
-        for (i = 0; i < count; i++) {
-                lk_buf = dht_lock_asprintf (lk_array[i]);
-                if (!lk_buf)
-                        goto out;
+    for (i = 0; i < count; i++) {
+        lk_buf = dht_lock_asprintf(lk_array[i]);
+        if (!lk_buf)
+            goto out;
 
-                gf_msg (name, log_level, 0, DHT_MSG_LK_ARRAY_INFO,
-                        "%d. %s", i, lk_buf);
-                GF_FREE (lk_buf);
-        }
+        gf_smsg(name, log_level, 0, DHT_MSG_LK_ARRAY_INFO, "index=%d", i,
+                "lk_buf=%s", lk_buf, NULL);
+        GF_FREE(lk_buf);
+    }
 
 out:
-        return;
+    return;
 }
 
 static void
-dht_lock_stack_destroy (call_frame_t *lock_frame, dht_lock_type_t lk)
+dht_lock_stack_destroy(call_frame_t *lock_frame, dht_lock_type_t lk)
 {
-        dht_local_t *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = lock_frame->local;
+    local = lock_frame->local;
 
-        if (lk == DHT_INODELK) {
-                local->lock[0].layout.my_layout.locks = NULL;
-                local->lock[0].layout.my_layout.lk_count = 0;
-        } else {
-                local->lock[0].ns.directory_ns.locks = NULL;
-                local->lock[0].ns.directory_ns.lk_count = 0;
-        }
+    if (lk == DHT_INODELK) {
+        local->lock[0].layout.my_layout.locks = NULL;
+        local->lock[0].layout.my_layout.lk_count = 0;
+    } else {
+        local->lock[0].ns.directory_ns.locks = NULL;
+        local->lock[0].ns.directory_ns.lk_count = 0;
+    }
 
-        DHT_STACK_DESTROY (lock_frame);
-        return;
+    DHT_STACK_DESTROY(lock_frame);
+    return;
 }
 
 static void
-dht_lock_free (dht_lock_t *lock)
+dht_lock_free(dht_lock_t *lock)
 {
-        if (lock == NULL)
-                goto out;
+    if (lock == NULL)
+        goto out;
 
-        loc_wipe (&lock->loc);
-        GF_FREE (lock->domain);
-        GF_FREE (lock->basename);
-        mem_put (lock);
+    loc_wipe(&lock->loc);
+    GF_FREE(lock->domain);
+    GF_FREE(lock->basename);
+    mem_put(lock);
 
 out:
-        return;
+    return;
 }
 
 static void
-dht_set_lkowner (dht_lock_t **lk_array, int count, gf_lkowner_t *lkowner)
+dht_set_lkowner(dht_lock_t **lk_array, int count, gf_lkowner_t *lkowner)
 {
-        int i = 0;
+    int i = 0;
 
-        if (!lk_array || !lkowner)
-                goto out;
+    if (!lk_array || !lkowner)
+        goto out;
 
-        for (i = 0; i < count; i++) {
-                lk_array[i]->lk_owner = *lkowner;
-        }
+    for (i = 0; i < count; i++) {
+        lk_array[i]->lk_owner = *lkowner;
+    }
 
 out:
-        return;
+    return;
 }
 
 static int
-dht_lock_request_cmp (const void *val1, const void *val2)
+dht_lock_request_cmp(const void *val1, const void *val2)
 {
-        dht_lock_t *lock1 = NULL;
-        dht_lock_t *lock2 = NULL;
-        int         ret   = -1;
+    dht_lock_t *lock1 = NULL;
+    dht_lock_t *lock2 = NULL;
+    int ret = -1;
 
-        lock1 = *(dht_lock_t **)val1;
-        lock2 = *(dht_lock_t **)val2;
+    lock1 = *(dht_lock_t **)val1;
+    lock2 = *(dht_lock_t **)val2;
 
-        GF_VALIDATE_OR_GOTO ("dht-locks", lock1, out);
-        GF_VALIDATE_OR_GOTO ("dht-locks", lock2, out);
+    GF_VALIDATE_OR_GOTO("dht-locks", lock1, out);
+    GF_VALIDATE_OR_GOTO("dht-locks", lock2, out);
 
-        ret = strcmp (lock1->xl->name, lock2->xl->name);
+    ret = strcmp(lock1->xl->name, lock2->xl->name);
 
-        if (ret == 0) {
-                ret = gf_uuid_compare (lock1->loc.gfid, lock2->loc.gfid);
-        }
+    if (ret == 0) {
+        ret = gf_uuid_compare(lock1->loc.gfid, lock2->loc.gfid);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-dht_lock_order_requests (dht_lock_t **locks, int count)
+dht_lock_order_requests(dht_lock_t **locks, int count)
 {
-        int        ret     = -1;
+    int ret = -1;
 
-        if (!locks || !count)
-                goto out;
+    if (!locks || !count)
+        goto out;
 
-        qsort (locks, count, sizeof (*locks), dht_lock_request_cmp);
-        ret = 0;
+    qsort(locks, count, sizeof(*locks), dht_lock_request_cmp);
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 void
-dht_lock_array_free (dht_lock_t **lk_array, int count)
+dht_lock_array_free(dht_lock_t **lk_array, int count)
 {
-        int            i       = 0;
-        dht_lock_t    *lock    = NULL;
+    int i = 0;
+    dht_lock_t *lock = NULL;
 
-        if (lk_array == NULL)
-                goto out;
+    if (lk_array == NULL)
+        goto out;
 
-        for (i = 0; i < count; i++) {
-                lock = lk_array[i];
-                lk_array[i] = NULL;
-                dht_lock_free (lock);
-        }
+    for (i = 0; i < count; i++) {
+        lock = lk_array[i];
+        lk_array[i] = NULL;
+        dht_lock_free(lock);
+    }
 
 out:
-        return;
+    return;
 }
 
 int32_t
-dht_lock_count (dht_lock_t **lk_array, int lk_count)
+dht_lock_count(dht_lock_t **lk_array, int lk_count)
 {
-        int i = 0, locked = 0;
+    int i = 0, locked = 0;
 
-        if ((lk_array == NULL) || (lk_count == 0))
-                goto out;
+    if ((lk_array == NULL) || (lk_count == 0))
+        goto out;
 
-        for (i = 0; i < lk_count; i++) {
-                if (lk_array[i]->locked)
-                        locked++;
-        }
+    for (i = 0; i < lk_count; i++) {
+        if (lk_array[i]->locked)
+            locked++;
+    }
 out:
-        return locked;
+    return locked;
 }
 
 static call_frame_t *
-dht_lock_frame (call_frame_t *parent_frame)
+dht_lock_frame(call_frame_t *parent_frame)
 {
-        call_frame_t *lock_frame = NULL;
+    call_frame_t *lock_frame = NULL;
 
-        lock_frame = copy_frame (parent_frame);
-        if (lock_frame == NULL)
-                goto out;
+    lock_frame = copy_frame(parent_frame);
+    if (lock_frame == NULL)
+        goto out;
 
-        set_lk_owner_from_ptr (&lock_frame->root->lk_owner, parent_frame->root);
+    set_lk_owner_from_ptr(&lock_frame->root->lk_owner, parent_frame->root);
 
 out:
-        return lock_frame;
+    return lock_frame;
 }
 
 dht_lock_t *
-dht_lock_new (xlator_t *this, xlator_t *xl, loc_t *loc, short type,
-              const char *domain, const char *basename,
-              dht_reaction_type_t do_on_failure)
+dht_lock_new(xlator_t *this, xlator_t *xl, loc_t *loc, short type,
+             const char *domain, const char *basename,
+             dht_reaction_type_t do_on_failure)
 {
-        dht_conf_t *conf = NULL;
-        dht_lock_t *lock = NULL;
-
-        conf = this->private;
-
-        lock = mem_get0 (conf->lock_pool);
-        if (lock == NULL)
-                goto out;
-
-        lock->xl = xl;
-        lock->type = type;
-        lock->do_on_failure = do_on_failure;
-
-        lock->domain = gf_strdup (domain);
-        if (lock->domain == NULL) {
-                dht_lock_free (lock);
-                lock = NULL;
-                goto out;
-        }
-
-        if (basename) {
-                lock->basename = gf_strdup (basename);
-                if (lock->basename == NULL) {
-                        dht_lock_free (lock);
-                        lock = NULL;
-                        goto out;
-                }
+    dht_conf_t *conf = NULL;
+    dht_lock_t *lock = NULL;
+
+    conf = this->private;
+
+    lock = mem_get0(conf->lock_pool);
+    if (lock == NULL)
+        goto out;
+
+    lock->xl = xl;
+    lock->type = type;
+    lock->do_on_failure = do_on_failure;
+
+    lock->domain = gf_strdup(domain);
+    if (lock->domain == NULL) {
+        dht_lock_free(lock);
+        lock = NULL;
+        goto out;
+    }
+
+    if (basename) {
+        lock->basename = gf_strdup(basename);
+        if (lock->basename == NULL) {
+            dht_lock_free(lock);
+            lock = NULL;
+            goto out;
         }
-
-        /* Fill only inode and gfid.
-           posix and protocol/server give preference to pargfid/basename over
-           gfid/inode for resolution if all the three parameters of loc_t are
-           present. I want to avoid the following hypothetical situation:
-
-           1. rebalance did a lookup on a dentry and got a gfid.
-           2. rebalance acquires lock on loc_t which was filled with gfid and
-              path (pargfid/bname) from step 1.
-           3. somebody deleted and recreated the same file
-           4. rename on the same path acquires lock on loc_t which now points
-              to a different inode (and hence gets the lock).
-           5. rebalance continues to migrate file (note that not all fops done
-              by rebalance during migration are inode/gfid based Eg., unlink)
-           6. rename continues.
-        */
-        lock->loc.inode = inode_ref (loc->inode);
-        loc_gfid (loc, lock->loc.gfid);
+    }
+
+    /* Fill only inode and gfid.
+       posix and protocol/server give preference to pargfid/basename over
+       gfid/inode for resolution if all the three parameters of loc_t are
+       present. I want to avoid the following hypothetical situation:
+
+       1. rebalance did a lookup on a dentry and got a gfid.
+       2. rebalance acquires lock on loc_t which was filled with gfid and
+          path (pargfid/bname) from step 1.
+       3. somebody deleted and recreated the same file
+       4. rename on the same path acquires lock on loc_t which now points
+          to a different inode (and hence gets the lock).
+       5. rebalance continues to migrate file (note that not all fops done
+          by rebalance during migration are inode/gfid based Eg., unlink)
+       6. rename continues.
+    */
+    lock->loc.inode = inode_ref(loc->inode);
+    loc_gfid(loc, lock->loc.gfid);
 
 out:
-        return lock;
+    return lock;
 }
 
 static int
-dht_local_entrylk_init (call_frame_t *frame, dht_lock_t **lk_array,
-                     int lk_count, fop_entrylk_cbk_t entrylk_cbk)
+dht_local_entrylk_init(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                       fop_entrylk_cbk_t entrylk_cbk)
 {
-        int          ret   = -1;
-        dht_local_t *local = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local == NULL) {
-                local = dht_local_init (frame, NULL, NULL, 0);
-        }
+    if (local == NULL) {
+        local = dht_local_init(frame, NULL, NULL, 0);
+    }
 
-        if (local == NULL) {
-                goto out;
-        }
+    if (local == NULL) {
+        goto out;
+    }
 
-        local->lock[0].ns.directory_ns.entrylk_cbk = entrylk_cbk;
-        local->lock[0].ns.directory_ns.locks = lk_array;
-        local->lock[0].ns.directory_ns.lk_count = lk_count;
+    local->lock[0].ns.directory_ns.entrylk_cbk = entrylk_cbk;
+    local->lock[0].ns.directory_ns.locks = lk_array;
+    local->lock[0].ns.directory_ns.lk_count = lk_count;
 
-        ret = dht_lock_order_requests (local->lock[0].ns.directory_ns.locks,
-                                       local->lock[0].ns.directory_ns.lk_count);
-        if (ret < 0)
-                goto out;
+    ret = dht_lock_order_requests(local->lock[0].ns.directory_ns.locks,
+                                  local->lock[0].ns.directory_ns.lk_count);
+    if (ret < 0)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static void
-dht_entrylk_done (call_frame_t *lock_frame)
+dht_entrylk_done(call_frame_t *lock_frame)
 {
-        fop_entrylk_cbk_t  entrylk_cbk = NULL;
-        call_frame_t      *main_frame  = NULL;
-        dht_local_t       *local       = NULL;
+    fop_entrylk_cbk_t entrylk_cbk = NULL;
+    call_frame_t *main_frame = NULL;
+    dht_local_t *local = NULL;
 
-        local = lock_frame->local;
-        main_frame = local->main_frame;
+    local = lock_frame->local;
+    main_frame = local->main_frame;
 
-        local->lock[0].ns.directory_ns.locks = NULL;
-        local->lock[0].ns.directory_ns.lk_count = 0;
+    local->lock[0].ns.directory_ns.locks = NULL;
+    local->lock[0].ns.directory_ns.lk_count = 0;
 
-        entrylk_cbk = local->lock[0].ns.directory_ns.entrylk_cbk;
-        local->lock[0].ns.directory_ns.entrylk_cbk = NULL;
+    entrylk_cbk = local->lock[0].ns.directory_ns.entrylk_cbk;
+    local->lock[0].ns.directory_ns.entrylk_cbk = NULL;
 
-        entrylk_cbk (main_frame, NULL, main_frame->this,
-                     local->lock[0].ns.directory_ns.op_ret,
-                     local->lock[0].ns.directory_ns.op_errno, NULL);
+    entrylk_cbk(main_frame, NULL, main_frame->this,
+                local->lock[0].ns.directory_ns.op_ret,
+                local->lock[0].ns.directory_ns.op_errno, NULL);
 
-        dht_lock_stack_destroy (lock_frame, DHT_ENTRYLK);
-        return;
+    dht_lock_stack_destroy(lock_frame, DHT_ENTRYLK);
+    return;
 }
 
 static int32_t
-dht_unlock_entrylk_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_unlock_entrylk_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local                   = NULL;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        local = frame->local;
-        gf_uuid_unparse (local->lock[0].ns.directory_ns.locks[0]->loc.inode->gfid, gfid);
+    local = frame->local;
+    gf_uuid_unparse(local->lock[0].ns.directory_ns.locks[0]->loc.inode->gfid,
+                    gfid);
 
-        if (op_ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "unlock failed on gfid: %s, stale lock might be left "
-                        "in DHT_LAYOUT_HEAL_DOMAIN", gfid);
-        }
+    if (op_ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                DHT_MSG_UNLOCK_GFID_FAILED, "gfid=%s", gfid,
+                "DHT_LAYOUT_HEAL_DOMAIN", NULL);
+    }
 
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
 static int32_t
-dht_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_unlock_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local                  = NULL;
-        int          lk_index               = 0, call_cnt = 0;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *local = NULL;
+    int lk_index = 0, call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        lk_index = (long) cookie;
+    lk_index = (long)cookie;
 
-        local = frame->local;
+    local = frame->local;
 
-        uuid_utoa_r (local->lock[0].ns.directory_ns.locks[lk_index]->loc.gfid, gfid);
+    uuid_utoa_r(local->lock[0].ns.directory_ns.locks[lk_index]->loc.gfid, gfid);
 
-        if (op_ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_UNLOCKING_FAILED,
-                        "unlocking failed on %s:%s",
-                        local->lock[0].ns.directory_ns.locks[lk_index]->xl->name,
-                        gfid);
-        } else {
-                local->lock[0].ns.directory_ns.locks[lk_index]->locked = 0;
-        }
+    if (op_ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLOCKING_FAILED,
+                "name=%s",
+                local->lock[0].ns.directory_ns.locks[lk_index]->xl->name,
+                "gfid=%s", gfid, NULL);
+    } else {
+        local->lock[0].ns.directory_ns.locks[lk_index]->locked = 0;
+    }
 
-        call_cnt = dht_frame_return (frame);
-        if (is_last_call (call_cnt)) {
-                dht_entrylk_done (frame);
-        }
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        dht_entrylk_done(frame);
+    }
 
-        return 0;
+    return 0;
 }
 
 static int32_t
-dht_unlock_entrylk (call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
-                     fop_entrylk_cbk_t entrylk_cbk)
+dht_unlock_entrylk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                   fop_entrylk_cbk_t entrylk_cbk)
 {
-        dht_local_t     *local      = NULL;
-        int              ret        = -1 , i = 0;
-        call_frame_t    *lock_frame = NULL;
-        int              call_cnt   = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht-locks", frame, done);
-        GF_VALIDATE_OR_GOTO (frame->this->name, lk_array, done);
-        GF_VALIDATE_OR_GOTO (frame->this->name, entrylk_cbk, done);
-
-        call_cnt = dht_lock_count (lk_array, lk_count);
-        if (call_cnt == 0) {
-                ret = 0;
-                goto done;
-        }
-
-        lock_frame = dht_lock_frame (frame);
-        if (lock_frame == NULL) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_UNLOCKING_FAILED,
-                        "cannot allocate a frame, not unlocking following "
-                        "entrylks:");
-
-                dht_log_lk_array (frame->this->name, GF_LOG_WARNING, lk_array,
-                                  lk_count);
-                goto done;
-        }
+    dht_local_t *local = NULL;
+    int ret = -1, i = 0;
+    call_frame_t *lock_frame = NULL;
+    int call_cnt = 0;
 
-        ret = dht_local_entrylk_init (lock_frame, lk_array, lk_count,
-                                      entrylk_cbk);
-        if (ret < 0) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_UNLOCKING_FAILED,
-                        "storing locks in local failed, not unlocking "
-                        "following entrylks:");
-
-                dht_log_lk_array (frame->this->name, GF_LOG_WARNING, lk_array,
-                                  lk_count);
-
-                goto done;
-        }
-
-        local = lock_frame->local;
-        local->main_frame = frame;
-        local->call_cnt = call_cnt;
-
-        for (i = 0; i < local->lock[0].ns.directory_ns.lk_count; i++) {
-                if (!local->lock[0].ns.directory_ns.locks[i]->locked)
-                        continue;
-
-                lock_frame->root->lk_owner = local->lock[0].ns.directory_ns.locks[i]->lk_owner;
-                STACK_WIND_COOKIE (lock_frame, dht_unlock_entrylk_cbk,
-                                   (void *)(long)i,
-                                   local->lock[0].ns.directory_ns.locks[i]->xl,
-                                   local->lock[0].ns.directory_ns.locks[i]->xl->fops->entrylk,
-                                   local->lock[0].ns.directory_ns.locks[i]->domain,
-                                   &local->lock[0].ns.directory_ns.locks[i]->loc,
-                                   local->lock[0].ns.directory_ns.locks[i]->basename,
-                                   ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
-                if (!--call_cnt)
-                        break;
-        }
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, entrylk_cbk, done);
 
-        return 0;
+    call_cnt = dht_lock_count(lk_array, lk_count);
+    if (call_cnt == 0) {
+        ret = 0;
+        goto done;
+    }
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+        goto done;
+    }
+
+    ret = dht_local_entrylk_init(lock_frame, lk_array, lk_count, entrylk_cbk);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+
+        goto done;
+    }
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+    local->call_cnt = call_cnt;
+
+    for (i = 0; i < local->lock[0].ns.directory_ns.lk_count; i++) {
+        if (!local->lock[0].ns.directory_ns.locks[i]->locked)
+            continue;
+
+        lock_frame->root
+            ->lk_owner = local->lock[0].ns.directory_ns.locks[i]->lk_owner;
+        STACK_WIND_COOKIE(
+            lock_frame, dht_unlock_entrylk_cbk, (void *)(long)i,
+            local->lock[0].ns.directory_ns.locks[i]->xl,
+            local->lock[0].ns.directory_ns.locks[i]->xl->fops->entrylk,
+            local->lock[0].ns.directory_ns.locks[i]->domain,
+            &local->lock[0].ns.directory_ns.locks[i]->loc,
+            local->lock[0].ns.directory_ns.locks[i]->basename, ENTRYLK_UNLOCK,
+            ENTRYLK_WRLCK, NULL);
+        if (!--call_cnt)
+            break;
+    }
+
+    return 0;
 
 done:
-        if (lock_frame)
-                dht_lock_stack_destroy (lock_frame, DHT_ENTRYLK);
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_ENTRYLK);
 
-        /* no locks acquired, invoke entrylk_cbk */
-        if (ret == 0)
-                entrylk_cbk (frame, NULL, frame->this, 0, 0, NULL);
+    /* no locks acquired, invoke entrylk_cbk */
+    if (ret == 0)
+        entrylk_cbk(frame, NULL, frame->this, 0, 0, NULL);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-dht_unlock_entrylk_wrapper (call_frame_t *frame, dht_elock_wrap_t *entrylk)
+dht_unlock_entrylk_wrapper(call_frame_t *frame, dht_elock_wrap_t *entrylk)
 {
-        dht_local_t  *local                   = NULL, *lock_local = NULL;
-        call_frame_t *lock_frame              = NULL;
-        char          pgfid[GF_UUID_BUF_SIZE] = {0};
-        int           ret                     = 0;
-
-        local = frame->local;
-
-        if (!entrylk || !entrylk->locks)
-                goto out;
-
-        gf_uuid_unparse (local->loc.parent->gfid, pgfid);
-
-        lock_frame = copy_frame (frame);
-        if (lock_frame == NULL) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, ENOMEM,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): "
-                        "copy frame failed", pgfid, local->loc.name,
-                        local->loc.path);
-                goto done;
-        }
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
 
-        lock_local = dht_local_init (lock_frame, NULL, NULL, 0);
-        if (lock_local == NULL) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, ENOMEM,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): "
-                        "local creation failed", pgfid, local->loc.name,
-                        local->loc.path);
-                goto done;
-        }
+    local = frame->local;
 
-        lock_frame->local = lock_local;
+    if (!entrylk || !entrylk->locks)
+        goto out;
 
-        lock_local->lock[0].ns.directory_ns.locks = entrylk->locks;
-        lock_local->lock[0].ns.directory_ns.lk_count = entrylk->lk_count;
-        entrylk->locks = NULL;
-        entrylk->lk_count = 0;
+    gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_COPY_FRAME_FAILED, "pgfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
 
-        ret = dht_unlock_entrylk (lock_frame,
-                                  lock_local->lock[0].ns.directory_ns.locks,
-                                  lock_local->lock[0].ns.directory_ns.lk_count,
-                                  dht_unlock_entrylk_done);
-        if (ret)
-                goto done;
+    lock_local = dht_local_init(lock_frame, NULL, NULL, 0);
+    if (lock_local == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_CREATE_FAILED, "local", "pgfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
 
-        lock_frame = NULL;
+    lock_frame->local = lock_local;
+
+    lock_local->lock[0].ns.directory_ns.locks = entrylk->locks;
+    lock_local->lock[0].ns.directory_ns.lk_count = entrylk->lk_count;
+    entrylk->locks = NULL;
+    entrylk->lk_count = 0;
+
+    ret = dht_unlock_entrylk(
+        lock_frame, lock_local->lock[0].ns.directory_ns.locks,
+        lock_local->lock[0].ns.directory_ns.lk_count, dht_unlock_entrylk_done);
+    if (ret)
+        goto done;
+
+    lock_frame = NULL;
 
 done:
-        if (lock_frame != NULL) {
-                DHT_STACK_DESTROY (lock_frame);
-        }
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 static int
-dht_entrylk_cleanup_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno,
-                         dict_t *xdata)
+dht_entrylk_cleanup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_entrylk_done (frame);
-        return 0;
+    dht_entrylk_done(frame);
+    return 0;
 }
 
 static void
-dht_entrylk_cleanup (call_frame_t *lock_frame)
+dht_entrylk_cleanup(call_frame_t *lock_frame)
 {
-        dht_lock_t  **lk_array = NULL;
-        int           lk_count = 0, lk_acquired = 0;
-        dht_local_t  *local    = NULL;
+    dht_lock_t **lk_array = NULL;
+    int lk_count = 0, lk_acquired = 0;
+    dht_local_t *local = NULL;
 
-        local = lock_frame->local;
+    local = lock_frame->local;
 
-        lk_array = local->lock[0].ns.directory_ns.locks;
-        lk_count = local->lock[0].ns.directory_ns.lk_count;
+    lk_array = local->lock[0].ns.directory_ns.locks;
+    lk_count = local->lock[0].ns.directory_ns.lk_count;
 
-        lk_acquired = dht_lock_count (lk_array, lk_count);
-        if (lk_acquired != 0) {
-                dht_unlock_entrylk (lock_frame, lk_array, lk_count,
-                                     dht_entrylk_cleanup_cbk);
-        } else {
-                dht_entrylk_done (lock_frame);
-        }
+    lk_acquired = dht_lock_count(lk_array, lk_count);
+    if (lk_acquired != 0) {
+        dht_unlock_entrylk(lock_frame, lk_array, lk_count,
+                           dht_entrylk_cleanup_cbk);
+    } else {
+        dht_entrylk_done(lock_frame);
+    }
 
-        return;
+    return;
 }
 
-
 static int32_t
-dht_blocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_blocking_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        int          lk_index = 0;
-        int          i        = 0;
-        dht_local_t *local    = NULL;
-
-        lk_index = (long) cookie;
-
-        local = frame->local;
-        if (op_ret == 0) {
-                local->lock[0].ns.directory_ns.locks[lk_index]->locked = _gf_true;
-        } else {
-                switch (op_errno) {
-                case ESTALE:
-                case ENOENT:
-                        if (local->lock[0].ns.directory_ns.locks[lk_index]->do_on_failure != IGNORE_ENOENT_ESTALE) {
-                                local->lock[0].ns.directory_ns.op_ret = -1;
-                                local->lock[0].ns.directory_ns.op_errno = op_errno;
-                                goto cleanup;
-                        }
-                        break;
-                default:
-                        local->lock[0].ns.directory_ns.op_ret = -1;
-                        local->lock[0].ns.directory_ns.op_errno = op_errno;
-                        goto cleanup;
+    int lk_index = 0;
+    int i = 0;
+    dht_local_t *local = NULL;
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+    if (op_ret == 0) {
+        local->lock[0].ns.directory_ns.locks[lk_index]->locked = _gf_true;
+    } else {
+        switch (op_errno) {
+            case ESTALE:
+            case ENOENT:
+                if (local->lock[0]
+                        .ns.directory_ns.locks[lk_index]
+                        ->do_on_failure != IGNORE_ENOENT_ESTALE) {
+                    local->lock[0].ns.directory_ns.op_ret = -1;
+                    local->lock[0].ns.directory_ns.op_errno = op_errno;
+                    goto cleanup;
                 }
+                break;
+            default:
+                local->lock[0].ns.directory_ns.op_ret = -1;
+                local->lock[0].ns.directory_ns.op_errno = op_errno;
+                goto cleanup;
         }
+    }
 
-        if (lk_index == (local->lock[0].ns.directory_ns.lk_count - 1)) {
-                for (i = 0; (i < local->lock[0].ns.directory_ns.lk_count) &&
-                     (!local->lock[0].ns.directory_ns.locks[i]->locked); i++)
-                        ;
-
-                if (i == local->lock[0].ns.directory_ns.lk_count) {
-                        local->lock[0].ns.directory_ns.op_ret = -1;
-                        local->lock[0].ns.directory_ns.op_errno = op_errno;
-                }
+    if (lk_index == (local->lock[0].ns.directory_ns.lk_count - 1)) {
+        for (i = 0; (i < local->lock[0].ns.directory_ns.lk_count) &&
+                    (!local->lock[0].ns.directory_ns.locks[i]->locked);
+             i++)
+            ;
 
-                dht_entrylk_done (frame);
-        } else {
-                dht_blocking_entrylk_rec (frame, ++lk_index);
+        if (i == local->lock[0].ns.directory_ns.lk_count) {
+            local->lock[0].ns.directory_ns.op_ret = -1;
+            local->lock[0].ns.directory_ns.op_errno = op_errno;
         }
 
-        return 0;
+        dht_entrylk_done(frame);
+    } else {
+        dht_blocking_entrylk_rec(frame, ++lk_index);
+    }
+
+    return 0;
 
 cleanup:
-        dht_entrylk_cleanup (frame);
+    dht_entrylk_cleanup(frame);
 
-        return 0;
+    return 0;
 }
 
 void
-dht_blocking_entrylk_rec (call_frame_t *frame, int i)
+dht_blocking_entrylk_rec(call_frame_t *frame, int i)
 {
-        dht_local_t     *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        STACK_WIND_COOKIE (frame, dht_blocking_entrylk_cbk,
-                           (void *) (long) i,
-                           local->lock[0].ns.directory_ns.locks[i]->xl,
-                           local->lock[0].ns.directory_ns.locks[i]->xl->fops->entrylk,
-                           local->lock[0].ns.directory_ns.locks[i]->domain,
-                           &local->lock[0].ns.directory_ns.locks[i]->loc,
-                           local->lock[0].ns.directory_ns.locks[i]->basename,
-                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+    STACK_WIND_COOKIE(
+        frame, dht_blocking_entrylk_cbk, (void *)(long)i,
+        local->lock[0].ns.directory_ns.locks[i]->xl,
+        local->lock[0].ns.directory_ns.locks[i]->xl->fops->entrylk,
+        local->lock[0].ns.directory_ns.locks[i]->domain,
+        &local->lock[0].ns.directory_ns.locks[i]->loc,
+        local->lock[0].ns.directory_ns.locks[i]->basename, ENTRYLK_LOCK,
+        ENTRYLK_WRLCK, NULL);
 
-        return;
+    return;
 }
 
 int
-dht_blocking_entrylk (call_frame_t *frame, dht_lock_t **lk_array,
-                      int lk_count, fop_entrylk_cbk_t entrylk_cbk)
+dht_blocking_entrylk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_entrylk_cbk_t entrylk_cbk)
 {
-        int           ret        = -1;
-        call_frame_t *lock_frame = NULL;
-        dht_local_t  *local      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht-locks", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, lk_array, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, entrylk_cbk, out);
-
-        lock_frame = dht_lock_frame (frame);
-        if (lock_frame == NULL)
-                goto out;
-
-        ret = dht_local_entrylk_init (lock_frame, lk_array, lk_count,
-                                      entrylk_cbk);
-        if (ret < 0) {
-                goto out;
-        }
+    int ret = -1;
+    call_frame_t *lock_frame = NULL;
+    dht_local_t *local = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, entrylk_cbk, out);
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL)
+        goto out;
 
-        dht_set_lkowner (lk_array, lk_count, &lock_frame->root->lk_owner);
+    ret = dht_local_entrylk_init(lock_frame, lk_array, lk_count, entrylk_cbk);
+    if (ret < 0) {
+        goto out;
+    }
 
-        local = lock_frame->local;
-        local->main_frame = frame;
+    dht_set_lkowner(lk_array, lk_count, &lock_frame->root->lk_owner);
 
-        dht_blocking_entrylk_rec (lock_frame, 0);
+    local = lock_frame->local;
+    local->main_frame = frame;
 
-        return 0;
+    dht_blocking_entrylk_rec(lock_frame, 0);
+
+    return 0;
 out:
-        if (lock_frame)
-                dht_lock_stack_destroy (lock_frame, DHT_ENTRYLK);
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_ENTRYLK);
 
-        return -1;
+    return -1;
 }
 
 static int
-dht_local_inodelk_init (call_frame_t *frame, dht_lock_t **lk_array,
-                     int lk_count, fop_inodelk_cbk_t inodelk_cbk)
+dht_local_inodelk_init(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                       fop_inodelk_cbk_t inodelk_cbk)
 {
-        int          ret   = -1;
-        dht_local_t *local = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local == NULL) {
-                local = dht_local_init (frame, NULL, NULL, 0);
-        }
+    if (local == NULL) {
+        local = dht_local_init(frame, NULL, NULL, 0);
+    }
 
-        if (local == NULL) {
-                goto out;
-        }
+    if (local == NULL) {
+        goto out;
+    }
 
-        local->lock[0].layout.my_layout.inodelk_cbk = inodelk_cbk;
-        local->lock[0].layout.my_layout.locks = lk_array;
-        local->lock[0].layout.my_layout.lk_count = lk_count;
+    local->lock[0].layout.my_layout.inodelk_cbk = inodelk_cbk;
+    local->lock[0].layout.my_layout.locks = lk_array;
+    local->lock[0].layout.my_layout.lk_count = lk_count;
 
-        ret = dht_lock_order_requests (local->lock[0].layout.my_layout.locks,
-                                       local->lock[0].layout.my_layout.lk_count);
-        if (ret < 0)
-                goto out;
+    ret = dht_lock_order_requests(local->lock[0].layout.my_layout.locks,
+                                  local->lock[0].layout.my_layout.lk_count);
+    if (ret < 0)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static void
-dht_inodelk_done (call_frame_t *lock_frame)
+dht_inodelk_done(call_frame_t *lock_frame)
 {
-        fop_inodelk_cbk_t  inodelk_cbk = NULL;
-        call_frame_t      *main_frame  = NULL;
-        dht_local_t       *local       = NULL;
+    fop_inodelk_cbk_t inodelk_cbk = NULL;
+    call_frame_t *main_frame = NULL;
+    dht_local_t *local = NULL;
 
-        local = lock_frame->local;
-        main_frame = local->main_frame;
+    local = lock_frame->local;
+    main_frame = local->main_frame;
 
-        local->lock[0].layout.my_layout.locks = NULL;
-        local->lock[0].layout.my_layout.lk_count = 0;
+    local->lock[0].layout.my_layout.locks = NULL;
+    local->lock[0].layout.my_layout.lk_count = 0;
 
-        inodelk_cbk = local->lock[0].layout.my_layout.inodelk_cbk;
-        local->lock[0].layout.my_layout.inodelk_cbk = NULL;
+    inodelk_cbk = local->lock[0].layout.my_layout.inodelk_cbk;
+    local->lock[0].layout.my_layout.inodelk_cbk = NULL;
 
-        inodelk_cbk (main_frame, NULL, main_frame->this,
-                     local->lock[0].layout.my_layout.op_ret,
-                     local->lock[0].layout.my_layout.op_errno, NULL);
+    inodelk_cbk(main_frame, NULL, main_frame->this,
+                local->lock[0].layout.my_layout.op_ret,
+                local->lock[0].layout.my_layout.op_errno, NULL);
 
-        dht_lock_stack_destroy (lock_frame, DHT_INODELK);
-        return;
+    dht_lock_stack_destroy(lock_frame, DHT_INODELK);
+    return;
 }
 
 static int32_t
-dht_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_unlock_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local                  = NULL;
-        int          lk_index               = 0, call_cnt = 0;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        lk_index = (long) cookie;
-
-        local = frame->local;
-        if (op_ret < 0) {
-                uuid_utoa_r (local->lock[0].layout.my_layout.locks[lk_index]->loc.gfid,
-                             gfid);
-
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_UNLOCKING_FAILED,
-                        "unlocking failed on %s:%s",
-                        local->lock[0].layout.my_layout.locks[lk_index]->xl->name,
-                        gfid);
-        } else {
-                local->lock[0].layout.my_layout.locks[lk_index]->locked = 0;
-        }
-
-        call_cnt = dht_frame_return (frame);
-        if (is_last_call (call_cnt)) {
-                dht_inodelk_done (frame);
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    int lk_index = 0, call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+    if (op_ret < 0) {
+        uuid_utoa_r(local->lock[0].layout.my_layout.locks[lk_index]->loc.gfid,
+                    gfid);
+
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLOCKING_FAILED,
+                "name=%s",
+                local->lock[0].layout.my_layout.locks[lk_index]->xl->name,
+                "gfid=%s", gfid, NULL);
+    } else {
+        local->lock[0].layout.my_layout.locks[lk_index]->locked = 0;
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        dht_inodelk_done(frame);
+    }
+
+    return 0;
 }
 
 static int32_t
-dht_unlock_inodelk_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_unlock_inodelk_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local                   = NULL;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        local = frame->local;
-        gf_uuid_unparse (local->lock[0].layout.my_layout.locks[0]->loc.inode->gfid, gfid);
+    local = frame->local;
+    gf_uuid_unparse(local->lock[0].layout.my_layout.locks[0]->loc.inode->gfid,
+                    gfid);
 
-        if (op_ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "unlock failed on gfid: %s, stale lock might be left "
-                        "in DHT_LAYOUT_HEAL_DOMAIN", gfid);
-        }
+    if (op_ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                DHT_MSG_UNLOCK_GFID_FAILED, "DHT_LAYOUT_HEAL_DOMAIN gfid=%s",
+                gfid, NULL);
+    }
 
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
 int32_t
-dht_unlock_inodelk (call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
-                     fop_inodelk_cbk_t inodelk_cbk)
+dht_unlock_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                   fop_inodelk_cbk_t inodelk_cbk)
 {
-        dht_local_t     *local      = NULL;
-        struct gf_flock  flock      = {0,};
-        int              ret        = -1 , i = 0;
-        call_frame_t    *lock_frame = NULL;
-        int              call_cnt   = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht-locks", frame, done);
-        GF_VALIDATE_OR_GOTO (frame->this->name, lk_array, done);
-        GF_VALIDATE_OR_GOTO (frame->this->name, inodelk_cbk, done);
-
-        call_cnt = dht_lock_count (lk_array, lk_count);
-        if (call_cnt == 0) {
-                ret = 0;
-                goto done;
-        }
-
-        lock_frame = dht_lock_frame (frame);
-        if (lock_frame == NULL) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_UNLOCKING_FAILED,
-                        "cannot allocate a frame, not unlocking following "
-                        "locks:");
-
-                dht_log_lk_array (frame->this->name, GF_LOG_WARNING, lk_array,
-                                  lk_count);
-                goto done;
-        }
-
-        ret = dht_local_inodelk_init (lock_frame, lk_array, lk_count,
-                                      inodelk_cbk);
-        if (ret < 0) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_UNLOCKING_FAILED,
-                        "storing locks in local failed, not unlocking "
-                        "following locks:");
-
-                dht_log_lk_array (frame->this->name, GF_LOG_WARNING, lk_array,
-                                  lk_count);
-
-                goto done;
-        }
-
-        local = lock_frame->local;
-        local->main_frame = frame;
-        local->call_cnt = call_cnt;
-
-        flock.l_type = F_UNLCK;
-
-        for (i = 0; i < local->lock[0].layout.my_layout.lk_count; i++) {
-                if (!local->lock[0].layout.my_layout.locks[i]->locked)
-                        continue;
-
-                lock_frame->root->lk_owner = local->lock[0].layout.my_layout.locks[i]->lk_owner;
-                STACK_WIND_COOKIE (lock_frame, dht_unlock_inodelk_cbk,
-                                   (void *)(long)i,
-                                   local->lock[0].layout.my_layout.locks[i]->xl,
-                                   local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
-                                   local->lock[0].layout.my_layout.locks[i]->domain,
-                                   &local->lock[0].layout.my_layout.locks[i]->loc, F_SETLK,
-                                   &flock, NULL);
-                if (!--call_cnt)
-                        break;
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = -1, i = 0;
+    call_frame_t *lock_frame = NULL;
+    int call_cnt = 0;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, inodelk_cbk, done);
+
+    call_cnt = dht_lock_count(lk_array, lk_count);
+    if (call_cnt == 0) {
+        ret = 0;
+        goto done;
+    }
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+        goto done;
+    }
+
+    ret = dht_local_inodelk_init(lock_frame, lk_array, lk_count, inodelk_cbk);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+
+        goto done;
+    }
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+    local->call_cnt = call_cnt;
+
+    flock.l_type = F_UNLCK;
+
+    for (i = 0; i < local->lock[0].layout.my_layout.lk_count; i++) {
+        if (!local->lock[0].layout.my_layout.locks[i]->locked)
+            continue;
+
+        lock_frame->root
+            ->lk_owner = local->lock[0].layout.my_layout.locks[i]->lk_owner;
+        STACK_WIND_COOKIE(
+            lock_frame, dht_unlock_inodelk_cbk, (void *)(long)i,
+            local->lock[0].layout.my_layout.locks[i]->xl,
+            local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
+            local->lock[0].layout.my_layout.locks[i]->domain,
+            &local->lock[0].layout.my_layout.locks[i]->loc, F_SETLK, &flock,
+            NULL);
+        if (!--call_cnt)
+            break;
+    }
+
+    return 0;
 
 done:
-        if (lock_frame)
-                dht_lock_stack_destroy (lock_frame, DHT_INODELK);
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_INODELK);
 
-        /* no locks acquired, invoke inodelk_cbk */
-        if (ret == 0)
-                inodelk_cbk (frame, NULL, frame->this, 0, 0, NULL);
+    /* no locks acquired, invoke inodelk_cbk */
+    if (ret == 0)
+        inodelk_cbk(frame, NULL, frame->this, 0, 0, NULL);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-dht_unlock_inodelk_wrapper (call_frame_t *frame, dht_ilock_wrap_t *inodelk)
+dht_unlock_inodelk_wrapper(call_frame_t *frame, dht_ilock_wrap_t *inodelk)
 {
-        dht_local_t  *local                   = NULL, *lock_local = NULL;
-        call_frame_t *lock_frame              = NULL;
-        char          pgfid[GF_UUID_BUF_SIZE] = {0};
-        int           ret                     = 0;
-
-        local = frame->local;
-
-        if (!inodelk || !inodelk->locks)
-                goto out;
-
-        gf_uuid_unparse (local->loc.parent->gfid, pgfid);
-
-        lock_frame = copy_frame (frame);
-        if (lock_frame == NULL) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, ENOMEM,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): "
-                        "copy frame failed", pgfid, local->loc.name,
-                        local->loc.path);
-                goto done;
-        }
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
 
-        lock_local = dht_local_init (lock_frame, NULL, NULL, 0);
-        if (lock_local == NULL) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, ENOMEM,
-                        DHT_MSG_PARENT_LAYOUT_CHANGED,
-                        "mkdir (%s/%s) (path: %s): "
-                        "local creation failed", pgfid, local->loc.name,
-                        local->loc.path);
-                goto done;
-        }
+    local = frame->local;
 
-        lock_frame->local = lock_local;
+    if (!inodelk || !inodelk->locks)
+        goto out;
 
-        lock_local->lock[0].layout.my_layout.locks = inodelk->locks;
-        lock_local->lock[0].layout.my_layout.lk_count = inodelk->lk_count;
-        inodelk->locks = NULL;
-        inodelk->lk_count = 0;
+    gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_COPY_FRAME_FAILED, "pgfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
 
-        ret = dht_unlock_inodelk (lock_frame,
-                                  lock_local->lock[0].layout.my_layout.locks,
-                                  lock_local->lock[0].layout.my_layout.lk_count,
-                                  dht_unlock_inodelk_done);
+    lock_local = dht_local_init(lock_frame, NULL, NULL, 0);
+    if (lock_local == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_CREATE_FAILED, "local", "gfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
 
-        if (ret)
-                goto done;
+    lock_frame->local = lock_local;
 
-        lock_frame = NULL;
+    lock_local->lock[0].layout.my_layout.locks = inodelk->locks;
+    lock_local->lock[0].layout.my_layout.lk_count = inodelk->lk_count;
+    inodelk->locks = NULL;
+    inodelk->lk_count = 0;
+
+    ret = dht_unlock_inodelk(
+        lock_frame, lock_local->lock[0].layout.my_layout.locks,
+        lock_local->lock[0].layout.my_layout.lk_count, dht_unlock_inodelk_done);
+
+    if (ret)
+        goto done;
+
+    lock_frame = NULL;
 
 done:
-        if (lock_frame != NULL) {
-                DHT_STACK_DESTROY (lock_frame);
-        }
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
 out:
-        return 0;
+    return 0;
 }
 
 static int
-dht_inodelk_cleanup_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno,
-                         dict_t *xdata)
+dht_inodelk_cleanup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_inodelk_done (frame);
-        return 0;
+    dht_inodelk_done(frame);
+    return 0;
 }
 
 static void
-dht_inodelk_cleanup (call_frame_t *lock_frame)
+dht_inodelk_cleanup(call_frame_t *lock_frame)
 {
-        dht_lock_t  **lk_array = NULL;
-        int           lk_count = 0, lk_acquired = 0;
-        dht_local_t  *local    = NULL;
+    dht_lock_t **lk_array = NULL;
+    int lk_count = 0, lk_acquired = 0;
+    dht_local_t *local = NULL;
 
-        local = lock_frame->local;
+    local = lock_frame->local;
 
-        lk_array = local->lock[0].layout.my_layout.locks;
-        lk_count = local->lock[0].layout.my_layout.lk_count;
+    lk_array = local->lock[0].layout.my_layout.locks;
+    lk_count = local->lock[0].layout.my_layout.lk_count;
 
-        lk_acquired = dht_lock_count (lk_array, lk_count);
-        if (lk_acquired != 0) {
-                dht_unlock_inodelk (lock_frame, lk_array, lk_count,
-                                     dht_inodelk_cleanup_cbk);
-        } else {
-                dht_inodelk_done (lock_frame);
-        }
+    lk_acquired = dht_lock_count(lk_array, lk_count);
+    if (lk_acquired != 0) {
+        dht_unlock_inodelk(lock_frame, lk_array, lk_count,
+                           dht_inodelk_cleanup_cbk);
+    } else {
+        dht_inodelk_done(lock_frame);
+    }
 
-        return;
+    return;
 }
 
 static int32_t
-dht_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_nonblocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local                   = NULL;
-        int          lk_index               = 0, call_cnt = 0;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *local = NULL;
+    int lk_index = 0, call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        local = frame->local;
-        lk_index = (long) cookie;
+    local = frame->local;
+    lk_index = (long)cookie;
 
-        if (op_ret == -1) {
-                local->lock[0].layout.my_layout.op_ret = -1;
-                local->lock[0].layout.my_layout.op_errno = op_errno;
+    if (op_ret == -1) {
+        local->lock[0].layout.my_layout.op_ret = -1;
+        local->lock[0].layout.my_layout.op_errno = op_errno;
 
-                if (local && local->lock[0].layout.my_layout.locks[lk_index]) {
-                        uuid_utoa_r (local->lock[0].layout.my_layout.locks[lk_index]->loc.inode->gfid,
-                                     gfid);
-
-                        gf_msg_debug (this->name, op_errno,
-                                      "inodelk failed on gfid: %s "
-                                      "subvolume: %s", gfid,
-                                      local->lock[0].layout.my_layout.locks[lk_index]->xl->name);
-                }
+        if (local && local->lock[0].layout.my_layout.locks[lk_index]) {
+            uuid_utoa_r(local->lock[0]
+                            .layout.my_layout.locks[lk_index]
+                            ->loc.inode->gfid,
+                        gfid);
 
-                goto out;
+            gf_msg_debug(
+                this->name, op_errno,
+                "inodelk failed on gfid: %s "
+                "subvolume: %s",
+                gfid,
+                local->lock[0].layout.my_layout.locks[lk_index]->xl->name);
         }
 
-        local->lock[0].layout.my_layout.locks[lk_index]->locked = _gf_true;
+        goto out;
+    }
 
-out:
-        call_cnt = dht_frame_return (frame);
-        if (is_last_call (call_cnt)) {
-                if (local->lock[0].layout.my_layout.op_ret < 0) {
-                        dht_inodelk_cleanup (frame);
-                        return 0;
-                }
+    local->lock[0].layout.my_layout.locks[lk_index]->locked = _gf_true;
 
-                dht_inodelk_done (frame);
+out:
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        if (local->lock[0].layout.my_layout.op_ret < 0) {
+            dht_inodelk_cleanup(frame);
+            return 0;
         }
 
-        return 0;
+        dht_inodelk_done(frame);
+    }
+
+    return 0;
 }
 
 int
-dht_nonblocking_inodelk (call_frame_t *frame, dht_lock_t **lk_array,
-                         int lk_count, fop_inodelk_cbk_t inodelk_cbk)
+dht_nonblocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array,
+                        int lk_count, fop_inodelk_cbk_t inodelk_cbk)
 {
-        struct gf_flock  flock      = {0,};
-        int              i          = 0, ret = 0;
-        dht_local_t     *local      = NULL;
-        call_frame_t    *lock_frame = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht-locks", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, lk_array, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, inodelk_cbk, out);
-
-        lock_frame = dht_lock_frame (frame);
-        if (lock_frame == NULL)
-                goto out;
-
-        ret = dht_local_inodelk_init (lock_frame, lk_array, lk_count,
-                                      inodelk_cbk);
-        if (ret < 0) {
-                goto out;
-        }
+    struct gf_flock flock = {
+        0,
+    };
+    int i = 0, ret = 0;
+    dht_local_t *local = NULL;
+    call_frame_t *lock_frame = NULL;
 
-        dht_set_lkowner (lk_array, lk_count, &lock_frame->root->lk_owner);
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, inodelk_cbk, out);
 
-        local = lock_frame->local;
-        local->main_frame = frame;
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL)
+        goto out;
 
-        local->call_cnt = lk_count;
+    ret = dht_local_inodelk_init(lock_frame, lk_array, lk_count, inodelk_cbk);
+    if (ret < 0) {
+        goto out;
+    }
 
-        for (i = 0; i < lk_count; i++) {
-                flock.l_type = local->lock[0].layout.my_layout.locks[i]->type;
+    dht_set_lkowner(lk_array, lk_count, &lock_frame->root->lk_owner);
 
-                STACK_WIND_COOKIE (lock_frame, dht_nonblocking_inodelk_cbk,
-                                   (void *) (long) i,
-                                   local->lock[0].layout.my_layout.locks[i]->xl,
-                                   local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
-                                   local->lock[0].layout.my_layout.locks[i]->domain,
-                                   &local->lock[0].layout.my_layout.locks[i]->loc,
-                                   F_SETLK,
-                                   &flock, NULL);
-        }
+    local = lock_frame->local;
+    local->main_frame = frame;
+
+    local->call_cnt = lk_count;
+
+    for (i = 0; i < lk_count; i++) {
+        flock.l_type = local->lock[0].layout.my_layout.locks[i]->type;
+
+        STACK_WIND_COOKIE(
+            lock_frame, dht_nonblocking_inodelk_cbk, (void *)(long)i,
+            local->lock[0].layout.my_layout.locks[i]->xl,
+            local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
+            local->lock[0].layout.my_layout.locks[i]->domain,
+            &local->lock[0].layout.my_layout.locks[i]->loc, F_SETLK, &flock,
+            NULL);
+    }
 
-        return 0;
+    return 0;
 
 out:
-        if (lock_frame)
-                dht_lock_stack_destroy (lock_frame, DHT_INODELK);
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_INODELK);
 
-        return -1;
+    return -1;
 }
 
 static int32_t
-dht_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_blocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        int                  lk_index       = 0;
-        int                  i              = 0;
-        dht_local_t         *local          = NULL;
-        char         gfid[GF_UUID_BUF_SIZE] = {0,};
-        dht_reaction_type_t  reaction       = 0;
-
-        lk_index = (long) cookie;
-
-        local = frame->local;
-        if (op_ret == 0) {
-                local->lock[0].layout.my_layout.locks[lk_index]->locked = _gf_true;
-        } else {
-                switch (op_errno) {
-                case ESTALE:
-                case ENOENT:
-                        reaction = local->lock[0].layout.my_layout.locks[lk_index]->do_on_failure;
-                        if ((reaction != IGNORE_ENOENT_ESTALE) &&
-                            (reaction != IGNORE_ENOENT_ESTALE_EIO)) {
-                                gf_uuid_unparse (local->lock[0].layout.my_layout.locks[lk_index]->loc.gfid, gfid);
-                                local->lock[0].layout.my_layout.op_ret = -1;
-                                local->lock[0].layout.my_layout.op_errno = op_errno;
-                                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                        DHT_MSG_INODELK_FAILED,
-                                        "inodelk failed on subvol %s. gfid:%s",
-                                        local->lock[0].layout.my_layout.locks[lk_index]->xl->name,
-                                        gfid);
-                                goto cleanup;
-                        }
-                        break;
-                case EIO:
-                        reaction = local->lock[0].layout.my_layout.locks[lk_index]->do_on_failure;
-                        if (reaction != IGNORE_ENOENT_ESTALE_EIO) {
-                                gf_uuid_unparse (local->lock[0].layout.my_layout.locks[lk_index]->loc.gfid, gfid);
-                                local->lock[0].layout.my_layout.op_ret = -1;
-                                local->lock[0].layout.my_layout.op_errno = op_errno;
-                                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                        DHT_MSG_INODELK_FAILED,
-                                        "inodelk failed on subvol %s. gfid:%s",
-                                        local->lock[0].layout.my_layout.locks[lk_index]->xl->name,
-                                        gfid);
-                                goto cleanup;
-                        }
-                        break;
-
-                default:
-                        gf_uuid_unparse (local->lock[0].layout.my_layout.locks[lk_index]->loc.gfid, gfid);
-                        local->lock[0].layout.my_layout.op_ret = -1;
-                        local->lock[0].layout.my_layout.op_errno = op_errno;
-                        gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                DHT_MSG_INODELK_FAILED,
-                                "inodelk failed on subvol %s, gfid:%s",
-                                local->lock[0].layout.my_layout.locks[lk_index]->xl->name, gfid);
-                        goto cleanup;
+    int lk_index = 0;
+    int i = 0;
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+    dht_reaction_type_t reaction = 0;
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+    if (op_ret == 0) {
+        local->lock[0].layout.my_layout.locks[lk_index]->locked = _gf_true;
+    } else {
+        switch (op_errno) {
+            case ESTALE:
+            case ENOENT:
+                reaction = local->lock[0]
+                               .layout.my_layout.locks[lk_index]
+                               ->do_on_failure;
+                if ((reaction != IGNORE_ENOENT_ESTALE) &&
+                    (reaction != IGNORE_ENOENT_ESTALE_EIO)) {
+                    gf_uuid_unparse(local->lock[0]
+                                        .layout.my_layout.locks[lk_index]
+                                        ->loc.gfid,
+                                    gfid);
+                    local->lock[0].layout.my_layout.op_ret = -1;
+                    local->lock[0].layout.my_layout.op_errno = op_errno;
+                    gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                            DHT_MSG_INODELK_FAILED, "subvol=%s",
+                            local->lock[0]
+                                .layout.my_layout.locks[lk_index]
+                                ->xl->name,
+                            "gfid=%s", gfid, NULL);
+                    goto cleanup;
                 }
-        }
+                break;
+            case EIO:
+                reaction = local->lock[0]
+                               .layout.my_layout.locks[lk_index]
+                               ->do_on_failure;
+                if (reaction != IGNORE_ENOENT_ESTALE_EIO) {
+                    gf_uuid_unparse(local->lock[0]
+                                        .layout.my_layout.locks[lk_index]
+                                        ->loc.gfid,
+                                    gfid);
+                    local->lock[0].layout.my_layout.op_ret = -1;
+                    local->lock[0].layout.my_layout.op_errno = op_errno;
+                    gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                            DHT_MSG_INODELK_FAILED, "subvol=%s",
+                            local->lock[0]
+                                .layout.my_layout.locks[lk_index]
+                                ->xl->name,
+                            "gfid=%s", gfid, NULL);
+                    goto cleanup;
+                }
+                break;
 
-        if (lk_index == (local->lock[0].layout.my_layout.lk_count - 1)) {
-                for (i = 0; (i < local->lock[0].layout.my_layout.lk_count) &&
-                     (!local->lock[0].layout.my_layout.locks[i]->locked); i++)
-                        ;
+            default:
+                gf_uuid_unparse(
+                    local->lock[0].layout.my_layout.locks[lk_index]->loc.gfid,
+                    gfid);
+                local->lock[0].layout.my_layout.op_ret = -1;
+                local->lock[0].layout.my_layout.op_errno = op_errno;
+                gf_smsg(
+                    this->name, GF_LOG_ERROR, op_errno, DHT_MSG_INODELK_FAILED,
+                    "subvol=%s",
+                    local->lock[0].layout.my_layout.locks[lk_index]->xl->name,
+                    "gfid=%s", gfid, NULL);
+                goto cleanup;
+        }
+    }
 
-                if (i == local->lock[0].layout.my_layout.lk_count) {
-                        local->lock[0].layout.my_layout.op_ret = -1;
-                        local->lock[0].layout.my_layout.op_errno = op_errno;
-                }
+    if (lk_index == (local->lock[0].layout.my_layout.lk_count - 1)) {
+        for (i = 0; (i < local->lock[0].layout.my_layout.lk_count) &&
+                    (!local->lock[0].layout.my_layout.locks[i]->locked);
+             i++)
+            ;
 
-                dht_inodelk_done (frame);
-        } else {
-                dht_blocking_inodelk_rec (frame, ++lk_index);
+        if (i == local->lock[0].layout.my_layout.lk_count) {
+            local->lock[0].layout.my_layout.op_ret = -1;
+            local->lock[0].layout.my_layout.op_errno = op_errno;
         }
 
-        return 0;
+        dht_inodelk_done(frame);
+    } else {
+        dht_blocking_inodelk_rec(frame, ++lk_index);
+    }
+
+    return 0;
 
 cleanup:
-        dht_inodelk_cleanup (frame);
+    dht_inodelk_cleanup(frame);
 
-        return 0;
+    return 0;
 }
 
 void
-dht_blocking_inodelk_rec (call_frame_t *frame, int i)
+dht_blocking_inodelk_rec(call_frame_t *frame, int i)
 {
-        dht_local_t     *local = NULL;
-        struct gf_flock  flock = {0,};
+    dht_local_t *local = NULL;
+    struct gf_flock flock = {
+        0,
+    };
 
-        local = frame->local;
+    local = frame->local;
 
-        flock.l_type = local->lock[0].layout.my_layout.locks[i]->type;
+    flock.l_type = local->lock[0].layout.my_layout.locks[i]->type;
 
-        STACK_WIND_COOKIE (frame, dht_blocking_inodelk_cbk,
-                           (void *) (long) i,
-                           local->lock[0].layout.my_layout.locks[i]->xl,
-                           local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
-                           local->lock[0].layout.my_layout.locks[i]->domain,
-                           &local->lock[0].layout.my_layout.locks[i]->loc,
-                           F_SETLKW,
-                           &flock, NULL);
+    STACK_WIND_COOKIE(
+        frame, dht_blocking_inodelk_cbk, (void *)(long)i,
+        local->lock[0].layout.my_layout.locks[i]->xl,
+        local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
+        local->lock[0].layout.my_layout.locks[i]->domain,
+        &local->lock[0].layout.my_layout.locks[i]->loc, F_SETLKW, &flock, NULL);
 
-        return;
+    return;
 }
 
 int
-dht_blocking_inodelk (call_frame_t *frame, dht_lock_t **lk_array,
-                      int lk_count, fop_inodelk_cbk_t inodelk_cbk)
+dht_blocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_inodelk_cbk_t inodelk_cbk)
 {
-        int           ret                       = -1;
-        call_frame_t *lock_frame                = NULL;
-        dht_local_t  *local                     = NULL;
-        dht_local_t  *tmp_local                 = NULL;
-        char          gfid[GF_UUID_BUF_SIZE]    = {0,};
-
-        GF_VALIDATE_OR_GOTO ("dht-locks", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, lk_array, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, inodelk_cbk, out);
-
-        tmp_local = frame->local;
-
-        lock_frame = dht_lock_frame (frame);
-        if (lock_frame == NULL) {
-                gf_uuid_unparse (tmp_local->loc.gfid, gfid);
-                gf_msg ("dht", GF_LOG_ERROR, ENOMEM,
-                        DHT_MSG_LOCK_FRAME_FAILED,
-                        "memory allocation failed for lock_frame. gfid:%s"
-                        " path:%s", gfid, tmp_local->loc.path);
-                goto out;
-        }
-
-        ret = dht_local_inodelk_init (lock_frame, lk_array, lk_count,
-                                      inodelk_cbk);
-        if (ret < 0) {
-                gf_uuid_unparse (tmp_local->loc.gfid, gfid);
-                gf_msg ("dht", GF_LOG_ERROR, ENOMEM,
-                        DHT_MSG_LOCAL_LOCK_INIT_FAILED,
-                        "dht_local_lock_init failed, gfid: %s path:%s", gfid,
-                        tmp_local->loc.path);
-                goto out;
-        }
-
-        dht_set_lkowner (lk_array, lk_count, &lock_frame->root->lk_owner);
-
-        local = lock_frame->local;
-        local->main_frame = frame;
-
-        dht_blocking_inodelk_rec (lock_frame, 0);
-
-        return 0;
+    int ret = -1;
+    call_frame_t *lock_frame = NULL;
+    dht_local_t *local = NULL;
+    dht_local_t *tmp_local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, inodelk_cbk, out);
+
+    tmp_local = frame->local;
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL) {
+        gf_uuid_unparse(tmp_local->loc.gfid, gfid);
+        gf_smsg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_LOCK_FRAME_FAILED,
+                "gfid=%s", gfid, "path=%s", tmp_local->loc.path, NULL);
+        goto out;
+    }
+
+    ret = dht_local_inodelk_init(lock_frame, lk_array, lk_count, inodelk_cbk);
+    if (ret < 0) {
+        gf_uuid_unparse(tmp_local->loc.gfid, gfid);
+        gf_smsg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_LOCAL_LOCK_INIT_FAILED,
+                "gfid=%s", gfid, "path=%s", tmp_local->loc.path, NULL);
+        goto out;
+    }
+
+    dht_set_lkowner(lk_array, lk_count, &lock_frame->root->lk_owner);
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+
+    dht_blocking_inodelk_rec(lock_frame, 0);
+
+    return 0;
 out:
-        if (lock_frame)
-                dht_lock_stack_destroy (lock_frame, DHT_INODELK);
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_INODELK);
 
-        return -1;
+    return -1;
 }
 
 void
-dht_unlock_namespace (call_frame_t *frame, dht_dir_transaction_t *lock)
+dht_unlock_namespace(call_frame_t *frame, dht_dir_transaction_t *lock)
 {
-        GF_VALIDATE_OR_GOTO ("dht-locks", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, lock, out);
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lock, out);
 
-        dht_unlock_entrylk_wrapper (frame, &lock->ns.directory_ns);
-        dht_unlock_inodelk_wrapper (frame, &lock->ns.parent_layout);
+    dht_unlock_entrylk_wrapper(frame, &lock->ns.directory_ns);
+    dht_unlock_inodelk_wrapper(frame, &lock->ns.parent_layout);
 
 out:
-        return;
+    return;
 }
 
 static int32_t
-dht_protect_namespace_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_protect_namespace_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local    = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
-        if (op_ret != 0)
-                dht_unlock_inodelk_wrapper (frame,
-                                            &local->current->ns.parent_layout);
+    local = frame->local;
+    if (op_ret != 0)
+        dht_unlock_inodelk_wrapper(frame, &local->current->ns.parent_layout);
 
-        local->current->ns.ns_cbk (frame, cookie, this, op_ret, op_errno,
-                                   xdata);
-        return 0;
+    local->current->ns.ns_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-dht_blocking_entrylk_after_inodelk (call_frame_t *frame, void *cookie,
-                                    xlator_t *this, int32_t op_ret,
-                                    int32_t op_errno, dict_t *xdata)
+dht_blocking_entrylk_after_inodelk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t           *local                   = NULL;
-        int                    ret                     = -1;
-        loc_t                 *loc                     = NULL;
-        dht_lock_t           **lk_array                = NULL;
-        char                   pgfid[GF_UUID_BUF_SIZE] = {0};
-        int                    count                   = 0;
-        dht_elock_wrap_t      *entrylk                 = NULL;
-
-        local = frame->local;
-        entrylk = &local->current->ns.directory_ns;
-
-        if (op_ret < 0) {
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-                goto err;
-        }
-
-        loc = &entrylk->locks[0]->loc;
-        gf_uuid_unparse (loc->gfid, pgfid);
-
-        local->op_ret = 0;
-        lk_array = entrylk->locks;
-        count = entrylk->lk_count;
-
-        ret = dht_blocking_entrylk (frame, lk_array, count,
-                                    dht_protect_namespace_cbk);
-
-        if (ret < 0) {
-                local->op_ret = -1;
-                local->op_errno = EIO;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_ENTRYLK_ERROR,
-                        "%s (%s/%s): "
-                        "dht_blocking_entrylk failed after taking inodelk",
-                        gf_fop_list[local->fop], pgfid,
-                        entrylk->locks[0]->basename);
-                goto err;
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    int ret = -1;
+    loc_t *loc = NULL;
+    dht_lock_t **lk_array = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int count = 0;
+    dht_elock_wrap_t *entrylk = NULL;
+
+    local = frame->local;
+    entrylk = &local->current->ns.directory_ns;
+
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    loc = &entrylk->locks[0]->loc;
+    gf_uuid_unparse(loc->gfid, pgfid);
+
+    local->op_ret = 0;
+    lk_array = entrylk->locks;
+    count = entrylk->lk_count;
+
+    ret = dht_blocking_entrylk(frame, lk_array, count,
+                               dht_protect_namespace_cbk);
+
+    if (ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_ENTRYLK_FAILED_AFT_INODELK, "fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "basename=%s",
+                entrylk->locks[0]->basename, NULL);
+        goto err;
+    }
+
+    return 0;
 
 err:
-        if (lk_array != NULL) {
-                dht_lock_array_free (lk_array, count);
-                GF_FREE (lk_array);
-                entrylk->locks = NULL;
-                entrylk->lk_count = 0;
-        }
-
-        /* Unlock inodelk. No harm calling unlock twice */
-        dht_unlock_inodelk_wrapper (frame, &local->current->ns.parent_layout);
-        /* Call ns_cbk. It will take care of unwinding */
-        local->current->ns.ns_cbk (frame, NULL, this, local->op_ret,
-                                   local->op_errno, NULL);
-        return 0;
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+        entrylk->locks = NULL;
+        entrylk->lk_count = 0;
+    }
+
+    /* Unlock inodelk. No harm calling unlock twice */
+    dht_unlock_inodelk_wrapper(frame, &local->current->ns.parent_layout);
+    /* Call ns_cbk. It will take care of unwinding */
+    local->current->ns.ns_cbk(frame, NULL, this, local->op_ret, local->op_errno,
+                              NULL);
+    return 0;
 }
 
 /* Given the loc and the subvol, this routine takes the inodelk on
@@ -1265,137 +1263,130 @@ err:
  * and then entrylk serially.
  */
 int
-dht_protect_namespace (call_frame_t *frame, loc_t *loc,
-                       xlator_t *subvol,
-                       struct dht_namespace *ns,
-                       fop_entrylk_cbk_t ns_cbk)
+dht_protect_namespace(call_frame_t *frame, loc_t *loc, xlator_t *subvol,
+                      struct dht_namespace *ns, fop_entrylk_cbk_t ns_cbk)
 {
-        dht_ilock_wrap_t  *inodelk                 = NULL;
-        dht_elock_wrap_t  *entrylk                 = NULL;
-        dht_lock_t       **lk_array                = NULL;
-        dht_local_t       *local                   = NULL;
-        xlator_t          *this                    = NULL;
-        loc_t              parent                  = {0,};
-        int                ret                     = -1;
-        char               pgfid[GF_UUID_BUF_SIZE] = {0};
-        int32_t            op_errno                = 0;
-        int                count                   = 1;
-
-        GF_VALIDATE_OR_GOTO ("dht-locks", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc->parent, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, subvol, out);
-
-        local = frame->local;
-        this = frame->this;
-
-        inodelk = &ns->parent_layout;
-        entrylk = &ns->directory_ns;
-
-        /* Initialize entrylk_cbk and parent loc */
-        ns->ns_cbk = ns_cbk;
-
-        ret = dht_build_parent_loc (this, &parent, loc, &op_errno);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        DHT_MSG_LOC_FAILED, "gfid:%s (name:%s) (path: %s): "
-                        "parent loc build failed", loc->gfid, loc->name,
-                         loc->path);
-                goto out;
-        }
-        gf_uuid_unparse (parent.gfid, pgfid);
-
-        /* Alloc inodelk */
-        inodelk->locks = GF_CALLOC (count, sizeof (*lk_array),
-                                    gf_common_mt_pointer);
-        if (inodelk->locks == NULL) {
-                local->op_errno = ENOMEM;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_NO_MEMORY,
-                        "%s (%s/%s) (path: %s): "
-                        "calloc failure",
-                        gf_fop_list[local->fop], pgfid, loc->name, loc->path);
-                goto out;
-        }
-
-        inodelk->locks[0] = dht_lock_new (this, subvol, &parent, F_RDLCK,
-                                          DHT_LAYOUT_HEAL_DOMAIN, NULL,
-                                          FAIL_ON_ANY_ERROR);
-        if (inodelk->locks[0] == NULL) {
-                local->op_errno = ENOMEM;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_NO_MEMORY,
-                        "%s (%s/%s) (path: %s): "
-                        "inodelk: lock allocation failed",
-                        gf_fop_list[local->fop], pgfid, loc->name, loc->path);
-                goto err;
-        }
-        inodelk->lk_count = count;
-
-        /* Allock entrylk */
-        entrylk->locks = GF_CALLOC (count, sizeof (*lk_array),
-                                    gf_common_mt_pointer);
-        if (entrylk->locks == NULL) {
-                local->op_errno = ENOMEM;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_NO_MEMORY,
-                        "%s (%s/%s) (path: %s): "
-                        "entrylk: calloc failure",
-                        gf_fop_list[local->fop], pgfid, loc->name, loc->path);
-
-                goto err;
-        }
-
-        entrylk->locks[0] = dht_lock_new (this, subvol, &parent, F_WRLCK,
-                                          DHT_ENTRY_SYNC_DOMAIN, loc->name,
-                                          FAIL_ON_ANY_ERROR);
-        if (entrylk->locks[0] == NULL) {
-                local->op_errno = ENOMEM;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_NO_MEMORY,
-                        "%s (%s/%s) (path: %s): "
-                        "entrylk: lock allocation failed",
-                        gf_fop_list[local->fop], pgfid, loc->name, loc->path);
-
-                goto err;
-        }
-        entrylk->lk_count = count;
-
-        /* Take read inodelk on parent. If it is successful, take write entrylk
-         * on name in cbk.
-         */
-        lk_array = inodelk->locks;
-        ret = dht_blocking_inodelk (frame, lk_array, count,
-                                    dht_blocking_entrylk_after_inodelk);
-        if (ret < 0) {
-                local->op_errno = EIO;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_INODELK_ERROR,
-                        "%s (%s/%s) (path: %s): "
-                        "dht_blocking_inodelk failed",
-                        gf_fop_list[local->fop], pgfid, loc->name, loc->path);
-                goto err;
-        }
-
-        loc_wipe (&parent);
-
-        return 0;
+    dht_ilock_wrap_t *inodelk = NULL;
+    dht_elock_wrap_t *entrylk = NULL;
+    dht_lock_t **lk_array = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    loc_t parent = {
+        0,
+    };
+    int ret = -1;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t op_errno = 0;
+    int count = 1;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, loc->parent, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, subvol, out);
+
+    local = frame->local;
+    this = frame->this;
+
+    inodelk = &ns->parent_layout;
+    entrylk = &ns->directory_ns;
+
+    /* Initialize entrylk_cbk and parent loc */
+    ns->ns_cbk = ns_cbk;
+
+    ret = dht_build_parent_loc(this, &parent, loc, &op_errno);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_LOC_FAILED,
+                "gfid=%s", loc->gfid, "name=%s", loc->name, "path=%s",
+                loc->path, NULL);
+        goto out;
+    }
+    gf_uuid_unparse(parent.gfid, pgfid);
+
+    /* Alloc inodelk */
+    inodelk->locks = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
+    if (inodelk->locks == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_CALLOC_FAILED, "fop=%s", gf_fop_list[local->fop],
+                "pgfid=%s", pgfid, "name=%s", loc->name, "path=%s", loc->path,
+                NULL);
+        goto out;
+    }
+
+    inodelk->locks[0] = dht_lock_new(this, subvol, &parent, F_RDLCK,
+                                     DHT_LAYOUT_HEAL_DOMAIN, NULL,
+                                     FAIL_ON_ANY_ERROR);
+    if (inodelk->locks[0] == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_LOCK_ALLOC_FAILED, "inodelk-fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s",
+                loc->name, "path=%s", loc->path, NULL);
+        goto err;
+    }
+    inodelk->lk_count = count;
+
+    /* Allock entrylk */
+    entrylk->locks = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
+    if (entrylk->locks == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_CALLOC_FAILED, "entrylk-fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s",
+                loc->name, "path=%s", loc->path, NULL);
+
+        goto err;
+    }
+
+    entrylk->locks[0] = dht_lock_new(this, subvol, &parent, F_WRLCK,
+                                     DHT_ENTRY_SYNC_DOMAIN, loc->name,
+                                     FAIL_ON_ANY_ERROR);
+    if (entrylk->locks[0] == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_LOCK_ALLOC_FAILED, "entrylk-fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s",
+                loc->name, "path=%s", loc->path, NULL);
+
+        goto err;
+    }
+    entrylk->lk_count = count;
+
+    /* Take read inodelk on parent. If it is successful, take write entrylk
+     * on name in cbk.
+     */
+    lk_array = inodelk->locks;
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_blocking_entrylk_after_inodelk);
+    if (ret < 0) {
+        local->op_errno = EIO;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_BLOCK_INODELK_FAILED, "fop=%s", gf_fop_list[local->fop],
+                "pgfid=%s", pgfid, "name=%s", loc->name, "path=%s", loc->path,
+                NULL);
+
+        goto err;
+    }
+
+    loc_wipe(&parent);
+
+    return 0;
 err:
-        if (entrylk->locks != NULL) {
-                dht_lock_array_free (entrylk->locks, count);
-                GF_FREE (entrylk->locks);
-                entrylk->locks = NULL;
-                entrylk->lk_count = 0;
-        }
+    if (entrylk->locks != NULL) {
+        dht_lock_array_free(entrylk->locks, count);
+        GF_FREE(entrylk->locks);
+        entrylk->locks = NULL;
+        entrylk->lk_count = 0;
+    }
 
-        if (inodelk->locks != NULL) {
-                dht_lock_array_free (inodelk->locks, count);
-                GF_FREE (inodelk->locks);
-                inodelk->locks = NULL;
-                inodelk->lk_count = 0;
-        }
+    if (inodelk->locks != NULL) {
+        dht_lock_array_free(inodelk->locks, count);
+        GF_FREE(inodelk->locks);
+        inodelk->locks = NULL;
+        inodelk->lk_count = 0;
+    }
 
-        loc_wipe (&parent);
+    loc_wipe(&parent);
 out:
-        return -1;
+    return -1;
 }
diff --git a/xlators/cluster/dht/src/dht-lock.h b/xlators/cluster/dht/src/dht-lock.h
index f36bf3fd685..6485c03fb6e 100644
--- a/xlators/cluster/dht/src/dht-lock.h
+++ b/xlators/cluster/dht/src/dht-lock.h
@@ -11,36 +11,35 @@
 #ifndef _DHT_LOCK_H
 #define _DHT_LOCK_H
 
-#include "xlator.h"
 #include "dht-common.h"
 
 void
-dht_lock_array_free (dht_lock_t **lk_array, int count);
+dht_lock_array_free(dht_lock_t **lk_array, int count);
 
 int32_t
-dht_lock_count (dht_lock_t **lk_array, int lk_count);
+dht_lock_count(dht_lock_t **lk_array, int lk_count);
 
 dht_lock_t *
-dht_lock_new (xlator_t *this, xlator_t *xl, loc_t *loc, short type,
-              const char *domain, const char *basename,
-              dht_reaction_type_t do_on_failure);
+dht_lock_new(xlator_t *this, xlator_t *xl, loc_t *loc, short type,
+             const char *domain, const char *basename,
+             dht_reaction_type_t do_on_failure);
 
 int32_t
-dht_unlock_entrylk_wrapper (call_frame_t *, dht_elock_wrap_t *);
+dht_unlock_entrylk_wrapper(call_frame_t *, dht_elock_wrap_t *);
 
 void
-dht_blocking_entrylk_rec (call_frame_t *frame, int i);
+dht_blocking_entrylk_rec(call_frame_t *frame, int i);
 
 int
-dht_blocking_entrylk (call_frame_t *frame, dht_lock_t **lk_array,
-                      int lk_count, fop_inodelk_cbk_t entrylk_cbk);
+dht_blocking_entrylk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_inodelk_cbk_t entrylk_cbk);
 
 int32_t
-dht_unlock_inodelk (call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
-                     fop_inodelk_cbk_t inodelk_cbk);
+dht_unlock_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                   fop_inodelk_cbk_t inodelk_cbk);
 
 int32_t
-dht_unlock_inodelk_wrapper (call_frame_t *, dht_ilock_wrap_t *);
+dht_unlock_inodelk_wrapper(call_frame_t *, dht_ilock_wrap_t *);
 
 /* Acquire non-blocking inodelk on a list of xlators.
  *
@@ -57,11 +56,11 @@ dht_unlock_inodelk_wrapper (call_frame_t *, dht_ilock_wrap_t *);
  */
 
 int
-dht_nonblocking_inodelk (call_frame_t *frame, dht_lock_t **lk_array,
-                         int lk_count, fop_inodelk_cbk_t inodelk_cbk);
+dht_nonblocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array,
+                        int lk_count, fop_inodelk_cbk_t inodelk_cbk);
 
 void
-dht_blocking_inodelk_rec (call_frame_t *frame, int i);
+dht_blocking_inodelk_rec(call_frame_t *frame, int i);
 
 /* same as dht_nonblocking_inodelk, but issues sequential blocking locks on
  * @lk_array directly. locks are issued on some order which remains same
@@ -69,25 +68,24 @@ dht_blocking_inodelk_rec (call_frame_t *frame, int i);
  */
 
 int
-dht_blocking_inodelk (call_frame_t *frame, dht_lock_t **lk_array,
-                      int lk_count, fop_inodelk_cbk_t inodelk_cbk);
+dht_blocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_inodelk_cbk_t inodelk_cbk);
 
 int32_t
-dht_blocking_entrylk_after_inodelk (call_frame_t *frame, void *cookie,
-                                    xlator_t *this, int32_t op_ret,
-                                    int32_t op_errno, dict_t *xdata);
+dht_blocking_entrylk_after_inodelk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, dict_t *xdata);
 
 int32_t
-dht_blocking_entrylk_after_inodelk_rename (call_frame_t *frame, void *cookie,
-                                           xlator_t *this, int32_t op_ret,
-                                           int32_t op_errno, dict_t *xdata);
+dht_blocking_entrylk_after_inodelk_rename(call_frame_t *frame, void *cookie,
+                                          xlator_t *this, int32_t op_ret,
+                                          int32_t op_errno, dict_t *xdata);
 
 void
-dht_unlock_namespace (call_frame_t *, dht_dir_transaction_t *);
+dht_unlock_namespace(call_frame_t *, dht_dir_transaction_t *);
 
 int
-dht_protect_namespace (call_frame_t *frame, loc_t *loc, xlator_t *subvol,
-                       struct dht_namespace *ns,
-                       fop_entrylk_cbk_t ns_cbk);
+dht_protect_namespace(call_frame_t *frame, loc_t *loc, xlator_t *subvol,
+                      struct dht_namespace *ns, fop_entrylk_cbk_t ns_cbk);
 
-#endif   /* _DHT_LOCK_H */
+#endif /* _DHT_LOCK_H */
diff --git a/xlators/cluster/dht/src/dht-mem-types.h b/xlators/cluster/dht/src/dht-mem-types.h
index 19cccef537b..e3c4471334a 100644
--- a/xlators/cluster/dht/src/dht-mem-types.h
+++ b/xlators/cluster/dht/src/dht-mem-types.h
@@ -8,38 +8,31 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __DHT_MEM_TYPES_H__
 #define __DHT_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_dht_mem_types_ {
-        gf_dht_mt_dht_du_t = gf_common_mt_end + 1,
-        gf_dht_mt_dht_conf_t,
-        gf_dht_mt_char,
-        gf_dht_mt_int32_t,
-        gf_dht_mt_xlator_t,
-        gf_dht_mt_dht_layout_t,
-        gf_switch_mt_dht_conf_t,
-        gf_switch_mt_dht_du_t,
-        gf_switch_mt_switch_sched_array,
-        gf_switch_mt_switch_struct,
-        gf_dht_mt_subvol_time,
-        gf_dht_mt_loc_t,
-        gf_defrag_info_mt,
-        gf_dht_mt_inode_ctx_t,
-        gf_dht_mt_ctx_stat_time_t,
-        gf_dht_mt_dirent_t,
-        gf_dht_mt_container_t,
-        gf_dht_mt_octx_t,
-        gf_dht_mt_miginfo_t,
-        gf_tier_mt_bricklist_t,
-        gf_tier_mt_ipc_ctr_params_t,
-        gf_dht_mt_fd_ctx_t,
-        gf_tier_mt_qfile_array_t,
-        gf_dht_ret_cache_t,
-        gf_dht_nodeuuids_t,
-        gf_dht_mt_end
+    gf_dht_mt_dht_du_t = gf_common_mt_end + 1,
+    gf_dht_mt_dht_conf_t,
+    gf_dht_mt_char,
+    gf_dht_mt_int32_t,
+    gf_dht_mt_xlator_t,
+    gf_dht_mt_dht_layout_t,
+    gf_switch_mt_switch_sched_array,
+    gf_switch_mt_switch_struct,
+    gf_dht_mt_subvol_time,
+    gf_dht_mt_loc_t,
+    gf_defrag_info_mt,
+    gf_dht_mt_inode_ctx_t,
+    gf_dht_mt_dirent_t,
+    gf_dht_mt_container_t,
+    gf_dht_mt_octx_t,
+    gf_dht_mt_miginfo_t,
+    gf_dht_mt_fd_ctx_t,
+    gf_dht_ret_cache_t,
+    gf_dht_nodeuuids_t,
+    gf_dht_mt_end
 };
 #endif
diff --git a/xlators/cluster/dht/src/dht-messages.h b/xlators/cluster/dht/src/dht-messages.h
index bdea5a092b7..601f8dad78b 100644
--- a/xlators/cluster/dht/src/dht-messages.h
+++ b/xlators/cluster/dht/src/dht-messages.h
@@ -10,7 +10,7 @@
 #ifndef _DHT_MESSAGES_H_
 #define _DHT_MESSAGES_H_
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
 
 /* To add new message IDs, append new identifiers at the end of the list.
  *
@@ -22,136 +22,365 @@
  * glfs-message-id.h.
  */
 
-GLFS_MSGID(DHT,
-        DHT_MSG_CACHED_SUBVOL_GET_FAILED,
-        DHT_MSG_CREATE_LINK_FAILED,
-        DHT_MSG_DICT_SET_FAILED,
-        DHT_MSG_DIR_ATTR_HEAL_FAILED,
-        DHT_MSG_DIR_SELFHEAL_FAILED,
-        DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-        DHT_MSG_FILE_ON_MULT_SUBVOL,
-        DHT_MSG_FILE_TYPE_MISMATCH,
-        DHT_MSG_GFID_MISMATCH,
-        DHT_MSG_GFID_NULL,
-        DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-        DHT_MSG_INIT_FAILED,
-        DHT_MSG_INVALID_CONFIGURATION,
-        DHT_MSG_INVALID_DISK_LAYOUT,
-        DHT_MSG_INVALID_OPTION,
-        DHT_MSG_LAYOUT_FIX_FAILED,
-        DHT_MSG_LAYOUT_MERGE_FAILED,
-        DHT_MSG_LAYOUT_MISMATCH,
-        DHT_MSG_LAYOUT_NULL,
-        DHT_MSG_MIGRATE_DATA_COMPLETE,
-        DHT_MSG_MIGRATE_DATA_FAILED,
-        DHT_MSG_MIGRATE_FILE_COMPLETE,
-        DHT_MSG_MIGRATE_FILE_FAILED,
-        DHT_MSG_NO_MEMORY,
-        DHT_MSG_OPENDIR_FAILED,
-        DHT_MSG_REBALANCE_FAILED,
-        DHT_MSG_REBALANCE_START_FAILED,
-        DHT_MSG_REBALANCE_STATUS,
-        DHT_MSG_REBALANCE_STOPPED,
-        DHT_MSG_RENAME_FAILED,
-        DHT_MSG_SETATTR_FAILED,
-        DHT_MSG_SUBVOL_INSUFF_INODES,
-        DHT_MSG_SUBVOL_INSUFF_SPACE,
-        DHT_MSG_UNLINK_FAILED,
-        DHT_MSG_LAYOUT_SET_FAILED,
-        DHT_MSG_LOG_FIXED_LAYOUT,
-        DHT_MSG_LOG_TIER_ERROR,
-        DHT_MSG_LOG_TIER_STATUS,
-        DHT_MSG_GET_XATTR_FAILED,
-        DHT_MSG_FILE_LOOKUP_FAILED,
-        DHT_MSG_OPEN_FD_FAILED,
-        DHT_MSG_SET_INODE_CTX_FAILED,
-        DHT_MSG_UNLOCKING_FAILED,
-        DHT_MSG_DISK_LAYOUT_NULL,
-        DHT_MSG_SUBVOL_INFO,
-        DHT_MSG_CHUNK_SIZE_INFO,
-        DHT_MSG_LAYOUT_FORM_FAILED,
-        DHT_MSG_SUBVOL_ERROR,
-        DHT_MSG_LAYOUT_SORT_FAILED,
-        DHT_MSG_REGEX_INFO,
-        DHT_MSG_FOPEN_FAILED,
-        DHT_MSG_SET_HOSTNAME_FAILED,
-        DHT_MSG_BRICK_ERROR,
-        DHT_MSG_SYNCOP_FAILED,
-        DHT_MSG_MIGRATE_INFO,
-        DHT_MSG_SOCKET_ERROR,
-        DHT_MSG_CREATE_FD_FAILED,
-        DHT_MSG_READDIR_ERROR,
-        DHT_MSG_CHILD_LOC_BUILD_FAILED,
-        DHT_MSG_SET_SWITCH_PATTERN_ERROR,
-        DHT_MSG_COMPUTE_HASH_FAILED,
-        DHT_MSG_FIND_LAYOUT_ANOMALIES_ERROR,
-        DHT_MSG_ANOMALIES_INFO,
-        DHT_MSG_LAYOUT_INFO,
-        DHT_MSG_INODE_LK_ERROR,
-        DHT_MSG_RENAME_INFO,
-        DHT_MSG_DATA_NULL,
-        DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED,
-        DHT_MSG_UNLINK_LOOKUP_INFO,
-        DHT_MSG_LINK_FILE_LOOKUP_INFO,
-        DHT_MSG_OPERATION_NOT_SUP,
-        DHT_MSG_NOT_LINK_FILE_ERROR,
-        DHT_MSG_CHILD_DOWN,
-        DHT_MSG_UUID_PARSE_ERROR,
-        DHT_MSG_GET_DISK_INFO_ERROR,
-        DHT_MSG_INVALID_VALUE,
-        DHT_MSG_SWITCH_PATTERN_INFO,
-        DHT_MSG_SUBVOL_OP_FAILED,
-        DHT_MSG_LAYOUT_PRESET_FAILED,
-        DHT_MSG_INVALID_LINKFILE,
-        DHT_MSG_FIX_LAYOUT_INFO,
-        DHT_MSG_GET_HOSTNAME_FAILED,
-        DHT_MSG_WRITE_FAILED,
-        DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED,
-        DHT_MSG_FSYNC_FAILED,
-        DHT_MSG_SUBVOL_DECOMMISSION_INFO,
-        DHT_MSG_BRICK_QUERY_FAILED,
-        DHT_MSG_SUBVOL_NO_LAYOUT_INFO,
-        DHT_MSG_OPEN_FD_ON_DST_FAILED,
-        DHT_MSG_SUBVOL_NOT_FOUND,
-        DHT_MSG_FILE_LOOKUP_ON_DST_FAILED,
-        DHT_MSG_DISK_LAYOUT_MISSING,
-        DHT_MSG_DICT_GET_FAILED,
-        DHT_MSG_REVALIDATE_CBK_INFO,
-        DHT_MSG_UPGRADE_BRICKS,
-        DHT_MSG_LK_ARRAY_INFO,
-        DHT_MSG_RENAME_NOT_LOCAL,
-        DHT_MSG_RECONFIGURE_INFO,
-        DHT_MSG_INIT_LOCAL_SUBVOL_FAILED,
-        DHT_MSG_SYS_CALL_GET_TIME_FAILED,
-        DHT_MSG_NO_DISK_USAGE_STATUS,
-        DHT_MSG_SUBVOL_DOWN_ERROR,
-        DHT_MSG_REBAL_THROTTLE_INFO,
-        DHT_MSG_COMMIT_HASH_INFO,
-        DHT_MSG_REBAL_STRUCT_SET,
-        DHT_MSG_HAS_MIGINFO,
-        DHT_MSG_LOG_IPC_TIER_ERROR,
-        DHT_MSG_TIER_PAUSED,
-        DHT_MSG_TIER_RESUME,
-        DHT_MSG_SETTLE_HASH_FAILED,
-        DHT_MSG_DEFRAG_PROCESS_DIR_FAILED,
-        DHT_MSG_FD_CTX_SET_FAILED,
-        DHT_MSG_STALE_LOOKUP,
-        DHT_MSG_PARENT_LAYOUT_CHANGED,
-        DHT_MSG_LOCK_MIGRATION_FAILED,
-        DHT_MSG_LOCK_INODE_UNREF_FAILED,
-        DHT_MSG_ASPRINTF_FAILED,
-        DHT_MSG_DIR_LOOKUP_FAILED,
-        DHT_MSG_INODELK_FAILED,
-        DHT_MSG_LOCK_FRAME_FAILED,
-        DHT_MSG_LOCAL_LOCK_INIT_FAILED,
-        DHT_MSG_ENTRYLK_ERROR,
-        DHT_MSG_INODELK_ERROR,
-        DHT_MSG_LOC_FAILED,
-        DHT_MSG_UNKNOWN_FOP,
-        DHT_MSG_MIGRATE_FILE_SKIPPED,
-        DHT_MSG_DIR_XATTR_HEAL_FAILED,
-        DHT_MSG_HASHED_SUBVOL_DOWN,
-        DHT_MSG_NON_HASHED_SUBVOL_DOWN
-);
+GLFS_MSGID(
+    DHT, DHT_MSG_CACHED_SUBVOL_GET_FAILED, DHT_MSG_CREATE_LINK_FAILED,
+    DHT_MSG_DICT_SET_FAILED, DHT_MSG_DIR_ATTR_HEAL_FAILED,
+    DHT_MSG_DIR_SELFHEAL_FAILED, DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
+    DHT_MSG_FILE_ON_MULT_SUBVOL, DHT_MSG_FILE_TYPE_MISMATCH,
+    DHT_MSG_GFID_MISMATCH, DHT_MSG_GFID_NULL, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+    DHT_MSG_INIT_FAILED, DHT_MSG_INVALID_CONFIGURATION,
+    DHT_MSG_INVALID_DISK_LAYOUT, DHT_MSG_INVALID_OPTION,
+    DHT_MSG_LAYOUT_FIX_FAILED, DHT_MSG_LAYOUT_MERGE_FAILED,
+    DHT_MSG_LAYOUT_MISMATCH, DHT_MSG_LAYOUT_NULL, DHT_MSG_MIGRATE_DATA_COMPLETE,
+    DHT_MSG_MIGRATE_DATA_FAILED, DHT_MSG_MIGRATE_FILE_COMPLETE,
+    DHT_MSG_MIGRATE_FILE_FAILED, DHT_MSG_NO_MEMORY, DHT_MSG_OPENDIR_FAILED,
+    DHT_MSG_REBALANCE_FAILED, DHT_MSG_REBALANCE_START_FAILED,
+    DHT_MSG_REBALANCE_STATUS, DHT_MSG_REBALANCE_STOPPED, DHT_MSG_RENAME_FAILED,
+    DHT_MSG_SETATTR_FAILED, DHT_MSG_SUBVOL_INSUFF_INODES,
+    DHT_MSG_SUBVOL_INSUFF_SPACE, DHT_MSG_UNLINK_FAILED,
+    DHT_MSG_LAYOUT_SET_FAILED, DHT_MSG_LOG_FIXED_LAYOUT,
+    DHT_MSG_GET_XATTR_FAILED, DHT_MSG_FILE_LOOKUP_FAILED,
+    DHT_MSG_OPEN_FD_FAILED, DHT_MSG_SET_INODE_CTX_FAILED,
+    DHT_MSG_UNLOCKING_FAILED, DHT_MSG_DISK_LAYOUT_NULL, DHT_MSG_SUBVOL_INFO,
+    DHT_MSG_CHUNK_SIZE_INFO, DHT_MSG_LAYOUT_FORM_FAILED, DHT_MSG_SUBVOL_ERROR,
+    DHT_MSG_LAYOUT_SORT_FAILED, DHT_MSG_REGEX_INFO, DHT_MSG_FOPEN_FAILED,
+    DHT_MSG_SET_HOSTNAME_FAILED, DHT_MSG_BRICK_ERROR, DHT_MSG_SYNCOP_FAILED,
+    DHT_MSG_MIGRATE_INFO, DHT_MSG_SOCKET_ERROR, DHT_MSG_CREATE_FD_FAILED,
+    DHT_MSG_READDIR_ERROR, DHT_MSG_CHILD_LOC_BUILD_FAILED,
+    DHT_MSG_SET_SWITCH_PATTERN_ERROR, DHT_MSG_COMPUTE_HASH_FAILED,
+    DHT_MSG_FIND_LAYOUT_ANOMALIES_ERROR, DHT_MSG_ANOMALIES_INFO,
+    DHT_MSG_LAYOUT_INFO, DHT_MSG_INODE_LK_ERROR, DHT_MSG_RENAME_INFO,
+    DHT_MSG_DATA_NULL, DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED,
+    DHT_MSG_UNLINK_LOOKUP_INFO, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+    DHT_MSG_OPERATION_NOT_SUP, DHT_MSG_NOT_LINK_FILE_ERROR, DHT_MSG_CHILD_DOWN,
+    DHT_MSG_UUID_PARSE_ERROR, DHT_MSG_GET_DISK_INFO_ERROR,
+    DHT_MSG_INVALID_VALUE, DHT_MSG_SWITCH_PATTERN_INFO,
+    DHT_MSG_SUBVOL_OP_FAILED, DHT_MSG_LAYOUT_PRESET_FAILED,
+    DHT_MSG_INVALID_LINKFILE, DHT_MSG_FIX_LAYOUT_INFO,
+    DHT_MSG_GET_HOSTNAME_FAILED, DHT_MSG_WRITE_FAILED,
+    DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED, DHT_MSG_FSYNC_FAILED,
+    DHT_MSG_SUBVOL_DECOMMISSION_INFO, DHT_MSG_BRICK_QUERY_FAILED,
+    DHT_MSG_SUBVOL_NO_LAYOUT_INFO, DHT_MSG_OPEN_FD_ON_DST_FAILED,
+    DHT_MSG_SUBVOL_NOT_FOUND, DHT_MSG_FILE_LOOKUP_ON_DST_FAILED,
+    DHT_MSG_DISK_LAYOUT_MISSING, DHT_MSG_DICT_GET_FAILED,
+    DHT_MSG_REVALIDATE_CBK_INFO, DHT_MSG_UPGRADE_BRICKS, DHT_MSG_LK_ARRAY_INFO,
+    DHT_MSG_RENAME_NOT_LOCAL, DHT_MSG_RECONFIGURE_INFO,
+    DHT_MSG_INIT_LOCAL_SUBVOL_FAILED, DHT_MSG_SYS_CALL_GET_TIME_FAILED,
+    DHT_MSG_NO_DISK_USAGE_STATUS, DHT_MSG_SUBVOL_DOWN_ERROR,
+    DHT_MSG_REBAL_THROTTLE_INFO, DHT_MSG_COMMIT_HASH_INFO,
+    DHT_MSG_REBAL_STRUCT_SET, DHT_MSG_HAS_MIGINFO, DHT_MSG_SETTLE_HASH_FAILED,
+    DHT_MSG_DEFRAG_PROCESS_DIR_FAILED, DHT_MSG_FD_CTX_SET_FAILED,
+    DHT_MSG_STALE_LOOKUP, DHT_MSG_PARENT_LAYOUT_CHANGED,
+    DHT_MSG_LOCK_MIGRATION_FAILED, DHT_MSG_LOCK_INODE_UNREF_FAILED,
+    DHT_MSG_ASPRINTF_FAILED, DHT_MSG_DIR_LOOKUP_FAILED, DHT_MSG_INODELK_FAILED,
+    DHT_MSG_LOCK_FRAME_FAILED, DHT_MSG_LOCAL_LOCK_INIT_FAILED,
+    DHT_MSG_ENTRYLK_ERROR, DHT_MSG_INODELK_ERROR, DHT_MSG_LOC_FAILED,
+    DHT_MSG_UNKNOWN_FOP, DHT_MSG_MIGRATE_FILE_SKIPPED,
+    DHT_MSG_DIR_XATTR_HEAL_FAILED, DHT_MSG_HASHED_SUBVOL_DOWN,
+    DHT_MSG_NON_HASHED_SUBVOL_DOWN, DHT_MSG_SYNCTASK_CREATE_FAILED,
+    DHT_MSG_DIR_HEAL_ABORT, DHT_MSG_MIGRATE_SKIP, DHT_MSG_FD_CREATE_FAILED,
+    DHT_MSG_DICT_NEW_FAILED, DHT_MSG_FAILED_TO_OPEN, DHT_MSG_CREATE_FAILED,
+    DHT_MSG_FILE_NOT_EXIST, DHT_MSG_CHOWN_FAILED, DHT_MSG_FALLOCATE_FAILED,
+    DHT_MSG_FTRUNCATE_FAILED, DHT_MSG_STATFS_FAILED, DHT_MSG_WRITE_CROSS,
+    DHT_MSG_NEW_TARGET_FOUND, DHT_MSG_INSUFF_MEMORY, DHT_MSG_SET_XATTR_FAILED,
+    DHT_MSG_SET_MODE_FAILED, DHT_MSG_FILE_EXISTS_IN_DEST,
+    DHT_MSG_SYMLINK_FAILED, DHT_MSG_LINKFILE_DEL_FAILED, DHT_MSG_MKNOD_FAILED,
+    DHT_MSG_MIGRATE_CLEANUP_FAILED, DHT_MSG_LOCK_MIGRATE,
+    DHT_MSG_PARENT_BUILD_FAILED, DHT_MSG_HASHED_SUBVOL_NOT_FOUND,
+    DHT_MSG_ACQUIRE_ENTRYLK_FAILED, DHT_MSG_CREATE_DST_FAILED,
+    DHT_MSG_MIGRATION_EXIT, DHT_MSG_CHANGED_DST, DHT_MSG_TRACE_FAILED,
+    DHT_MSG_WRITE_LOCK_FAILED, DHT_MSG_GETACTIVELK_FAILED, DHT_MSG_STAT_FAILED,
+    DHT_MSG_UNLINK_PERFORM_FAILED, DHT_MSG_CLANUP_SOURCE_FILE_FAILED,
+    DHT_MSG_UNLOCK_FILE_FAILED, DHT_MSG_REMOVE_XATTR_FAILED,
+    DHT_MSG_DATA_MIGRATE_ABORT, DHT_MSG_DEFRAG_NULL, DHT_MSG_PARENT_NULL,
+    DHT_MSG_GFID_NOT_PRESENT, DHT_MSG_CHILD_LOC_FAILED,
+    DHT_MSG_SET_LOOKUP_FAILED, DHT_MSG_DIR_REMOVED, DHT_MSG_FIX_NOT_COMP,
+    DHT_MSG_SUBVOL_DETER_FAILED, DHT_MSG_LOCAL_SUBVOL, DHT_MSG_NODE_UUID,
+    DHT_MSG_SIZE_FILE, DHT_MSG_GET_DATA_SIZE_FAILED,
+    DHT_MSG_PTHREAD_JOIN_FAILED, DHT_MSG_COUNTER_THREAD_CREATE_FAILED,
+    DHT_MSG_MIGRATION_INIT_QUEUE_FAILED, DHT_MSG_PAUSED_TIMEOUT, DHT_MSG_WOKE,
+    DHT_MSG_ABORT_REBALANCE, DHT_MSG_CREATE_TASK_REBAL_FAILED,
+    DHT_MSG_REBAL_ESTIMATE_NOT_AVAIL, DHT_MSG_ADD_CHOICES_ERROR,
+    DHT_MSG_GET_CHOICES_ERROR, DHT_MSG_PREPARE_STATUS_ERROR,
+    DHT_MSG_SET_CHOICE_FAILED, DHT_MSG_SET_HASHED_SUBVOL_FAILED,
+    DHT_MSG_XATTR_HEAL_NOT_POSS, DHT_MSG_LINKTO_FILE_FAILED,
+    DHT_MSG_STALE_LINKFILE_DELETE, DHT_MSG_NO_SUBVOL_FOR_LINKTO,
+    DHT_MSG_SUBVOL_RETURNED, DHT_MSG_UNKNOWN_LOCAL_XSEL, DHT_MSG_GET_XATTR_ERR,
+    DHT_MSG_ALLOC_OR_FILL_FAILED, DHT_MSG_GET_REAL_NAME_FAILED,
+    DHT_MSG_COPY_UUID_FAILED, DHT_MSG_MDS_DETER_FAILED,
+    DHT_MSG_CREATE_REBAL_FAILED, DHT_MSG_LINK_LAYOUT_FAILED,
+    DHT_MSG_NO_SUBVOL_IN_LAYOUT, DHT_MSG_MEM_ALLOC_FAILED,
+    DHT_MSG_SET_IN_PARAMS_DICT_FAILED, DHT_MSG_LOC_COPY_FAILED,
+    DHT_MSG_PARENT_LOC_FAILED, DHT_MSG_CREATE_LOCK_FAILED,
+    DHT_MSG_PREV_ATTEMPT_FAILED, DHT_MSG_REFRESH_ATTEMPT,
+    DHT_MSG_ACQUIRE_LOCK_FAILED, DHT_MSG_CREATE_STUB_FAILED,
+    DHT_MSG_WIND_LOCK_REQ_FAILED, DHT_MSG_REFRESH_FAILED,
+    DHT_MSG_CACHED_SUBVOL_ERROR, DHT_MSG_NO_LINK_SUBVOL, DHT_MSG_SET_KEY_FAILED,
+    DHT_MSG_REMOVE_LINKTO_FAILED, DHT_MSG_LAYOUT_DICT_SET_FAILED,
+    DHT_MSG_XATTR_DICT_NULL, DHT_MSG_DUMMY_ALLOC_FAILED, DHT_MSG_DICT_IS_NULL,
+    DHT_MSG_LINK_INODE_FAILED, DHT_MSG_SELFHEAL_FAILED, DHT_MSG_NO_MDS_SUBVOL,
+    DHT_MSG_LIST_XATTRS_FAILED, DHT_MSG_RESET_INTER_XATTR_FAILED,
+    DHT_MSG_MDS_DOWN_UNABLE_TO_SET, DHT_MSG_WIND_UNLOCK_FAILED,
+    DHT_MSG_COMMIT_HASH_FAILED, DHT_MSG_UNLOCK_GFID_FAILED,
+    DHT_MSG_UNLOCK_FOLLOW_ENTRYLK, DHT_MSG_COPY_FRAME_FAILED,
+    DHT_MSG_UNLOCK_FOLLOW_LOCKS, DHT_MSG_ENTRYLK_FAILED_AFT_INODELK,
+    DHT_MSG_CALLOC_FAILED, DHT_MSG_LOCK_ALLOC_FAILED,
+    DHT_MSG_BLOCK_INODELK_FAILED,
+    DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK,
+    DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS,
+    DHT_MSG_DST_NULL_SET_FAILED);
+
+#define DHT_MSG_FD_CTX_SET_FAILED_STR "Failed to set fd ctx"
+#define DHT_MSG_INVALID_VALUE_STR "Different dst found in the fd ctx"
+#define DHT_MSG_UNKNOWN_FOP_STR "Unknown FOP on file"
+#define DHT_MSG_OPEN_FD_ON_DST_FAILED_STR "Failed to open the fd on file"
+#define DHT_MSG_SYNCTASK_CREATE_FAILED_STR "Failed to create synctask"
+#define DHT_MSG_ASPRINTF_FAILED_STR                                            \
+    "asprintf failed while fetching subvol from the id"
+#define DHT_MSG_HAS_MIGINFO_STR "Found miginfo in the inode ctx"
+#define DHT_MSG_FILE_LOOKUP_FAILED_STR "failed to lookup the file"
+#define DHT_MSG_INVALID_LINKFILE_STR                                           \
+    "linkto target is different from cached-subvol. treating as destination "  \
+    "subvol"
+#define DHT_MSG_GFID_MISMATCH_STR "gfid different on the target file"
+#define DHT_MSG_GET_XATTR_FAILED_STR "failed to get 'linkto' xattr"
+#define DHT_MSG_SET_INODE_CTX_FAILED_STR "failed to set inode-ctx target file"
+#define DHT_MSG_DIR_SELFHEAL_FAILED_STR "Healing of path failed"
+#define DHT_MSG_DIR_HEAL_ABORT_STR                                             \
+    "Failed to get path from subvol. Aborting directory healing"
+#define DHT_MSG_DIR_XATTR_HEAL_FAILED_STR "xattr heal failed for directory"
+#define DHT_MSG_LOCK_INODE_UNREF_FAILED_STR                                    \
+    "Found a NULL inode. Failed to unref the inode"
+#define DHT_MSG_DICT_SET_FAILED_STR "Failed to set dictionary value"
+#define DHT_MSG_NOT_LINK_FILE_ERROR_STR "got non-linkfile"
+#define DHT_MSG_CREATE_LINK_FAILED_STR "failed to initialize linkfile data"
+#define DHT_MSG_UNLINK_FAILED_STR "Unlinking linkfile on subvolume failed"
+#define DHT_MSG_MIGRATE_FILE_FAILED_STR "Migrate file failed"
+#define DHT_MSG_NO_MEMORY_STR "could not allocate memory for dict"
+#define DHT_MSG_SUBVOL_ERROR_STR "Failed to get linkto subvol"
+#define DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED_STR "link failed on subvol"
+#define DHT_MSG_MIGRATE_FILE_SKIPPED_STR "Migration skipped"
+#define DHT_MSG_FD_CREATE_FAILED_STR "fd create failed"
+#define DHT_MSG_DICT_NEW_FAILED_STR "dict_new failed"
+#define DHT_MSG_FAILED_TO_OPEN_STR "failed to open"
+#define DHT_MSG_CREATE_FAILED_STR "failed to create"
+#define DHT_MSG_FILE_NOT_EXIST_STR "file does not exist"
+#define DHT_MSG_CHOWN_FAILED_STR "chown failed"
+#define DHT_MSG_FALLOCATE_FAILED_STR "fallocate failed"
+#define DHT_MSG_FTRUNCATE_FAILED_STR "ftruncate failed"
+#define DHT_MSG_STATFS_FAILED_STR "failed to get statfs"
+#define DHT_MSG_WRITE_CROSS_STR                                                \
+    "write will cross min-fre-disk for file on subvol. looking for new subvol"
+#define DHT_MSG_SUBVOL_INSUFF_SPACE_STR                                        \
+    "Could not find any subvol with space accommodating the file. Cosider "    \
+    "adding bricks"
+#define DHT_MSG_NEW_TARGET_FOUND_STR "New target found for file"
+#define DHT_MSG_INSUFF_MEMORY_STR "insufficient memory"
+#define DHT_MSG_SET_XATTR_FAILED_STR "failed to set xattr"
+#define DHT_MSG_SET_MODE_FAILED_STR "failed to set mode"
+#define DHT_MSG_FILE_EXISTS_IN_DEST_STR "file exists in destination"
+#define DHT_MSG_LINKFILE_DEL_FAILED_STR "failed to delete the linkfile"
+#define DHT_MSG_SYMLINK_FAILED_STR "symlink failed"
+#define DHT_MSG_MKNOD_FAILED_STR "mknod failed"
+#define DHT_MSG_SETATTR_FAILED_STR "failed to perform setattr"
+#define DHT_MSG_MIGRATE_CLEANUP_FAILED_STR                                     \
+    "Migrate file cleanup failed: failed to fstat file"
+#define DHT_MSG_LOCK_MIGRATE_STR "locks will be migrated for file"
+#define DHT_MSG_PARENT_BUILD_FAILED_STR                                        \
+    "failed to build parent loc, which is needed to acquire entrylk to "       \
+    "synchronize with renames on this path. Skipping migration"
+#define DHT_MSG_HASHED_SUBVOL_NOT_FOUND_STR                                    \
+    "cannot find hashed subvol which is needed to synchronize with renames "   \
+    "on this path. Skipping migration"
+#define DHT_MSG_ACQUIRE_ENTRYLK_FAILED_STR "failed to acquire entrylk on subvol"
+#define DHT_MSG_CREATE_DST_FAILED_STR "create dst failed for file"
+#define DHT_MSG_MIGRATION_EXIT_STR "Exiting migration"
+#define DHT_MSG_CHANGED_DST_STR "destination changed fo file"
+#define DHT_MSG_TRACE_FAILED_STR "Trace failed"
+#define DHT_MSG_WRITE_LOCK_FAILED_STR "write lock failed"
+#define DHT_MSG_GETACTIVELK_FAILED_STR "getactivelk failed for file"
+#define DHT_MSG_STAT_FAILED_STR "failed to do a stat"
+#define DHT_MSG_UNLINK_PERFORM_FAILED_STR "failed to perform unlink"
+#define DHT_MSG_MIGRATE_FILE_COMPLETE_STR "completed migration"
+#define DHT_MSG_CLANUP_SOURCE_FILE_FAILED_STR "failed to cleanup source file"
+#define DHT_MSG_UNLOCK_FILE_FAILED_STR "failed to unlock file"
+#define DHT_MSG_REMOVE_XATTR_FAILED_STR "remove xattr failed"
+#define DHT_MSG_SOCKET_ERROR_STR "Failed to unlink listener socket"
+#define DHT_MSG_HASHED_SUBVOL_GET_FAILED_STR "Failed to get hashed subvolume"
+#define DHT_MSG_CACHED_SUBVOL_GET_FAILED_STR "Failed to get cached subvolume"
+#define DHT_MSG_MIGRATE_DATA_FAILED_STR "migrate-data failed"
+#define DHT_MSG_DEFRAG_NULL_STR "defrag is NULL"
+#define DHT_MSG_DATA_MIGRATE_ABORT_STR                                         \
+    "Readdirp failed. Aborting data migration for dict"
+#define DHT_MSG_LAYOUT_FIX_FAILED_STR "fix layout failed"
+#define DHT_MSG_PARENT_NULL_STR "parent is NULL"
+#define DHT_MSG_GFID_NOT_PRESENT_STR "gfid not present"
+#define DHT_MSG_CHILD_LOC_FAILED_STR "Child loc build failed"
+#define DHT_MSG_SET_LOOKUP_FAILED_STR "Failed to set lookup"
+#define DHT_MSG_DIR_LOOKUP_FAILED_STR "lookup failed"
+#define DHT_MSG_DIR_REMOVED_STR "Dir renamed or removed. Skipping"
+#define DHT_MSG_READDIR_ERROR_STR "readdir failed, Aborting fix-layout"
+#define DHT_MSG_SETTLE_HASH_FAILED_STR "Settle hash failed"
+#define DHT_MSG_DEFRAG_PROCESS_DIR_FAILED_STR "gf_defrag_process_dir failed"
+#define DHT_MSG_FIX_NOT_COMP_STR                                               \
+    "Unable to retrieve fixlayout xattr. Assume background fix layout not "    \
+    "complete"
+#define DHT_MSG_SUBVOL_DETER_FAILED_STR                                        \
+    "local subvolume determination failed with error"
+#define DHT_MSG_LOCAL_SUBVOL_STR "local subvol"
+#define DHT_MSG_NODE_UUID_STR "node uuid"
+#define DHT_MSG_SIZE_FILE_STR "Total size files"
+#define DHT_MSG_GET_DATA_SIZE_FAILED_STR                                       \
+    "Failed to get the total data size. Unable to estimate time to complete "  \
+    "rebalance"
+#define DHT_MSG_PTHREAD_JOIN_FAILED_STR                                        \
+    "file_counter_thread: pthread_join failed"
+#define DHT_MSG_COUNTER_THREAD_CREATE_FAILED_STR                               \
+    "Failed to create the file counter thread"
+#define DHT_MSG_MIGRATION_INIT_QUEUE_FAILED_STR                                \
+    "Failed to initialise migration queue"
+#define DHT_MSG_REBALANCE_STOPPED_STR "Received stop command on rebalance"
+#define DHT_MSG_PAUSED_TIMEOUT_STR "Request pause timer timeout"
+#define DHT_MSG_WOKE_STR "woken"
+#define DHT_MSG_ABORT_REBALANCE_STR "Aborting rebalance"
+#define DHT_MSG_REBALANCE_START_FAILED_STR                                     \
+    "Failed to start rebalance: look up on / failed"
+#define DHT_MSG_CREATE_TASK_REBAL_FAILED_STR                                   \
+    "Could not create task for rebalance"
+#define DHT_MSG_REBAL_ESTIMATE_NOT_AVAIL_STR                                   \
+    "Rebalance estimates will not be available"
+#define DHT_MSG_REBALANCE_STATUS_STR "Rebalance status"
+#define DHT_MSG_DATA_NULL_STR "data value is NULL"
+#define DHT_MSG_ADD_CHOICES_ERROR_STR "Error to add choices in buffer"
+#define DHT_MSG_GET_CHOICES_ERROR_STR "Error to get choices"
+#define DHT_MSG_PREPARE_STATUS_ERROR_STR "Error to prepare status"
+#define DHT_MSG_SET_CHOICE_FAILED_STR "Failed to set full choice"
+#define DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED_STR                               \
+    "Failed to aggregate quota xattr"
+#define DHT_MSG_FILE_TYPE_MISMATCH_STR                                         \
+    "path exists as a file on one subvolume and directory on another. Please " \
+    "fix it manually"
+#define DHT_MSG_LAYOUT_SET_FAILED_STR "failed to set layout for subvolume"
+#define DHT_MSG_LAYOUT_MERGE_FAILED_STR "failed to merge layouts for subvolume"
+#define DHT_MSG_SET_HASHED_SUBVOL_FAILED_STR "Failed to set hashed subvolume"
+#define DHT_MSG_XATTR_HEAL_NOT_POSS_STR                                        \
+    "No gfid exists for path. so healing xattr is not possible"
+#define DHT_MSG_REVALIDATE_CBK_INFO_STR "Revalidate: subvolume returned -1"
+#define DHT_MSG_LAYOUT_MISMATCH_STR "Mismatching layouts"
+#define DHT_MSG_UNLINK_LOOKUP_INFO_STR "lookup_unlink retuened"
+#define DHT_MSG_LINKTO_FILE_FAILED_STR                                         \
+    "Could not unlink the linkto file as either fd is open and/or linkto "     \
+    "xattr is set"
+#define DHT_MSG_LAYOUT_PRESET_FAILED_STR                                       \
+    "Could not set pre-set layout for subvolume"
+#define DHT_MSG_FILE_ON_MULT_SUBVOL_STR                                        \
+    "multiple subvolumes have file (preferably rename the file in the "        \
+    "backend, and do a fresh lookup"
+#define DHT_MSG_STALE_LINKFILE_DELETE_STR                                      \
+    "attempting deletion of stale linkfile"
+#define DHT_MSG_LINK_FILE_LOOKUP_INFO_STR "Lookup on following linkfile"
+#define DHT_MSG_NO_SUBVOL_FOR_LINKTO_STR "No link subvolume for linkto"
+#define DHT_MSG_SUBVOL_RETURNED_STR "Subvolume returned -1"
+#define DHT_MSG_UNKNOWN_LOCAL_XSEL_STR "Unknown local->xsel"
+#define DHT_MSG_DICT_GET_FAILED_STR "Failed to get"
+#define DHT_MSG_UUID_PARSE_ERROR_STR "Failed to parse uuid"
+#define DHT_MSG_GET_XATTR_ERR_STR "getxattr err for dir"
+#define DHT_MSG_ALLOC_OR_FILL_FAILED_STR "alloc or fill failed"
+#define DHT_MSG_UPGRADE_BRICKS_STR                                             \
+    "At least one of the bricks does not support this operation. Please "      \
+    "upgrade all bricks"
+#define DHT_MSG_GET_REAL_NAME_FAILED_STR "Failed to get real filename"
+#define DHT_MSG_LAYOUT_NULL_STR "Layout is NULL"
+#define DHT_MSG_COPY_UUID_FAILED_STR "Failed to copy node uuid key"
+#define DHT_MSG_MDS_DETER_FAILED_STR                                           \
+    "Cannot determine MDS, fetching xattr randomly from a subvol"
+#define DHT_MSG_HASHED_SUBVOL_DOWN_STR                                         \
+    "MDS is down for path, so fetching xattr randomly from subvol"
+#define DHT_MSG_CREATE_REBAL_FAILED_STR                                        \
+    "failed to create a new rebalance synctask"
+#define DHT_MSG_FIX_LAYOUT_INFO_STR "fixing the layout"
+#define DHT_MSG_OPERATION_NOT_SUP_STR "wrong directory-spread-count value"
+#define DHT_MSG_LINK_LAYOUT_FAILED_STR "failed to link the layout in inode"
+#define DHT_MSG_NO_SUBVOL_IN_LAYOUT_STR "no subvolume in layout for path"
+#define DHT_MSG_INODE_LK_ERROR_STR "mknod lock failed for file"
+#define DHT_MSG_MEM_ALLOC_FAILED_STR "mem allocation failed"
+#define DHT_MSG_PARENT_LAYOUT_CHANGED_STR                                      \
+    "extracting in-memory layout of parent failed"
+#define DHT_MSG_SET_IN_PARAMS_DICT_FAILED_STR                                  \
+    "setting in params dictionary failed"
+#define DHT_MSG_LOC_COPY_FAILED_STR "loc_copy failed"
+#define DHT_MSG_LOC_FAILED_STR "parent loc build failed"
+#define DHT_MSG_PARENT_LOC_FAILED_STR "locking parent failed"
+#define DHT_MSG_CREATE_LOCK_FAILED_STR "Create lock failed"
+#define DHT_MSG_PREV_ATTEMPT_FAILED_STR                                        \
+    "mkdir loop detected. parent layout didn't change even though previous "   \
+    "attempt of mkdir failed because of in-memory layout not matching with "   \
+    "that on disk."
+#define DHT_MSG_REFRESH_ATTEMPT_STR                                            \
+    "mkdir parent layout changed. Attempting a refresh and then a retry"
+#define DHT_MSG_ACQUIRE_LOCK_FAILED_STR                                        \
+    "Acquiring lock on parent to guard against layout-change failed"
+#define DHT_MSG_CREATE_STUB_FAILED_STR "creating stub failed"
+#define DHT_MSG_WIND_LOCK_REQ_FAILED_STR                                       \
+    "cannot wind lock request to guard parent layout"
+#define DHT_MSG_REFRESH_FAILED_STR "refreshing parent layout failed."
+#define DHT_MSG_CACHED_SUBVOL_ERROR_STR "On cached subvol"
+#define DHT_MSG_NO_LINK_SUBVOL_STR "Linkfile does not have link subvolume"
+#define DHT_MSG_SET_KEY_FAILED_STR "failed to set key"
+#define DHT_MSG_CHILD_DOWN_STR "Received CHILD_DOWN. Exiting"
+#define DHT_MSG_LOG_FIXED_LAYOUT_STR "log layout fixed"
+#define DHT_MSG_REBAL_STRUCT_SET_STR "local->rebalance already set"
+#define DHT_MSG_REMOVE_LINKTO_FAILED_STR "Removal of linkto failed at subvol"
+#define DHT_MSG_LAYOUT_DICT_SET_FAILED_STR "dht layout dict set failed"
+#define DHT_MSG_SUBVOL_INFO_STR "creating subvolume"
+#define DHT_MSG_COMPUTE_HASH_FAILED_STR "hash computation failed"
+#define DHT_MSG_INVALID_DISK_LAYOUT_STR                                        \
+    "Invalid disk layout: Catastrophic error layout with unknown type found"
+#define DHT_MSG_LAYOUT_SORT_FAILED_STR "layout sort failed"
+#define DHT_MSG_ANOMALIES_INFO_STR "Found anomalies"
+#define DHT_MSG_XATTR_DICT_NULL_STR "xattr dictionary is NULL"
+#define DHT_MSG_DISK_LAYOUT_MISSING_STR "Disk layout missing"
+#define DHT_MSG_LAYOUT_INFO_STR "layout info"
+#define DHT_MSG_SUBVOL_NO_LAYOUT_INFO_STR "no pre-set layout for subvol"
+#define DHT_MSG_SELFHEAL_XATTR_FAILED_STR "layout setxattr failed"
+#define DHT_MSG_DIR_SELFHEAL_XATTR_FAILED_STR "Directory self heal xattr failed"
+#define DHT_MSG_DUMMY_ALLOC_FAILED_STR "failed to allocate dummy layout"
+#define DHT_MSG_DICT_IS_NULL_STR                                               \
+    "dict is NULL, need to make sure gfids are same"
+#define DHT_MSG_ENTRYLK_ERROR_STR "acquiring entrylk after inodelk failed"
+#define DHT_MSG_NO_DISK_USAGE_STATUS_STR "no du stats"
+#define DHT_MSG_LINK_INODE_FAILED_STR "linking inode failed"
+#define DHT_MSG_SELFHEAL_FAILED_STR "Directory selfheal failed"
+#define DHT_MSG_NO_MDS_SUBVOL_STR "No mds subvol"
+#define DHT_MSG_LIST_XATTRS_FAILED_STR "failed to list xattrs"
+#define DHT_MSG_RESET_INTER_XATTR_FAILED_STR "Failed to reset internal xattr"
+#define DHT_MSG_MDS_DOWN_UNABLE_TO_SET_STR                                     \
+    "mds subvol is down, unable to set xattr"
+#define DHT_MSG_DIR_ATTR_HEAL_FAILED_STR                                       \
+    "Directory attr heal failed. Failed to set uid/gid"
+#define DHT_MSG_WIND_UNLOCK_FAILED_STR                                         \
+    "Winding unlock failed: stale locks left on brick"
+#define DHT_MSG_COMMIT_HASH_FAILED_STR "Directory commit hash updaten failed"
+#define DHT_MSG_LK_ARRAY_INFO_STR "lk info"
+#define DHT_MSG_UNLOCK_GFID_FAILED_STR                                         \
+    "unlock failed on gfid: stale lock might be left"
+#define DHT_MSG_UNLOCKING_FAILED_STR "unlocking failed"
+#define DHT_MSG_UNLOCK_FOLLOW_ENTRYLK_STR "not unlocking following entrylks"
+#define DHT_MSG_COPY_FRAME_FAILED_STR "copy frame failed"
+#define DHT_MSG_UNLOCK_FOLLOW_LOCKS_STR "not unlocking following locks"
+#define DHT_MSG_INODELK_FAILED_STR "inodelk failed on subvol"
+#define DHT_MSG_LOCK_FRAME_FAILED_STR "memory allocation failed for lock_frame"
+#define DHT_MSG_LOCAL_LOCK_INIT_FAILED_STR "dht_local_lock_init failed"
+#define DHT_MSG_ENTRYLK_FAILED_AFT_INODELK_STR                                 \
+    "dht_blocking_entrylk failed after taking inodelk"
+#define DHT_MSG_BLOCK_INODELK_FAILED_STR "dht_blocking_inodelk failed"
+#define DHT_MSG_CALLOC_FAILED_STR "calloc failed"
+#define DHT_MSG_LOCK_ALLOC_FAILED_STR "lock allocation failed"
+#define DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS_STR        \
+    "cannot allocate a frame, not unlocking following entrylks"
+#define DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK_STR       \
+    "storing locks in local failed, not unlocking following entrylks"
+#define DHT_MSG_DST_NULL_SET_FAILED_STR                                        \
+    "src or dst is NULL, Failed to set dictionary value"
 
 #endif /* _DHT_MESSAGES_H_ */
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index cc03eaa1195..8ba8082bd86 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -8,254 +8,143 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include "tier.h"
 #include "dht-common.h"
-#include "xlator.h"
-#include "syscall.h"
-#include <signal.h>
+#include <glusterfs/syscall.h>
 #include <fnmatch.h>
 #include <signal.h>
-#include "events.h"
-
-#define GF_DISK_SECTOR_SIZE              512
-#define DHT_REBALANCE_PID               4242 /* Change it if required */
-#define DHT_REBALANCE_BLKSIZE           (1024 * 1024)  /* 1 MB */
-#define MAX_MIGRATE_QUEUE_COUNT          500
-#define MIN_MIGRATE_QUEUE_COUNT          200
-#define MAX_REBAL_TYPE_SIZE               16
-#define FILE_CNT_INTERVAL                600 /* 10 mins */
-#define ESTIMATE_START_INTERVAL          600 /* 10 mins */
-#define HARDLINK_MIG_INPROGRESS          -2
-#define SKIP_MIGRATION_FD_POSITIVE       -3
+#include <glusterfs/events.h>
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
+
+#define GF_DISK_SECTOR_SIZE 512
+#define DHT_REBALANCE_PID 4242        /* Change it if required */
+#define DHT_REBALANCE_BLKSIZE 1048576 /* 1 MB */
+#define MAX_MIGRATE_QUEUE_COUNT 500
+#define MIN_MIGRATE_QUEUE_COUNT 200
+#define MAX_REBAL_TYPE_SIZE 16
+#define FILE_CNT_INTERVAL 600       /* 10 mins */
+#define ESTIMATE_START_INTERVAL 600 /* 10 mins */
+#define HARDLINK_MIG_INPROGRESS -2
+#define SKIP_MIGRATION_FD_POSITIVE -3
 #ifndef MAX
-#define MAX(a, b) (((a) > (b))?(a):(b))
+#define MAX(a, b) (((a) > (b)) ? (a) : (b))
 #endif
 
-
-#define GF_CRAWL_INDEX_MOVE(idx, sv_cnt)  {     \
-                idx++;                          \
-                idx %= sv_cnt;                  \
-        }
+#define GF_CRAWL_INDEX_MOVE(idx, sv_cnt)                                       \
+    {                                                                          \
+        idx++;                                                                 \
+        idx %= sv_cnt;                                                         \
+    }
 
 uint64_t g_totalfiles = 0;
 uint64_t g_totalsize = 0;
 
-
 void
-gf_defrag_free_dir_dfmeta (struct dir_dfmeta *meta, int local_subvols_cnt)
+gf_defrag_free_dir_dfmeta(struct dir_dfmeta *meta, int local_subvols_cnt)
 {
-        int     i = 0;
-
-        if (meta) {
-                for (i = 0; i < local_subvols_cnt; i++) {
-                        gf_dirent_free (&meta->equeue[i]);
-                }
-
-                GF_FREE (meta->equeue);
-                GF_FREE (meta->head);
-                GF_FREE (meta->iterator);
-                GF_FREE (meta->offset_var);
-                GF_FREE (meta->fetch_entries);
-                GF_FREE (meta);
-        }
+    int i = 0;
+
+    if (meta) {
+        for (i = 0; i < local_subvols_cnt; i++) {
+            if (meta->equeue)
+                gf_dirent_free(&meta->equeue[i]);
+            if (meta->lfd && meta->lfd[i])
+                fd_unref(meta->lfd[i]);
+        }
+
+        GF_FREE(meta->equeue);
+        GF_FREE(meta->head);
+        GF_FREE(meta->iterator);
+        GF_FREE(meta->offset_var);
+        GF_FREE(meta->fetch_entries);
+        GF_FREE(meta->lfd);
+        GF_FREE(meta);
+    }
 }
 
 void
-gf_defrag_free_container (struct dht_container *container)
+gf_defrag_free_container(struct dht_container *container)
 {
-        if (container) {
-                gf_dirent_entry_free (container->df_entry);
+    if (container) {
+        gf_dirent_entry_free(container->df_entry);
 
-                if (container->parent_loc) {
-                        loc_wipe (container->parent_loc);
-                }
+        if (container->parent_loc) {
+            loc_wipe(container->parent_loc);
+        }
 
-                GF_FREE (container->parent_loc);
+        GF_FREE(container->parent_loc);
 
-                GF_FREE (container);
-        }
+        GF_FREE(container);
+    }
 }
 
 void
-dht_set_global_defrag_error (gf_defrag_info_t *defrag, int ret)
+dht_set_global_defrag_error(gf_defrag_info_t *defrag, int ret)
 {
-        LOCK (&defrag->lock);
-        {
-                defrag->global_error = ret;
-        }
-        UNLOCK (&defrag->lock);
-        return;
+    LOCK(&defrag->lock);
+    {
+        defrag->global_error = ret;
+    }
+    UNLOCK(&defrag->lock);
+    return;
 }
 
-
-static gf_boolean_t
-dht_is_tier_command (int cmd) {
-
-        gf_boolean_t is_tier = _gf_false;
-
-        switch (cmd) {
-        case GF_DEFRAG_CMD_START_TIER:
-        case GF_DEFRAG_CMD_STATUS_TIER:
-        case GF_DEFRAG_CMD_START_DETACH_TIER:
-        case GF_DEFRAG_CMD_STOP_DETACH_TIER:
-        case GF_DEFRAG_CMD_PAUSE_TIER:
-        case GF_DEFRAG_CMD_RESUME_TIER:
-                is_tier = _gf_true;
-                break;
-        default:
-                break;
-        }
-        return is_tier;
-
-}
-
-
 static int
-dht_send_rebalance_event (xlator_t *this, int cmd, gf_defrag_status_t status)
+dht_send_rebalance_event(xlator_t *this, int cmd, gf_defrag_status_t status)
 {
-        int ret = -1;
-        char *volname = NULL;
-        char *tmpstr  = NULL;
-        char *ptr = NULL;
-        char *suffix = "-dht";
-        dht_conf_t   *conf = NULL;
-        gf_defrag_info_t *defrag = NULL;
-        int len = 0;
-
-        eventtypes_t event = EVENT_LAST;
-
-        switch (status) {
+    int ret = -1;
+    char *volname = NULL;
+    char *tmpstr = NULL;
+    char *ptr = NULL;
+    char *suffix = "-dht";
+    int len = 0;
+
+    eventtypes_t event = EVENT_LAST;
+
+    switch (status) {
         case GF_DEFRAG_STATUS_COMPLETE:
-                event = EVENT_VOLUME_REBALANCE_COMPLETE;
-                break;
+            event = EVENT_VOLUME_REBALANCE_COMPLETE;
+            break;
         case GF_DEFRAG_STATUS_FAILED:
-                event = EVENT_VOLUME_REBALANCE_FAILED;
-                break;
+            event = EVENT_VOLUME_REBALANCE_FAILED;
+            break;
         case GF_DEFRAG_STATUS_STOPPED:
-                event = EVENT_VOLUME_REBALANCE_STOP;
-                break;
+            event = EVENT_VOLUME_REBALANCE_STOP;
+            break;
         default:
-                break;
-
-        }
-
-        if (dht_is_tier_command (cmd)) {
-                /* We should have the tier volume name*/
-                conf = this->private;
-                defrag = conf->defrag;
-                volname = defrag->tier_conf.volname;
-        } else {
-                /* DHT volume */
-                len = strlen (this->name);
-                tmpstr = gf_strdup (this->name);
-                if (tmpstr) {
-                        ptr = tmpstr + (len - strlen (suffix));
-                        if (!strcmp (ptr, suffix)) {
-                                tmpstr[len - strlen (suffix)] = '\0';
-                                volname = tmpstr;
-                        }
-                }
-        }
-
-        if (!volname) {
-                /* Better than nothing */
-                volname = this->name;
-        }
-
-        if (event != EVENT_LAST) {
-                gf_event (event, "volume=%s", volname);
-        }
-
-        GF_FREE (tmpstr);
-        return ret;
+            break;
+    }
+
+    /* DHT volume */
+    len = strlen(this->name) - strlen(suffix);
+    tmpstr = gf_strdup(this->name);
+    if (tmpstr) {
+        ptr = tmpstr + len;
+        if (!strcmp(ptr, suffix)) {
+            tmpstr[len] = '\0';
+            volname = tmpstr;
+        }
+    }
+
+    if (!volname) {
+        /* Better than nothing */
+        volname = this->name;
+    }
+
+    if (event != EVENT_LAST) {
+        gf_event(event, "volume=%s", volname);
+    }
+
+    GF_FREE(tmpstr);
+    return ret;
 }
 
-
 static void
-dht_strip_out_acls (dict_t *dict)
+dht_strip_out_acls(dict_t *dict)
 {
-        if (dict) {
-                dict_del (dict, "trusted.SGI_ACL_FILE");
-                dict_del (dict, POSIX_ACL_ACCESS_XATTR);
-        }
-}
-
-
-
-static int
-dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count,
-                      int32_t size, off_t offset, struct iobref *iobref,
-                      int *fop_errno)
-{
-        int i            = 0;
-        int ret          = -1;
-        int start_idx    = 0;
-        int tmp_offset   = 0;
-        int write_needed = 0;
-        int buf_len      = 0;
-        int size_pending = 0;
-        char *buf        = NULL;
-
-        /* loop through each vector */
-        for (i = 0; i < count; i++) {
-                buf = vec[i].iov_base;
-                buf_len = vec[i].iov_len;
-
-                for (start_idx = 0; (start_idx + GF_DISK_SECTOR_SIZE) <= buf_len;
-                     start_idx += GF_DISK_SECTOR_SIZE) {
-
-                        if (mem_0filled (buf + start_idx, GF_DISK_SECTOR_SIZE) != 0) {
-                                write_needed = 1;
-                                continue;
-                        }
-
-                        if (write_needed) {
-                                ret = syncop_write (to, fd, (buf + tmp_offset),
-                                                    (start_idx - tmp_offset),
-                                                    (offset + tmp_offset),
-                                                    iobref, 0, NULL, NULL);
-                                /* 'path' will be logged in calling function */
-                                if (ret < 0) {
-                                        gf_log (THIS->name, GF_LOG_WARNING,
-                                                "failed to write (%s)",
-                                                strerror (-ret));
-                                        *fop_errno = -ret;
-                                        ret = -1;
-                                        goto out;
-                                }
-
-                                write_needed = 0;
-                        }
-                        tmp_offset = start_idx + GF_DISK_SECTOR_SIZE;
-                }
-
-                if ((start_idx < buf_len) || write_needed) {
-                        /* This means, last chunk is not yet written.. write it */
-                        ret = syncop_write (to, fd, (buf + tmp_offset),
-                                            (buf_len - tmp_offset),
-                                            (offset + tmp_offset), iobref, 0,
-                                            NULL, NULL);
-                        if (ret < 0) {
-                                /* 'path' will be logged in calling function */
-                                gf_log (THIS->name, GF_LOG_WARNING,
-                                        "failed to write (%s)",
-                                        strerror (-ret));
-                                *fop_errno = -ret;
-                                ret = -1;
-                                goto out;
-                        }
-                }
-
-                size_pending = (size - buf_len);
-                if (!size_pending)
-                        break;
-        }
-
-        ret = size;
-out:
-        return ret;
-
+    if (dict) {
+        dict_del(dict, "trusted.SGI_ACL_FILE");
+        dict_del(dict, POSIX_ACL_ACCESS_XATTR);
+    }
 }
 
 /*
@@ -294,323 +183,318 @@ be converted to "0" in dht_migrate_file.
 */
 
 int32_t
-gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, int *fop_errno)
+gf_defrag_handle_hardlink(xlator_t *this, loc_t *loc, int *fop_errno)
 {
-        int32_t                 ret             = -1;
-        xlator_t               *cached_subvol   = NULL;
-        xlator_t               *hashed_subvol   = NULL;
-        xlator_t               *linkto_subvol   = NULL;
-        data_t                 *data            = NULL;
-        struct iatt             iatt            = {0,};
-        int32_t                 op_errno        = 0;
-        dht_conf_t             *conf            = NULL;
-        gf_loglevel_t          loglevel         = 0;
-        dict_t                 *link_xattr      = NULL;
-        dict_t                 *dict            = NULL;
-        dict_t                 *xattr_rsp       = NULL;
-        struct iatt             stbuf                   = {0,};
-
+    int32_t ret = -1;
+    xlator_t *cached_subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *linkto_subvol = NULL;
+    data_t *data = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    int32_t op_errno = 0;
+    dht_conf_t *conf = NULL;
+    gf_loglevel_t loglevel = 0;
+    dict_t *link_xattr = NULL;
+    dict_t *dict = NULL;
+    dict_t *xattr_rsp = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+
+    *fop_errno = EINVAL;
+
+    GF_VALIDATE_OR_GOTO("defrag", loc, out);
+    GF_VALIDATE_OR_GOTO("defrag", loc->name, out);
+    GF_VALIDATE_OR_GOTO("defrag", this, out);
+    GF_VALIDATE_OR_GOTO("defrag", this->private, out);
+
+    conf = this->private;
+
+    if (gf_uuid_is_null(loc->pargfid)) {
+        gf_msg("", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "loc->pargfid is NULL for %s",
+               loc->path);
         *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    if (gf_uuid_is_null(loc->gfid)) {
+        gf_msg("", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "loc->gfid is NULL for %s",
+               loc->path);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
 
-        GF_VALIDATE_OR_GOTO ("defrag", loc, out);
-        GF_VALIDATE_OR_GOTO ("defrag", loc->name, out);
-        GF_VALIDATE_OR_GOTO ("defrag", this, out);
-        GF_VALIDATE_OR_GOTO ("defrag", this->private, out);
-
-        conf = this->private;
-
-        if (gf_uuid_is_null (loc->pargfid)) {
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :"
-                        "loc->pargfid is NULL for %s", loc->path);
-                *fop_errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        if (gf_uuid_is_null (loc->gfid)) {
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :"
-                        "loc->gfid is NULL for %s", loc->path);
-                *fop_errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        link_xattr = dict_new ();
-        if (!link_xattr) {
-                ret = -1;
-                *fop_errno = ENOMEM;
-                goto out;
-        }
-
-        /*
-          Parallel migration can lead to migration of the hard link multiple
-          times which can lead to data loss. Hence, adding a fresh lookup to
-          decide whether migration is required or not.
-
-          Elaborating the scenario for let say 10 hardlinks [link{1..10}]:
-              Let say the first hard link "link1"  does the setxattr of the
-          new hashed subvolume info on the cached file. As there are multiple
-          threads working, we might have already all the links created on the
-          new hashed by the time we reach hardlink let say link5. Now the
-          number of links on hashed is equal to that of cached. Hence, file
-          migration will happen for link6.
-
-                 Cached                                 Hashed
-          --------T link6                        rwxrwxrwx   link6
-
-          Now post above state all the link file on the cached will be zero
-          byte linkto files. Hence, if we still do migration for the following
-          files link{7..10}, we will end up migrating 0 data leading to data
-          loss.
-                Hence, a lookup can make sure whether we need to migrate the
-          file or not.
-        */
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                *fop_errno = ENOMEM;
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
-                        "could not allocate memory for dict");
-                goto out;
-        }
-
-        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
+    link_xattr = dict_new();
+    if (!link_xattr) {
+        ret = -1;
+        *fop_errno = ENOMEM;
+        goto out;
+    }
+
+    /*
+      Parallel migration can lead to migration of the hard link multiple
+      times which can lead to data loss. Hence, adding a fresh lookup to
+      decide whether migration is required or not.
+
+      Elaborating the scenario for let say 10 hardlinks [link{1..10}]:
+          Let say the first hard link "link1"  does the setxattr of the
+      new hashed subvolume info on the cached file. As there are multiple
+      threads working, we might have already all the links created on the
+      new hashed by the time we reach hardlink let say link5. Now the
+      number of links on hashed is equal to that of cached. Hence, file
+      migration will happen for link6.
+
+             Cached                                 Hashed
+      --------T link6                        rwxrwxrwx   link6
+
+      Now post above state all the link file on the cached will be zero
+      byte linkto files. Hence, if we still do migration for the following
+      files link{7..10}, we will end up migrating 0 data leading to data
+      loss.
+            Hence, a lookup can make sure whether we need to migrate the
+      file or not.
+    */
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        *fop_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "could not allocate memory for dict");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, conf->link_xattr_name, 256);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to set 'linkto' key in dict",
+               loc->path);
+        goto out;
+    }
+
+    ret = syncop_lookup(this, loc, &stbuf, NULL, dict, &xattr_rsp);
+    if (ret) {
+        /*Ignore ENOENT and ESTALE as file might have been
+          migrated already*/
+        if (-ret == ENOENT || -ret == ESTALE) {
+            ret = -2;
+            goto out;
+        }
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:%s lookup failed with ret = %d", loc->path,
+               ret);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    cached_subvol = dht_subvol_get_cached(this, loc->inode);
+    if (!cached_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "Failed to get cached subvol"
+               " for %s on %s",
+               loc->name, this->name);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (!hashed_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "Failed to get hashed subvol"
+               " for %s on %s",
+               loc->name, this->name);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    /* Hardlink migration happens only with remove-brick. So this condition will
+     * be true only when the migration has happened. In case hardlinks are
+     * migrated for rebalance case, remove this check. Having this check here
+     * avoid redundant calls below*/
+    if (hashed_subvol == cached_subvol) {
+        ret = -2;
+        goto out;
+    }
+
+    gf_log(this->name, GF_LOG_INFO,
+           "Attempting to migrate hardlink %s "
+           "with gfid %s from %s -> %s",
+           loc->name, uuid_utoa(loc->gfid), cached_subvol->name,
+           hashed_subvol->name);
+
+    data = dict_get(xattr_rsp, conf->link_xattr_name);
+    /* set linkto on cached -> hashed if not present, else link it */
+    if (!data) {
+        ret = dict_set_str(link_xattr, conf->link_xattr_name,
+                           hashed_subvol->name);
         if (ret) {
-                *fop_errno = ENOMEM;
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: failed to set 'linkto' key in dict", loc->path);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed :"
+                   "Failed to set dictionary value:"
+                   " key = %s for %s",
+                   conf->link_xattr_name, loc->name);
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        ret = syncop_lookup (this, loc, &stbuf, NULL, dict, &xattr_rsp);
+        ret = syncop_setxattr(cached_subvol, loc, link_xattr, 0, NULL, NULL);
         if (ret) {
-                /*Ignore ENOENT and ESTALE as file might have been
-                  migrated already*/
-                if (-ret == ENOENT || -ret == ESTALE) {
-                        ret = -2;
-                        goto out;
-                }
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:%s lookup failed with ret = %d",
-                        loc->path, ret);
-                *fop_errno = -ret;
-                ret = -1;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed :"
+                   "Linkto setxattr failed %s -> %s",
+                   cached_subvol->name, loc->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
 
-        cached_subvol = dht_subvol_get_cached (this, loc->inode);
-        if (!cached_subvol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :"
-                        "Failed to get cached subvol"
-                        " for %s on %s", loc->name, this->name);
-                *fop_errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        if (!hashed_subvol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :"
-                        "Failed to get hashed subvol"
-                        " for %s on %s", loc->name, this->name);
-                *fop_errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        /* Hardlink migration happens only with remove-brick. So this condition will
-         * be true only when the migration has happened. In case hardlinks are migrated
-         * for rebalance case, remove this check. Having this check here avoid redundant
-         * calls below*/
-        if (hashed_subvol == cached_subvol) {
-                ret = -2;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_INFO, "Attempting to migrate hardlink %s "
-                "with gfid %s from %s -> %s", loc->name, uuid_utoa (loc->gfid),
-                cached_subvol->name, hashed_subvol->name);
-
-        data = dict_get (xattr_rsp, conf->link_xattr_name);
-        /* set linkto on cached -> hashed if not present, else link it */
-        if (!data) {
-                ret = dict_set_str (link_xattr, conf->link_xattr_name,
-                                    hashed_subvol->name);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed :"
-                                "Failed to set dictionary value:"
-                                " key = %s for %s",
-                                conf->link_xattr_name, loc->name);
-                        *fop_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
+        gf_msg_debug(this->name, 0,
+                     "hardlink target subvol created on %s "
+                     ",cached %s, file %s",
+                     hashed_subvol->name, cached_subvol->name, loc->path);
 
-                ret = syncop_setxattr (cached_subvol, loc, link_xattr, 0, NULL,
-                                       NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed :"
-                                "Linkto setxattr failed %s -> %s",
-                                cached_subvol->name,
-                                loc->name);
-                        *fop_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
-
-                gf_msg_debug (this->name, 0, "hardlink target subvol created on %s "
-                              ",cached %s, file %s",
-                              hashed_subvol->name, cached_subvol->name, loc->path);
-
-                ret = -2;
-                goto out;
+        ret = -2;
+        goto out;
+    } else {
+        linkto_subvol = dht_linkfile_subvol(this, NULL, NULL, xattr_rsp);
+        if (!linkto_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SUBVOL_ERROR,
+                   "Failed to get "
+                   "linkto subvol for %s",
+                   loc->name);
         } else {
-                linkto_subvol = dht_linkfile_subvol (this, NULL, NULL, xattr_rsp);
-                if (!linkto_subvol) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_SUBVOL_ERROR,
-                                "Failed to get "
-                                "linkto subvol for %s", loc->name);
-                } else {
-                        hashed_subvol = linkto_subvol;
-                }
-
-                ret = syncop_link (hashed_subvol, loc, loc, &iatt, NULL, NULL);
-                if  (ret) {
-                        op_errno = -ret;
-                        ret = -1;
-
-                        loglevel = (op_errno == EEXIST) ? GF_LOG_DEBUG : \
-                                    GF_LOG_ERROR;
-                        gf_msg (this->name, loglevel, op_errno,
-                                DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED,
-                                "link of %s -> %s"
-                                " failed on  subvol %s", loc->name,
-                                uuid_utoa(loc->gfid),
-                                hashed_subvol->name);
-                        if (op_errno != EEXIST) {
-                                *fop_errno = op_errno;
-                                goto out;
-                        }
-                } else {
-                        gf_msg_debug (this->name, 0, "syncop_link successful for"
-                                      " hardlink %s on subvol %s, cached %s", loc->path,
-                                      hashed_subvol->name, cached_subvol->name);
-
-                }
+            hashed_subvol = linkto_subvol;
         }
 
-        ret = syncop_lookup (hashed_subvol, loc, &iatt, NULL, NULL, NULL);
+        ret = syncop_link(hashed_subvol, loc, loc, &iatt, NULL, NULL);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :Failed lookup %s on %s ",
-                        loc->name, hashed_subvol->name);
-
-                *fop_errno = -ret;
-                ret = -1;
-                goto out;
+            op_errno = -ret;
+            ret = -1;
+
+            loglevel = (op_errno == EEXIST) ? GF_LOG_DEBUG : GF_LOG_ERROR;
+            gf_msg(this->name, loglevel, op_errno,
+                   DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED,
+                   "link of %s -> %s"
+                   " failed on  subvol %s",
+                   loc->name, uuid_utoa(loc->gfid), hashed_subvol->name);
+            if (op_errno != EEXIST) {
+                *fop_errno = op_errno;
+                goto out;
+            }
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "syncop_link successful for"
+                         " hardlink %s on subvol %s, cached %s",
+                         loc->path, hashed_subvol->name, cached_subvol->name);
         }
+    }
 
-        /* There is a race where on the target subvol for the hardlink
-         * (note: hash subvol for the hardlink might differ from this), some
-         * other client(non-rebalance) would have created a linkto file for that
-         * hardlink as part of lookup. So let say there are 10 hardlinks, on the
-         * 5th hardlink it self the hardlinks might have migrated. Now for
-         * (6..10th) hardlinks the cached and target would be same as the file
-         * has already migrated. Hence this check is needed  */
-        if (cached_subvol == hashed_subvol) {
-                gf_msg_debug (this->name, 0, "source %s and destination %s "
-                             "for hardlink %s are same", cached_subvol->name,
-                             hashed_subvol->name, loc->path);
-                ret = -2;
-                goto out;
-        }
+    ret = syncop_lookup(hashed_subvol, loc, &iatt, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :Failed lookup %s on %s ", loc->name,
+               hashed_subvol->name);
 
-        if (iatt.ia_nlink == stbuf.ia_nlink) {
-                ret = dht_migrate_file (this, loc, cached_subvol, hashed_subvol,
-                                        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS,
-                                        fop_errno);
-                if (ret) {
-                        goto out;
-                }
-        }
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* There is a race where on the target subvol for the hardlink
+     * (note: hash subvol for the hardlink might differ from this), some
+     * other client(non-rebalance) would have created a linkto file for that
+     * hardlink as part of lookup. So let say there are 10 hardlinks, on the
+     * 5th hardlink it self the hardlinks might have migrated. Now for
+     * (6..10th) hardlinks the cached and target would be same as the file
+     * has already migrated. Hence this check is needed  */
+    if (cached_subvol == hashed_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "source %s and destination %s "
+                     "for hardlink %s are same",
+                     cached_subvol->name, hashed_subvol->name, loc->path);
         ret = -2;
+        goto out;
+    }
+
+    if (iatt.ia_nlink == stbuf.ia_nlink) {
+        ret = dht_migrate_file(this, loc, cached_subvol, hashed_subvol,
+                               GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS, fop_errno);
+        if (ret) {
+            goto out;
+        }
+    }
+    ret = -2;
 out:
-        if (link_xattr)
-                dict_unref (link_xattr);
+    if (link_xattr)
+        dict_unref(link_xattr);
 
-        if (xattr_rsp)
-                dict_unref (xattr_rsp);
+    if (xattr_rsp)
+        dict_unref(xattr_rsp);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
-
-
 static int
-__check_file_has_hardlink (xlator_t *this, loc_t *loc,
-                           struct iatt *stbuf, dict_t *xattrs, int flags,
-                           gf_defrag_info_t *defrag, dht_conf_t *conf, int *fop_errno)
+__check_file_has_hardlink(xlator_t *this, loc_t *loc, struct iatt *stbuf,
+                          dict_t *xattrs, int flags, gf_defrag_info_t *defrag,
+                          dht_conf_t *conf, int *fop_errno)
 {
-       int ret = 0;
+    int ret = 0;
 
-       if (flags == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS) {
-                ret = 0;
-                return ret;
-       }
-       if (stbuf->ia_nlink > 1) {
-                /* support for decomission */
-                if (flags == GF_DHT_MIGRATE_HARDLINK) {
-                        synclock_lock (&conf->link_lock);
-                        ret = gf_defrag_handle_hardlink
-                                (this, loc, fop_errno);
-                        synclock_unlock (&conf->link_lock);
-                        /*
-                        Returning zero will force the file to be remigrated.
-                        Checkout gf_defrag_handle_hardlink for more information.
-                        */
-                        if (ret && ret != -2) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_MIGRATE_FILE_FAILED,
-                                        "Migrate file failed:"
-                                        "%s: failed to migrate file with link",
-                                        loc->path);
-                        }
-                } else {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migration skipped for:"
-                                "%s: file has hardlinks", loc->path);
-                        *fop_errno = ENOTSUP;
-                        ret = 1;
-                }
-       }
+    if (flags == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS) {
+        ret = 0;
+        return ret;
+    }
+    if (stbuf->ia_nlink > 1) {
+        /* support for decomission */
+        if (flags == GF_DHT_MIGRATE_HARDLINK) {
+            synclock_lock(&conf->link_lock);
+            ret = gf_defrag_handle_hardlink(this, loc, fop_errno);
+            synclock_unlock(&conf->link_lock);
+            /*
+            Returning zero will force the file to be remigrated.
+            Checkout gf_defrag_handle_hardlink for more information.
+            */
+            if (ret && ret != -2) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       DHT_MSG_MIGRATE_FILE_FAILED,
+                       "Migrate file failed:"
+                       "%s: failed to migrate file with link",
+                       loc->path);
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migration skipped for:"
+                   "%s: file has hardlinks",
+                   loc->path);
+            *fop_errno = ENOTSUP;
+            ret = 1;
+        }
+    }
 
-       return ret;
+    return ret;
 }
 
-
 /*
      return values
      0 : File will be migrated
@@ -620,964 +504,963 @@ __check_file_has_hardlink (xlator_t *this, loc_t *loc,
     -1 : failure
 */
 static int
-__is_file_migratable (xlator_t *this, loc_t *loc,
-                      struct iatt *stbuf, dict_t *xattrs, int flags,
-                      gf_defrag_info_t *defrag, dht_conf_t *conf,
-                      int *fop_errno)
+__is_file_migratable(xlator_t *this, loc_t *loc, struct iatt *stbuf,
+                     dict_t *xattrs, int flags, gf_defrag_info_t *defrag,
+                     dht_conf_t *conf, int *fop_errno)
 {
-        int ret = -1;
-        int lock_count = 0;
-
-        if (IA_ISDIR (stbuf->ia_type)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: migrate-file called on directory", loc->path);
-                *fop_errno = EISDIR;
-                ret = -1;
-                goto out;
-        }
-
-        if (!conf->lock_migration_enabled) {
-                ret = dict_get_int32 (xattrs, GLUSTERFS_POSIXLK_COUNT,
-                                      &lock_count);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed:"
-                                "%s: Unable to get lock count for file",
-                                loc->path);
-                        *fop_errno = EINVAL;
-                        ret = -1;
-                        goto out;
-                }
-
-                if (lock_count) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed: %s: File has locks."
-                                " Skipping file migration", loc->path);
-                        *fop_errno = ENOTSUP;
-                        ret = 1;
-                        goto out;
-                }
-        }
+    int ret = -1;
+    int lock_count = 0;
+
+    if (IA_ISDIR(stbuf->ia_type)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: migrate-file called on directory",
+               loc->path);
+        *fop_errno = EISDIR;
+        ret = -1;
+        goto out;
+    }
 
-        /* Check if file has hardlink*/
-        ret = __check_file_has_hardlink (this, loc, stbuf, xattrs,
-                                         flags, defrag, conf, fop_errno);
+    if (!conf->lock_migration_enabled) {
+        ret = dict_get_int32(xattrs, GLUSTERFS_POSIXLK_COUNT, &lock_count);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: Unable to get lock count for file",
+                   loc->path);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
+        }
+
+        if (lock_count) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed: %s: File has locks."
+                   " Skipping file migration",
+                   loc->path);
+            *fop_errno = ENOTSUP;
+            ret = 1;
+            goto out;
+        }
+    }
+
+    /* Check if file has hardlink*/
+    ret = __check_file_has_hardlink(this, loc, stbuf, xattrs, flags, defrag,
+                                    conf, fop_errno);
 out:
-        return ret;
+    return ret;
 }
 
-
 static int
-__dht_rebalance_create_dst_file (xlator_t *this, xlator_t *to, xlator_t *from,
-                                 loc_t *loc, struct iatt *stbuf, fd_t **dst_fd,
-                                 int *fop_errno)
+__dht_rebalance_create_dst_file(xlator_t *this, xlator_t *to, xlator_t *from,
+                                loc_t *loc, struct iatt *stbuf, fd_t **dst_fd,
+                                int *fop_errno, int file_has_holes)
 {
-        int          ret  = -1;
-        int          ret2 = -1;
-        fd_t        *fd   = NULL;
-        struct iatt  new_stbuf = {0,};
-        struct iatt  check_stbuf= {0,};
-        dht_conf_t  *conf = NULL;
-        dict_t      *dict = NULL;
-        dict_t      *xdata = NULL;
-
-        conf = this->private;
-
-        dict = dict_new ();
-        if (!dict) {
-                *fop_errno = ENOMEM;
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        DHT_MSG_NO_MEMORY, "dictionary allocation failed for"
-                        "path:%s", loc->path);
-                goto out;
-        }
-        ret = dict_set_gfuuid (dict, "gfid-req", stbuf->ia_gfid, true);
-        if (ret) {
-                *fop_errno = ENOMEM;
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: failed to set dictionary value: key = gfid-req",
-                        loc->path);
-                goto out;
-        }
-
-        ret = dict_set_str (dict, conf->link_xattr_name, from->name);
-        if (ret) {
-                *fop_errno = ENOMEM;
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: failed to set dictionary value: key = %s ",
-                        loc->path, conf->link_xattr_name);
-                goto out;
-        }
-
-        fd = fd_create (loc->inode, DHT_REBALANCE_PID);
-        if (!fd) {
-                *fop_errno = ENOMEM;
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: fd create failed (destination)",
-                        loc->path);
-                goto out;
-        }
-
-        if (!!dht_is_tier_xlator (this)) {
-                xdata = dict_new ();
-                if (!xdata) {
-                        *fop_errno = ENOMEM;
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: dict_new failed)",
-                                loc->path);
-                        goto out;
-                }
-
-                ret = dict_set_int32 (xdata, GF_CLEAN_WRITE_PROTECTION, 1);
-                if (ret) {
-                        *fop_errno = ENOMEM;
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "%s: failed to set dictionary value: key = %s ",
-                                loc->path, GF_CLEAN_WRITE_PROTECTION);
-                        goto out;
-                }
-        }
-
-        ret = syncop_lookup (to, loc, &new_stbuf, NULL, xdata, NULL);
-        if (!ret) {
-                /* File exits in the destination, check if gfid matches */
-                if (gf_uuid_compare (stbuf->ia_gfid, new_stbuf.ia_gfid) != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_GFID_MISMATCH,
-                                "file %s exists in %s with different gfid",
-                                loc->path, to->name);
-                        *fop_errno = EINVAL;
-                        ret = -1;
-                        goto out;
-                }
-        }
-        if ((ret < 0) && (-ret != ENOENT)) {
-                /* File exists in destination, but not accessible */
-                gf_msg (THIS->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: failed to lookup file",
-                        loc->path);
-                *fop_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-
-        /* Create the destination with LINKFILE mode, and linkto xattr,
-           if the linkfile already exists, just open the file */
-        if (!ret) {
-                /*
-                 * File already present, just open the file.
-                 */
-                ret = syncop_open (to, loc, O_RDWR, fd, NULL, NULL);
-                 if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "failed to open %s on %s",
-                                loc->path, to->name);
-                        *fop_errno = -ret;
-                        ret = -1;
-                        goto out;
-                 }
-        } else {
-                ret = syncop_create (to, loc, O_RDWR, DHT_LINKFILE_MODE, fd,
-                                     &new_stbuf, dict, NULL);
-                 if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "failed to create %s on %s",
-                                loc->path, to->name);
-                        *fop_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
-
-        }
-
-        fd_bind (fd);
+    int ret = -1;
+    int ret2 = -1;
+    fd_t *fd = NULL;
+    struct iatt new_stbuf = {
+        0,
+    };
+    struct iatt check_stbuf = {
+        0,
+    };
+    dht_conf_t *conf = NULL;
+    dict_t *dict = NULL;
+    dict_t *xdata = NULL;
+
+    conf = this->private;
+
+    dict = dict_new();
+    if (!dict) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "dictionary allocation failed for"
+               "path:%s",
+               loc->path);
+        goto out;
+    }
+    ret = dict_set_gfuuid(dict, "gfid-req", stbuf->ia_gfid, true);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: failed to set dictionary value: key = gfid-req", loc->path);
+        goto out;
+    }
+
+    ret = dict_set_str(dict, conf->link_xattr_name, from->name);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: failed to set dictionary value: key = %s ", loc->path,
+               conf->link_xattr_name);
+        goto out;
+    }
+
+    fd = fd_create(loc->inode, DHT_REBALANCE_PID);
+    if (!fd) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: fd create failed (destination)", loc->path);
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: dict_new failed)", loc->path);
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(xdata, GF_CLEAN_WRITE_PROTECTION, 1);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: failed to set dictionary value: key = %s ", loc->path,
+               GF_CLEAN_WRITE_PROTECTION);
+        goto out;
+    }
+
+    ret = syncop_lookup(to, loc, &new_stbuf, NULL, xdata, NULL);
+    if (!ret) {
+        /* File exits in the destination, check if gfid matches */
+        if (gf_uuid_compare(stbuf->ia_gfid, new_stbuf.ia_gfid) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH,
+                   "file %s exists in %s with different gfid", loc->path,
+                   to->name);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
+        }
+    }
+    if ((ret < 0) && (-ret != ENOENT)) {
+        /* File exists in destination, but not accessible */
+        gf_msg(THIS->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to lookup file", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
-        /*Reason of doing lookup after create again:
-         *In the create, there is some time-gap between opening fd at the
-         *server (posix_layer) and binding it in server (incrementing fd count),
-         *so if in that time-gap, if other process sends unlink considering it
-         *as a linkto file, because inode->fd count will be 0, so file will be
-         *unlinked at the backend. And because furthur operations are performed
-         *on fd, so though migration will be done but will end with no file
-         *at  the backend.
+    /* Create the destination with LINKFILE mode, and linkto xattr,
+       if the linkfile already exists, just open the file */
+    if (!ret) {
+        /*
+         * File already present, just open the file.
          */
-
-        ret = syncop_lookup (to, loc, &check_stbuf, NULL, NULL, NULL);
-        if (!ret) {
-
-                if (gf_uuid_compare (stbuf->ia_gfid, check_stbuf.ia_gfid) != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_GFID_MISMATCH,
-                                "file %s exists in %s with different gfid,"
-                                "found in lookup after create",
-                                loc->path, to->name);
-                        *fop_errno = EINVAL;
-                        ret = -1;
-                        goto out;
-                }
-
-        }
-
-        if (-ret == ENOENT) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED, "%s: file does not exist"
-                        "on %s", loc->path, to->name);
-                *fop_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-
-        ret = syncop_fsetattr (to, fd, stbuf,
-                               (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
-                                NULL, NULL, NULL, NULL);
+        ret = syncop_open(to, loc, O_RDWR, fd, NULL, NULL);
         if (ret < 0) {
-                *fop_errno = -ret;
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "chown failed for %s on %s",
-                        loc->path, to->name);
-        }
-
-        /* No need to bother about 0 byte size files */
-        if (stbuf->ia_size > 0) {
-                if (conf->use_fallocate) {
-                        ret = syncop_fallocate (to, fd, 0, 0, stbuf->ia_size,
-                                                NULL, NULL);
-                        if (ret < 0) {
-                                if (ret == -EOPNOTSUPP || ret == -EINVAL ||
-                                    ret == -ENOSYS) {
-                                        conf->use_fallocate = _gf_false;
-                                } else {
-                                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                                "fallocate failed for %s on %s",
-                                                loc->path, to->name);
-
-                                        *fop_errno = -ret;
-
-                                        /* fallocate does not release the space
-                                         * in some cases
-                                         */
-                                        ret2 = syncop_ftruncate (to, fd, 0,
-                                                                 NULL, NULL,
-                                                                 NULL, NULL);
-                                        if (ret2 < 0) {
-                                                gf_msg (this->name,
-                                                        GF_LOG_WARNING, -ret2,
-                                                        DHT_MSG_MIGRATE_FILE_FAILED,
-                                                        "ftruncate failed for "
-                                                        "%s on %s",
-                                                        loc->path, to->name);
-                                        }
-                                        goto out;
-                                }
-                        }
-                }
-
-                if (!conf->use_fallocate) {
-                        ret = syncop_ftruncate (to, fd, stbuf->ia_size, NULL,
-                                                NULL, NULL, NULL);
-                        if (ret < 0) {
-                                *fop_errno = -ret;
-                                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                                        DHT_MSG_MIGRATE_FILE_FAILED,
-                                        "ftruncate failed for %s on %s",
-                                        loc->path, to->name);
-                        }
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "failed to open %s on %s", loc->path, to->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+    } else {
+        ret = syncop_create(to, loc, O_RDWR, DHT_LINKFILE_MODE, fd, &new_stbuf,
+                            dict, NULL);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "failed to create %s on %s", loc->path, to->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+    }
+
+    fd_bind(fd);
+
+    /*Reason of doing lookup after create again:
+     *In the create, there is some time-gap between opening fd at the
+     *server (posix_layer) and binding it in server (incrementing fd count),
+     *so if in that time-gap, if other process sends unlink considering it
+     *as a linkto file, because inode->fd count will be 0, so file will be
+     *unlinked at the backend. And because further operations are performed
+     *on fd, so though migration will be done but will end with no file
+     *at  the backend.
+     */
+
+    ret = syncop_lookup(to, loc, &check_stbuf, NULL, NULL, NULL);
+    if (!ret) {
+        if (gf_uuid_compare(stbuf->ia_gfid, check_stbuf.ia_gfid) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH,
+                   "file %s exists in %s with different gfid,"
+                   "found in lookup after create",
+                   loc->path, to->name);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (-ret == ENOENT) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: file does not exist"
+               "on %s",
+               loc->path, to->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_fsetattr(to, fd, stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+                          NULL, NULL, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "chown failed for %s on %s", loc->path, to->name);
+    }
+
+    /* No need to bother about 0 byte size files */
+    if (stbuf->ia_size > 0) {
+        if (conf->use_fallocate && !file_has_holes) {
+            ret = syncop_fallocate(to, fd, 0, 0, stbuf->ia_size, NULL, NULL);
+            if (ret < 0) {
+                if (ret == -EOPNOTSUPP || ret == -EINVAL || ret == -ENOSYS) {
+                    conf->use_fallocate = _gf_false;
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, -ret,
+                           DHT_MSG_MIGRATE_FILE_FAILED,
+                           "fallocate failed for %s on %s", loc->path,
+                           to->name);
+
+                    *fop_errno = -ret;
+
+                    /* fallocate does not release the space
+                     * in some cases
+                     */
+                    ret2 = syncop_ftruncate(to, fd, 0, NULL, NULL, NULL, NULL);
+                    if (ret2 < 0) {
+                        gf_msg(this->name, GF_LOG_WARNING, -ret2,
+                               DHT_MSG_MIGRATE_FILE_FAILED,
+                               "ftruncate failed for "
+                               "%s on %s",
+                               loc->path, to->name);
+                    }
+                    goto out;
                 }
+            }
+        } else {
+            ret = syncop_ftruncate(to, fd, stbuf->ia_size, NULL, NULL, NULL,
+                                   NULL);
+            if (ret < 0) {
+                *fop_errno = -ret;
+                gf_msg(this->name, GF_LOG_WARNING, -ret,
+                       DHT_MSG_MIGRATE_FILE_FAILED,
+                       "ftruncate failed for %s on %s", loc->path, to->name);
+            }
         }
+    }
 
-        /* success */
-        ret = 0;
+    /* success */
+    ret = 0;
 
-        if (dst_fd)
-                *dst_fd = fd;
+    if (dst_fd)
+        *dst_fd = fd;
 
 out:
-        if (ret) {
-                if (fd) {
-                        fd_unref (fd);
-                }
+    if (ret) {
+        if (fd) {
+            fd_unref(fd);
         }
-        if (dict)
-                dict_unref (dict);
+    }
+    if (dict)
+        dict_unref(dict);
 
-        if (xdata)
-                dict_unref (dict);
+    if (xdata)
+        dict_unref(xdata);
 
-
-        return ret;
+    return ret;
 }
 
 static int
-__dht_check_free_space (xlator_t *this, xlator_t *to, xlator_t *from,
-                        loc_t *loc, struct iatt *stbuf, int flag,
-                        dht_conf_t *conf, gf_boolean_t *target_changed,
-                        xlator_t **new_subvol, int *fop_errno)
+__dht_check_free_space(xlator_t *this, xlator_t *to, xlator_t *from, loc_t *loc,
+                       struct iatt *stbuf, int flag, dht_conf_t *conf,
+                       gf_boolean_t *target_changed, xlator_t **new_subvol,
+                       int *fop_errno)
 {
-        struct statvfs  src_statfs = {0,};
-        struct statvfs  dst_statfs = {0,};
-        int             ret        = -1;
-        dict_t         *xdata      = NULL;
-        dht_layout_t   *layout     = NULL;
-        uint64_t        src_statfs_blocks = 1;
-        uint64_t        dst_statfs_blocks = 1;
-        double          dst_post_availspacepercent = 0;
-        double          src_post_availspacepercent = 0;
-        uint64_t        file_blocks = 0;
-        uint64_t        src_total_blocks = 0;
-        uint64_t        dst_total_blocks = 0;
-
-        xdata = dict_new ();
-        if (!xdata) {
-                *fop_errno = ENOMEM;
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        DHT_MSG_NO_MEMORY,
-                        "failed to allocate dictionary");
-                goto out;
-        }
-
-        ret = dict_set_int8 (xdata, GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set "
-                        GF_INTERNAL_IGNORE_DEEM_STATFS" in dict");
-                ret = -1;
-                *fop_errno = ENOMEM;
-                goto out;
-        }
-
-        ret = syncop_statfs (from, loc, &src_statfs, xdata, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to get statfs of %s on %s",
-                        loc->path, from->name);
-                *fop_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-
-        ret = syncop_statfs (to, loc, &dst_statfs, xdata, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to get statfs of %s on %s",
-                        loc->path, to->name);
-                *fop_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-
-        gf_msg_debug (this->name, 0, "min_free_disk - %f , block available - "
-                      "%lu , block size - %lu ", conf->min_free_disk,
-                      dst_statfs.f_bavail, dst_statfs.f_bsize);
-
-        dst_statfs_blocks = dst_statfs.f_bavail *
-                            (dst_statfs.f_frsize /
-                            GF_DISK_SECTOR_SIZE);
-
-        src_statfs_blocks = src_statfs.f_bavail *
-                            (src_statfs.f_frsize /
-                            GF_DISK_SECTOR_SIZE);
-
-        dst_total_blocks = dst_statfs.f_blocks *
-                           (dst_statfs.f_frsize /
-                           GF_DISK_SECTOR_SIZE);
-
-        src_total_blocks = src_statfs.f_blocks *
-                           (src_statfs.f_frsize /
-                           GF_DISK_SECTOR_SIZE);
-
-        /* if force option is given, do not check for space @ dst.
-         * Check only if space is avail for the file */
-        if (flag != GF_DHT_MIGRATE_DATA)
-                goto check_avail_space;
-
-        /* Check:
-           During rebalance `migrate-data` - Destination subvol experiences
-           a `reduction` in 'blocks' of free space, at the same time source
-           subvol gains certain 'blocks' of free space. A valid check is
-           necessary here to avoid errorneous move to destination where
-           the space could be scantily available.
-           With heterogenous brick support, an actual space comparison could
-           prevent any files being migrated to newly added bricks if they are
-           smaller then the free space available on the existing bricks.
-         */
-        if (stbuf) {
-                if (!conf->use_fallocate) {
-                        file_blocks = stbuf->ia_size + GF_DISK_SECTOR_SIZE - 1;
-                        file_blocks /= GF_DISK_SECTOR_SIZE;
-
-                        if (file_blocks >= dst_statfs_blocks) {
-                                dst_statfs_blocks = 0;
-                        } else {
-                                dst_statfs_blocks -= file_blocks;
-                        }
-                }
-
-                src_post_availspacepercent =
-                        ((src_statfs_blocks + file_blocks) * 100) / src_total_blocks;
-
-                dst_post_availspacepercent =
-                        (dst_statfs_blocks * 100) / dst_total_blocks;
-
-                if (dst_post_availspacepercent < src_post_availspacepercent) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "data movement of file "
-                                "{blocks:%"PRIu64" name:(%s)} would result in "
-                                "dst node (%s:%"PRIu64") having lower disk "
-                                "space than the source node (%s:%"PRIu64")"
-                                ".Skipping file.", stbuf->ia_blocks, loc->path,
-                                to->name, dst_statfs_blocks, from->name,
-                                src_statfs_blocks);
-
-                        /* this is not a 'failure', but we don't want to
-                           consider this as 'success' too :-/ */
-                        *fop_errno = ENOSPC;
-                        ret = 1;
-                        goto out;
-                }
-        }
+    struct statvfs src_statfs = {
+        0,
+    };
+    struct statvfs dst_statfs = {
+        0,
+    };
+    int ret = -1;
+    dict_t *xdata = NULL;
+    dht_layout_t *layout = NULL;
+    uint64_t src_statfs_blocks = 1;
+    uint64_t dst_statfs_blocks = 1;
+    double dst_post_availspacepercent = 0;
+    double src_post_availspacepercent = 0;
+    uint64_t file_blocks = 0;
+    uint64_t src_total_blocks = 0;
+    uint64_t dst_total_blocks = 0;
+
+    xdata = dict_new();
+    if (!xdata) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "failed to allocate dictionary");
+        goto out;
+    }
+
+    ret = dict_set_int8(xdata, GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to set " GF_INTERNAL_IGNORE_DEEM_STATFS " in dict");
+        ret = -1;
+        *fop_errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_statfs(from, loc, &src_statfs, xdata, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to get statfs of %s on %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_statfs(to, loc, &dst_statfs, xdata, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to get statfs of %s on %s", loc->path, to->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "min_free_disk - %f , block available - %" PRId64
+                 ", block size - %lu",
+                 conf->min_free_disk, dst_statfs.f_bavail, dst_statfs.f_bsize);
+
+    dst_statfs_blocks = dst_statfs.f_bavail *
+                        (dst_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    src_statfs_blocks = src_statfs.f_bavail *
+                        (src_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    dst_total_blocks = dst_statfs.f_blocks *
+                       (dst_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    src_total_blocks = src_statfs.f_blocks *
+                       (src_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    /* if force option is given, do not check for space @ dst.
+     * Check only if space is avail for the file */
+    if (flag != GF_DHT_MIGRATE_DATA)
+        goto check_avail_space;
+
+    /* Check:
+       During rebalance `migrate-data` - Destination subvol experiences
+       a `reduction` in 'blocks' of free space, at the same time source
+       subvol gains certain 'blocks' of free space. A valid check is
+       necessary here to avoid erroneous move to destination where
+       the space could be scantily available.
+       With heterogeneous brick support, an actual space comparison could
+       prevent any files being migrated to newly added bricks if they are
+       smaller then the free space available on the existing bricks.
+     */
+    if (!conf->use_fallocate) {
+        file_blocks = stbuf->ia_size + GF_DISK_SECTOR_SIZE - 1;
+        file_blocks /= GF_DISK_SECTOR_SIZE;
+
+        if (file_blocks >= dst_statfs_blocks) {
+            dst_statfs_blocks = 0;
+        } else {
+            dst_statfs_blocks -= file_blocks;
+        }
+    }
+
+    src_post_availspacepercent = ((src_statfs_blocks + file_blocks) * 100) /
+                                 src_total_blocks;
+
+    dst_post_availspacepercent = (dst_statfs_blocks * 100) / dst_total_blocks;
+
+    if (dst_post_availspacepercent < src_post_availspacepercent) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "data movement of file "
+               "{blocks:%" PRIu64
+               " name:(%s)} would result in "
+               "dst node (%s:%" PRIu64
+               ") having lower disk "
+               "space than the source node (%s:%" PRIu64
+               ")"
+               ".Skipping file.",
+               stbuf->ia_blocks, loc->path, to->name, dst_statfs_blocks,
+               from->name, src_statfs_blocks);
+
+        /* this is not a 'failure', but we don't want to
+           consider this as 'success' too :-/ */
+        *fop_errno = ENOSPC;
+        ret = 1;
+        goto out;
+    }
 
 check_avail_space:
-        if (conf->disk_unit == 'p' && dst_statfs.f_blocks) {
-                dst_post_availspacepercent =
-                        (dst_statfs_blocks * 100) / dst_total_blocks;
-
-                gf_msg_debug (this->name, 0, "file : %s, post_availspacepercent"
-                              " : %lf f_bavail : %lu min-free-disk: %lf",
-                              loc->path, dst_post_availspacepercent,
-                              dst_statfs.f_bavail, conf->min_free_disk);
-
-                if (dst_post_availspacepercent < conf->min_free_disk) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0, 0,
-                                "Write will cross min-free-disk for "
-                                "file - %s on subvol - %s. Looking "
-                                "for new subvol", loc->path, to->name);
-
-                        goto find_new_subvol;
-                } else {
-                        ret = 0;
-                        goto out;
-                }
+    if (conf->disk_unit == 'p' && dst_statfs.f_blocks) {
+        dst_post_availspacepercent = (dst_statfs_blocks * 100) /
+                                     dst_total_blocks;
+
+        gf_msg_debug(this->name, 0,
+                     "file : %s, post_availspacepercent"
+                     " : %lf f_bavail : %" PRIu64 " min-free-disk: %lf",
+                     loc->path, dst_post_availspacepercent, dst_statfs.f_bavail,
+                     conf->min_free_disk);
+
+        if (dst_post_availspacepercent < conf->min_free_disk) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                   "Write will cross min-free-disk for "
+                   "file - %s on subvol - %s. Looking "
+                   "for new subvol",
+                   loc->path, to->name);
+
+            goto find_new_subvol;
+        } else {
+            ret = 0;
+            goto out;
         }
+    }
 
-        if (conf->disk_unit != 'p') {
-                if ((dst_statfs_blocks * GF_DISK_SECTOR_SIZE) <
-                                                      conf->min_free_disk) {
-                        gf_msg_debug (this->name, 0, "file : %s,  destination "
-                                      "frsize: %lu f_bavail : %lu "
-                                      "min-free-disk: %lf", loc->path,
-                                      dst_statfs.f_frsize, dst_statfs.f_bavail,
-                                      conf->min_free_disk);
+    if (conf->disk_unit != 'p') {
+        if ((dst_statfs_blocks * GF_DISK_SECTOR_SIZE) < conf->min_free_disk) {
+            gf_msg_debug(this->name, 0,
+                         "file : %s,  destination frsize: %lu "
+                         "f_bavail : %" PRIu64 " min-free-disk: %lf",
+                         loc->path, dst_statfs.f_frsize, dst_statfs.f_bavail,
+                         conf->min_free_disk);
 
-                        gf_msg (this->name, GF_LOG_WARNING, 0, 0, "write will"
-                                " cross min-free-disk for file - %s on subvol -"
-                                " %s. looking for new subvol", loc->path,
-                                to->name);
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                   "write will"
+                   " cross min-free-disk for file - %s on subvol -"
+                   " %s. looking for new subvol",
+                   loc->path, to->name);
 
-                        goto find_new_subvol;
+            goto find_new_subvol;
 
-                } else {
-                        ret = 0;
-                        goto out;
-                }
+        } else {
+            ret = 0;
+            goto out;
         }
+    }
 
 find_new_subvol:
-        layout = dht_layout_get (this, loc->parent);
-        if (!layout) {
-                gf_log (this->name, GF_LOG_ERROR, "Layout is NULL");
-                *fop_errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
-
-        *new_subvol = dht_subvol_with_free_space_inodes (this, to, from, layout,
-                                                         stbuf->ia_size);
-        if ((!(*new_subvol)) || (*new_subvol == from)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_SUBVOL_INSUFF_SPACE, "Could not find any subvol"
-                        " with space accomodating the file - %s. Consider "
-                        "adding bricks", loc->path);
-
-                *target_changed = _gf_false;
-                *fop_errno = ENOSPC;
-                ret = -1;
-        } else {
-                gf_msg (this->name, GF_LOG_INFO, 0, 0, "new target found - %s"
-                        " for file - %s", (*new_subvol)->name, loc->path);
-                *target_changed = _gf_true;
-                ret = 0;
-        }
+    layout = dht_layout_get(this, loc->parent);
+    if (!layout) {
+        gf_log(this->name, GF_LOG_ERROR, "Layout is NULL");
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    *new_subvol = dht_subvol_with_free_space_inodes(this, to, from, layout,
+                                                    stbuf->ia_size);
+    if ((!(*new_subvol)) || (*new_subvol == from)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SUBVOL_INSUFF_SPACE,
+               "Could not find any subvol"
+               " with space accommodating the file - %s. Consider "
+               "adding bricks",
+               loc->path);
+
+        *target_changed = _gf_false;
+        *fop_errno = ENOSPC;
+        ret = -1;
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "new target found - %s"
+               " for file - %s",
+               (*new_subvol)->name, loc->path);
+        *target_changed = _gf_true;
+        ret = 0;
+    }
 
 out:
-        if (xdata)
-                dict_unref (xdata);
-        return ret;
+    if (xdata)
+        dict_unref(xdata);
+    return ret;
 }
 
 static int
-__dht_rebalance_migrate_data (xlator_t *this, gf_defrag_info_t *defrag,
-                              xlator_t *from, xlator_t *to, fd_t *src,
-                              fd_t *dst, uint64_t ia_size, int hole_exists,
-                              int *fop_errno)
+__dht_rebalance_migrate_data(xlator_t *this, gf_defrag_info_t *defrag,
+                             xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst,
+                             uint64_t ia_size, int hole_exists, int *fop_errno)
 {
-        int            ret    = 0;
-        int            count  = 0;
-        off_t          offset = 0;
-        struct iovec  *vector = NULL;
-        struct iobref *iobref = NULL;
-        uint64_t       total  = 0;
-        size_t         read_size = 0;
-        dict_t        *xdata = NULL;
-        dht_conf_t    *conf  = NULL;
-
-        conf = this->private;
-        /* if file size is '0', no need to enter this loop */
-        while (total < ia_size) {
-                read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE) ?
-                             DHT_REBALANCE_BLKSIZE : (ia_size - total));
-
-                ret = syncop_readv (from, src, read_size,
-                                    offset, 0, &vector, &count, &iobref, NULL,
-                                    NULL, NULL);
-                if (!ret || (ret < 0)) {
-                        *fop_errno = -ret;
-                        break;
-                }
-
-                if (hole_exists) {
-                        ret = dht_write_with_holes (to, dst, vector, count,
-                                                    ret, offset, iobref,
-                                                    fop_errno);
-                } else {
-                        if (!conf->force_migration &&
-                            !dht_is_tier_xlator (this)) {
-                                xdata = dict_new ();
-                                if (!xdata) {
-                                        gf_msg ("dht", GF_LOG_ERROR, 0,
-                                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                                "insufficient memory");
-                                        ret = -1;
-                                        *fop_errno = ENOMEM;
-                                        break;
-                                }
-
-                                /* Fail this write and abort rebalance if we
-                                 * detect a write from client since migration of
-                                 * this file started. This is done to avoid
-                                 * potential data corruption due to out of order
-                                 * writes from rebalance and client to the same
-                                 * region (as compared between src and dst
-                                 * files). See
-                                 * https://github.com/gluster/glusterfs/issues/308
-                                 * for more details.
-                                 */
-                                ret = dict_set_int32 (xdata,
-                                                      GF_AVOID_OVERWRITE, 1);
-                                if (ret) {
-                                        gf_msg ("dht", GF_LOG_ERROR, 0,
-                                                ENOMEM, "failed to set dict");
-                                        ret = -1;
-                                        *fop_errno = ENOMEM;
-                                        break;
-                                }
-
-                        }
-
-                        ret = syncop_writev (to, dst, vector, count,
-                                             offset, iobref, 0, NULL, NULL,
-                                             xdata, NULL);
-                        if (ret < 0) {
-                                *fop_errno = -ret;
-                        }
-                }
+    int ret = 0;
+    int count = 0;
+    off_t offset = 0;
+    off_t data_offset = 0;
+    off_t hole_offset = 0;
+    struct iovec *vector = NULL;
+    struct iobref *iobref = NULL;
+    uint64_t total = 0;
+    size_t read_size = 0;
+    size_t data_block_size = 0;
+    dict_t *xdata = NULL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    /* if file size is '0', no need to enter this loop */
+    while (total < ia_size) {
+        /* This is a regular file - read it sequentially */
+        if (!hole_exists) {
+            read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE)
+                             ? DHT_REBALANCE_BLKSIZE
+                             : (ia_size - total));
+        } else {
+            /* This is a sparse file - read only the data segments in the file
+             */
+
+            /* If the previous data block is fully copied, find the next data
+             * segment
+             * starting at the offset of the last read and written byte,  */
+            if (data_block_size <= 0) {
+                ret = syncop_seek(from, src, offset, GF_SEEK_DATA, NULL,
+                                  &data_offset);
+                if (ret) {
+                    if (ret == -ENXIO)
+                        ret = 0; /* No more data segments */
+                    else
+                        *fop_errno = -ret; /* Error occurred */
 
-                if ((defrag && defrag->cmd == GF_DEFRAG_CMD_START_TIER) &&
-                    (gf_defrag_get_pause_state (&defrag->tier_conf) != TIER_RUNNING)) {
-                        gf_msg ("tier", GF_LOG_INFO, 0,
-                                DHT_MSG_TIER_PAUSED,
-                                "Migrate file paused");
-                        ret = -1;
+                    break;
                 }
 
-                if (ret < 0) {
+                /* If the position of the current data segment is greater than
+                 * the position of the next hole, find the next hole in order to
+                 * calculate the length of the new data segment */
+                if (data_offset > hole_offset) {
+                    /* Starting at the offset of the last data segment, find the
+                     * next hole */
+                    ret = syncop_seek(from, src, data_offset, GF_SEEK_HOLE,
+                                      NULL, &hole_offset);
+                    if (ret) {
+                        /* If an error occurred here it's a real error because
+                         * if the seek for a data segment was successful then
+                         * necessarily another hole must exist (EOF is a hole)
+                         */
+                        *fop_errno = -ret;
                         break;
-                }
-
-                offset += ret;
-                total += ret;
+                    }
 
-                GF_FREE (vector);
-                if (iobref)
-                        iobref_unref (iobref);
-                iobref = NULL;
-                vector = NULL;
-        }
-        if (iobref)
-                iobref_unref (iobref);
-        GF_FREE (vector);
-
-        if (ret >= 0)
-                ret = 0;
-        else
+                    /* Calculate the total size of the current data block */
+                    data_block_size = hole_offset - data_offset;
+                }
+            } else {
+                /* There is still data in the current segment, move the
+                 * data_offset to the position of the last written byte */
+                data_offset = offset;
+            }
+
+            /* Calculate how much data needs to be read and written. If the data
+             * segment's length is bigger than DHT_REBALANCE_BLKSIZE, read and
+             * write DHT_REBALANCE_BLKSIZE data length and the rest in the
+             * next iteration(s) */
+            read_size = ((data_block_size > DHT_REBALANCE_BLKSIZE)
+                             ? DHT_REBALANCE_BLKSIZE
+                             : data_block_size);
+
+            /* Calculate the remaining size of the data block - maybe there's no
+             * need to seek for data in the next iteration */
+            data_block_size -= read_size;
+
+            /* Set offset to the offset of the data segment so read and write
+             * will have the correct position */
+            offset = data_offset;
+        }
+
+        ret = syncop_readv(from, src, read_size, offset, 0, &vector, &count,
+                           &iobref, NULL, NULL, NULL);
+
+        if (!ret || (ret < 0)) {
+            if (!ret) {
+                /* File was probably truncated*/
                 ret = -1;
-
-        if (xdata) {
-                dict_unref (xdata);
+                *fop_errno = ENOSPC;
+            } else {
+                *fop_errno = -ret;
+            }
+            break;
         }
 
-        return ret;
-}
-
-
-static int
-__dht_rebalance_open_src_file (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
-                               struct iatt *stbuf, fd_t **src_fd,
-                               gf_boolean_t *clean_src, int *fop_errno)
-{
-
-        int          ret  = 0;
-        fd_t        *fd   = NULL;
-        dict_t      *dict = NULL;
-        struct iatt  iatt = {0,};
-        dht_conf_t  *conf = NULL;
-
-        conf = this->private;
-
-        *clean_src = _gf_false;
+        if (!conf->force_migration) {
+            if (!xdata) {
+                xdata = dict_new();
+                if (!xdata) {
+                    gf_msg("dht", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                           "insufficient memory");
+                    ret = -1;
+                    *fop_errno = ENOMEM;
+                    break;
+                }
 
-        fd = fd_create (loc->inode, DHT_REBALANCE_PID);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: fd create failed (source)", loc->path);
-                *fop_errno = ENOMEM;
-                ret = -1;
-                goto out;
+                /* Fail this write and abort rebalance if we
+                 * detect a write from client since migration of
+                 * this file started. This is done to avoid
+                 * potential data corruption due to out of order
+                 * writes from rebalance and client to the same
+                 * region (as compared between src and dst
+                 * files). See
+                 * https://github.com/gluster/glusterfs/issues/308
+                 * for more details.
+                 */
+                ret = dict_set_int32_sizen(xdata, GF_AVOID_OVERWRITE, 1);
+                if (ret) {
+                    gf_msg("dht", GF_LOG_ERROR, 0, ENOMEM,
+                           "failed to set dict");
+                    ret = -1;
+                    *fop_errno = ENOMEM;
+                    break;
+                }
+            }
         }
 
-        ret = syncop_open (from, loc, O_RDWR, fd, NULL, NULL);
+        ret = syncop_writev(to, dst, vector, count, offset, iobref, 0, NULL,
+                            NULL, xdata, NULL);
         if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to open file %s on %s",
-                        loc->path, from->name);
-                *fop_errno = -ret;
-                ret = -1;
-                goto out;
+            *fop_errno = -ret;
+            break;
         }
 
-        fd_bind (fd);
-
-        if (src_fd)
-                *src_fd = fd;
+        offset += ret;
+        total += ret;
 
+        GF_FREE(vector);
+        if (iobref)
+            iobref_unref(iobref);
+        iobref = NULL;
+        vector = NULL;
+    }
+    if (iobref)
+        iobref_unref(iobref);
+    GF_FREE(vector);
+
+    if (ret >= 0)
+        ret = 0;
+    else
         ret = -1;
-        dict = dict_new ();
-        if (!dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: Could not allocate memory for dict", loc->path);
-                *fop_errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
 
-        ret = dict_set_str (dict, conf->link_xattr_name, to->name);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to set xattr in dict for %s (linkto:%s)",
-                        loc->path, to->name);
-                *fop_errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    if (xdata) {
+        dict_unref(xdata);
+    }
 
-        /* Once the migration starts, the source should have 'linkto' key set
-           to show which is the target, so other clients can work around it */
-        ret = syncop_setxattr (from, loc, dict, 0, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to set xattr on %s in %s",
-                        loc->path, from->name);
-                *fop_errno = -ret;
-                ret = -1;
-                goto out;
-        }
+    return ret;
+}
 
-        /* Reset source mode/xattr if migration fails*/
-        *clean_src = _gf_true;
+static int
+__dht_rebalance_open_src_file(xlator_t *this, xlator_t *from, xlator_t *to,
+                              loc_t *loc, struct iatt *stbuf, fd_t **src_fd,
+                              gf_boolean_t *clean_src, int *fop_errno)
+{
+    int ret = 0;
+    fd_t *fd = NULL;
+    dict_t *dict = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    *clean_src = _gf_false;
+
+    fd = fd_create(loc->inode, DHT_REBALANCE_PID);
+    if (!fd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: fd create failed (source)", loc->path);
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_open(from, loc, O_RDWR, fd, NULL, NULL);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to open file %s on %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
-        /* mode should be (+S+T) to indicate migration is in progress */
-        iatt.ia_prot = stbuf->ia_prot;
-        iatt.ia_type = stbuf->ia_type;
-        iatt.ia_prot.sticky = 1;
-        iatt.ia_prot.sgid = 1;
+    fd_bind(fd);
 
-        ret = syncop_setattr (from, loc, &iatt, GF_SET_ATTR_MODE, NULL, NULL,
-                              NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to set mode on %s in %s",
-                        loc->path, from->name);
-                *fop_errno = -ret;
-                ret = -1;
-                goto out;
-        }
+    if (src_fd)
+        *src_fd = fd;
 
-        /* success */
-        ret = 0;
+    ret = -1;
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: Could not allocate memory for dict", loc->path);
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, conf->link_xattr_name, to->name);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set xattr in dict for %s (linkto:%s)", loc->path,
+               to->name);
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    /* Once the migration starts, the source should have 'linkto' key set
+       to show which is the target, so other clients can work around it */
+    ret = syncop_setxattr(from, loc, dict, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to set xattr on %s in %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* Reset source mode/xattr if migration fails*/
+    *clean_src = _gf_true;
+
+    /* mode should be (+S+T) to indicate migration is in progress */
+    iatt.ia_prot = stbuf->ia_prot;
+    iatt.ia_type = stbuf->ia_type;
+    iatt.ia_prot.sticky = 1;
+    iatt.ia_prot.sgid = 1;
+
+    ret = syncop_setattr(from, loc, &iatt, GF_SET_ATTR_MODE, NULL, NULL, NULL,
+                         NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to set mode on %s in %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* success */
+    ret = 0;
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int
-migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
-                       struct iatt *buf, int *fop_errno)
+migrate_special_files(xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
+                      struct iatt *buf, int *fop_errno)
 {
-        int          ret      = -1;
-        dict_t      *rsp_dict = NULL;
-        dict_t      *dict     = NULL;
-        char        *link     = NULL;
-        struct iatt  stbuf    = {0,};
-        dht_conf_t  *conf     = this->private;
-
-        dict = dict_new ();
-        if (!dict) {
-                *fop_errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
-        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
-        if (ret) {
-                *fop_errno = ENOMEM;
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set 'linkto' key in dict", loc->path);
-                goto out;
-        }
-
-        /* check in the destination if the file is link file */
-        ret = syncop_lookup (to, loc, &stbuf, NULL, dict, &rsp_dict);
-        if ((ret < 0) && (-ret != ENOENT)) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: lookup failed",
-                        loc->path);
-                *fop_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-
-        /* we no more require this key */
-        dict_del (dict, conf->link_xattr_name);
-
-        /* file exists in target node, only if it is 'linkfile' its valid,
-           otherwise, error out */
-        if (!ret) {
-                if (!check_is_linkfile (loc->inode, &stbuf, rsp_dict,
-                                        conf->link_xattr_name)) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: file exists in destination", loc->path);
-                        *fop_errno = EINVAL;
-                        ret = -1;
-                        goto out;
-                }
-
-                /* as file is linkfile, delete it */
-                ret = syncop_unlink (to, loc, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: failed to delete the linkfile",
-                                loc->path);
-                        *fop_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        /* Set the gfid of the source file in dict */
-        ret = dict_set_gfuuid (dict, "gfid-req", buf->ia_gfid, true);
+    int ret = -1;
+    dict_t *rsp_dict = NULL;
+    dict_t *dict = NULL;
+    char *link = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    dht_conf_t *conf = this->private;
+
+    dict = dict_new();
+    if (!dict) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_int32(dict, conf->link_xattr_name, 256);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_log(this->name, GF_LOG_ERROR,
+               "%s: failed to set 'linkto' key in dict", loc->path);
+        goto out;
+    }
+
+    /* check in the destination if the file is link file */
+    ret = syncop_lookup(to, loc, &stbuf, NULL, dict, &rsp_dict);
+    if ((ret < 0) && (-ret != ENOENT)) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: lookup failed", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* we no more require this key */
+    dict_del(dict, conf->link_xattr_name);
+
+    /* file exists in target node, only if it is 'linkfile' its valid,
+       otherwise, error out */
+    if (!ret) {
+        if (!check_is_linkfile(loc->inode, &stbuf, rsp_dict,
+                               conf->link_xattr_name)) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: file exists in destination", loc->path);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
+        }
+
+        /* as file is linkfile, delete it */
+        ret = syncop_unlink(to, loc, NULL, NULL);
         if (ret) {
-                *fop_errno = ENOMEM;
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set gfid in dict for create", loc->path);
-                goto out;
-        }
-
-        /* Create the file in target */
-        if (IA_ISLNK (buf->ia_type)) {
-                /* Handle symlinks separately */
-                ret = syncop_readlink (from, loc, &link, buf->ia_size, NULL,
-                                       NULL);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: readlink on symlink failed",
-                                loc->path);
-                        *fop_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = syncop_symlink (to, loc, link, 0, dict, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: creating symlink failed",
-                                loc->path);
-                        *fop_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
-
-                goto done;
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to delete the linkfile", loc->path);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+    }
+
+    /* Set the gfid of the source file in dict */
+    ret = dict_set_gfuuid(dict, "gfid-req", buf->ia_gfid, true);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_log(this->name, GF_LOG_ERROR,
+               "%s: failed to set gfid in dict for create", loc->path);
+        goto out;
+    }
+
+    /* Create the file in target */
+    if (IA_ISLNK(buf->ia_type)) {
+        /* Handle symlinks separately */
+        ret = syncop_readlink(from, loc, &link, buf->ia_size, NULL, NULL);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: readlink on symlink failed", loc->path);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
 
-        ret = syncop_mknod (to, loc, st_mode_from_ia (buf->ia_prot,
-                                                      buf->ia_type),
-                            makedev (ia_major (buf->ia_rdev),
-                                     ia_minor (buf->ia_rdev)), 0, dict, NULL);
+        ret = syncop_symlink(to, loc, link, 0, dict, NULL);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: mknod failed",
-                        loc->path);
-                *fop_errno = -ret;
-                ret = -1;
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED, "%s: creating symlink failed",
+                   loc->path);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+
+        goto done;
+    }
+
+    ret = syncop_mknod(to, loc, st_mode_from_ia(buf->ia_prot, buf->ia_type),
+                       makedev(ia_major(buf->ia_rdev), ia_minor(buf->ia_rdev)),
+                       0, dict, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: mknod failed", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
 done:
-        ret = syncop_setattr (to, loc, buf,
-                              (GF_SET_ATTR_MTIME |
-                               GF_SET_ATTR_UID | GF_SET_ATTR_GID |
-                               GF_SET_ATTR_MODE), NULL, NULL, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: failed to perform setattr on %s",
-                        loc->path, to->name);
-                *fop_errno = -ret;
-        }
-
-        ret = syncop_unlink (from, loc, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: unlink failed",
-                        loc->path);
-                *fop_errno = -ret;
-                ret = -1;
-        }
+    ret = syncop_setattr(to, loc, buf,
+                         (GF_SET_ATTR_MTIME | GF_SET_ATTR_UID |
+                          GF_SET_ATTR_GID | GF_SET_ATTR_MODE),
+                         NULL, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to perform setattr on %s", loc->path, to->name);
+        *fop_errno = -ret;
+    }
+
+    ret = syncop_unlink(from, loc, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: unlink failed", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+    }
 
 out:
-        GF_FREE (link);
-        if (dict)
-                dict_unref (dict);
+    GF_FREE(link);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        return ret;
+    return ret;
 }
 
 static int
-__dht_migration_cleanup_src_file (xlator_t *this, loc_t *loc, fd_t *fd,
-                                  xlator_t *from, ia_prot_t *src_ia_prot)
+__dht_migration_cleanup_src_file(xlator_t *this, loc_t *loc, fd_t *fd,
+                                 xlator_t *from, ia_prot_t *src_ia_prot)
 {
-        int ret                       = -1;
-        dht_conf_t     *conf          = NULL;
-        struct iatt     new_stbuf     = {0,};
-
-        if (!this || !fd || !from || !src_ia_prot) {
-                goto out;
-        }
-
-        conf = this->private;
-
-        /*Revert source mode and xattr changes*/
-        ret = syncop_fstat (from, fd, &new_stbuf, NULL, NULL);
-        if (ret < 0) {
-                /* Failed to get the stat info */
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file cleanup failed: failed to fstat "
-                        "file %s on %s ", loc->path, from->name);
-                ret = -1;
-                goto out;
-        }
-
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    struct iatt new_stbuf = {
+        0,
+    };
+
+    if (!this || !fd || !from || !src_ia_prot) {
+        goto out;
+    }
+
+    conf = this->private;
+
+    /*Revert source mode and xattr changes*/
+    ret = syncop_fstat(from, fd, &new_stbuf, NULL, NULL);
+    if (ret < 0) {
+        /* Failed to get the stat info */
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file cleanup failed: failed to fstat "
+               "file %s on %s ",
+               loc->path, from->name);
+        ret = -1;
+        goto out;
+    }
 
-        /* Remove the sticky bit and sgid bit set, reset it to 0*/
-        if (!src_ia_prot->sticky)
-                new_stbuf.ia_prot.sticky = 0;
+    /* Remove the sticky bit and sgid bit set, reset it to 0*/
+    if (!src_ia_prot->sticky)
+        new_stbuf.ia_prot.sticky = 0;
 
-        if (!src_ia_prot->sgid)
-                new_stbuf.ia_prot.sgid = 0;
+    if (!src_ia_prot->sgid)
+        new_stbuf.ia_prot.sgid = 0;
 
-        ret = syncop_fsetattr (from, fd, &new_stbuf,
-                               (GF_SET_ATTR_GID | GF_SET_ATTR_MODE),
-                               NULL, NULL, NULL, NULL);
+    ret = syncop_fsetattr(from, fd, &new_stbuf,
+                          (GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL, NULL,
+                          NULL, NULL);
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file cleanup failed:"
-                        "%s: failed to perform fsetattr on %s ",
-                        loc->path, from->name);
-                ret = -1;
-                goto out;
-        }
-
-        ret = syncop_fremovexattr (from, fd, conf->link_xattr_name, 0, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to remove linkto xattr on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file cleanup failed:"
+               "%s: failed to perform fsetattr on %s ",
+               loc->path, from->name);
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_fremovexattr(from, fd, conf->link_xattr_name, 0, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to remove linkto xattr on %s (%s)", loc->path,
+               from->name, strerror(-ret));
+        ret = -1;
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
-
 /*
   return values:
 
@@ -1586,1099 +1469,1066 @@ out:
     1 : not a failure, but we can't migrate data as of now
 */
 int
-dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
-                  int flag, int *fop_errno)
+dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
+                 int flag, int *fop_errno)
 {
-        int                     ret                     = -1;
-        struct iatt             new_stbuf               = {0,};
-        struct iatt             stbuf                   = {0,};
-        struct iatt             empty_iatt              = {0,};
-        ia_prot_t               src_ia_prot             = {0,};
-        fd_t                    *src_fd                 = NULL;
-        fd_t                    *dst_fd                 = NULL;
-        dict_t                  *dict                   = NULL;
-        dict_t                  *xattr                  = NULL;
-        dict_t                  *xattr_rsp              = NULL;
-        int                     file_has_holes          = 0;
-        dht_conf_t              *conf                   = this->private;
-        int                     rcvd_enoent_from_src    = 0;
-        struct gf_flock         flock                   = {0, };
-        struct gf_flock         plock                   = {0, };
-        loc_t                   tmp_loc                 = {0, };
-        loc_t                   parent_loc              = {0, };
-        gf_boolean_t            inodelk_locked          = _gf_false;
-        gf_boolean_t            entrylk_locked          = _gf_false;
-        gf_boolean_t            p_locked                = _gf_false;
-        int                     lk_ret                  = -1;
-        gf_defrag_info_t        *defrag                 =  NULL;
-        gf_boolean_t            clean_src               = _gf_false;
-        gf_boolean_t            clean_dst               = _gf_false;
-        int                     log_level               = GF_LOG_INFO;
-        gf_boolean_t            delete_src_linkto       = _gf_true;
-        lock_migration_info_t   locklist;
-        dict_t                  *meta_dict              = NULL;
-        gf_boolean_t            meta_locked             = _gf_false;
-        gf_boolean_t            target_changed          = _gf_false;
-        xlator_t                *new_target             = NULL;
-        xlator_t                *old_target             = NULL;
-        xlator_t                *hashed_subvol          = NULL;
-        fd_t                    *linkto_fd              = NULL;
-
-
-        if (from == to) {
-                gf_msg_debug (this->name, 0, "destination and source are same. file %s"
-                              " might have migrated already", loc->path);
-                ret = 0;
-                goto out;
-        }
-
-        /* If defrag is NULL, it should be assumed that migration is triggered
-         * from client */
-        defrag = conf->defrag;
-
-        /* migration of files from clients is restricted to non-tiered clients
-         * for now */
-        if (!defrag && dht_is_tier_xlator (this)) {
-                ret = ENOTSUP;
-                goto out;
-        }
-
-        if (defrag && defrag->tier_conf.is_tier)
-                log_level = GF_LOG_TRACE;
+    int ret = -1;
+    struct iatt new_stbuf = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt empty_iatt = {
+        0,
+    };
+    ia_prot_t src_ia_prot = {
+        0,
+    };
+    fd_t *src_fd = NULL;
+    fd_t *dst_fd = NULL;
+    dict_t *dict = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xattr_rsp = NULL;
+    int file_has_holes = 0;
+    dht_conf_t *conf = this->private;
+    int rcvd_enoent_from_src = 0;
+    struct gf_flock flock = {
+        0,
+    };
+    struct gf_flock plock = {
+        0,
+    };
+    loc_t tmp_loc = {
+        0,
+    };
+    loc_t parent_loc = {
+        0,
+    };
+    gf_boolean_t inodelk_locked = _gf_false;
+    gf_boolean_t entrylk_locked = _gf_false;
+    gf_boolean_t p_locked = _gf_false;
+    int lk_ret = -1;
+    gf_defrag_info_t *defrag = NULL;
+    gf_boolean_t clean_src = _gf_false;
+    gf_boolean_t clean_dst = _gf_false;
+    int log_level = GF_LOG_INFO;
+    gf_boolean_t delete_src_linkto = _gf_true;
+    lock_migration_info_t locklist;
+    dict_t *meta_dict = NULL;
+    gf_boolean_t meta_locked = _gf_false;
+    gf_boolean_t target_changed = _gf_false;
+    xlator_t *new_target = NULL;
+    xlator_t *old_target = NULL;
+    xlator_t *hashed_subvol = NULL;
+    fd_t *linkto_fd = NULL;
+    dict_t *xdata = NULL;
+
+    if (from == to) {
+        gf_msg_debug(this->name, 0,
+                     "destination and source are same. file %s"
+                     " might have migrated already",
+                     loc->path);
+        ret = 0;
+        goto out;
+    }
 
-        gf_log (this->name,
-                log_level, "%s: attempting to move from %s to %s",
-                loc->path, from->name, to->name);
+    gf_log(this->name, log_level, "%s: attempting to move from %s to %s",
+           loc->path, from->name, to->name);
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                *fop_errno = ENOMEM;
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
-                        "Could not allocate memory for dict");
-                goto out;
-        }
-        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        *fop_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "Could not allocate memory for dict");
+        goto out;
+    }
+    ret = dict_set_int32(dict, conf->link_xattr_name, 256);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to set 'linkto' key in dict",
+               loc->path);
+        goto out;
+    }
+
+    /* Do not migrate file in case lock migration is not enabled on the
+     * volume*/
+    if (!conf->lock_migration_enabled) {
+        ret = dict_set_int32(dict, GLUSTERFS_POSIXLK_COUNT, sizeof(int32_t));
         if (ret) {
-                *fop_errno = ENOMEM;
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: failed to set 'linkto' key in dict", loc->path);
-                goto out;
-        }
-
-        /* Do not migrate file in case lock migration is not enabled on the
-         * volume*/
-        if (!conf->lock_migration_enabled) {
-                ret = dict_set_int32 (dict,
-                                 GLUSTERFS_POSIXLK_COUNT, sizeof(int32_t));
-                if (ret) {
-                        *fop_errno = ENOMEM;
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed: %s: failed to "
-                                "set "GLUSTERFS_POSIXLK_COUNT" key in dict",
-                                loc->path);
-                        goto out;
-                }
-        } else {
-                gf_msg (this->name, GF_LOG_INFO, 0, 0, "locks will be migrated"
-                        " for file: %s", loc->path);
-        }
-
-        ret = dht_build_parent_loc (this, &parent_loc, loc, fop_errno);
-        if (ret < 0) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_WARNING, *fop_errno,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: failed to build parent loc, which is needed to "
-                        "acquire entrylk to synchronize with renames on this "
-                        "path. Skipping migration", loc->path);
-                goto out;
-        }
-
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        if (hashed_subvol == NULL) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: cannot find hashed subvol which is needed to "
-                        "synchronize with renames on this path. "
-                        "Skipping migration", loc->path);
-                goto out;
-        }
-
-        flock.l_type = F_WRLCK;
-
-        tmp_loc.inode = inode_ref (loc->inode);
-        gf_uuid_copy (tmp_loc.gfid, loc->gfid);
-        tmp_loc.path = gf_strdup(loc->path);
-
-        /* this inodelk happens with flock.owner being zero. But to synchronize
-         * hardlink migration we need to have different lkowner for each migration
-         * Filed a bug here: https://bugzilla.redhat.com/show_bug.cgi?id=1468202 to
-         * track the fix for this. Currently synclock takes care of synchronizing
-         * hardlink migration. Once this bug is fixed we can avoid taking synclock */
-        ret = syncop_inodelk (from, DHT_FILE_MIGRATE_DOMAIN, &tmp_loc, F_SETLKW,
-                              &flock, NULL, NULL);
-        if (ret < 0) {
-                *fop_errno = -ret;
-                ret = -1;
-                gf_msg (this->name, GF_LOG_WARNING, *fop_errno,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "migrate file failed: "
-                        "%s: failed to lock file on %s",
-                        loc->path, from->name);
-                goto out;
-        }
-
-        inodelk_locked = _gf_true;
-
-        /* dht_rename has changed to use entrylk on hashed subvol for
-         * synchronization. So, rebalance too has to acquire an entrylk on
-         * hashed subvol.
-         */
-        ret = syncop_entrylk (hashed_subvol, DHT_ENTRY_SYNC_DOMAIN, &parent_loc,
-                              loc->name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL,
-                              NULL);
-        if (ret < 0) {
-                *fop_errno = -ret;
-                ret = -1;
-                gf_msg (this->name, GF_LOG_WARNING, *fop_errno,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: failed to acquire entrylk on subvol %s",
-                        loc->path, hashed_subvol->name);
-                goto out;
-        }
-
-        entrylk_locked = _gf_true;
-
-        /* Phase 1 - Data migration is in progress from now on */
-        ret = syncop_lookup (from, loc, &stbuf, NULL, dict, &xattr_rsp);
+            *fop_errno = ENOMEM;
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed: %s: failed to "
+                   "set " GLUSTERFS_POSIXLK_COUNT " key in dict",
+                   loc->path);
+            goto out;
+        }
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "locks will be migrated"
+               " for file: %s",
+               loc->path);
+    }
+
+    /* The file is locked to prevent a rename during a migration. Renames
+     * and migrations on the file at the same time can lead to data loss.
+     */
+
+    ret = dht_build_parent_loc(this, &parent_loc, loc, fop_errno);
+    if (ret < 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to build parent loc, which is needed to "
+               "acquire entrylk to synchronize with renames on this "
+               "path. Skipping migration",
+               loc->path);
+        goto out;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: cannot find hashed subvol which is needed to "
+               "synchronize with renames on this path. "
+               "Skipping migration",
+               loc->path);
+        goto out;
+    }
+
+    flock.l_type = F_WRLCK;
+
+    tmp_loc.inode = inode_ref(loc->inode);
+    gf_uuid_copy(tmp_loc.gfid, loc->gfid);
+    tmp_loc.path = gf_strdup(loc->path);
+
+    /* this inodelk happens with flock.owner being zero. But to synchronize
+     * hardlink migration we need to have different lkowner for each migration
+     * Filed a bug here: https://bugzilla.redhat.com/show_bug.cgi?id=1468202 to
+     * track the fix for this. Currently synclock takes care of synchronizing
+     * hardlink migration. Once this bug is fixed we can avoid taking synclock
+     */
+    ret = syncop_inodelk(from, DHT_FILE_MIGRATE_DOMAIN, &tmp_loc, F_SETLKW,
+                         &flock, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "migrate file failed: "
+               "%s: failed to lock file on %s",
+               loc->path, from->name);
+        goto out;
+    }
+
+    inodelk_locked = _gf_true;
+
+    /* dht_rename has changed to use entrylk on hashed subvol for
+     * synchronization. So, rebalance too has to acquire an entrylk on
+     * hashed subvol.
+     */
+    ret = syncop_entrylk(hashed_subvol, DHT_ENTRY_SYNC_DOMAIN, &parent_loc,
+                         loc->name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to acquire entrylk on subvol %s", loc->path,
+               hashed_subvol->name);
+        goto out;
+    }
+
+    entrylk_locked = _gf_true;
+
+    /* Phase 1 - Data migration is in progress from now on */
+    ret = syncop_lookup(from, loc, &stbuf, NULL, dict, &xattr_rsp);
+    if (ret) {
+        *fop_errno = -ret;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: lookup failed on %s",
+               loc->path, from->name);
+        goto out;
+    }
+
+    /* preserve source mode, so set the same to the destination */
+    src_ia_prot = stbuf.ia_prot;
+
+    /* Check if file can be migrated */
+    ret = __is_file_migratable(this, loc, &stbuf, xattr_rsp, flag, defrag, conf,
+                               fop_errno);
+    if (ret) {
+        if (ret == HARDLINK_MIG_INPROGRESS)
+            ret = 0;
+        goto out;
+    }
+
+    /* Take care of the special files */
+    if (!IA_ISREG(stbuf.ia_type)) {
+        /* Special files */
+        ret = migrate_special_files(this, from, to, loc, &stbuf, fop_errno);
+        goto out;
+    }
+
+    /* Try to preserve 'holes' while migrating data */
+    if (stbuf.ia_size > (stbuf.ia_blocks * GF_DISK_SECTOR_SIZE))
+        file_has_holes = 1;
+
+    /* create the destination, with required modes/xattr */
+    ret = __dht_rebalance_create_dst_file(this, to, from, loc, &stbuf, &dst_fd,
+                                          fop_errno, file_has_holes);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "Create dst failed"
+               " on - %s for file - %s",
+               to->name, loc->path);
+        goto out;
+    }
+
+    clean_dst = _gf_true;
+
+    ret = __dht_check_free_space(this, to, from, loc, &stbuf, flag, conf,
+                                 &target_changed, &new_target, fop_errno);
+    if (target_changed) {
+        /* Can't handle for hardlinks. Marking this as failure */
+        if (flag == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS || stbuf.ia_nlink > 1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SUBVOL_INSUFF_SPACE,
+                   "Exiting migration for"
+                   " file - %s. flag - %d, stbuf.ia_nlink - %d",
+                   loc->path, flag, stbuf.ia_nlink);
+            ret = -1;
+            goto out;
+        }
+
+        ret = syncop_ftruncate(to, dst_fd, 0, NULL, NULL, NULL, NULL);
         if (ret) {
-                *fop_errno = -ret;
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, *fop_errno,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: lookup failed on %s",
-                        loc->path, from->name);
-               goto out;
+            gf_log(this->name, GF_LOG_WARNING,
+                   "%s: failed to perform truncate on %s (%s)", loc->path,
+                   to->name, strerror(-ret));
         }
 
-        /* preserve source mode, so set the same to the destination */
-        src_ia_prot = stbuf.ia_prot;
+        syncop_close(dst_fd);
+        dst_fd = NULL;
 
-        /* Check if file can be migrated */
-        ret = __is_file_migratable (this, loc, &stbuf, xattr_rsp, flag, defrag, conf,
-                                    fop_errno);
-        if (ret) {
-                if (ret == HARDLINK_MIG_INPROGRESS)
-                        ret = 0;
-                goto out;
-        }
+        old_target = to;
+        to = new_target;
 
-        /* Take care of the special files */
-        if (!IA_ISREG (stbuf.ia_type)) {
-                /* Special files */
-                ret = migrate_special_files (this, from, to, loc, &stbuf,
-                                             fop_errno);
-                goto out;
-        }
+        clean_dst = _gf_false;
 
-        /* create the destination, with required modes/xattr */
-        ret = __dht_rebalance_create_dst_file (this, to, from, loc, &stbuf,
-                                               &dst_fd, fop_errno);
+        /* if the file migration is successful to this new target, then
+         * update the xattr on the old destination to point the new
+         * destination. We need to do update this only post migration
+         * as in case of failure the linkto needs to point to the source
+         * subvol */
+        ret = __dht_rebalance_create_dst_file(
+            this, to, from, loc, &stbuf, &dst_fd, fop_errno, file_has_holes);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "Create dst failed"
-                        " on - %s for file - %s", to->name, loc->path);
-                goto out;
-        }
-
-        clean_dst = _gf_true;
-
-        ret = __dht_check_free_space (this, to, from, loc, &stbuf, flag, conf,
-                                      &target_changed, &new_target, fop_errno);
-        if (target_changed) {
-                /* Can't handle for hardlinks. Marking this as failure */
-                if (flag == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS || stbuf.ia_nlink > 1) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_SUBVOL_INSUFF_SPACE, "Exiting migration for"
-                                " file - %s. flag - %d, stbuf.ia_nlink - %d",
-                               loc->path,  flag, stbuf.ia_nlink);
-                        ret = -1;
-                        goto out;
-                }
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Create dst failed"
+                   " on - %s for file - %s",
+                   to->name, loc->path);
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                   "destination for file "
+                   "- %s is changed to - %s",
+                   loc->path, to->name);
+            clean_dst = _gf_true;
+        }
+    }
+
+    if (ret) {
+        goto out;
+    }
+
+    /* Open the source, and also update mode/xattr */
+    ret = __dht_rebalance_open_src_file(this, from, to, loc, &stbuf, &src_fd,
+                                        &clean_src, fop_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: failed to open %s on %s", loc->path,
+               from->name);
+        goto out;
+    }
+
+    /* TODO: move all xattr related operations to fd based operations */
+    ret = syncop_listxattr(from, loc, &xattr, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to get xattr from %s",
+               loc->path, from->name);
+        ret = -1;
+        goto out;
+    }
+
+    /* Copying posix acls to the linkto file messes up the permissions*/
+    dht_strip_out_acls(xattr);
+
+    /* Remove the linkto xattr as we don't want to overwrite the value
+     * set on the dst.
+     */
+    dict_del(xattr, conf->link_xattr_name);
+
+    /* We need to error out if this fails as having the wrong shard xattrs
+     * set on the dst could cause data corruption
+     */
+    ret = syncop_fsetxattr(to, dst_fd, xattr, 0, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to set xattr on %s", loc->path, to->name);
+        ret = -1;
+        goto out;
+    }
 
+    if (xattr_rsp) {
+        /* we no more require this key */
+        dict_del(dict, conf->link_xattr_name);
+        dict_unref(xattr_rsp);
+    }
+
+    ret = syncop_fstat(from, src_fd, &stbuf, dict, &xattr_rsp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:failed to lookup %s on %s ", loc->path,
+               from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* Check again if file has hardlink */
+    ret = __check_file_has_hardlink(this, loc, &stbuf, xattr_rsp, flag, defrag,
+                                    conf, fop_errno);
+    if (ret) {
+        if (ret == HARDLINK_MIG_INPROGRESS)
+            ret = 0;
+        goto out;
+    }
+
+    ret = __dht_rebalance_migrate_data(this, defrag, from, to, src_fd, dst_fd,
+                                       stbuf.ia_size, file_has_holes,
+                                       fop_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: %s: failed to migrate data", loc->path);
 
-                ret = syncop_ftruncate (to, dst_fd, 0, NULL, NULL, NULL, NULL);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: failed to perform truncate on %s (%s)",
-                                loc->path, to->name, strerror (-ret));
-                }
+        ret = -1;
+        goto out;
+    }
+
+    /* TODO: Sync the locks */
+
+    xdata = dict_new();
+    if (!xdata || dict_set_int8(xdata, "last-fsync", 1)) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "%s: failed to set last-fsync flag on "
+               "%s (%s)",
+               loc->path, to->name, strerror(ENOMEM));
+    }
+
+    ret = syncop_fsync(to, dst_fd, 0, NULL, NULL, xdata, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING, "%s: failed to fsync on %s (%s)",
+               loc->path, to->name, strerror(-ret));
+        *fop_errno = -ret;
+    }
+
+    /* Phase 2 - Data-Migration Complete, Housekeeping updates pending */
+
+    ret = syncop_fstat(from, src_fd, &new_stbuf, NULL, NULL);
+    if (ret < 0) {
+        /* Failed to get the stat info */
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: failed to fstat file %s on %s ", loc->path,
+               from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
-                syncop_close (dst_fd);
-                dst_fd = NULL;
+    /* Lock the entire source file to prevent clients from taking a
+       lock on it as dht_lk does not handle file migration.
 
-                old_target = to;
-                to = new_target;
+       This still leaves a small window where conflicting locks can
+       be granted to different clients. If client1 requests a blocking
+       lock on the src file, it will be granted after the migrating
+       process releases its lock. If client2 requests a lock on the dst
+       data file, it will also be granted, but all FOPs will be redirected
+       to the dst data file.
+    */
 
-                clean_dst = _gf_false;
+    /* Take meta lock  */
 
+    if (conf->lock_migration_enabled) {
+        meta_dict = dict_new();
+        if (!meta_dict) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "dict_new failed");
 
-                /* if the file migration is successful to this new target, then
-                 * update the xattr on the old destination to point the new
-                 * destination. We need to do update this only post migration
-                 * as in case of failure the linkto needs to point to the source
-                 * subvol */
-                ret = __dht_rebalance_create_dst_file (this, to, from, loc, &stbuf,
-                                                       &dst_fd, fop_errno);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Create dst failed"
-                                " on - %s for file - %s", to->name, loc->path);
-                        goto out;
-                } else {
-                        gf_msg (this->name, GF_LOG_INFO, 0, 0, "destination for file "
-                                "- %s is changed to - %s", loc->path, to->name);
-                        clean_dst = _gf_true;
-                }
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
+        ret = dict_set_str(meta_dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
         if (ret) {
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary value: key = %s,"
+                   " path = %s",
+                   GLUSTERFS_INTERNAL_FOP_KEY, loc->path);
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        /* Open the source, and also update mode/xattr */
-        ret = __dht_rebalance_open_src_file (this, from, to, loc, &stbuf, &src_fd,
-                                             &clean_src, fop_errno);
+        ret = dict_set_int32(meta_dict, GF_META_LOCK_KEY, 1);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed: failed to open %s on %s",
-                        loc->path, from->name);
-                goto out;
-        }
-
-        /* TODO: move all xattr related operations to fd based operations */
-        ret = syncop_listxattr (from, loc, &xattr, NULL, NULL);
-        if (ret < 0) {
-                *fop_errno = -ret;
-                gf_msg (this->name, GF_LOG_WARNING, *fop_errno,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: failed to get xattr from %s",
-                        loc->path, from->name);
-                ret = -1;
-                goto out;
-        }
-
-        /* Copying posix acls to the linkto file messes up the permissions*/
-        dht_strip_out_acls (xattr);
-
-        /* Remove the linkto xattr as we don't want to overwrite the value
-         * set on the dst.
-         */
-        dict_del (xattr, conf->link_xattr_name);
-
-        /* We need to error out if this fails as having the wrong shard xattrs
-         * set on the dst could cause data corruption
-         */
-        ret = syncop_fsetxattr (to, dst_fd, xattr, 0, NULL, NULL);
-        if (ret < 0) {
-                *fop_errno = -ret;
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: failed to set xattr on %s",
-                        loc->path, to->name);
-                ret = -1;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace dict_set failed");
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        if (xattr_rsp) {
-                /* we no more require this key */
-                dict_del (dict, conf->link_xattr_name);
-                dict_unref (xattr_rsp);
-        }
-
-        ret = syncop_fstat (from, src_fd, &stbuf, dict, &xattr_rsp);
+        ret = syncop_setxattr(from, loc, meta_dict, 0, NULL, NULL);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:failed to lookup %s on %s ",
-                        loc->path, from->name);
-                *fop_errno = -ret;
-                ret = -1;
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace syncop_setxattr metalock failed");
 
-        /* Check again if file has hardlink */
-        ret = __check_file_has_hardlink (this, loc, &stbuf, xattr_rsp,
-                                         flag, defrag, conf, fop_errno);
-        if (ret) {
-                if (ret == HARDLINK_MIG_INPROGRESS)
-                        ret = 0;
-                goto out;
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        } else {
+            meta_locked = _gf_true;
         }
-        /* Try to preserve 'holes' while migrating data */
-        if (stbuf.ia_size > (stbuf.ia_blocks * GF_DISK_SECTOR_SIZE))
-                file_has_holes = 1;
+    }
 
+    if (!conf->lock_migration_enabled) {
+        plock.l_type = F_WRLCK;
+        plock.l_start = 0;
+        plock.l_len = 0;
+        plock.l_whence = SEEK_SET;
 
-        ret = __dht_rebalance_migrate_data (this, defrag, from, to,
-                                            src_fd, dst_fd, stbuf.ia_size,
-                                            file_has_holes, fop_errno);
+        ret = syncop_lk(from, src_fd, F_SETLK, &plock, NULL, NULL);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed: %s: failed to migrate data",
-                        loc->path);
-
-                ret = -1;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: Failed to lock on %s",
+                   loc->path, from->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
 
-        /* TODO: Sync the locks */
+        p_locked = _gf_true;
 
-        ret = syncop_fsync (to, dst_fd, 0, NULL, NULL, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to fsync on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-                *fop_errno = -ret;
-        }
+    } else {
+        INIT_LIST_HEAD(&locklist.list);
 
+        ret = syncop_getactivelk(from, loc, &locklist, NULL, NULL);
+        if (ret == 0) {
+            gf_log(this->name, GF_LOG_INFO, "No active locks on:%s", loc->path);
 
-        /* Phase 2 - Data-Migration Complete, Housekeeping updates pending */
+        } else if (ret > 0) {
+            ret = syncop_setactivelk(to, loc, &locklist, NULL, NULL);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, -ret,
+                       DHT_MSG_LOCK_MIGRATION_FAILED, "write lock failed on:%s",
+                       loc->path);
 
-        ret = syncop_fstat (from, src_fd, &new_stbuf, NULL, NULL);
-        if (ret < 0) {
-                /* Failed to get the stat info */
-                gf_msg ( this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed: failed to fstat file %s on %s ",
-                        loc->path, from->name);
                 *fop_errno = -ret;
                 ret = -1;
-                goto out;
-        }
-
-        /* Lock the entire source file to prevent clients from taking a
-           lock on it as dht_lk does not handle file migration.
-
-           This still leaves a small window where conflicting locks can
-           be granted to different clients. If client1 requests a blocking
-           lock on the src file, it will be granted after the migrating
-           process releases its lock. If client2 requests a lock on the dst
-           data file, it will also be granted, but all FOPs will be redirected
-           to the dst data file.
-        */
-
-        /* Take meta lock  */
-
-        if (conf->lock_migration_enabled) {
-                meta_dict = dict_new ();
-                if (!meta_dict) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "dict_new failed");
-
-                        *fop_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_str (meta_dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value: key = %s,"
-                                " path = %s", GLUSTERFS_INTERNAL_FOP_KEY,
-                                 loc->path);
-                        *fop_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_int32 (meta_dict, GF_META_LOCK_KEY, 1);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Trace dict_set failed");
-                        *fop_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = syncop_setxattr (from, loc, meta_dict, 0, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Trace syncop_setxattr metalock failed");
-
-                        *fop_errno = -ret;
-                        ret = -1;
-                        goto out;
-                } else {
-                        meta_locked = _gf_true;
-                }
-        }
-
-        if (!conf->lock_migration_enabled) {
-                plock.l_type = F_WRLCK;
-                plock.l_start = 0;
-                plock.l_len = 0;
-                plock.l_whence = SEEK_SET;
-
-                ret = syncop_lk (from, src_fd, F_SETLK, &plock, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed:"
-                                "%s: Failed to lock on %s",
-                                loc->path, from->name);
-                        *fop_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
-
-                p_locked = _gf_true;
-
+                goto metaunlock;
+            }
         } else {
-
-                INIT_LIST_HEAD (&locklist.list);
-
-                ret = syncop_getactivelk (from, loc, &locklist, NULL, NULL);
-                if (ret == 0) {
-                        gf_log (this->name, GF_LOG_INFO, "No active locks on:%s"
-                                , loc->path);
-
-                } else if (ret > 0) {
-
-                        ret = syncop_setactivelk (to, loc, &locklist, NULL,
-                                                  NULL);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                        DHT_MSG_LOCK_MIGRATION_FAILED,
-                                        "write lock failed on:%s", loc->path);
-
-                                *fop_errno = -ret;
-                                ret = -1;
-                                goto metaunlock;
-                        }
-                } else {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_LOCK_MIGRATION_FAILED,
-                                "getactivelk failed for file: %s", loc->path);
-                        *fop_errno = -ret;
-                }
-        }
-
-
-        /* source would have both sticky bit and sgid bit set, reset it to 0,
-           and set the source permission on destination, if it was not set
-           prior to setting rebalance-modes in source  */
-        if (!src_ia_prot.sticky)
-                new_stbuf.ia_prot.sticky = 0;
-
-        if (!src_ia_prot.sgid)
-                new_stbuf.ia_prot.sgid = 0;
-
-        /* TODO: if the source actually had sticky bit, or sgid bit set,
-           we are not handling it */
-
-        ret = syncop_fsetattr (to, dst_fd, &new_stbuf,
-                               (GF_SET_ATTR_UID | GF_SET_ATTR_GID |
-                                GF_SET_ATTR_MODE), NULL, NULL, NULL, NULL);
+            gf_msg(this->name, GF_LOG_ERROR, -ret,
+                   DHT_MSG_LOCK_MIGRATION_FAILED,
+                   "getactivelk failed for file: %s", loc->path);
+            *fop_errno = -ret;
+        }
+    }
+
+    /* source would have both sticky bit and sgid bit set, reset it to 0,
+       and set the source permission on destination, if it was not set
+       prior to setting rebalance-modes in source  */
+    if (!src_ia_prot.sticky)
+        new_stbuf.ia_prot.sticky = 0;
+
+    if (!src_ia_prot.sgid)
+        new_stbuf.ia_prot.sgid = 0;
+
+    /* TODO: if the source actually had sticky bit, or sgid bit set,
+       we are not handling it */
+
+    ret = syncop_fsetattr(
+        to, dst_fd, &new_stbuf,
+        (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL, NULL,
+        NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to perform setattr on %s ",
+               loc->path, to->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto metaunlock;
+    }
+
+    /* Because 'futimes' is not portable */
+    ret = syncop_setattr(to, loc, &new_stbuf,
+                         (GF_SET_ATTR_MTIME | GF_SET_ATTR_ATIME), NULL, NULL,
+                         NULL, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to perform setattr on %s ", loc->path, to->name);
+        *fop_errno = -ret;
+    }
+
+    if (target_changed) {
+        dict_del(dict, GLUSTERFS_POSIXLK_COUNT);
+        ret = dict_set_str(dict, conf->link_xattr_name, to->name);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: failed to perform setattr on %s ",
-                        loc->path, to->name);
-                *fop_errno = -ret;
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set xattr in dict for %s (linkto:%s)", loc->path,
+                   to->name);
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
+        }
+
+        ret = syncop_setxattr(old_target, loc, dict, 0, NULL, NULL);
+        if (ret && -ret != ESTALE && -ret != ENOENT) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "failed to set xattr on %s in %s", loc->path,
+                   old_target->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        } else if (-ret == ESTALE || -ret == ENOENT) {
+            /* The failure ESTALE indicates that the linkto
+             * file on the hashed subvol might have been deleted.
+             * In this case will create a linkto file with new target
+             * as linkto xattr value*/
+            linkto_fd = fd_create(loc->inode, DHT_REBALANCE_PID);
+            if (!linkto_fd) {
+                gf_msg(this->name, GF_LOG_ERROR, errno,
+                       DHT_MSG_MIGRATE_FILE_FAILED, "%s: fd create failed",
+                       loc->path);
+                *fop_errno = ENOMEM;
                 ret = -1;
-                goto metaunlock;
-        }
-
-        /* Because 'futimes' is not portable */
-        ret = syncop_setattr (to, loc, &new_stbuf,
-                              (GF_SET_ATTR_MTIME | GF_SET_ATTR_ATIME),
-                              NULL, NULL, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform setattr on %s ",
-                        loc->path, to->name);
+                goto out;
+            }
+            ret = syncop_create(old_target, loc, O_RDWR, DHT_LINKFILE_MODE,
+                                linkto_fd, NULL, dict, NULL);
+            if (ret != 0 && -ret != EEXIST && -ret != ESTALE) {
                 *fop_errno = -ret;
-        }
-
-        if (target_changed) {
-                dict_del (dict, conf->link_xattr_name);
-                dict_del (dict, GLUSTERFS_POSIXLK_COUNT);
-                ret = dict_set_str (dict, conf->link_xattr_name, to->name);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to set xattr in dict for %s (linkto:%s)",
-                                loc->path, to->name);
-                        *fop_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = syncop_setxattr (old_target, loc, dict, 0, NULL, NULL);
-                if (ret && -ret != ESTALE && -ret != ENOENT) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "failed to set xattr on %s in %s",
-                                loc->path, old_target->name);
-                        *fop_errno = -ret;
-                        ret = -1;
-                        goto out;
-                } else if (-ret == ESTALE || -ret == ENOENT) {
-                       /* The failure ESTALE indicates that the linkto
-                        * file on the hashed subvol might have been deleted.
-                        * In this case will create a linkto file with new target
-                        * as linkto xattr value*/
-                        linkto_fd = fd_create (loc->inode, DHT_REBALANCE_PID);
-                        if (!linkto_fd) {
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        DHT_MSG_MIGRATE_FILE_FAILED,
-                                        "%s: fd create failed", loc->path);
-                                *fop_errno = ENOMEM;
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = syncop_create (old_target, loc, O_RDWR,
-                                             DHT_LINKFILE_MODE, linkto_fd,
-                                             NULL, dict, NULL);
-                        if (ret != 0 && -ret != EEXIST && -ret != ESTALE) {
-                                *fop_errno = -ret;
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                        DHT_MSG_MIGRATE_FILE_FAILED,
-                                        "failed to create linkto file on %s in %s",
-                                        loc->path, old_target->name);
-                                goto out;
-                        } else if (ret == 0) {
-                                ret = syncop_fsetattr (old_target, linkto_fd, &stbuf,
-                                                       (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
-                                                       NULL, NULL, NULL, NULL);
-                                if (ret < 0) {
-                                        *fop_errno = -ret;
-                                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                                "chown failed for %s on %s",
-                                                loc->path, old_target->name);
-                                }
-                        }
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, -ret,
+                       DHT_MSG_MIGRATE_FILE_FAILED,
+                       "failed to create linkto file on %s in %s", loc->path,
+                       old_target->name);
+                goto out;
+            } else if (ret == 0) {
+                ret = syncop_fsetattr(old_target, linkto_fd, &stbuf,
+                                      (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL,
+                                      NULL, NULL, NULL);
+                if (ret < 0) {
+                    *fop_errno = -ret;
+                    gf_msg(this->name, GF_LOG_ERROR, -ret,
+                           DHT_MSG_MIGRATE_FILE_FAILED,
+                           "chown failed for %s on %s", loc->path,
+                           old_target->name);
                 }
+            }
+        }
+    }
+
+    clean_dst = _gf_false;
+
+    /* Posix acls are not set on DHT linkto files as part of the initial
+     * initial xattrs set on the dst file, so these need
+     * to be set on the dst file after the linkto attrs are removed.
+     * TODO: Optimize this.
+     */
+    if (xattr) {
+        dict_unref(xattr);
+        xattr = NULL;
+    }
+
+    /* Set only the Posix ACLs this time */
+    ret = syncop_getxattr(from, loc, &xattr, POSIX_ACL_ACCESS_XATTR, NULL,
+                          NULL);
+    if (ret < 0) {
+        if ((-ret != ENODATA) && (-ret != ENOATTR)) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: failed to get xattr from %s",
+                   loc->path, from->name);
+            *fop_errno = -ret;
+        }
+    } else {
+        ret = syncop_setxattr(to, loc, xattr, 0, NULL, NULL);
+        if (ret < 0) {
+            /* Potential problem here where Posix ACLs will
+             * not be set on the target file */
+
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: failed to set xattr on %s",
+                   loc->path, to->name);
+            *fop_errno = -ret;
+        }
+    }
+
+    /* The src file is being unlinked after this so we don't need
+       to clean it up */
+    clean_src = _gf_false;
+
+    /* Make the source as a linkfile first before deleting it */
+    empty_iatt.ia_prot.sticky = 1;
+    ret = syncop_fsetattr(from, src_fd, &empty_iatt, GF_SET_ATTR_MODE, NULL,
+                          NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to perform setattr on %s ",
+               loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto metaunlock;
+    }
+
+    /* Free up the data blocks on the source node, as the whole
+        file is migrated */
+    ret = syncop_ftruncate(from, src_fd, 0, NULL, NULL, NULL, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to perform truncate on %s (%s)", loc->path,
+               from->name, strerror(-ret));
+        *fop_errno = -ret;
+    }
+
+    /* remove the 'linkto' xattr from the destination */
+    ret = syncop_fremovexattr(to, dst_fd, conf->link_xattr_name, 0, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to perform removexattr on %s (%s)", loc->path,
+               to->name, strerror(-ret));
+        *fop_errno = -ret;
+    }
+
+    /* Do a stat and check the gfid before unlink */
+
+    /*
+     * Cached file changes its state from non-linkto to linkto file after
+     * migrating data. If lookup from any other mount-point is performed,
+     * converted-linkto-cached file will be treated as a stale and will be
+     * unlinked. But by this time, file is already migrated. So further
+     * failure because of ENOENT should  not be treated as error
+     */
+
+    ret = syncop_stat(from, loc, &empty_iatt, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to do a stat on %s", loc->path, from->name);
+
+        if (-ret != ENOENT) {
+            *fop_errno = -ret;
+            ret = -1;
+            goto metaunlock;
+        }
+
+        rcvd_enoent_from_src = 1;
+    }
+
+    if ((gf_uuid_compare(empty_iatt.ia_gfid, loc->gfid) == 0) &&
+        (!rcvd_enoent_from_src) && delete_src_linkto) {
+        /* take out the source from namespace */
+        ret = syncop_unlink(from, loc, NULL, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to perform unlink on %s", loc->path, from->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto metaunlock;
         }
+    }
 
-        clean_dst = _gf_false;
+    ret = syncop_lookup(this, loc, NULL, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg_debug(this->name, -ret,
+                     "%s: failed to lookup the file on subvolumes", loc->path);
+        *fop_errno = -ret;
+    }
 
-        /* Posix acls are not set on DHT linkto files as part of the initial
-         * initial xattrs set on the dst file, so these need
-         * to be set on the dst file after the linkto attrs are removed.
-         * TODO: Optimize this.
-         */
-        if (xattr) {
-                dict_unref (xattr);
-                xattr = NULL;
-        }
+    gf_msg(this->name, log_level, 0, DHT_MSG_MIGRATE_FILE_COMPLETE,
+           "completed migration of %s from subvolume %s to %s", loc->path,
+           from->name, to->name);
 
-        /* Set only the Posix ACLs this time */
-        ret = syncop_getxattr (from, loc, &xattr, POSIX_ACL_ACCESS_XATTR,
-                               NULL, NULL);
-        if (ret < 0) {
-                if ((-ret != ENODATA) && (-ret != ENOATTR)) {
-                        gf_msg (this->name, GF_LOG_WARNING, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed:"
-                                "%s: failed to get xattr from %s",
-                                loc->path, from->name);
-                        *fop_errno = -ret;
-                }
-        } else {
-                ret = syncop_setxattr (to, loc, xattr, 0, NULL, NULL);
-                if (ret < 0) {
-                        /* Potential problem here where Posix ACLs will
-                         * not be set on the target file */
-
-                        gf_msg (this->name, GF_LOG_WARNING, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed:"
-                                "%s: failed to set xattr on %s",
-                                loc->path, to->name);
-                        *fop_errno = -ret;
-                }
-        }
+    ret = 0;
 
-        /* store size of previous migrated file  */
-        if (defrag && defrag->tier_conf.is_tier) {
-                if (from != TIER_HASHED_SUBVOL) {
-                        defrag->tier_conf.st_last_promoted_size = stbuf.ia_size;
-                } else {
-                        /* Don't delete the linkto file on the hashed subvol */
-                        delete_src_linkto = _gf_false;
-                        defrag->tier_conf.st_last_demoted_size = stbuf.ia_size;
-                }
-        }
+metaunlock:
 
-        /* The src file is being unlinked after this so we don't need
-           to clean it up */
-        clean_src = _gf_false;
+    if (conf->lock_migration_enabled && meta_locked) {
+        dict_del(meta_dict, GF_META_LOCK_KEY);
 
-        /* Make the source as a linkfile first before deleting it */
-        empty_iatt.ia_prot.sticky = 1;
-        ret = syncop_fsetattr (from, src_fd, &empty_iatt,
-                               GF_SET_ATTR_MODE, NULL, NULL, NULL, NULL);
+        ret = dict_set_int32(meta_dict, GF_META_UNLOCK_KEY, 1);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: failed to perform setattr on %s ",
-                        loc->path, from->name);
-                *fop_errno = -ret;
-                ret = -1;
-                goto metaunlock;
-        }
-
-       /* Free up the data blocks on the source node, as the whole
-           file is migrated */
-        ret = syncop_ftruncate (from, src_fd, 0, NULL, NULL, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform truncate on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                *fop_errno = -ret;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace dict_set failed");
 
-        /* remove the 'linkto' xattr from the destination */
-        ret = syncop_fremovexattr (to, dst_fd, conf->link_xattr_name, 0, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform removexattr on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-                *fop_errno = -ret;
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        /* Do a stat and check the gfid before unlink */
-
-        /*
-         * Cached file changes its state from non-linkto to linkto file after
-         * migrating data. If lookup from any other mount-point is performed,
-         * converted-linkto-cached file will be treated as a stale and will be
-         * unlinked. But by this time, file is already migrated. So further
-         * failure because of ENOENT should  not be treated as error
-         */
+        if (clean_dst == _gf_false)
+            ret = dict_set_int32(meta_dict, "status", 1);
+        else
+            ret = dict_set_int32(meta_dict, "status", 0);
 
-        ret = syncop_stat (from, loc, &empty_iatt, NULL, NULL);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: failed to do a stat on %s",
-                        loc->path, from->name);
-
-                if (-ret != ENOENT) {
-                        *fop_errno = -ret;
-                        ret = -1;
-                        goto metaunlock;
-                }
-
-                rcvd_enoent_from_src = 1;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace dict_set failed");
 
-
-        if ((gf_uuid_compare (empty_iatt.ia_gfid, loc->gfid) == 0 ) &&
-            (!rcvd_enoent_from_src) && delete_src_linkto) {
-                /* take out the source from namespace */
-                ret = syncop_unlink (from, loc, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: failed to perform unlink on %s",
-                                loc->path, from->name);
-                        *fop_errno = -ret;
-                        ret = -1;
-                        goto metaunlock;
-                }
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        ret = syncop_lookup (this, loc, NULL, NULL, NULL, NULL);
+        ret = syncop_setxattr(from, loc, meta_dict, 0, NULL, NULL);
         if (ret) {
-                gf_msg_debug (this->name, -ret,
-                              "%s: failed to lookup the file on subvolumes",
-                              loc->path);
-                *fop_errno = -ret;
-        }
-
-        gf_msg (this->name, log_level, 0,
-                DHT_MSG_MIGRATE_FILE_COMPLETE,
-                "completed migration of %s from subvolume %s to %s",
-                loc->path, from->name, to->name);
-
-        ret = 0;
-
-metaunlock:
-
-        if (conf->lock_migration_enabled && meta_locked) {
-
-                dict_del (meta_dict, GF_META_LOCK_KEY);
-
-                ret = dict_set_int32 (meta_dict, GF_META_UNLOCK_KEY, 1);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Trace dict_set failed");
-
-                        *fop_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-
-                if (clean_dst == _gf_false)
-                        ret = dict_set_int32 (meta_dict, "status", 1);
-                else
-                        ret = dict_set_int32 (meta_dict, "status", 0);
-
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Trace dict_set failed");
-
-                        *fop_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace syncop_setxattr meta unlock failed");
 
-                ret = syncop_setxattr (from, loc, meta_dict, 0, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Trace syncop_setxattr meta unlock failed");
-
-                        *fop_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
+    }
 
 out:
-        if (clean_src) {
-                /* Revert source mode and xattr changes*/
-                lk_ret = __dht_migration_cleanup_src_file (this, loc, src_fd,
-                                                        from, &src_ia_prot);
-                if (lk_ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: failed to cleanup source file on %s",
-                                loc->path, from->name);
-                }
-        }
-
-        /* reset the destination back to 0 */
-        if (clean_dst) {
-                lk_ret = syncop_ftruncate (to, dst_fd, 0, NULL, NULL,
-                                           NULL, NULL);
-                if (lk_ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, -lk_ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed: "
-                                "%s: failed to reset target size back to 0",
-                                loc->path);
-                }
-        }
-
-        if (inodelk_locked) {
-                flock.l_type = F_UNLCK;
-
-                lk_ret = syncop_inodelk (from, DHT_FILE_MIGRATE_DOMAIN,
-                                         &tmp_loc, F_SETLK, &flock, NULL, NULL);
-                if (lk_ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, -lk_ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: failed to unlock file on %s",
-                                loc->path, from->name);
-                }
-        }
-
-        if (entrylk_locked) {
-                lk_ret = syncop_entrylk (hashed_subvol, DHT_ENTRY_SYNC_DOMAIN,
-                                         &parent_loc, loc->name, ENTRYLK_UNLOCK,
-                                         ENTRYLK_UNLOCK, NULL, NULL);
-                if (lk_ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, -lk_ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: failed to unlock entrylk on %s",
-                                loc->path, hashed_subvol->name);
-                }
-        }
-
-        if (p_locked) {
-                plock.l_type = F_UNLCK;
-                lk_ret = syncop_lk (from, src_fd, F_SETLK, &plock, NULL, NULL);
-
-                if (lk_ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, -lk_ret,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: failed to unlock file on %s",
-                                loc->path, from->name);
-                }
-        }
-
-        if (!dht_is_tier_xlator (this)) {
-                lk_ret = syncop_removexattr (to, loc,
-                                             GF_PROTECT_FROM_EXTERNAL_WRITES,
-                                             NULL, NULL);
-                if (lk_ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, -lk_ret, 0,
-                                "%s: removexattr failed key %s", loc->path,
-                                GF_CLEAN_WRITE_PROTECTION);
-                }
-        }
-
-        if (dict)
-                dict_unref (dict);
-
-        if (xattr)
-                dict_unref (xattr);
-        if (xattr_rsp)
-                dict_unref (xattr_rsp);
-
-        if (dst_fd)
-                syncop_close (dst_fd);
-        if (src_fd)
-                syncop_close (src_fd);
-        if (linkto_fd)
-                syncop_close (linkto_fd);
-
-        loc_wipe (&tmp_loc);
-        loc_wipe (&parent_loc);
-
-        return ret;
+    if (clean_src) {
+        /* Revert source mode and xattr changes*/
+        lk_ret = __dht_migration_cleanup_src_file(this, loc, src_fd, from,
+                                                  &src_ia_prot);
+        if (lk_ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to cleanup source file on %s", loc->path,
+                   from->name);
+        }
+    }
+
+    /* reset the destination back to 0 */
+    if (clean_dst) {
+        lk_ret = syncop_ftruncate(to, dst_fd, 0, NULL, NULL, NULL, NULL);
+        if (lk_ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed: "
+                   "%s: failed to reset target size back to 0",
+                   loc->path);
+        }
+    }
+
+    if (inodelk_locked) {
+        flock.l_type = F_UNLCK;
+
+        lk_ret = syncop_inodelk(from, DHT_FILE_MIGRATE_DOMAIN, &tmp_loc,
+                                F_SETLK, &flock, NULL, NULL);
+        if (lk_ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to unlock file on %s", loc->path, from->name);
+        }
+    }
+
+    if (entrylk_locked) {
+        lk_ret = syncop_entrylk(hashed_subvol, DHT_ENTRY_SYNC_DOMAIN,
+                                &parent_loc, loc->name, ENTRYLK_UNLOCK,
+                                ENTRYLK_UNLOCK, NULL, NULL);
+        if (lk_ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to unlock entrylk on %s", loc->path,
+                   hashed_subvol->name);
+        }
+    }
+
+    if (p_locked) {
+        plock.l_type = F_UNLCK;
+        lk_ret = syncop_lk(from, src_fd, F_SETLK, &plock, NULL, NULL);
+
+        if (lk_ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to unlock file on %s", loc->path, from->name);
+        }
+    }
+
+    lk_ret = syncop_removexattr(to, loc, GF_PROTECT_FROM_EXTERNAL_WRITES, NULL,
+                                NULL);
+    if (lk_ret && (lk_ret != -ENODATA) && (lk_ret != -ENOATTR)) {
+        gf_msg(this->name, GF_LOG_WARNING, -lk_ret, 0,
+               "%s: removexattr failed key %s", loc->path,
+               GF_PROTECT_FROM_EXTERNAL_WRITES);
+    }
+
+    if (dict)
+        dict_unref(dict);
+
+    if (xattr)
+        dict_unref(xattr);
+    if (xattr_rsp)
+        dict_unref(xattr_rsp);
+
+    if (dst_fd)
+        syncop_close(dst_fd);
+
+    if (src_fd)
+        syncop_close(src_fd);
+    if (linkto_fd)
+        syncop_close(linkto_fd);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    loc_wipe(&tmp_loc);
+    loc_wipe(&parent_loc);
+
+    return ret;
 }
 
 static int
-rebalance_task (void *data)
+rebalance_task(void *data)
 {
-        int           ret   = -1;
-        dht_local_t  *local = NULL;
-        call_frame_t *frame = NULL;
-        int           fop_errno = 0;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    int fop_errno = 0;
 
-        frame = data;
+    frame = data;
 
-        local = frame->local;
+    local = frame->local;
 
-        /* This function is 'synchrounous', hence if it returns,
-           we are done with the task */
-        ret = dht_migrate_file (THIS, &local->loc, local->rebalance.from_subvol,
-                                local->rebalance.target_node, local->flags,
-                                &fop_errno);
+    /* This function is 'synchrounous', hence if it returns,
+       we are done with the task */
+    ret = dht_migrate_file(THIS, &local->loc, local->rebalance.from_subvol,
+                           local->rebalance.target_node, local->flags,
+                           &fop_errno);
 
-        return ret;
+    return ret;
 }
 
 static int
-rebalance_task_completion (int op_ret, call_frame_t *sync_frame, void *data)
+rebalance_task_completion(int op_ret, call_frame_t *sync_frame, void *data)
 {
-        int32_t       op_errno   = EINVAL;
-
-        if (op_ret == -1) {
-                /* Failure of migration process, mostly due to write process.
-                   as we can't preserve the exact errno, lets say there was
-                   no space to migrate-data
-                */
-                op_errno = ENOSPC;
-        } else if (op_ret == 1) {
-                /* migration didn't happen, but is not a failure, let the user
-                   understand that he doesn't have permission to migrate the
-                   file.
-                */
-                op_ret = -1;
-                op_errno = EPERM;
-        } else if (op_ret != 0) {
-                op_errno = -op_ret;
-                op_ret = -1;
-        }
+    int32_t op_errno = EINVAL;
 
-        DHT_STACK_UNWIND (setxattr, sync_frame, op_ret, op_errno, NULL);
-        return 0;
+    if (op_ret == -1) {
+        /* Failure of migration process, mostly due to write process.
+           as we can't preserve the exact errno, lets say there was
+           no space to migrate-data
+        */
+        op_errno = ENOSPC;
+    } else if (op_ret == 1) {
+        /* migration didn't happen, but is not a failure, let the user
+           understand that he doesn't have permission to migrate the
+           file.
+        */
+        op_ret = -1;
+        op_errno = EPERM;
+    } else if (op_ret != 0) {
+        op_errno = -op_ret;
+        op_ret = -1;
+    }
+
+    DHT_STACK_UNWIND(setxattr, sync_frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
 int
-dht_start_rebalance_task (xlator_t *this, call_frame_t *frame)
+dht_start_rebalance_task(xlator_t *this, call_frame_t *frame)
 {
-        int           ret   = -1;
+    int ret = -1;
 
-        ret = synctask_new (this->ctx->env, rebalance_task,
-                            rebalance_task_completion,
-                            frame, frame);
-        return ret;
+    ret = synctask_new(this->ctx->env, rebalance_task,
+                       rebalance_task_completion, frame, frame);
+    return ret;
 }
 
-
 int
-gf_listener_stop (xlator_t *this)
+gf_listener_stop(xlator_t *this)
 {
-        glusterfs_ctx_t  *ctx = NULL;
-        cmd_args_t       *cmd_args = NULL;
-        int              ret = 0;
-
-        ctx = this->ctx;
-        GF_ASSERT (ctx);
-        cmd_args = &ctx->cmd_args;
-        if (cmd_args->sock_file) {
-                ret = sys_unlink (cmd_args->sock_file);
-                if (ret && (ENOENT == errno)) {
-                        ret = 0;
-                }
-        }
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        DHT_MSG_SOCKET_ERROR,
-                        "Failed to unlink listener "
-                        "socket %s", cmd_args->sock_file);
-        }
-        return ret;
+    glusterfs_ctx_t *ctx = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
+
+    ctx = this->ctx;
+    GF_ASSERT(ctx);
+    cmd_args = &ctx->cmd_args;
+    if (cmd_args->sock_file) {
+        ret = sys_unlink(cmd_args->sock_file);
+        if (ret && (ENOENT == errno)) {
+            ret = 0;
+        }
+    }
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, DHT_MSG_SOCKET_ERROR,
+               "Failed to unlink listener "
+               "socket %s",
+               cmd_args->sock_file);
+    }
+    return ret;
 }
 
 void
-dht_build_root_inode (xlator_t *this, inode_t **inode)
+dht_build_root_inode(xlator_t *this, inode_t **inode)
 {
-        inode_table_t    *itable        = NULL;
-        uuid_t            root_gfid     = {0, };
+    inode_table_t *itable = NULL;
+    static uuid_t root_gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
 
-        itable = inode_table_new (0, this);
-        if (!itable)
-                return;
+    itable = inode_table_new(0, this);
+    if (!itable)
+        return;
 
-        root_gfid[15] = 1;
-        *inode = inode_find (itable, root_gfid);
+    *inode = inode_find(itable, root_gfid);
 }
 
 void
-dht_build_root_loc (inode_t *inode, loc_t *loc)
+dht_build_root_loc(inode_t *inode, loc_t *loc)
 {
-        loc->path = "/";
-        loc->inode = inode;
-        loc->inode->ia_type = IA_IFDIR;
-        memset (loc->gfid, 0, 16);
-        loc->gfid[15] = 1;
+    loc->path = "/";
+    loc->inode = inode;
+    loc->inode->ia_type = IA_IFDIR;
+    memset(loc->gfid, 0, 16);
+    loc->gfid[15] = 1;
 }
 
 /* return values: 1 -> error, bug ignore and continue
                   0 -> proceed
                  -1 -> error, handle it */
 int32_t
-gf_defrag_handle_migrate_error (int32_t op_errno, gf_defrag_info_t *defrag)
+gf_defrag_handle_migrate_error(int32_t op_errno, gf_defrag_info_t *defrag)
 {
-        int ret = 0;
-        /* if errno is not ENOTCONN, we can still continue
-           with rebalance process */
-        if (op_errno != ENOTCONN) {
-                ret = 1;
-                goto out;
-        }
-
-        if (op_errno == ENOTCONN) {
-                /* Most probably mount point went missing (mostly due
-                   to a brick down), say rebalance failure to user,
-                   let him restart it if everything is fine */
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                ret = -1;
-                goto out;
-        }
+    int ret = 0;
+    /* if errno is not ENOTCONN, we can still continue
+       with rebalance process */
+    if (op_errno != ENOTCONN) {
+        ret = 1;
+        goto out;
+    }
+
+    if (op_errno == ENOTCONN) {
+        /* Most probably mount point went missing (mostly due
+           to a brick down), say rebalance failure to user,
+           let him restart it if everything is fine */
+        defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+        ret = -1;
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 static gf_boolean_t
-gf_defrag_pattern_match (gf_defrag_info_t *defrag, char *name, uint64_t size)
+gf_defrag_pattern_match(gf_defrag_info_t *defrag, char *name, uint64_t size)
 {
-        gf_defrag_pattern_list_t *trav = NULL;
-        gf_boolean_t               match = _gf_false;
-        gf_boolean_t               ret = _gf_false;
+    gf_defrag_pattern_list_t *trav = NULL;
+    gf_boolean_t match = _gf_false;
+    gf_boolean_t ret = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("dht", defrag, out);
+    GF_VALIDATE_OR_GOTO("dht", defrag, out);
 
-        trav = defrag->defrag_pattern;
-        while (trav) {
-                if (!fnmatch (trav->path_pattern, name, FNM_NOESCAPE)) {
-                        match = _gf_true;
-                        break;
-                }
-                trav = trav->next;
+    trav = defrag->defrag_pattern;
+    while (trav) {
+        if (!fnmatch(trav->path_pattern, name, FNM_NOESCAPE)) {
+            match = _gf_true;
+            break;
         }
+        trav = trav->next;
+    }
 
-        if ((match == _gf_true) && (size >= trav->size))
-                ret = _gf_true;
+    if ((match == _gf_true) && (size >= trav->size))
+        ret = _gf_true;
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
-int dht_dfreaddirp_done (dht_dfoffset_ctx_t *offset_var, int cnt) {
-
-        int i;
-        int result = 1;
+int
+dht_dfreaddirp_done(dht_dfoffset_ctx_t *offset_var, int cnt)
+{
+    int i;
+    int result = 1;
 
-        for (i = 0; i < cnt; i++) {
-                if (offset_var[i].readdir_done == 0) {
-                        result = 0;
-                        break;
-                }
+    for (i = 0; i < cnt; i++) {
+        if (offset_var[i].readdir_done == 0) {
+            result = 0;
+            break;
         }
-        return result;
+    }
+    return result;
 }
 
-int static
-gf_defrag_ctx_subvols_init (dht_dfoffset_ctx_t *offset_var, xlator_t *this) {
-
-        int i;
-        dht_conf_t *conf = NULL;
+int static gf_defrag_ctx_subvols_init(dht_dfoffset_ctx_t *offset_var,
+                                      xlator_t *this)
+{
+    int i;
+    dht_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf)
-               return -1;
+    if (!conf)
+        return -1;
 
-        for (i = 0; i < conf->local_subvols_cnt; i++) {
-               offset_var[i].this = conf->local_subvols[i];
-               offset_var[i].offset = (off_t) 0;
-               offset_var[i].readdir_done = 0;
-        }
+    for (i = 0; i < conf->local_subvols_cnt; i++) {
+        offset_var[i].this = conf->local_subvols[i];
+        offset_var[i].offset = (off_t)0;
+        offset_var[i].readdir_done = 0;
+    }
 
-        return 0;
+    return 0;
 }
 
-
 static int
-dht_get_first_non_null_index (subvol_nodeuuids_info_t *entry)
+dht_get_first_non_null_index(subvol_nodeuuids_info_t *entry)
 {
-        int      i        = 0;
-        int      index    = 0;
+    int i = 0;
+    int index = 0;
 
-        for (i = 0; i < entry->count; i++) {
-                if (!gf_uuid_is_null (entry->elements[i].uuid)) {
-                        index = i;
-                        goto out;
-                }
+    for (i = 0; i < entry->count; i++) {
+        if (!gf_uuid_is_null(entry->elements[i].uuid)) {
+            index = i;
+            goto out;
         }
+    }
 
-        if (i == entry->count) {
-                index = -1;
-        }
+    if (i == entry->count) {
+        index = -1;
+    }
 out:
-        return index;
+    return index;
 }
 
-
 /* Return value
  * 0 : this node does not migrate the file
  * 1 : this node migrates the file
@@ -2688,2779 +2538,2165 @@ out:
  * all hardlinks.
  */
 
-int
-gf_defrag_should_i_migrate (xlator_t *this, int local_subvol_index, uuid_t gfid)
+gf_boolean_t
+gf_defrag_should_i_migrate(xlator_t *this, int local_subvol_index, uuid_t gfid)
 {
-        int         ret               = 0;
-        int         i                 = local_subvol_index;
-        char       *str               = NULL;
-        uint32_t    hashval           = 0;
-        int32_t     index             = 0;
-        dht_conf_t *conf              = NULL;
-        char        buf[UUID_CANONICAL_FORM_LEN + 1] = {0, };
-        subvol_nodeuuids_info_t *entry = NULL;
-
-
-        conf = this->private;
-
-        /* Pure distribute. A subvol in this case
-            will be handled by only one node */
-
-        entry = &(conf->local_nodeuuids[i]);
-        if (entry->count == 1) {
-                return 1;
-        }
-
-        str = uuid_utoa_r (gfid, buf);
-        ret = dht_hash_compute (this, 0, str, &hashval);
-        if (ret == 0) {
-                index = (hashval % entry->count);
-                if (entry->elements[index].info
-                                 == REBAL_NODEUUID_MINE) {
-                        /* Index matches this node's nodeuuid.*/
-                        ret = 1;
-                        goto out;
-                }
-
-                /* Brick down - some other node has to migrate these files*/
-                if (gf_uuid_is_null (entry->elements[index].uuid)) {
-                        /* Fall back to the first non-null index */
-                        index = dht_get_first_non_null_index (entry);
-
-                        if (index == -1) {
-                                /* None of the bricks in the subvol are up.
-                                 * CHILD_DOWN will kill the process soon */
-
-                                return 0;
-                        }
-
-                        if (entry->elements[index].info == REBAL_NODEUUID_MINE) {
-                                /* Index matches this node's nodeuuid.*/
-                                ret = 1;
-                                goto out;
-                        }
-                }
+    gf_boolean_t ret = _gf_false;
+    int i = local_subvol_index;
+    char *str = NULL;
+    uint32_t hashval = 0;
+    int32_t index = 0;
+    dht_conf_t *conf = NULL;
+    char buf[UUID_CANONICAL_FORM_LEN + 1] = {
+        0,
+    };
+    subvol_nodeuuids_info_t *entry = NULL;
+
+    conf = this->private;
+
+    /* Pure distribute. A subvol in this case
+        will be handled by only one node */
+
+    entry = &(conf->local_nodeuuids[i]);
+    if (entry->count == 1) {
+        return 1;
+    }
+
+    str = uuid_utoa_r(gfid, buf);
+    if (dht_hash_compute(this, 0, str, &hashval) == 0) {
+        index = (hashval % entry->count);
+        if (entry->elements[index].info == REBAL_NODEUUID_MINE) {
+            /* Index matches this node's nodeuuid.*/
+            ret = _gf_true;
+            goto out;
+        }
+
+        /* Brick down - some other node has to migrate these files*/
+        if (gf_uuid_is_null(entry->elements[index].uuid)) {
+            /* Fall back to the first non-null index */
+            index = dht_get_first_non_null_index(entry);
+
+            if (index == -1) {
+                /* None of the bricks in the subvol are up.
+                 * CHILD_DOWN will kill the process soon */
+
+                return _gf_false;
+            }
+
+            if (entry->elements[index].info == REBAL_NODEUUID_MINE) {
+                /* Index matches this node's nodeuuid.*/
+                ret = _gf_true;
+                goto out;
+            }
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-gf_defrag_migrate_single_file (void *opaque)
+gf_defrag_migrate_single_file(void *opaque)
 {
-        xlator_t                *this           = NULL;
-        dht_conf_t              *conf           = NULL;
-        gf_defrag_info_t        *defrag         = NULL;
-        int                      ret            = 0;
-        gf_dirent_t             *entry          = NULL;
-        struct timeval           start          = {0,};
-        loc_t                    entry_loc      = {0,};
-        loc_t                   *loc            = NULL;
-        struct iatt              iatt           = {0,};
-        dict_t                  *migrate_data   = NULL;
-        struct timeval           end            = {0,};
-        double                   elapsed        = {0,};
-        struct dht_container    *rebal_entry    = NULL;
-        inode_t                 *inode          = NULL;
-        xlator_t                *hashed_subvol  = NULL;
-        xlator_t                *cached_subvol  = NULL;
-        call_frame_t            *statfs_frame   = NULL;
-        xlator_t                *old_THIS       = NULL;
-        data_t                  *tmp            = NULL;
-        int                      fop_errno      = 0;
-        gf_dht_migrate_data_type_t rebal_type   = GF_DHT_MIGRATE_DATA;
-        char                     value[MAX_REBAL_TYPE_SIZE]    = {0,};
-        struct iatt             *iatt_ptr       = NULL;
-        gf_boolean_t            update_skippedcount = _gf_true;
-        int                     i = 0;
-
-        rebal_entry = (struct dht_container *)opaque;
-        if (!rebal_entry) {
-                gf_log ("DHT", GF_LOG_ERROR, "rebal_entry is NULL");
-                ret = -1;
-                goto out;
-        }
-
-        this = rebal_entry->this;
-
-        conf = this->private;
-
-        defrag = conf->defrag;
-
-        loc = rebal_entry->parent_loc;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    int ret = 0;
+    gf_dirent_t *entry = NULL;
+    struct timeval start = {
+        0,
+    };
+    loc_t entry_loc = {
+        0,
+    };
+    loc_t *loc = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *migrate_data = NULL;
+    struct timeval end = {
+        0,
+    };
+    double elapsed = {
+        0,
+    };
+    struct dht_container *rebal_entry = NULL;
+    inode_t *inode = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    call_frame_t *statfs_frame = NULL;
+    xlator_t *old_THIS = NULL;
+    data_t *tmp = NULL;
+    int fop_errno = 0;
+    gf_dht_migrate_data_type_t rebal_type = GF_DHT_MIGRATE_DATA;
+    char value[MAX_REBAL_TYPE_SIZE] = {
+        0,
+    };
+    struct iatt *iatt_ptr = NULL;
+    gf_boolean_t update_skippedcount = _gf_true;
+    int i = 0;
+    gf_boolean_t should_i_migrate = 0;
+
+    rebal_entry = (struct dht_container *)opaque;
+    if (!rebal_entry) {
+        gf_log("DHT", GF_LOG_ERROR, "rebal_entry is NULL");
+        ret = -1;
+        goto out;
+    }
 
-        migrate_data = rebal_entry->migrate_data;
+    this = rebal_entry->this;
 
-        entry = rebal_entry->df_entry;
-        iatt_ptr = &entry->d_stat;
+    conf = this->private;
 
-        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                ret = -1;
-                goto out;
-        }
+    defrag = conf->defrag;
 
-        if (defrag->stats == _gf_true) {
-                gettimeofday (&start, NULL);
-        }
+    loc = rebal_entry->parent_loc;
 
-        if (defrag->defrag_pattern &&
-            (gf_defrag_pattern_match (defrag, entry->d_name,
-                                      entry->d_stat.ia_size) == _gf_false)) {
-                gf_log (this->name, GF_LOG_ERROR, "pattern_match failed");
-                goto out;
-        }
+    migrate_data = rebal_entry->migrate_data;
 
-        memset (&entry_loc, 0, sizeof (entry_loc));
+    entry = rebal_entry->df_entry;
+    iatt_ptr = &entry->d_stat;
 
-        ret = dht_build_child_loc (this, &entry_loc, loc, entry->d_name);
-        if (ret) {
-                LOCK (&defrag->lock);
-                {
-                        defrag->total_failures += 1;
-                }
-                UNLOCK (&defrag->lock);
+    if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+        ret = -1;
+        goto out;
+    }
 
-                ret = 0;
+    if (defrag->stats == _gf_true) {
+        gettimeofday(&start, NULL);
+    }
 
-                gf_log (this->name, GF_LOG_ERROR, "Child loc build failed");
+    if (defrag->defrag_pattern &&
+        (gf_defrag_pattern_match(defrag, entry->d_name,
+                                 entry->d_stat.ia_size) == _gf_false)) {
+        gf_log(this->name, GF_LOG_ERROR, "pattern_match failed");
+        goto out;
+    }
 
-                goto out;
-        }
+    memset(&entry_loc, 0, sizeof(entry_loc));
 
-        if (!gf_defrag_should_i_migrate (this, rebal_entry->local_subvol_index,
-                                         entry->d_stat.ia_gfid)) {
-                gf_msg_debug (this->name, 0, "Don't migrate %s ",
-                              entry_loc.path);
-                goto out;
+    ret = dht_build_child_loc(this, &entry_loc, loc, entry->d_name);
+    if (ret) {
+        LOCK(&defrag->lock);
+        {
+            defrag->total_failures += 1;
         }
+        UNLOCK(&defrag->lock);
 
-        gf_uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
+        ret = 0;
 
-        gf_uuid_copy (entry_loc.pargfid, loc->gfid);
+        gf_log(this->name, GF_LOG_ERROR, "Child loc build failed");
 
-        ret = syncop_lookup (this, &entry_loc, &iatt, NULL, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed: %s lookup failed",
-                        entry_loc.path);
-                ret = 0;
-                goto out;
-        }
+        goto out;
+    }
 
-        iatt_ptr = &iatt;
+    should_i_migrate = gf_defrag_should_i_migrate(
+        this, rebal_entry->local_subvol_index, entry->d_stat.ia_gfid);
 
-        hashed_subvol = dht_subvol_get_hashed (this, &entry_loc);
-        if (!hashed_subvol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                        "Failed to get hashed subvol for %s",
-                        entry_loc.path);
-                ret = 0;
-                goto out;
-        }
+    gf_uuid_copy(entry_loc.gfid, entry->d_stat.ia_gfid);
 
-        cached_subvol = dht_subvol_get_cached (this, entry_loc.inode);
-        if (!cached_subvol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_CACHED_SUBVOL_GET_FAILED,
-                        "Failed to get cached subvol for %s",
-                        entry_loc.path);
+    gf_uuid_copy(entry_loc.pargfid, loc->gfid);
 
-                ret = 0;
-                goto out;
-        }
+    ret = syncop_lookup(this, &entry_loc, &iatt, NULL, NULL, NULL);
 
-        if (hashed_subvol == cached_subvol) {
-                ret = 0;
-                goto out;
-        }
-
-        inode = inode_link (entry_loc.inode, entry_loc.parent, entry->d_name, &iatt);
-        inode_unref (entry_loc.inode);
-        /* use the inode returned by inode_link */
-        entry_loc.inode = inode;
-
-        old_THIS = THIS;
-        THIS = this;
-        statfs_frame = create_frame (this, this->ctx->pool);
-        if (!statfs_frame) {
-                gf_msg (this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM,
-                        "Insufficient memory. Frame creation failed");
-                ret = -1;
-                goto out;
+    if (!should_i_migrate) {
+        /* this node isn't supposed to migrate the file. suppressing any
+         * potential error from lookup as this file is under migration by
+         * another node */
+        if (ret) {
+            gf_msg_debug(this->name, -ret,
+                         "Ignoring lookup failure: node isn't migrating %s",
+                         entry_loc.path);
+            ret = 0;
+        }
+        gf_msg_debug(this->name, 0, "Don't migrate %s ", entry_loc.path);
+        goto out;
+    }
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: %s lookup failed", entry_loc.path);
+
+        /* Increase failure count only for remove-brick op, so that
+         * user is warned to check the removed-brick for any files left
+         * unmigrated
+         */
+        if (conf->decommission_subvols_cnt) {
+            LOCK(&defrag->lock);
+            {
+                defrag->total_failures += 1;
+            }
+            UNLOCK(&defrag->lock);
         }
 
-        /* async statfs information for honoring min-free-disk */
-        dht_get_du_info (statfs_frame, this, loc);
-        THIS = old_THIS;
+        ret = 0;
+        goto out;
+    }
 
-        tmp = dict_get (migrate_data, GF_XATTR_FILE_MIGRATE_KEY);
-        if (tmp) {
-                memcpy (value, tmp->data, tmp->len);
-                if (strcmp (value, "force") == 0)
-                        rebal_type = GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
+    iatt_ptr = &iatt;
 
-                if (conf->decommission_in_progress)
-                        rebal_type = GF_DHT_MIGRATE_HARDLINK;
-        }
+    hashed_subvol = dht_subvol_get_hashed(this, &entry_loc);
+    if (!hashed_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "Failed to get hashed subvol for %s", entry_loc.path);
+        ret = 0;
+        goto out;
+    }
 
-        ret = dht_migrate_file (this, &entry_loc, cached_subvol,
-                                hashed_subvol, rebal_type, &fop_errno);
-        if (ret == 1) {
-                if (fop_errno == ENOSPC) {
-                        gf_msg_debug (this->name, 0, "migrate-data skipped for"
-                                      " %s due to space constraints",
-                                      entry_loc.path);
-
-                        /* For remove-brick case if the source is not one of the
-                        * removed-brick, do not mark the error as failure */
-                        if (conf->decommission_subvols_cnt) {
-                                for (i = 0; i < conf->subvolume_cnt; i++) {
-                                        if (conf->decommissioned_bricks[i] == cached_subvol) {
-                                                LOCK (&defrag->lock);
-                                                {
-                                                    defrag->total_failures += 1;
-                                                    update_skippedcount = _gf_false;
-                                                }
-                                                UNLOCK (&defrag->lock);
-
-                                                break;
-                                        }
-                                }
-                        }
+    cached_subvol = dht_subvol_get_cached(this, entry_loc.inode);
+    if (!cached_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CACHED_SUBVOL_GET_FAILED,
+               "Failed to get cached subvol for %s", entry_loc.path);
 
-                        if (update_skippedcount) {
-                                LOCK (&defrag->lock);
-                                {
-                                        defrag->skipped += 1;
-                                }
-                                UNLOCK (&defrag->lock);
-
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_MIGRATE_FILE_SKIPPED,
-                                        "File migration skipped for %s.",
-                                        entry_loc.path);
-                        }
+        ret = 0;
+        goto out;
+    }
 
-                } else if (fop_errno == ENOTSUP) {
-                        gf_msg_debug (this->name, 0, "migrate-data skipped for"
-                                      " hardlink %s ", entry_loc.path);
-                        LOCK (&defrag->lock);
+    if (hashed_subvol == cached_subvol) {
+        ret = 0;
+        goto out;
+    }
+
+    inode = inode_link(entry_loc.inode, entry_loc.parent, entry->d_name, &iatt);
+    inode_unref(entry_loc.inode);
+    /* use the inode returned by inode_link */
+    entry_loc.inode = inode;
+
+    old_THIS = THIS;
+    THIS = this;
+    statfs_frame = create_frame(this, this->ctx->pool);
+    if (!statfs_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM,
+               "Insufficient memory. Frame creation failed");
+        ret = -1;
+        goto out;
+    }
+
+    /* async statfs information for honoring min-free-disk */
+    dht_get_du_info(statfs_frame, this, loc);
+    THIS = old_THIS;
+
+    tmp = dict_get(migrate_data, GF_XATTR_FILE_MIGRATE_KEY);
+    if (tmp) {
+        memcpy(value, tmp->data, tmp->len);
+        if (strcmp(value, "force") == 0)
+            rebal_type = GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
+
+        if (conf->decommission_in_progress)
+            rebal_type = GF_DHT_MIGRATE_HARDLINK;
+    }
+
+    ret = dht_migrate_file(this, &entry_loc, cached_subvol, hashed_subvol,
+                           rebal_type, &fop_errno);
+    if (ret == 1) {
+        if (fop_errno == ENOSPC) {
+            gf_msg_debug(this->name, 0,
+                         "migrate-data skipped for"
+                         " %s due to space constraints",
+                         entry_loc.path);
+
+            /* For remove-brick case if the source is not one of the
+             * removed-brick, do not mark the error as failure */
+            if (conf->decommission_subvols_cnt) {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                    if (conf->decommissioned_bricks[i] == cached_subvol) {
+                        LOCK(&defrag->lock);
                         {
-                                defrag->skipped += 1;
+                            defrag->total_failures += 1;
+                            update_skippedcount = _gf_false;
                         }
-                        UNLOCK (&defrag->lock);
+                        UNLOCK(&defrag->lock);
 
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_MIGRATE_FILE_SKIPPED,
-                                "File migration skipped for %s.",
-                                entry_loc.path);
+                        break;
+                    }
                 }
+            }
 
-                ret = 0;
-
-        } else if (ret < 0) {
-                if (fop_errno != EEXIST) {
-                        gf_msg (this->name, GF_LOG_ERROR, fop_errno,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "migrate-data failed for %s", entry_loc.path);
-
-                        LOCK (&defrag->lock);
-                        {
-                                defrag->total_failures += 1;
-                        }
-                        UNLOCK (&defrag->lock);
-
+            if (update_skippedcount) {
+                LOCK(&defrag->lock);
+                {
+                    defrag->skipped += 1;
                 }
+                UNLOCK(&defrag->lock);
 
-                ret = gf_defrag_handle_migrate_error (fop_errno, defrag);
+                gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_MIGRATE_FILE_SKIPPED,
+                       "File migration skipped for %s.", entry_loc.path);
+            }
 
-                if (!ret) {
-                        gf_msg(this->name, GF_LOG_ERROR, fop_errno,
-                               DHT_MSG_MIGRATE_FILE_FAILED,
-                               "migrate-data on %s failed:", entry_loc.path);
-                } else if (ret == 1) {
-                        ret = 0;
-                }
+        } else if (fop_errno == ENOTSUP) {
+            gf_msg_debug(this->name, 0,
+                         "migrate-data skipped for"
+                         " hardlink %s ",
+                         entry_loc.path);
+            LOCK(&defrag->lock);
+            {
+                defrag->skipped += 1;
+            }
+            UNLOCK(&defrag->lock);
 
-                goto out;
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_MIGRATE_FILE_SKIPPED,
+                   "File migration skipped for %s.", entry_loc.path);
         }
 
-        LOCK (&defrag->lock);
-        {
-                defrag->total_files += 1;
-                defrag->total_data += iatt.ia_size;
-        }
-        UNLOCK (&defrag->lock);
-
-        if (defrag->stats == _gf_true) {
-                gettimeofday (&end, NULL);
-                elapsed = (end.tv_sec - start.tv_sec) * 1e6 +
-                          (end.tv_usec - start.tv_usec);
-                gf_log (this->name, GF_LOG_INFO, "Migration of "
-                        "file:%s size:%"PRIu64" bytes took %.2f"
-                        "secs and ret: %d", entry_loc.name,
-                        iatt.ia_size, elapsed/1e6, ret);
+        ret = 0;
+        goto out;
+    } else if (ret < 0) {
+        if (fop_errno != EEXIST) {
+            gf_msg(this->name, GF_LOG_ERROR, fop_errno,
+                   DHT_MSG_MIGRATE_FILE_FAILED, "migrate-data failed for %s",
+                   entry_loc.path);
+
+            LOCK(&defrag->lock);
+            {
+                defrag->total_failures += 1;
+            }
+            UNLOCK(&defrag->lock);
         }
 
+        ret = gf_defrag_handle_migrate_error(fop_errno, defrag);
+
+        if (!ret) {
+            gf_msg(this->name, GF_LOG_ERROR, fop_errno,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "migrate-data on %s failed:", entry_loc.path);
+        } else if (ret == 1) {
+            ret = 0;
+        }
+
+        goto out;
+    }
+
+    LOCK(&defrag->lock);
+    {
+        defrag->total_files += 1;
+        defrag->total_data += iatt.ia_size;
+    }
+    UNLOCK(&defrag->lock);
+
+    if (defrag->stats == _gf_true) {
+        gettimeofday(&end, NULL);
+        elapsed = gf_tvdiff(&start, &end);
+        gf_log(this->name, GF_LOG_INFO,
+               "Migration of "
+               "file:%s size:%" PRIu64
+               " bytes took %.2f"
+               "secs and ret: %d",
+               entry_loc.name, iatt.ia_size, elapsed / 1e6, ret);
+    }
+
 out:
-        if (statfs_frame) {
-                STACK_DESTROY (statfs_frame->root);
-        }
+    if (statfs_frame) {
+        STACK_DESTROY(statfs_frame->root);
+    }
 
-        if (iatt_ptr) {
-                LOCK (&defrag->lock);
-                {
-                        defrag->size_processed += iatt_ptr->ia_size;
-                }
-                UNLOCK (&defrag->lock);
+    if (iatt_ptr) {
+        LOCK(&defrag->lock);
+        {
+            defrag->size_processed += iatt_ptr->ia_size;
         }
-        loc_wipe (&entry_loc);
-
-        return ret;
+        UNLOCK(&defrag->lock);
+    }
+    loc_wipe(&entry_loc);
 
+    return ret;
 }
 
 void *
-gf_defrag_task (void *opaque)
+gf_defrag_task(void *opaque)
 {
-        struct list_head        *q_head         = NULL;
-        struct dht_container    *iterator       = NULL;
-        gf_defrag_info_t        *defrag         = NULL;
-        int                      ret            = 0;
-        pid_t                    pid            = GF_CLIENT_PID_DEFRAG;
-
-        defrag = (gf_defrag_info_t *)opaque;
-        if (!defrag) {
-                gf_msg ("dht", GF_LOG_ERROR, 0, 0, "defrag is NULL");
-                goto out;
+    struct list_head *q_head = NULL;
+    struct dht_container *iterator = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    int ret = 0;
+    pid_t pid = GF_CLIENT_PID_DEFRAG;
+
+    defrag = (gf_defrag_info_t *)opaque;
+    if (!defrag) {
+        gf_msg("dht", GF_LOG_ERROR, 0, 0, "defrag is NULL");
+        goto out;
+    }
+
+    syncopctx_setfspid(&pid);
+
+    q_head = &(defrag->queue[0].list);
+
+    /* The following while loop will dequeue one entry from the defrag->queue
+       under lock. We will update the defrag->global_error only when there
+       is an error which is critical to stop the rebalance process. The stop
+       message will be intimated to other migrator threads by setting the
+       defrag->defrag_status to GF_DEFRAG_STATUS_FAILED.
+
+       In defrag->queue, a low watermark (MIN_MIGRATE_QUEUE_COUNT) is
+       maintained so that crawler does not starve the file migration
+       workers and a high watermark (MAX_MIGRATE_QUEUE_COUNT) so that
+       crawler does not go far ahead in filling up the queue.
+     */
+
+    while (_gf_true) {
+        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+            pthread_cond_broadcast(&defrag->rebalance_crawler_alarm);
+            pthread_cond_broadcast(&defrag->parallel_migration_cond);
+            goto out;
         }
 
-        syncopctx_setfspid (&pid);
+        pthread_mutex_lock(&defrag->dfq_mutex);
+        {
+            /*Throttle down:
+              If the reconfigured count is less than current thread
+              count, then the current thread will sleep */
+
+            /*TODO: Need to refactor the following block to work
+             *under defrag->lock. For now access
+             * defrag->current_thread_count and rthcount under
+             * dfq_mutex lock */
+            while (!defrag->crawl_done && (defrag->recon_thread_count <
+                                           defrag->current_thread_count)) {
+                defrag->current_thread_count--;
+                gf_msg_debug("DHT", 0,
+                             "Thread sleeping. "
+                             "current thread count: %d",
+                             defrag->current_thread_count);
+
+                pthread_cond_wait(&defrag->df_wakeup_thread,
+                                  &defrag->dfq_mutex);
+
+                defrag->current_thread_count++;
+                gf_msg_debug("DHT", 0,
+                             "Thread wokeup. "
+                             "current thread count: %d",
+                             defrag->current_thread_count);
+            }
+
+            if (defrag->q_entry_count) {
+                iterator = list_entry(q_head->next, typeof(*iterator), list);
+
+                gf_msg_debug("DHT", 0,
+                             "picking entry "
+                             "%s",
+                             iterator->df_entry->d_name);
+
+                list_del_init(&(iterator->list));
+
+                defrag->q_entry_count--;
+
+                if ((defrag->q_entry_count < MIN_MIGRATE_QUEUE_COUNT) &&
+                    defrag->wakeup_crawler) {
+                    pthread_cond_broadcast(&defrag->rebalance_crawler_alarm);
+                }
+                pthread_mutex_unlock(&defrag->dfq_mutex);
+                ret = gf_defrag_migrate_single_file((void *)iterator);
 
-        q_head = &(defrag->queue[0].list);
+                /*Critical errors: ENOTCONN and ENOSPACE*/
+                if (ret) {
+                    dht_set_global_defrag_error(defrag, ret);
 
-       /* The following while loop will dequeue one entry from the defrag->queue
-          under lock. We will update the defrag->global_error only when there
-          is an error which is critical to stop the rebalance process. The stop
-          message will be intimated to other migrator threads by setting the
-          defrag->defrag_status to GF_DEFRAG_STATUS_FAILED.
+                    defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
 
-          In defrag->queue, a low watermark (MIN_MIGRATE_QUEUE_COUNT) is
-          maintained so that crawler does not starve the file migration
-          workers and a high watermark (MAX_MIGRATE_QUEUE_COUNT) so that
-          crawler does not go far ahead in filling up the queue.
-        */
+                    pthread_cond_broadcast(&defrag->rebalance_crawler_alarm);
 
-        while (_gf_true) {
+                    pthread_cond_broadcast(&defrag->parallel_migration_cond);
 
-                if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                        pthread_cond_broadcast (
-                                &defrag->rebalance_crawler_alarm);
-                        pthread_cond_broadcast (
-                                &defrag->parallel_migration_cond);
-                        goto out;
+                    goto out;
                 }
 
-                pthread_mutex_lock (&defrag->dfq_mutex);
-                {
-
-                        /*Throttle down:
-                          If the reconfigured count is less than current thread
-                          count, then the current thread will sleep */
-
-                        /*TODO: Need to refactor the following block to work
-                         *under defrag->lock. For now access
-                         * defrag->current_thread_count and rthcount under
-                         * dfq_mutex lock */
-                        while (!defrag->crawl_done &&
-                              (defrag->recon_thread_count <
-                                        defrag->current_thread_count)) {
-                                defrag->current_thread_count--;
-                                gf_msg_debug ("DHT", 0, "Thread sleeping. "
-                                              "current thread count: %d",
-                                              defrag->current_thread_count);
-
-                                pthread_cond_wait (
-                                           &defrag->df_wakeup_thread,
-                                           &defrag->dfq_mutex);
-
-                                defrag->current_thread_count++;
-                                gf_msg_debug ("DHT", 0, "Thread wokeup. "
-                                              "current thread count: %d",
-                                              defrag->current_thread_count);
-
-                        }
+                gf_defrag_free_container(iterator);
+
+                continue;
+            } else {
+                /* defrag->crawl_done flag is set means crawling
+                 file system is done and hence a list_empty when
+                 the above flag is set indicates there are no more
+                 entries to be added to the queue and rebalance is
+                 finished */
+
+                if (!defrag->crawl_done) {
+                    defrag->current_thread_count--;
+                    gf_msg_debug("DHT", 0,
+                                 "Thread "
+                                 "sleeping while  waiting "
+                                 "for migration entries. "
+                                 "current thread  count:%d",
+                                 defrag->current_thread_count);
+
+                    pthread_cond_wait(&defrag->parallel_migration_cond,
+                                      &defrag->dfq_mutex);
+                }
 
-                        if (defrag->q_entry_count) {
-                                iterator = list_entry (q_head->next,
-                                                typeof(*iterator), list);
-
-                                gf_msg_debug ("DHT", 0, "picking entry "
-                                              "%s", iterator->df_entry->d_name);
-
-                                list_del_init (&(iterator->list));
-
-                                defrag->q_entry_count--;
-
-                                if ((defrag->q_entry_count <
-                                        MIN_MIGRATE_QUEUE_COUNT) &&
-                                        defrag->wakeup_crawler) {
-                                        pthread_cond_broadcast (
-                                              &defrag->rebalance_crawler_alarm);
-                                }
-                                pthread_mutex_unlock (&defrag->dfq_mutex);
-                                ret = gf_defrag_migrate_single_file
-                                                        ((void *)iterator);
-
-                                /*Critical errors: ENOTCONN and ENOSPACE*/
-                                if (ret) {
-                                        dht_set_global_defrag_error
-                                                         (defrag, ret);
-
-                                        defrag->defrag_status =
-                                                       GF_DEFRAG_STATUS_FAILED;
-
-                                        pthread_cond_broadcast (
-                                             &defrag->rebalance_crawler_alarm);
-
-                                        pthread_cond_broadcast (
-                                             &defrag->parallel_migration_cond);
-
-                                        goto out;
-                                }
-
-                                gf_defrag_free_container (iterator);
-
-                                continue;
-                        } else {
-
-                        /* defrag->crawl_done flag is set means crawling
-                         file system is done and hence a list_empty when
-                         the above flag is set indicates there are no more
-                         entries to be added to the queue and rebalance is
-                         finished */
-
-                                if (!defrag->crawl_done) {
-
-                                        defrag->current_thread_count--;
-                                        gf_msg_debug ("DHT", 0, "Thread "
-                                                      "sleeping while  waiting "
-                                                      "for migration entries. "
-                                                      "current thread  count:%d",
-                                                      defrag->current_thread_count);
-
-                                        pthread_cond_wait (
-                                           &defrag->parallel_migration_cond,
-                                           &defrag->dfq_mutex);
-                                }
-
-                                if (defrag->crawl_done &&
-                                                 !defrag->q_entry_count) {
-                                        defrag->current_thread_count++;
-                                        gf_msg_debug ("DHT", 0, "Exiting thread");
-
-                                        pthread_cond_broadcast (
-                                             &defrag->parallel_migration_cond);
-                                        goto unlock;
-                                } else {
-                                        defrag->current_thread_count++;
-                                        gf_msg_debug ("DHT", 0, "Thread woke up"
-                                                      " as found migrating entries. "
-                                                      "current thread count:%d",
-                                                      defrag->current_thread_count);
-
-                                        pthread_mutex_unlock
-                                                 (&defrag->dfq_mutex);
-                                        continue;
-                                }
-                        }
+                if (defrag->crawl_done && !defrag->q_entry_count) {
+                    defrag->current_thread_count++;
+                    gf_msg_debug("DHT", 0, "Exiting thread");
 
+                    pthread_cond_broadcast(&defrag->parallel_migration_cond);
+                    goto unlock;
+                } else {
+                    defrag->current_thread_count++;
+                    gf_msg_debug("DHT", 0,
+                                 "Thread woke up"
+                                 " as found migrating entries. "
+                                 "current thread count:%d",
+                                 defrag->current_thread_count);
+
+                    pthread_mutex_unlock(&defrag->dfq_mutex);
+                    continue;
                 }
-unlock:
-                pthread_mutex_unlock (&defrag->dfq_mutex);
-                break;
+            }
         }
+    unlock:
+        pthread_mutex_unlock(&defrag->dfq_mutex);
+        break;
+    }
 out:
-        return NULL;
+    return NULL;
 }
 
-int static
-gf_defrag_get_entry (xlator_t *this, int i, struct dht_container **container,
-                     loc_t *loc, dht_conf_t *conf, gf_defrag_info_t *defrag,
-                     fd_t *fd, dict_t *migrate_data,
-                     struct dir_dfmeta *dir_dfmeta, dict_t *xattr_req,
-                     int *should_commit_hash, int *perrno)
+int static gf_defrag_get_entry(xlator_t *this, int i,
+                               struct dht_container **container, loc_t *loc,
+                               dht_conf_t *conf, gf_defrag_info_t *defrag,
+                               fd_t *fd, dict_t *migrate_data,
+                               struct dir_dfmeta *dir_dfmeta, dict_t *xattr_req,
+                               int *perrno)
 {
-        int                     ret             = -1;
-        char                    is_linkfile     = 0;
-        gf_dirent_t            *df_entry        = NULL;
-        struct dht_container   *tmp_container   = NULL;
+    int ret = 0;
+    char is_linkfile = 0;
+    gf_dirent_t *df_entry = NULL;
+    struct dht_container *tmp_container = NULL;
 
-        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                ret = -1;
-                goto out;
-        }
-
-        if (dir_dfmeta->offset_var[i].readdir_done == 1) {
-                ret = 0;
-                goto out;
-        }
-
-        if (dir_dfmeta->fetch_entries[i] == 1) {
-                ret = syncop_readdirp (conf->local_subvols[i], fd, 131072,
-                                       dir_dfmeta->offset_var[i].offset,
-                                       &(dir_dfmeta->equeue[i]),
-                                       xattr_req, NULL);
-                if (ret == 0) {
-                        dir_dfmeta->offset_var[i].readdir_done = 1;
-                        ret = 0;
-                        goto out;
-                }
+    if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+        ret = -1;
+        goto out;
+    }
 
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, -ret,
-                                DHT_MSG_MIGRATE_DATA_FAILED,
-                                "Readdirp failed. Aborting data migration for "
-                                "directory: %s", loc->path);
-                        *perrno = -ret;
-                        ret = -1;
-                        goto out;
-                }
+    if (dir_dfmeta->offset_var[i].readdir_done == 1) {
+        ret = 0;
+        goto out;
+    }
 
-                if (list_empty (&(dir_dfmeta->equeue[i].list))) {
-                        dir_dfmeta->offset_var[i].readdir_done = 1;
-                        ret = 0;
-                        goto out;
-                }
+    if (dir_dfmeta->fetch_entries[i] == 1) {
+        if (!fd) {
+            dir_dfmeta->fetch_entries[i] = 0;
+            dir_dfmeta->offset_var[i].readdir_done = 1;
+            ret = 0;
+            goto out;
+        }
 
-                dir_dfmeta->fetch_entries[i] = 0;
+        ret = syncop_readdirp(conf->local_subvols[i], fd, 131072,
+                              dir_dfmeta->offset_var[i].offset,
+                              &(dir_dfmeta->equeue[i]), xattr_req, NULL);
+        if (ret == 0) {
+            dir_dfmeta->offset_var[i].readdir_done = 1;
+            ret = 0;
+            goto out;
         }
 
-        while (1) {
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_DATA_FAILED,
+                   "Readdirp failed. Aborting data migration for "
+                   "directory: %s",
+                   loc->path);
+            *perrno = -ret;
+            ret = -1;
+            goto out;
+        }
 
-                if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                        ret = -1;
-                        goto out;
-                }
+        if (list_empty(&(dir_dfmeta->equeue[i].list))) {
+            dir_dfmeta->offset_var[i].readdir_done = 1;
+            ret = 0;
+            goto out;
+        }
 
-                df_entry = list_entry (dir_dfmeta->iterator[i]->next,
-                                       typeof (*df_entry), list);
+        dir_dfmeta->fetch_entries[i] = 0;
+    }
 
-                if (&df_entry->list == dir_dfmeta->head[i]) {
-                        gf_dirent_free (&(dir_dfmeta->equeue[i]));
-                        INIT_LIST_HEAD (&(dir_dfmeta->equeue[i].list));
-                        dir_dfmeta->fetch_entries[i] = 1;
-                        dir_dfmeta->iterator[i] = dir_dfmeta->head[i];
-                        ret = 0;
-                        goto out;
-                }
+    while (1) {
+        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+            ret = -1;
+            goto out;
+        }
 
-                dir_dfmeta->iterator[i] = dir_dfmeta->iterator[i]->next;
+        df_entry = list_entry(dir_dfmeta->iterator[i]->next, typeof(*df_entry),
+                              list);
 
-                dir_dfmeta->offset_var[i].offset = df_entry->d_off;
-                if (!strcmp (df_entry->d_name, ".") ||
-                    !strcmp (df_entry->d_name, ".."))
-                        continue;
+        if (&df_entry->list == dir_dfmeta->head[i]) {
+            gf_dirent_free(&(dir_dfmeta->equeue[i]));
+            INIT_LIST_HEAD(&(dir_dfmeta->equeue[i].list));
+            dir_dfmeta->fetch_entries[i] = 1;
+            dir_dfmeta->iterator[i] = dir_dfmeta->head[i];
+            ret = 0;
+            goto out;
+        }
 
-                if (IA_ISDIR (df_entry->d_stat.ia_type)) {
-                        defrag->size_processed += df_entry->d_stat.ia_size;
-                        continue;
-                }
+        dir_dfmeta->iterator[i] = dir_dfmeta->iterator[i]->next;
 
-                defrag->num_files_lookedup++;
+        dir_dfmeta->offset_var[i].offset = df_entry->d_off;
+        if (!strcmp(df_entry->d_name, ".") || !strcmp(df_entry->d_name, ".."))
+            continue;
 
-                if (defrag->defrag_pattern &&
-                    (gf_defrag_pattern_match (defrag, df_entry->d_name,
-                                              df_entry->d_stat.ia_size)
-                     == _gf_false)) {
-                        defrag->size_processed += df_entry->d_stat.ia_size;
-                        continue;
-                }
+        if (IA_ISDIR(df_entry->d_stat.ia_type)) {
+            defrag->size_processed += df_entry->d_stat.ia_size;
+            continue;
+        }
 
-                is_linkfile = check_is_linkfile (NULL, &df_entry->d_stat,
-                                                 df_entry->dict,
-                                                 conf->link_xattr_name);
+        defrag->num_files_lookedup++;
 
-                if (is_linkfile) {
-                        /* No need to add linkto file to the queue for
-                           migration. Only the actual data file need to
-                           be checked for migration criteria.
-                        */
+        if (defrag->defrag_pattern &&
+            (gf_defrag_pattern_match(defrag, df_entry->d_name,
+                                     df_entry->d_stat.ia_size) == _gf_false)) {
+            defrag->size_processed += df_entry->d_stat.ia_size;
+            continue;
+        }
 
-                        gf_msg_debug (this->name, 0, "Skipping linkfile"
-                                      " %s on subvol: %s", df_entry->d_name,
-                                      conf->local_subvols[i]->name);
-                        continue;
-                }
+        is_linkfile = check_is_linkfile(NULL, &df_entry->d_stat, df_entry->dict,
+                                        conf->link_xattr_name);
 
-                /*Build Container Structure */
+        if (is_linkfile) {
+            /* No need to add linkto file to the queue for
+               migration. Only the actual data file need to
+               be checked for migration criteria.
+            */
 
-                tmp_container =  GF_CALLOC (1, sizeof(struct dht_container),
-                                            gf_dht_mt_container_t);
-                if (!tmp_container) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to allocate "
-                                "memory for container");
-                        ret = -1;
-                        goto out;
-                }
-                tmp_container->df_entry = gf_dirent_for_name (df_entry->d_name);
-                if (!tmp_container->df_entry) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to allocate "
-                                "memory for df_entry");
-                        ret = -1;
-                        goto out;
-                }
+            gf_msg_debug(this->name, 0,
+                         "Skipping linkfile"
+                         " %s on subvol: %s",
+                         df_entry->d_name, conf->local_subvols[i]->name);
+            continue;
+        }
 
-                tmp_container->local_subvol_index = i;
+        /*Build Container Structure */
 
-                tmp_container->df_entry->d_stat = df_entry->d_stat;
+        tmp_container = GF_CALLOC(1, sizeof(struct dht_container),
+                                  gf_dht_mt_container_t);
+        if (!tmp_container) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to allocate "
+                   "memory for container");
+            ret = -1;
+            goto out;
+        }
+        tmp_container->df_entry = gf_dirent_for_name(df_entry->d_name);
+        if (!tmp_container->df_entry) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to allocate "
+                   "memory for df_entry");
+            ret = -1;
+            goto out;
+        }
 
-                tmp_container->df_entry->d_ino  = df_entry->d_ino;
+        tmp_container->local_subvol_index = i;
 
-                tmp_container->df_entry->d_type = df_entry->d_type;
+        tmp_container->df_entry->d_stat = df_entry->d_stat;
 
-                tmp_container->df_entry->d_len  = df_entry->d_len;
+        tmp_container->df_entry->d_ino = df_entry->d_ino;
 
-                tmp_container->parent_loc = GF_CALLOC(1, sizeof(*loc),
-                                                      gf_dht_mt_loc_t);
-                if (!tmp_container->parent_loc) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to allocate "
-                                "memory for loc");
-                        ret = -1;
-                        goto out;
-                }
+        tmp_container->df_entry->d_type = df_entry->d_type;
 
+        tmp_container->df_entry->d_len = df_entry->d_len;
 
-                ret = loc_copy (tmp_container->parent_loc, loc);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "loc_copy failed");
-                        ret = -1;
-                        goto out;
-                }
+        tmp_container->parent_loc = GF_CALLOC(1, sizeof(*loc), gf_dht_mt_loc_t);
+        if (!tmp_container->parent_loc) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to allocate "
+                   "memory for loc");
+            ret = -1;
+            goto out;
+        }
 
-                tmp_container->migrate_data = migrate_data;
+        ret = loc_copy(tmp_container->parent_loc, loc);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "loc_copy failed");
+            ret = -1;
+            goto out;
+        }
 
-                tmp_container->this = this;
+        tmp_container->migrate_data = migrate_data;
 
-                if (df_entry->dict)
-                        tmp_container->df_entry->dict =
-                                        dict_ref (df_entry->dict);
+        tmp_container->this = this;
 
-               /*Build Container Structue >> END*/
+        if (df_entry->dict)
+            tmp_container->df_entry->dict = dict_ref(df_entry->dict);
 
-                ret = 0;
-                goto out;
+        /*Build Container Structure >> END*/
 
-        }
+        ret = 0;
+        goto out;
+    }
 
 out:
-        if (ret == 0) {
-                *container = tmp_container;
-        } else {
-                if (tmp_container) {
-                        gf_defrag_free_container (tmp_container);
-                }
+    if (ret == 0) {
+        *container = tmp_container;
+    } else {
+        if (tmp_container) {
+            gf_defrag_free_container(tmp_container);
         }
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-gf_defrag_process_dir (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
-                       dict_t *migrate_data, int *perrno)
+gf_defrag_process_dir(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                      dict_t *migrate_data, int *perrno)
 {
-        int                      ret               = -1;
-        fd_t                    *fd                = NULL;
-        dht_conf_t              *conf              = NULL;
-        gf_dirent_t              entries;
-        dict_t                  *xattr_req         = NULL;
-        struct timeval           dir_start         = {0,};
-        struct timeval           end               = {0,};
-        double                   elapsed           = {0,};
-        int                      local_subvols_cnt = 0;
-        int                      i                 = 0;
-        int                      j                 = 0;
-        struct  dht_container   *container         = NULL;
-        int                      ldfq_count        = 0;
-        int                      dfc_index         = 0;
-        int                      throttle_up       = 0;
-        struct dir_dfmeta       *dir_dfmeta        = NULL;
-        int                      should_commit_hash = 1;
-
-        gf_log (this->name, GF_LOG_INFO, "migrate data called on %s",
-                loc->path);
-        gettimeofday (&dir_start, NULL);
-
-        conf = this->private;
-        local_subvols_cnt = conf->local_subvols_cnt;
-
-        if (!local_subvols_cnt) {
-                ret = 0;
-                goto out;
-        }
-
-        fd = fd_create (loc->inode, defrag->pid);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
-                ret = -1;
-                goto out;
-        }
-
-        ret = syncop_opendir (this, loc, fd, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_DATA_FAILED,
-                        "Migrate data failed: Failed to open dir %s",
-                        loc->path);
-                *perrno = -ret;
-                ret = -1;
-                goto out;
-        }
-
-        fd_bind (fd);
-        dir_dfmeta = GF_CALLOC (1, sizeof (*dir_dfmeta),
-                                                gf_common_mt_pointer);
-        if (!dir_dfmeta) {
-                gf_log (this->name, GF_LOG_ERROR, "dir_dfmeta is NULL");
-                ret = -1;
-                goto out;
-        }
-
-
-        dir_dfmeta->head = GF_CALLOC (local_subvols_cnt,
-                                      sizeof (*(dir_dfmeta->head)),
-                                      gf_common_mt_pointer);
-        if (!dir_dfmeta->head) {
-                gf_log (this->name, GF_LOG_ERROR, "dir_dfmeta->head is NULL");
-                ret = -1;
-                goto out;
-        }
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    gf_dirent_t entries;
+    dict_t *xattr_req = NULL;
+    struct timeval dir_start = {
+        0,
+    };
+    struct timeval end = {
+        0,
+    };
+    double elapsed = {
+        0,
+    };
+    int local_subvols_cnt = 0;
+    int i = 0;
+    int j = 0;
+    struct dht_container *container = NULL;
+    int ldfq_count = 0;
+    int dfc_index = 0;
+    int throttle_up = 0;
+    struct dir_dfmeta *dir_dfmeta = NULL;
+    xlator_t *old_THIS = NULL;
+
+    gf_log(this->name, GF_LOG_INFO, "migrate data called on %s", loc->path);
+    gettimeofday(&dir_start, NULL);
+
+    conf = this->private;
+    local_subvols_cnt = conf->local_subvols_cnt;
+
+    if (!local_subvols_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        dir_dfmeta->iterator = GF_CALLOC (local_subvols_cnt,
-                                          sizeof (*(dir_dfmeta->iterator)),
-                                          gf_common_mt_pointer);
-        if (!dir_dfmeta->iterator) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "dir_dfmeta->iterator is NULL");
-                ret = -1;
-                goto out;
-        }
+    old_THIS = THIS;
+    THIS = this;
 
-        dir_dfmeta->equeue = GF_CALLOC (local_subvols_cnt, sizeof (entries),
-                                        gf_dht_mt_dirent_t);
-        if (!dir_dfmeta->equeue) {
-                gf_log (this->name, GF_LOG_ERROR, "dir_dfmeta->equeue is NULL");
-                ret = -1;
-                goto out;
-        }
-
-        dir_dfmeta->offset_var = GF_CALLOC (local_subvols_cnt,
-                                            sizeof (dht_dfoffset_ctx_t),
-                                            gf_dht_mt_octx_t);
-        if (!dir_dfmeta->offset_var) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "dir_dfmeta->offset_var is NULL");
-                ret = -1;
-                goto out;
-        }
-        ret = gf_defrag_ctx_subvols_init (dir_dfmeta->offset_var, this);
+    dir_dfmeta = GF_CALLOC(1, sizeof(*dir_dfmeta), gf_common_mt_pointer);
+    if (!dir_dfmeta) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    dir_dfmeta->lfd = GF_CALLOC(local_subvols_cnt, sizeof(fd_t *),
+                                gf_common_mt_pointer);
+    if (!dir_dfmeta->lfd) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_INSUFF_MEMORY,
+                "for dir_dfmeta", NULL);
+        ret = -1;
+        *perrno = ENOMEM;
+        goto out;
+    }
+
+    for (i = 0; i < local_subvols_cnt; i++) {
+        dir_dfmeta->lfd[i] = fd_create(loc->inode, defrag->pid);
+        if (!dir_dfmeta->lfd[i]) {
+            gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_FD_CREATE_FAILED,
+                    NULL);
+            *perrno = ENOMEM;
+            ret = -1;
+            goto out;
+        }
+
+        ret = syncop_opendir(conf->local_subvols[i], loc, dir_dfmeta->lfd[i],
+                             NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "dht_dfoffset_ctx_t"
-                        "initialization failed");
-                ret = -1;
-                goto out;
-        }
+            fd_unref(dir_dfmeta->lfd[i]);
+            dir_dfmeta->lfd[i] = NULL;
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FAILED_TO_OPEN,
+                    "dir: %s", loc->path, "subvol: %s",
+                    conf->local_subvols[i]->name, NULL);
 
-        dir_dfmeta->fetch_entries = GF_CALLOC (local_subvols_cnt,
-                                               sizeof (int), gf_common_mt_int);
-        if (!dir_dfmeta->fetch_entries) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "dir_dfmeta->fetch_entries is NULL");
+            if (conf->decommission_in_progress) {
+                *perrno = -ret;
                 ret = -1;
                 goto out;
+            }
+        } else {
+            fd_bind(dir_dfmeta->lfd[i]);
         }
+    }
 
-        for (i = 0; i < local_subvols_cnt ; i++) {
-                INIT_LIST_HEAD (&(dir_dfmeta->equeue[i].list));
-                dir_dfmeta->head[i]          = &(dir_dfmeta->equeue[i].list);
-                dir_dfmeta->iterator[i]      = dir_dfmeta->head[i];
-                dir_dfmeta->fetch_entries[i] = 1;
-        }
-
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-               gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-               ret = -1;
-               goto out;
-        }
-
-        ret = dict_set_uint32 (xattr_req,
-                               conf->link_xattr_name, 256);
-        if (ret) {
-               gf_log (this->name, GF_LOG_ERROR, "failed to set dict for "
-                       "key: %s", conf->link_xattr_name);
-               ret = -1;
-               goto out;
-        }
-
-        /*
-         Job: Read entries from each local subvol and store the entries
-              in equeue array of linked list. Now pick one entry from the
-              equeue array in a round robin basis and add them to defrag Queue.
-        */
-
-        while (!dht_dfreaddirp_done(dir_dfmeta->offset_var,
-                                         local_subvols_cnt)) {
-
-                pthread_mutex_lock (&defrag->dfq_mutex);
-                {
-
-                      /*Throttle up: If reconfigured count is higher than
-                        current thread count, wake up the sleeping threads
-                        TODO: Need to refactor this. Instead of making the
-                        thread sleep and wake, we should terminate and spawn
-                        threads on-demand*/
-
-                        if (defrag->recon_thread_count >
-                                         defrag->current_thread_count) {
-                                throttle_up =
-                                        (defrag->recon_thread_count -
-                                            defrag->current_thread_count);
-                                for (j = 0; j < throttle_up; j++) {
-                                        pthread_cond_signal (
-                                             &defrag->df_wakeup_thread);
-                                }
-
-                        }
+    dir_dfmeta->head = GF_CALLOC(local_subvols_cnt, sizeof(*(dir_dfmeta->head)),
+                                 gf_common_mt_pointer);
+    if (!dir_dfmeta->head) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->head is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    dir_dfmeta->iterator = GF_CALLOC(local_subvols_cnt,
+                                     sizeof(*(dir_dfmeta->iterator)),
+                                     gf_common_mt_pointer);
+    if (!dir_dfmeta->iterator) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->iterator is NULL");
+        ret = -1;
+        goto out;
+    }
 
-                        while (defrag->q_entry_count >
-                                        MAX_MIGRATE_QUEUE_COUNT) {
-                                defrag->wakeup_crawler = 1;
-                                pthread_cond_wait (
-                                        &defrag->rebalance_crawler_alarm,
-                                        &defrag->dfq_mutex);
-                        }
+    dir_dfmeta->equeue = GF_CALLOC(local_subvols_cnt, sizeof(entries),
+                                   gf_dht_mt_dirent_t);
+    if (!dir_dfmeta->equeue) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->equeue is NULL");
+        ret = -1;
+        goto out;
+    }
 
-                       ldfq_count = defrag->q_entry_count;
+    dir_dfmeta->offset_var = GF_CALLOC(
+        local_subvols_cnt, sizeof(dht_dfoffset_ctx_t), gf_dht_mt_octx_t);
+    if (!dir_dfmeta->offset_var) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->offset_var is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_defrag_ctx_subvols_init(dir_dfmeta->offset_var, this);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "dht_dfoffset_ctx_t"
+               "initialization failed");
+        ret = -1;
+        goto out;
+    }
+
+    dir_dfmeta->fetch_entries = GF_CALLOC(local_subvols_cnt, sizeof(int),
+                                          gf_common_mt_int);
+    if (!dir_dfmeta->fetch_entries) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_INSUFF_MEMORY,
+                "for dir_dfmeta->fetch_entries", NULL);
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < local_subvols_cnt; i++) {
+        INIT_LIST_HEAD(&(dir_dfmeta->equeue[i].list));
+        dir_dfmeta->head[i] = &(dir_dfmeta->equeue[i].list);
+        dir_dfmeta->iterator[i] = dir_dfmeta->head[i];
+        dir_dfmeta->fetch_entries[i] = 1;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set dict for "
+               "key: %s",
+               conf->link_xattr_name);
+        ret = -1;
+        goto out;
+    }
 
-                       if (defrag->wakeup_crawler) {
-                               defrag->wakeup_crawler = 0;
-                       }
+    /*
+     Job: Read entries from each local subvol and store the entries
+          in equeue array of linked list. Now pick one entry from the
+          equeue array in a round robin basis and add them to defrag Queue.
+    */
 
+    while (!dht_dfreaddirp_done(dir_dfmeta->offset_var, local_subvols_cnt)) {
+        pthread_mutex_lock(&defrag->dfq_mutex);
+        {
+            /*Throttle up: If reconfigured count is higher than
+              current thread count, wake up the sleeping threads
+              TODO: Need to refactor this. Instead of making the
+              thread sleep and wake, we should terminate and spawn
+              threads on-demand*/
+
+            if (defrag->recon_thread_count > defrag->current_thread_count) {
+                throttle_up = (defrag->recon_thread_count -
+                               defrag->current_thread_count);
+                for (j = 0; j < throttle_up; j++) {
+                    pthread_cond_signal(&defrag->df_wakeup_thread);
                 }
-                pthread_mutex_unlock (&defrag->dfq_mutex);
-
-                while (ldfq_count <= MAX_MIGRATE_QUEUE_COUNT &&
-                       !dht_dfreaddirp_done(dir_dfmeta->offset_var,
-                                               local_subvols_cnt)) {
-
-                        ret = gf_defrag_get_entry (this, dfc_index, &container,
-                                                   loc, conf, defrag, fd,
-                                                   migrate_data, dir_dfmeta,
-                                                   xattr_req,
-                                                   &should_commit_hash, perrno);
-
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING, "Found "
-                                        "error from gf_defrag_get_entry");
-
-                                ret = -1;
-                                goto out;
-                        }
+            }
 
-                        /* Check if we got an entry, else we need to move the
-                           index to the next subvol */
-                        if (!container) {
-                                GF_CRAWL_INDEX_MOVE(dfc_index,
-                                                    local_subvols_cnt);
-                                continue;
-                        }
+            while (defrag->q_entry_count > MAX_MIGRATE_QUEUE_COUNT) {
+                defrag->wakeup_crawler = 1;
+                pthread_cond_wait(&defrag->rebalance_crawler_alarm,
+                                  &defrag->dfq_mutex);
+            }
 
-                        /* Q this entry in the dfq */
-                        pthread_mutex_lock (&defrag->dfq_mutex);
-                        {
-                                list_add_tail (&container->list,
-                                        &(defrag->queue[0].list));
-                                defrag->q_entry_count++;
-                                ldfq_count = defrag->q_entry_count;
-
-                                gf_msg_debug (this->name, 0, "added "
-                                              "file:%s parent:%s to the queue ",
-                                              container->df_entry->d_name,
-                                              container->parent_loc->path);
-
-                                pthread_cond_signal (
-                                        &defrag->parallel_migration_cond);
-                        }
-                        pthread_mutex_unlock (&defrag->dfq_mutex);
+            ldfq_count = defrag->q_entry_count;
 
-                        GF_CRAWL_INDEX_MOVE(dfc_index, local_subvols_cnt);
-                }
+            if (defrag->wakeup_crawler) {
+                defrag->wakeup_crawler = 0;
+            }
         }
+        pthread_mutex_unlock(&defrag->dfq_mutex);
 
-        gettimeofday (&end, NULL);
-        elapsed = (end.tv_sec - dir_start.tv_sec) * 1e6 +
-                  (end.tv_usec - dir_start.tv_usec);
-        gf_log (this->name, GF_LOG_INFO, "Migration operation on dir %s took "
-                "%.2f secs", loc->path, elapsed/1e6);
-        ret = 0;
-out:
+        while (
+            ldfq_count <= MAX_MIGRATE_QUEUE_COUNT &&
+            !dht_dfreaddirp_done(dir_dfmeta->offset_var, local_subvols_cnt)) {
+            ret = gf_defrag_get_entry(this, dfc_index, &container, loc, conf,
+                                      defrag, dir_dfmeta->lfd[dfc_index],
+                                      migrate_data, dir_dfmeta, xattr_req,
+                                      perrno);
 
-        gf_defrag_free_dir_dfmeta (dir_dfmeta, local_subvols_cnt);
-
-        if (xattr_req)
-                dict_unref(xattr_req);
-
-        if (fd)
-                fd_unref (fd);
+            if (defrag->defrag_status == GF_DEFRAG_STATUS_STOPPED) {
+                goto out;
+            }
 
-        if (ret == 0 && should_commit_hash == 0) {
-                ret = 2;
-        }
+            if (ret) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Found "
+                       "error from gf_defrag_get_entry");
 
-        /* It does not matter if it errored out - this number is
-         * used to calculate rebalance estimated time to complete.
-         * No locking required as dirs are processed by a single thread.
-         */
-        defrag->num_dirs_processed++;
-        return ret;
+                ret = -1;
+                goto out;
+            }
+
+            /* Check if we got an entry, else we need to move the
+               index to the next subvol */
+            if (!container) {
+                GF_CRAWL_INDEX_MOVE(dfc_index, local_subvols_cnt);
+                continue;
+            }
+
+            /* Q this entry in the dfq */
+            pthread_mutex_lock(&defrag->dfq_mutex);
+            {
+                list_add_tail(&container->list, &(defrag->queue[0].list));
+                defrag->q_entry_count++;
+                ldfq_count = defrag->q_entry_count;
+
+                gf_msg_debug(this->name, 0,
+                             "added "
+                             "file:%s parent:%s to the queue ",
+                             container->df_entry->d_name,
+                             container->parent_loc->path);
+
+                pthread_cond_signal(&defrag->parallel_migration_cond);
+            }
+            pthread_mutex_unlock(&defrag->dfq_mutex);
+
+            GF_CRAWL_INDEX_MOVE(dfc_index, local_subvols_cnt);
+        }
+    }
+
+    gettimeofday(&end, NULL);
+    elapsed = gf_tvdiff(&dir_start, &end);
+    gf_log(this->name, GF_LOG_INFO,
+           "Migration operation on dir %s took "
+           "%.2f secs",
+           loc->path, elapsed / 1e6);
+    ret = 0;
+out:
+    THIS = old_THIS;
+    gf_defrag_free_dir_dfmeta(dir_dfmeta, local_subvols_cnt);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    /* It does not matter if it errored out - this number is
+     * used to calculate rebalance estimated time to complete.
+     * No locking required as dirs are processed by a single thread.
+     */
+    defrag->num_dirs_processed++;
+    return ret;
 }
+
 int
-gf_defrag_settle_hash (xlator_t *this, gf_defrag_info_t *defrag,
-                       loc_t *loc, dict_t *fix_layout)
+gf_defrag_settle_hash(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                      dict_t *fix_layout)
 {
-        int     ret;
-        dht_conf_t *conf = NULL;
-        /*
-         * Now we're ready to update the directory commit hash for the volume
-         * root, so that hash miscompares and broadcast lookups can stop.
-         * However, we want to skip that if fix-layout is all we did.  In
-         * that case, we want the miscompares etc. to continue until a real
-         * rebalance is complete.
-         */
-        if (defrag->cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX
-            || defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER ||
-            defrag->cmd == GF_DEFRAG_CMD_DETACH_START) {
-                return 0;
-        }
-
-        conf = this->private;
-        if (!conf) {
-                /*Uh oh
-                 */
-                return -1;
-        }
+    int ret;
+    dht_conf_t *conf = NULL;
+    /*
+     * Now we're ready to update the directory commit hash for the volume
+     * root, so that hash miscompares and broadcast lookups can stop.
+     * However, we want to skip that if fix-layout is all we did.  In
+     * that case, we want the miscompares etc. to continue until a real
+     * rebalance is complete.
+     */
+    if (defrag->cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX ||
+        defrag->cmd == GF_DEFRAG_CMD_DETACH_START) {
+        return 0;
+    }
 
-        if (conf->local_subvols_cnt == 0 || !conf->lookup_optimize) {
-                /* Commit hash updates are only done on local subvolumes and
-                 * only when lookup optmization is needed (for older client
-                 * support)
-                 */
-                return 0;
-        }
+    conf = this->private;
+    if (!conf) {
+        /*Uh oh
+         */
+        return -1;
+    }
 
-        ret = dict_set_uint32 (fix_layout, "new-commit-hash",
-                               defrag->new_commit_hash);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set new-commit-hash");
-                return -1;
-        }
+    if (conf->local_subvols_cnt == 0 || !conf->lookup_optimize) {
+        /* Commit hash updates are only done on local subvolumes and
+         * only when lookup optimization is needed (for older client
+         * support)
+         */
+        return 0;
+    }
 
-        ret = syncop_setxattr (this, loc, fix_layout, 0, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_LAYOUT_FIX_FAILED,
-                        "fix layout on %s failed", loc->path);
+    ret = dict_set_uint32(fix_layout, "new-commit-hash",
+                          defrag->new_commit_hash);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Failed to set new-commit-hash");
+        return -1;
+    }
 
-                if (-ret == ENOENT || -ret == ESTALE) {
-                        /* Dir most likely is deleted */
-                        return 0;
-                }
+    ret = syncop_setxattr(this, loc, fix_layout, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LAYOUT_FIX_FAILED,
+               "fix layout on %s failed", loc->path);
 
-                return -1;
+        if (-ret == ENOENT || -ret == ESTALE) {
+            /* Dir most likely is deleted */
+            return 0;
         }
 
-        /* TBD: find more efficient solution than adding/deleting every time */
-        dict_del(fix_layout, "new-commit-hash");
-
-        return 0;
-}
+        return -1;
+    }
 
+    /* TBD: find more efficient solution than adding/deleting every time */
+    dict_del(fix_layout, "new-commit-hash");
 
+    return 0;
+}
 
-/* Function for doing a named lookup on file inodes during an attach tier
- * So that a hardlink lookup heal i.e gfid to parent gfid lookup heal
- * happens on pre-existing data. This is required so that the ctr database has
- * hardlinks of all the exisitng file in the volume. CTR xlator on the
- * brick/server side does db update/insert of the hardlink on a namelookup.
- * Currently the namedlookup is done synchronous to the fixlayout that is
- * triggered by attach tier. This is not performant, adding more time to
- * fixlayout. The performant approach is record the hardlinks on a compressed
- * datastore and then do the namelookup asynchronously later, giving the ctr db
- * eventual consistency
- * */
 int
-gf_fix_layout_tier_attach_lookup (xlator_t *this,
-                                 loc_t *parent_loc,
-                                 gf_dirent_t *file_dentry)
+gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                     dict_t *fix_layout, dict_t *migrate_data)
 {
-        int                      ret            = -1;
-        dict_t                  *lookup_xdata   = NULL;
-        dht_conf_t              *conf           = NULL;
-        loc_t                    file_loc       = {0,};
-        struct iatt              iatt           = {0,};
-
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, parent_loc, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, file_dentry, out);
+    int ret = -1;
+    loc_t entry_loc = {
+        0,
+    };
+    fd_t *fd = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *tmp = NULL;
+    gf_dirent_t *entry = NULL;
+    gf_boolean_t free_entries = _gf_false;
+    off_t offset = 0;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *linked_inode = NULL, *inode = NULL;
+    dht_conf_t *conf = NULL;
+    int perrno = 0;
+
+    conf = this->private;
+    if (!conf) {
+        ret = -1;
+        goto out;
+    }
 
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    ret = syncop_lookup(this, loc, &iatt, NULL, NULL, NULL);
+    if (ret) {
+        if (strcmp(loc->path, "/") == 0) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_DIR_LOOKUP_FAILED,
+                   "lookup failed for:%s", loc->path);
 
-        if (!parent_loc->inode) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "%s/%s parent is NULL", parent_loc->path,
-                        file_dentry->d_name);
-                goto out;
+            defrag->total_failures++;
+            ret = -1;
+            goto out;
         }
 
+        if (-ret == ENOENT || -ret == ESTALE) {
+            gf_msg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_LOOKUP_FAILED,
+                   "Dir:%s renamed or removed. Skipping", loc->path);
+            if (conf->decommission_subvols_cnt) {
+                defrag->total_failures++;
+            }
+            ret = 0;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_DIR_LOOKUP_FAILED,
+                   "lookup failed for:%s", loc->path);
 
-        conf   = this->private;
-
-        loc_wipe (&file_loc);
-
-        if (gf_uuid_is_null (file_dentry->d_stat.ia_gfid)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "%s/%s gfid not present", parent_loc->path,
-                        file_dentry->d_name);
-                goto out;
+            defrag->total_failures++;
+            goto out;
         }
+    }
 
-        gf_uuid_copy (file_loc.gfid, file_dentry->d_stat.ia_gfid);
+    fd = fd_create(loc->inode, defrag->pid);
+    if (!fd) {
+        gf_log(this->name, GF_LOG_ERROR, "Failed to create fd");
+        ret = -1;
+        goto out;
+    }
 
-        if (gf_uuid_is_null (parent_loc->gfid)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "%s/%s"
-                        " gfid not present", parent_loc->path,
-                        file_dentry->d_name);
-                goto out;
+    ret = syncop_opendir(this, loc, fd, NULL, NULL);
+    if (ret) {
+        if (-ret == ENOENT || -ret == ESTALE) {
+            if (conf->decommission_subvols_cnt) {
+                defrag->total_failures++;
+            }
+            ret = 0;
+            goto out;
         }
 
-        gf_uuid_copy (file_loc.pargfid, parent_loc->gfid);
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to open dir %s, "
+               "err:%d",
+               loc->path, -ret);
 
+        ret = -1;
+        goto out;
+    }
 
-        ret = dht_build_child_loc (this, &file_loc, parent_loc,
-                                                file_dentry->d_name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Child loc build failed");
-                ret = -1;
-                goto out;
-        }
-
-        lookup_xdata = dict_new ();
-        if (!lookup_xdata) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Failed creating lookup dict for %s",
-                        file_dentry->d_name);
-                goto out;
-        }
+    fd_bind(fd);
+    INIT_LIST_HEAD(&entries.list);
 
-        ret = dict_set_int32 (lookup_xdata, CTR_ATTACH_TIER_LOOKUP, 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to set lookup flag");
+    while ((ret = syncop_readdirp(this, fd, 131072, offset, &entries, NULL,
+                                  NULL)) != 0) {
+        if (ret < 0) {
+            if (-ret == ENOENT || -ret == ESTALE) {
+                if (conf->decommission_subvols_cnt) {
+                    defrag->total_failures++;
+                }
+                ret = 0;
                 goto out;
-        }
+            }
 
-        gf_uuid_copy (file_loc.parent->gfid, parent_loc->gfid);
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_READDIR_ERROR,
+                   "readdirp failed for "
+                   "path %s. Aborting fix-layout",
+                   loc->path);
 
-        /* Sending lookup to cold tier only */
-        ret = syncop_lookup (conf->subvolumes[0], &file_loc, &iatt,
-                        NULL, lookup_xdata, NULL);
-        if (ret) {
-                /* If the file does not exist on the cold tier than it must */
-                /* have been discovered on the hot tier. This is not an error. */
-                gf_msg (this->name, GF_LOG_INFO, 0, DHT_MSG_LOG_TIER_STATUS,
-                        "%s lookup to cold tier on attach heal failed", file_loc.path);
-                goto out;
+            ret = -1;
+            goto out;
         }
 
-        ret = 0;
+        if (list_empty(&entries.list))
+            break;
 
-out:
+        free_entries = _gf_true;
 
-        loc_wipe (&file_loc);
+        list_for_each_entry_safe(entry, tmp, &entries.list, list)
+        {
+            if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+                ret = 1;
+                goto out;
+            }
 
-        if (lookup_xdata)
-                dict_unref (lookup_xdata);
+            offset = entry->d_off;
 
-        return ret;
-}
+            if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+                continue;
+            if (!IA_ISDIR(entry->d_stat.ia_type)) {
+                continue;
+            }
+            loc_wipe(&entry_loc);
 
-int
-gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
-                  dict_t *fix_layout, dict_t *migrate_data)
-{
-        int                      ret            = -1;
-        loc_t                    entry_loc      = {0,};
-        fd_t                    *fd             = NULL;
-        gf_dirent_t              entries;
-        gf_dirent_t             *tmp            = NULL;
-        gf_dirent_t             *entry          = NULL;
-        gf_boolean_t             free_entries   = _gf_false;
-        off_t                    offset         = 0;
-        struct iatt              iatt           = {0,};
-        inode_t                 *linked_inode   = NULL, *inode = NULL;
-        dht_conf_t              *conf           = NULL;
-        int                      should_commit_hash = 1;
-        int                      perrno         = 0;
-
-        conf = this->private;
-        if (!conf) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = syncop_lookup (this, loc, &iatt, NULL, NULL, NULL);
-        if (ret) {
-                if (strcmp (loc->path, "/") == 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_DIR_LOOKUP_FAILED,
-                                "lookup failed for:%s", loc->path);
+            ret = dht_build_child_loc(this, &entry_loc, loc, entry->d_name);
+            if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "Child loc"
+                       " build failed for entry: %s",
+                       entry->d_name);
 
-                        defrag->total_failures++;
-                        ret = -1;
-                        goto out;
-                }
+                if (conf->decommission_in_progress) {
+                    defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
 
-                if (-ret == ENOENT || -ret == ESTALE) {
-                        gf_msg (this->name, GF_LOG_INFO, -ret,
-                                DHT_MSG_DIR_LOOKUP_FAILED,
-                                "Dir:%s renamed or removed. Skipping",
-                                loc->path);
-                                ret = 0;
-                                goto out;
+                    goto out;
                 } else {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_DIR_LOOKUP_FAILED,
-                                "lookup failed for:%s", loc->path);
-
-                        defrag->total_failures++;
-                        goto out;
+                    continue;
                 }
-        }
-
-        fd = fd_create (loc->inode, defrag->pid);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
-                ret = -1;
-                goto out;
-        }
-
-        ret = syncop_opendir (this, loc, fd, NULL, NULL);
-        if (ret) {
+            }
+
+            if (gf_uuid_is_null(entry->d_stat.ia_gfid)) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "%s/%s"
+                       " gfid not present",
+                       loc->path, entry->d_name);
+                continue;
+            }
+
+            gf_uuid_copy(entry_loc.gfid, entry->d_stat.ia_gfid);
+
+            /*In case the gfid stored in the inode by inode_link
+             * and the gfid obtained in the lookup differs, then
+             * client3_3_lookup_cbk will return ESTALE and proper
+             * error will be captured
+             */
+
+            linked_inode = inode_link(entry_loc.inode, loc->inode,
+                                      entry->d_name, &entry->d_stat);
+
+            inode = entry_loc.inode;
+            entry_loc.inode = linked_inode;
+            inode_unref(inode);
+
+            if (gf_uuid_is_null(loc->gfid)) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "%s/%s"
+                       " gfid not present",
+                       loc->path, entry->d_name);
+                continue;
+            }
+
+            gf_uuid_copy(entry_loc.pargfid, loc->gfid);
+
+            ret = syncop_lookup(this, &entry_loc, &iatt, NULL, NULL, NULL);
+            if (ret) {
                 if (-ret == ENOENT || -ret == ESTALE) {
-                        ret = 0;
-                        goto out;
-                }
-
-                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s, "
-                        "err:%d", loc->path, -ret);
-
-                ret = -1;
-                goto out;
-        }
-
-        fd_bind (fd);
-        INIT_LIST_HEAD (&entries.list);
-
-        while ((ret = syncop_readdirp (this, fd, 131072, offset, &entries,
-                                       NULL, NULL)) != 0)
-        {
-                if (ret < 0) {
-                        if (-ret == ENOENT || -ret == ESTALE) {
-                                ret = 0;
-                                goto out;
-                        }
+                    gf_msg(this->name, GF_LOG_INFO, -ret,
+                           DHT_MSG_DIR_LOOKUP_FAILED,
+                           "Dir:%s renamed or removed. "
+                           "Skipping",
+                           loc->path);
+                    ret = 0;
+                    if (conf->decommission_subvols_cnt) {
+                        defrag->total_failures++;
+                    }
+                    continue;
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, -ret,
+                           DHT_MSG_DIR_LOOKUP_FAILED, "lookup failed for:%s",
+                           entry_loc.path);
 
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_READDIR_ERROR, "readdirp failed for "
-                                "path %s. Aborting fix-layout", loc->path);
+                    defrag->total_failures++;
 
+                    if (conf->decommission_in_progress) {
+                        defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
                         ret = -1;
                         goto out;
+                    } else {
+                        continue;
+                    }
                 }
+            }
 
-                if (list_empty (&entries.list))
-                        break;
-
-                free_entries = _gf_true;
-
-                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
-                        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                                ret = 1;
-                                goto out;
-                        }
-
-                        offset = entry->d_off;
-
-                        if (!strcmp (entry->d_name, ".") ||
-                            !strcmp (entry->d_name, ".."))
-                                continue;
-                        if (!IA_ISDIR (entry->d_stat.ia_type)) {
-
-                                /* If its a fix layout during the attach
-                                 * tier operation do lookups on files
-                                 * on cold subvolume so that there is a
-                                 * CTR DB Lookup Heal triggered on existing
-                                 * data.
-                                 * */
-                                if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) {
-                                        gf_fix_layout_tier_attach_lookup
-                                                (this, loc, entry);
-                                }
-
-                                continue;
-                        }
-                        loc_wipe (&entry_loc);
-
-                        ret = dht_build_child_loc (this, &entry_loc, loc,
-                                                  entry->d_name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Child loc"
-                                        " build failed for entry: %s",
-                                        entry->d_name);
-
-                                if (conf->decommission_in_progress) {
-                                        defrag->defrag_status =
-                                        GF_DEFRAG_STATUS_FAILED;
-
-                                        goto out;
-                                } else {
-                                        should_commit_hash = 0;
-
-                                        continue;
-                                }
-                        }
-
-                        if (gf_uuid_is_null (entry->d_stat.ia_gfid)) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
-                                        " gfid not present", loc->path,
-                                         entry->d_name);
-                                continue;
-                        }
-
-
-                        gf_uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
-
-                        /*In case the gfid stored in the inode by inode_link
-                         * and the gfid obtained in the lookup differs, then
-                         * client3_3_lookup_cbk will return ESTALE and proper
-                         * error will be captured
-                         */
-
-                        linked_inode = inode_link (entry_loc.inode, loc->inode,
-                                                   entry->d_name,
-                                                   &entry->d_stat);
-
-                        inode = entry_loc.inode;
-                        entry_loc.inode = linked_inode;
-                        inode_unref (inode);
-
-                        if (gf_uuid_is_null (loc->gfid)) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
-                                        " gfid not present", loc->path,
-                                         entry->d_name);
-                                continue;
-                        }
-
-                        gf_uuid_copy (entry_loc.pargfid, loc->gfid);
-
-                        ret = syncop_lookup (this, &entry_loc, &iatt, NULL,
-                                             NULL, NULL);
-                        if (ret) {
-                                if (-ret == ENOENT || -ret == ESTALE) {
-                                        gf_msg (this->name, GF_LOG_INFO, -ret,
-                                                DHT_MSG_DIR_LOOKUP_FAILED,
-                                                "Dir:%s renamed or removed. "
-                                                "Skipping", loc->path);
-                                                ret = 0;
-                                        continue;
-                                } else {
-                                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                                DHT_MSG_DIR_LOOKUP_FAILED,
-                                                "lookup failed for:%s",
-                                                entry_loc.path);
-
-                                        defrag->total_failures++;
-
-                                        if (conf->decommission_in_progress) {
-                                                defrag->defrag_status =
-                                                GF_DEFRAG_STATUS_FAILED;
-                                                ret = -1;
-                                                goto out;
-                                        } else {
-                                                should_commit_hash = 0;
-                                                continue;
-                                        }
-                                }
-                        }
-
-                        /* A return value of 2 means, either process_dir or
-                         * lookup of a dir failed. Hence, don't commit hash
-                         * for the current directory*/
+            /* A return value of 2 means, either process_dir or
+             * lookup of a dir failed. Hence, don't commit hash
+             * for the current directory*/
 
-                        ret = gf_defrag_fix_layout (this, defrag, &entry_loc,
-                                                    fix_layout, migrate_data);
+            ret = gf_defrag_fix_layout(this, defrag, &entry_loc, fix_layout,
+                                       migrate_data);
 
-                        if (ret && ret != 2) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LAYOUT_FIX_FAILED,
-                                        "Fix layout failed for %s",
-                                        entry_loc.path);
-
-                                defrag->total_failures++;
+            if (defrag->defrag_status == GF_DEFRAG_STATUS_STOPPED ||
+                defrag->defrag_status == GF_DEFRAG_STATUS_FAILED) {
+                goto out;
+            }
 
-                                if (conf->decommission_in_progress) {
-                                        defrag->defrag_status =
-                                        GF_DEFRAG_STATUS_FAILED;
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LAYOUT_FIX_FAILED,
+                       "Fix layout failed for %s", entry_loc.path);
 
-                                        goto out;
-                                } else {
-                                        /* Let's not commit-hash if
-                                         * gf_defrag_fix_layout failed*/
-                                        continue;
-                                }
-                        }
-                }
+                defrag->total_failures++;
 
-                gf_dirent_free (&entries);
-                free_entries = _gf_false;
-                INIT_LIST_HEAD (&entries.list);
-        }
+                if (conf->decommission_in_progress) {
+                    defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
 
-        ret = syncop_setxattr (this, loc, fix_layout, 0, NULL, NULL);
-        if (ret) {
-                if (-ret == ENOENT || -ret == ESTALE) {
-                        gf_msg (this->name, GF_LOG_INFO, -ret,
-                                DHT_MSG_LAYOUT_FIX_FAILED,
-                                "Setxattr failed. Dir %s "
-                                "renamed or removed",
-                                loc->path);
-                        ret = 0;
+                    goto out;
                 } else {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_LAYOUT_FIX_FAILED,
-                                "Setxattr failed for %s",
-                                loc->path);
-
-                        defrag->total_failures++;
-
-                        if (conf->decommission_in_progress) {
-                                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                                ret = -1;
-                                goto out;
-                        }
-                 }
-        }
-
-        if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER) &&
-            (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX)) {
-                ret = gf_defrag_process_dir (this, defrag, loc, migrate_data,
-                                             &perrno);
-
-                if (ret && (ret != 2)) {
-                        if (perrno == ENOENT || perrno == ESTALE) {
-                                ret = 0;
-                                goto out;
-                        } else {
-
-                                defrag->total_failures++;
-
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_DEFRAG_PROCESS_DIR_FAILED,
-                                        "gf_defrag_process_dir failed for "
-                                        "directory: %s", loc->path);
-
-                                if (conf->decommission_in_progress) {
-                                        goto out;
-                                }
-
-                                should_commit_hash = 0;
-                        }
-                } else if (ret == 2) {
-                        should_commit_hash = 0;
+                    /* Let's not commit-hash if
+                     * gf_defrag_fix_layout failed*/
+                    continue;
                 }
-        }
-
-        gf_msg_trace (this->name, 0, "fix layout called on %s", loc->path);
-
-        if (should_commit_hash &&
-            gf_defrag_settle_hash (this, defrag, loc, fix_layout) != 0) {
-
+            }
+        }
+
+        gf_dirent_free(&entries);
+        free_entries = _gf_false;
+        INIT_LIST_HEAD(&entries.list);
+    }
+
+    /* A directory layout is fixed only after its subdirs are healed to
+     * any newly added bricks. If the layout is fixed before subdirs are
+     * healed, the newly added brick will get a non-null layout.
+     * Any subdirs which hash to that layout will no longer show up
+     * in a directory listing until they are healed.
+     */
+
+    ret = syncop_setxattr(this, loc, fix_layout, 0, NULL, NULL);
+
+    /* In case of a race where the directory is deleted just before
+     * layout setxattr, the errors are updated in the layout structure.
+     * We can use this information to make a decision whether the directory
+     * is deleted entirely.
+     */
+    if (ret == 0) {
+        ret = dht_dir_layout_error_check(this, loc->inode);
+        ret = -ret;
+    }
+
+    if (ret) {
+        if (-ret == ENOENT || -ret == ESTALE) {
+            gf_msg(this->name, GF_LOG_INFO, -ret, DHT_MSG_LAYOUT_FIX_FAILED,
+                   "Setxattr failed. Dir %s "
+                   "renamed or removed",
+                   loc->path);
+            if (conf->decommission_subvols_cnt) {
                 defrag->total_failures++;
+            }
+            ret = 0;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LAYOUT_FIX_FAILED,
+                   "Setxattr failed for %s", loc->path);
 
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_SETTLE_HASH_FAILED,
-                        "Settle hash failed for %s",
-                         loc->path);
+            defrag->total_failures++;
 
+            if (conf->decommission_in_progress) {
+                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
                 ret = -1;
-
-                if (conf->decommission_in_progress) {
-                        defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
-        if (free_entries)
-                gf_dirent_free (&entries);
-
-        loc_wipe (&entry_loc);
-
-        if (fd)
-                fd_unref (fd);
-
-        if (ret == 0 && should_commit_hash == 0) {
-                ret = 2;
+                goto out;
+            }
         }
+    }
 
-        return ret;
+    if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+        ret = gf_defrag_process_dir(this, defrag, loc, migrate_data, &perrno);
 
-}
-
-
-
-/******************************************************************************
- *                      Tier background Fix layout functions
- ******************************************************************************/
-/* This is the background tier fixlayout thread */
-void *
-gf_tier_do_fix_layout (void *args)
-{
-        gf_tier_fix_layout_arg_t *tier_fix_layout_arg   =  args;
-        int                 ret                         = -1;
-        xlator_t            *this                       = NULL;
-        dht_conf_t          *conf                       = NULL;
-        gf_defrag_info_t    *defrag                     = NULL;
-        dict_t              *dict                       = NULL;
-        loc_t               loc                         = {0,};
-        struct iatt         iatt                        = {0,};
-        struct iatt         parent                      = {0,};
-
-        GF_VALIDATE_OR_GOTO ("tier", tier_fix_layout_arg, out);
-        GF_VALIDATE_OR_GOTO ("tier", tier_fix_layout_arg->this, out);
-        this = tier_fix_layout_arg->this;
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-
-        defrag = conf->defrag;
-        GF_VALIDATE_OR_GOTO (this->name, defrag, out);
-        GF_VALIDATE_OR_GOTO (this->name, defrag->root_inode, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, tier_fix_layout_arg->fix_layout, out);
-
-
-        /* Get Root loc_t */
-        dht_build_root_loc (defrag->root_inode, &loc);
-        ret = syncop_lookup (this, &loc, &iatt, &parent, NULL, NULL);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_REBALANCE_START_FAILED,
-                        "Lookup on root failed.");
-                ret = -1;
+            if (perrno == ENOENT || perrno == ESTALE) {
+                ret = 0;
                 goto out;
-        }
-
+            } else {
+                defrag->total_failures++;
 
-        /* Start the crawl */
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LOG_TIER_STATUS, "Tiering Fixlayout started");
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       DHT_MSG_DEFRAG_PROCESS_DIR_FAILED,
+                       "gf_defrag_process_dir failed for "
+                       "directory: %s",
+                       loc->path);
 
-        ret = gf_defrag_fix_layout (this, defrag, &loc,
-                                    tier_fix_layout_arg->fix_layout, NULL);
-        if (ret && ret != 2) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_FAILED,
-                        "Tiering fixlayout failed.");
-                ret = -1;
-                goto out;
+                if (conf->decommission_in_progress) {
+                    goto out;
+                }
+            }
         }
+    }
 
-        if (ret != 2 && gf_defrag_settle_hash
-                        (this, defrag, &loc,
-                                tier_fix_layout_arg->fix_layout) != 0) {
-                defrag->total_failures++;
-                ret = -1;
-                goto out;
-        }
+    gf_msg_trace(this->name, 0, "fix layout called on %s", loc->path);
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    if (gf_defrag_settle_hash(this, defrag, loc, fix_layout) != 0) {
+        defrag->total_failures++;
 
-        ret = dict_set_str (dict, GF_XATTR_TIER_LAYOUT_FIXED_KEY, "yes");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_REBALANCE_FAILED,
-                        "Failed to set dictionary value: key = %s",
-                        GF_XATTR_TIER_LAYOUT_FIXED_KEY);
-                ret = -1;
-                goto out;
-        }
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SETTLE_HASH_FAILED,
+               "Settle hash failed for %s", loc->path);
 
-        /* Marking the completion of tiering fix layout via a xattr on root */
-        ret = syncop_setxattr (this, &loc, dict, 0, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set tiering fix "
-                        "layout completed xattr on %s", loc.path);
-                ret = -1;
-                goto out;
+        ret = -1;
+
+        if (conf->decommission_in_progress) {
+            defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret && defrag)
-                defrag->total_failures++;
+    if (free_entries)
+        gf_dirent_free(&entries);
 
-        if (dict)
-                dict_unref (dict);
+    loc_wipe(&entry_loc);
 
-        return NULL;
+    if (fd)
+        fd_unref(fd);
+
+    return ret;
 }
 
 int
-gf_tier_start_fix_layout (xlator_t *this,
-                         loc_t *loc,
-                         gf_defrag_info_t *defrag,
-                         dict_t *fix_layout)
+dht_init_local_subvols_and_nodeuuids(xlator_t *this, dht_conf_t *conf,
+                                     loc_t *loc)
 {
-        int ret                                       = -1;
-        dict_t  *tier_dict                            = NULL;
-        gf_tier_fix_layout_arg_t *tier_fix_layout_arg = NULL;
-
-        tier_dict = dict_new ();
-        if (!tier_dict) {
-                gf_log ("tier", GF_LOG_ERROR, "Tier fix layout failed :"
-                        "Creation of tier_dict failed");
-                ret = -1;
-                goto out;
-        }
-
-        /* Check if layout is fixed already */
-        ret = syncop_getxattr (this, loc, &tier_dict,
-                                GF_XATTR_TIER_LAYOUT_FIXED_KEY,
-                                NULL, NULL);
-        if (ret != 0) {
-
-                tier_fix_layout_arg = &defrag->tier_conf.tier_fix_layout_arg;
-
-                /*Fill crawl arguments */
-                tier_fix_layout_arg->this = this;
-                tier_fix_layout_arg->fix_layout = fix_layout;
+    dict_t *dict = NULL;
+    uuid_t *uuid_ptr = NULL;
+    int ret = -1;
+    int i = 0;
+    int j = 0;
+
+    /* Find local subvolumes */
+    ret = syncop_getxattr(this, loc, &dict, GF_REBAL_FIND_LOCAL_SUBVOL, NULL,
+                          NULL);
+    if (ret && (ret != -ENODATA)) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0,
+               "local "
+               "subvolume determination failed with error: %d",
+               -ret);
+        ret = -1;
+        goto out;
+    }
+
+    if (!ret)
+        goto out;
+
+    ret = syncop_getxattr(this, loc, &dict, GF_REBAL_OLD_FIND_LOCAL_SUBVOL,
+                          NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0,
+               "local "
+               "subvolume determination failed with error: %d",
+               -ret);
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 
-                /* Spawn the fix layout thread so that its done in the
-                 * background */
-                ret = gf_thread_create (&tier_fix_layout_arg->thread_id, NULL,
-                                        gf_tier_do_fix_layout,
-                                        tier_fix_layout_arg, "tierfixl");
-                if (ret) {
-                        gf_log ("tier", GF_LOG_ERROR, "Thread creation failed. "
-                                "Background fix layout for tiering will not "
-                                "work.");
-                        defrag->total_failures++;
-                        goto out;
-                }
-        }
-        ret = 0;
 out:
-        if (tier_dict)
-                dict_unref (tier_dict);
-
+    if (ret) {
         return ret;
-}
+    }
 
-void
-gf_tier_clear_fix_layout (xlator_t *this, loc_t *loc, gf_defrag_info_t *defrag)
-{
-        int ret         = -1;
-        dict_t *dict    = NULL;
-
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, defrag, out);
-
-        /* Check if background fixlayout is completed. This is not
-         * multi-process safe i.e there is a possibility that by the time
-         * we move to remove the xattr there it might have been cleared by some
-         * other detach process from other node. We ignore the error if such
-         * a thing happens */
-        ret = syncop_getxattr (this, loc, &dict,
-                        GF_XATTR_TIER_LAYOUT_FIXED_KEY, NULL, NULL);
-        if (ret) {
-                /* Background fixlayout not complete - nothing to clear*/
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Unable to retrieve fixlayout xattr."
-                        "Assume background fix layout not complete");
-                goto out;
-        }
+    for (i = 0; i < conf->local_subvols_cnt; i++) {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "local subvol: "
+               "%s",
+               conf->local_subvols[i]->name);
 
-        ret = syncop_removexattr (this, loc, GF_XATTR_TIER_LAYOUT_FIXED_KEY,
-                                  NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Failed removing tier fix layout "
-                        "xattr from %s", loc->path);
-                goto out;
+        for (j = 0; j < conf->local_nodeuuids[i].count; j++) {
+            uuid_ptr = &(conf->local_nodeuuids[i].elements[j].uuid);
+            gf_msg(this->name, GF_LOG_INFO, 0, 0, "node uuid : %s",
+                   uuid_utoa(*uuid_ptr));
         }
-        ret = 0;
-out:
-        if (dict)
-                dict_unref (dict);
-}
+    }
 
-void
-gf_tier_wait_fix_lookup (gf_defrag_info_t *defrag) {
-        if (defrag->tier_conf.tier_fix_layout_arg.thread_id) {
-                pthread_join (defrag->tier_conf.tier_fix_layout_arg.thread_id,
-                        NULL);
-        }
+    return ret;
 }
-/******************Tier background Fix layout functions END********************/
 
+/* Functions for the rebalance estimates feature */
 
 uint64_t
-gf_defrag_subvol_file_size (xlator_t *this, loc_t *root_loc)
+gf_defrag_subvol_file_size(xlator_t *this, loc_t *root_loc)
 {
-        int ret = -1;
-        struct statvfs buf = {0,};
-
-        if (!this)
-                return 0;
-
-        ret = syncop_statfs (this, root_loc, &buf, NULL, NULL);
-        if (ret) {
-                /* Aargh! */
-                return 0;
-        }
-        return ((buf.f_blocks - buf.f_bfree) * buf.f_frsize);
+    int ret = -1;
+    struct statvfs buf = {
+        0,
+    };
+
+    ret = syncop_statfs(this, root_loc, &buf, NULL, NULL);
+    if (ret) {
+        /* Aargh! */
+        return 0;
+    }
+    return ((buf.f_blocks - buf.f_bfree) * buf.f_frsize);
 }
 
 uint64_t
-gf_defrag_subvol_file_cnt (xlator_t *this, loc_t *root_loc)
+gf_defrag_total_file_size(xlator_t *this, loc_t *root_loc)
 {
-        int ret = -1;
-        struct statvfs buf = {0,};
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    uint64_t size_files = 0;
+    uint64_t total_size = 0;
 
-        if (!this)
-                return 0;
-
-        ret = syncop_statfs (this, root_loc, &buf, NULL, NULL);
-        if (ret) {
-                /* Aargh! */
-                return 0;
-        }
-        return (buf.f_files - buf.f_ffree);
+    conf = this->private;
+    if (!conf) {
+        return 0;
+    }
+
+    for (i = 0; i < conf->local_subvols_cnt; i++) {
+        size_files = gf_defrag_subvol_file_size(conf->local_subvols[i],
+                                                root_loc);
+        total_size += size_files;
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "local subvol: %s,"
+               "cnt = %" PRIu64,
+               conf->local_subvols[i]->name, size_files);
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, 0, "Total size files = %" PRIu64,
+           total_size);
+
+    return total_size;
 }
 
-
-uint64_t
-gf_defrag_total_file_size (xlator_t *this, loc_t *root_loc)
+static void *
+dht_file_counter_thread(void *args)
 {
-        dht_conf_t    *conf  = NULL;
-        int            i     = 0;
-        uint64_t       size_files = 0;
-        uint64_t       total_size = 0;
-
-        conf = this->private;
-        if (!conf) {
-                return 0;
-        }
+    gf_defrag_info_t *defrag = NULL;
+    loc_t root_loc = {
+        0,
+    };
+    struct timespec time_to_wait = {
+        0,
+    };
+    uint64_t tmp_size = 0;
+
+    if (!args)
+        return NULL;
 
-        for (i = 0 ; i < conf->local_subvols_cnt; i++) {
-                size_files = gf_defrag_subvol_file_size (conf->local_subvols[i],
-                                                       root_loc);
-                total_size += size_files;
-                gf_msg (this->name, GF_LOG_INFO, 0, 0, "local subvol: %s,"
-                        "cnt = %"PRIu64, conf->local_subvols[i]->name,
-                        size_files);
-        }
+    defrag = (gf_defrag_info_t *)args;
+    dht_build_root_loc(defrag->root_inode, &root_loc);
 
-        gf_msg (this->name, GF_LOG_INFO, 0, 0,
-                "Total size files = %"PRIu64, total_size);
+    while (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED) {
+        timespec_now(&time_to_wait);
+        time_to_wait.tv_sec += 600;
 
-        return total_size;
-}
+        pthread_mutex_lock(&defrag->fc_mutex);
+        pthread_cond_timedwait(&defrag->fc_wakeup_cond, &defrag->fc_mutex,
+                               &time_to_wait);
 
+        pthread_mutex_unlock(&defrag->fc_mutex);
 
-uint64_t
-gf_defrag_total_file_cnt (xlator_t *this, loc_t *root_loc)
-{
-        dht_conf_t    *conf  = NULL;
-        int            i     = 0;
-        uint64_t       num_files = 0;
-        uint64_t       total_entries = 0;
-
-        conf = this->private;
-        if (!conf) {
-                return 0;
-        }
-
-        for (i = 0 ; i < conf->local_subvols_cnt; i++) {
-                num_files = gf_defrag_subvol_file_cnt (conf->local_subvols[i],
-                                                       root_loc);
-                total_entries += num_files;
-                gf_msg (this->name, GF_LOG_INFO, 0, 0, "local subvol: %s,"
-                        "cnt = %"PRIu64, conf->local_subvols[i]->name,
-                        num_files);
-        }
+        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED)
+            break;
 
-        /* FIXFIXFIX: halve the number of files to negate .glusterfs contents
-           We need a better way to figure this out */
+        tmp_size = gf_defrag_total_file_size(defrag->this, &root_loc);
 
-        total_entries = total_entries/2;
-        if (total_entries > 20000)
-                total_entries += 10000;
+        gf_log("dht", GF_LOG_INFO, "tmp data size =%" PRIu64, tmp_size);
 
-        gf_msg (this->name, GF_LOG_INFO, 0, 0,
-                "Total number of files = %"PRIu64, total_entries);
+        if (!tmp_size) {
+            gf_msg("dht", GF_LOG_ERROR, 0, 0,
+                   "Failed to get "
+                   "the total data size. Unable to estimate "
+                   "time to complete rebalance.");
+        } else {
+            g_totalsize = tmp_size;
+            gf_msg_debug("dht", 0, "total data size =%" PRIu64, g_totalsize);
+        }
+    }
 
-        return total_entries;
+    return NULL;
 }
 
+int
+gf_defrag_estimates_cleanup(xlator_t *this, gf_defrag_info_t *defrag,
+                            pthread_t filecnt_thread)
+{
+    int ret = -1;
+
+    /* Wake up the filecounter thread.
+     * By now the defrag status will no longer be
+     * GF_DEFRAG_STATUS_STARTED so the thread will exit the loop.
+     */
+    pthread_mutex_lock(&defrag->fc_mutex);
+    {
+        pthread_cond_broadcast(&defrag->fc_wakeup_cond);
+    }
+    pthread_mutex_unlock(&defrag->fc_mutex);
+
+    ret = pthread_join(filecnt_thread, NULL);
+    if (ret) {
+        gf_msg("dht", GF_LOG_ERROR, ret, 0,
+               "file_counter_thread: pthread_join failed.");
+        ret = -1;
+    }
+    return ret;
+}
 
+int
+gf_defrag_estimates_init(xlator_t *this, loc_t *loc, pthread_t *filecnt_thread)
+{
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+
+    conf = this->private;
+    defrag = conf->defrag;
+
+    g_totalsize = gf_defrag_total_file_size(this, loc);
+    if (!g_totalsize) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "Failed to get "
+               "the total data size. Unable to estimate "
+               "time to complete rebalance.");
+        goto out;
+    }
+
+    ret = gf_thread_create(filecnt_thread, NULL, dht_file_counter_thread,
+                           (void *)defrag, "dhtfcnt");
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ret, 0,
+               "Failed to "
+               "create the file counter thread ");
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+out:
+    return ret;
+}
 
+/* Init and cleanup functions for parallel file migration*/
 int
-dht_get_local_subvols_and_nodeuuids (xlator_t *this, dht_conf_t *conf,
-                                     loc_t *loc)
+gf_defrag_parallel_migration_init(xlator_t *this, gf_defrag_info_t *defrag,
+                                  pthread_t **tid_array, int *thread_index)
 {
+    int ret = -1;
+    int thread_spawn_count = 0;
+    int index = 0;
+    pthread_t *tid = NULL;
 
-        dict_t                  *dict         = NULL;
-        gf_defrag_info_t        *defrag       = NULL;
-        int                      ret          = -1;
+    if (!defrag)
+        goto out;
 
-        defrag = conf->defrag;
+    /* Initialize global entry queue */
+    defrag->queue = GF_CALLOC(1, sizeof(struct dht_container),
+                              gf_dht_mt_container_t);
 
-        if (defrag->cmd != GF_DEFRAG_CMD_START_TIER) {
-                /* Find local subvolumes */
-                ret = syncop_getxattr (this, loc, &dict,
-                                       GF_REBAL_FIND_LOCAL_SUBVOL,
-                                       NULL, NULL);
-                if (ret && (ret != -ENODATA)) {
+    if (!defrag->queue) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+               "Failed to initialise migration queue");
+        ret = -1;
+        goto out;
+    }
 
-                        gf_msg (this->name, GF_LOG_ERROR, -ret, 0, "local "
-                                "subvolume determination failed with error: %d",
-                                -ret);
-                        ret = -1;
-                        goto out;
-                 }
+    INIT_LIST_HEAD(&(defrag->queue[0].list));
 
-        if (!ret)
-                goto out;
-        }
+    thread_spawn_count = MAX(MAX_REBAL_THREADS, 4);
 
-        ret = syncop_getxattr (this, loc, &dict,
-                               GF_REBAL_OLD_FIND_LOCAL_SUBVOL,
-                               NULL, NULL);
-        if (ret) {
+    gf_msg_debug(this->name, 0, "thread_spawn_count: %d", thread_spawn_count);
 
-                gf_msg (this->name, GF_LOG_ERROR, -ret, 0, "local "
-                        "subvolume determination failed with error: %d",
-                        -ret);
-                ret = -1;
-                goto out;
+    tid = GF_CALLOC(thread_spawn_count, sizeof(pthread_t),
+                    gf_common_mt_pthread_t);
+    if (!tid) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+               "Failed to create migration threads");
+        ret = -1;
+        goto out;
+    }
+    defrag->current_thread_count = thread_spawn_count;
+
+    /*Spawn Threads Here*/
+    while (index < thread_spawn_count) {
+        ret = gf_thread_create(&(tid[index]), NULL, gf_defrag_task,
+                               (void *)defrag, "dhtmig%d", (index + 1) & 0x3ff);
+        if (ret != 0) {
+            gf_msg("DHT", GF_LOG_ERROR, ret, 0, "Thread[%d] creation failed. ",
+                   index);
+            ret = -1;
+            goto out;
+        } else {
+            gf_log("DHT", GF_LOG_INFO,
+                   "Thread[%d] "
+                   "creation successful",
+                   index);
         }
-        ret = 0;
+        index++;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    *thread_index = index;
+    *tid_array = tid;
+
+    return ret;
 }
 
-static void*
-dht_file_counter_thread (void *args)
+int
+gf_defrag_parallel_migration_cleanup(gf_defrag_info_t *defrag,
+                                     pthread_t *tid_array, int thread_index)
 {
-        gf_defrag_info_t *defrag      = NULL;
-        loc_t root_loc                = {0,};
-        struct timespec time_to_wait  = {0,};
-        struct timeval now            = {0,};
-        uint64_t tmp_size             = 0;
-
-
-        if (!args)
-                return NULL;
+    int ret = -1;
+    int i = 0;
 
-        defrag = (gf_defrag_info_t *) args;
-        dht_build_root_loc (defrag->root_inode, &root_loc);
+    if (!defrag)
+        goto out;
 
-        while (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED) {
+    /* Wake up all migration threads */
+    pthread_mutex_lock(&defrag->dfq_mutex);
+    {
+        defrag->crawl_done = 1;
 
-                gettimeofday (&now, NULL);
-                time_to_wait.tv_sec = now.tv_sec + 600;
-                time_to_wait.tv_nsec = 0;
+        pthread_cond_broadcast(&defrag->parallel_migration_cond);
+        pthread_cond_broadcast(&defrag->df_wakeup_thread);
+    }
+    pthread_mutex_unlock(&defrag->dfq_mutex);
 
+    /*Wait for all the threads to complete their task*/
+    for (i = 0; i < thread_index; i++) {
+        pthread_join(tid_array[i], NULL);
+    }
 
-                pthread_mutex_lock (&defrag->fc_mutex);
-                pthread_cond_timedwait (&defrag->fc_wakeup_cond,
-                                        &defrag->fc_mutex,
-                                        &time_to_wait);
+    GF_FREE(tid_array);
 
-                pthread_mutex_unlock (&defrag->fc_mutex);
+    /* Cleanup the migration queue */
+    if (defrag->queue) {
+        gf_dirent_free(defrag->queue[0].df_entry);
+        INIT_LIST_HEAD(&(defrag->queue[0].list));
+    }
 
+    GF_FREE(defrag->queue);
 
-                if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED)
-                        break;
-
-                tmp_size = gf_defrag_total_file_size (defrag->this,
-                                                         &root_loc);
-
-                gf_log ("dht", GF_LOG_INFO,
-                        "tmp data size =%"PRIu64,
-                        tmp_size);
-
-                if (!tmp_size) {
-                        gf_msg ("dht", GF_LOG_ERROR, 0, 0, "Failed to get "
-                                "the total data size. Unable to estimate "
-                                "time to complete rebalance.");
-                } else {
-                        g_totalsize = tmp_size;
-                        gf_msg_debug ("dht", 0,
-                                      "total data size =%"PRIu64,
-                                      g_totalsize);
-                }
-        }
-
-        return NULL;
+    ret = 0;
+out:
+    return ret;
 }
 
-
-
 int
-gf_defrag_start_crawl (void *data)
+gf_defrag_start_crawl(void *data)
 {
-        xlator_t                *this                   = NULL;
-        dht_conf_t              *conf                   = NULL;
-        gf_defrag_info_t        *defrag                 = NULL;
-        int                      ret                    = -1;
-        loc_t                    loc                    = {0,};
-        struct iatt              iatt                   = {0,};
-        struct iatt              parent                 = {0,};
-        dict_t                  *fix_layout             = NULL;
-        dict_t                  *migrate_data           = NULL;
-        dict_t                  *status                 = NULL;
-        glusterfs_ctx_t         *ctx                    = NULL;
-        dht_methods_t           *methods                = NULL;
-        int                      i                      = 0;
-        int                      thread_index           = 0;
-        int                      err                    = 0;
-        int                      thread_spawn_count     = 0;
-        pthread_t               *tid                    = NULL;
-        char                    thread_name[GF_THREAD_NAMEMAX] = {0,};
-        pthread_t                filecnt_thread;
-        gf_boolean_t             is_tier_detach         = _gf_false;
-        call_frame_t            *statfs_frame           = NULL;
-        xlator_t                *old_THIS               = NULL;
-        int                      j                      = 0;
-        gf_boolean_t             fc_thread_started      = _gf_false;
-        uuid_t                  *uuid_ptr               = NULL;
-
-        this = data;
-        if (!this)
-                goto exit;
-
-        ctx = this->ctx;
-        if (!ctx)
-                goto exit;
-
-        conf = this->private;
-        if (!conf)
-                goto exit;
-
-        defrag = conf->defrag;
-        if (!defrag)
-                goto exit;
-
-        gettimeofday (&defrag->start_time, NULL);
-        dht_build_root_inode (this, &defrag->root_inode);
-        if (!defrag->root_inode)
-                goto out;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    dict_t *fix_layout = NULL;
+    dict_t *migrate_data = NULL;
+    dict_t *status = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    call_frame_t *statfs_frame = NULL;
+    xlator_t *old_THIS = NULL;
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    struct iatt parent = {
+        0,
+    };
+    int thread_index = 0;
+    pthread_t *tid = NULL;
+    pthread_t filecnt_thread;
+    gf_boolean_t fc_thread_started = _gf_false;
+
+    this = data;
+    if (!this)
+        goto exit;
+
+    ctx = this->ctx;
+    if (!ctx)
+        goto exit;
+
+    conf = this->private;
+    if (!conf)
+        goto exit;
+
+    defrag = conf->defrag;
+    if (!defrag)
+        goto exit;
+
+    defrag->start_time = gf_time();
+
+    dht_build_root_inode(this, &defrag->root_inode);
+    if (!defrag->root_inode)
+        goto out;
+
+    dht_build_root_loc(defrag->root_inode, &loc);
+
+    /* fix-layout on '/' first */
+
+    ret = syncop_lookup(this, &loc, &iatt, &parent, NULL, NULL);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_REBALANCE_START_FAILED,
+               "Failed to start rebalance: look up on / failed");
+        ret = -1;
+        goto out;
+    }
 
-        dht_build_root_loc (defrag->root_inode, &loc);
+    old_THIS = THIS;
+    THIS = this;
 
-        /* fix-layout on '/' first */
+    statfs_frame = create_frame(this, this->ctx->pool);
+    if (!statfs_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM,
+               "Insufficient memory. Frame creation failed");
+        ret = -1;
+        goto out;
+    }
 
-        ret = syncop_lookup (this, &loc, &iatt, &parent, NULL, NULL);
+    /* async statfs update for honoring min-free-disk */
+    dht_get_du_info(statfs_frame, this, &loc);
+    THIS = old_THIS;
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_REBALANCE_START_FAILED,
-                        "Failed to start rebalance: look up on / failed");
-                ret = -1;
-                goto out;
-        }
+    fix_layout = dict_new();
+    if (!fix_layout) {
+        ret = -1;
+        goto out;
+    }
+
+    /*
+     * Unfortunately, we can't do special xattrs (like fix.layout) and
+     * real ones in the same call currently, and changing it seems
+     * riskier than just doing two calls.
+     */
+
+    gf_log(this->name, GF_LOG_INFO, "%s using commit hash %u", __func__,
+           conf->vol_commit_hash);
+
+    ret = dict_set_uint32(fix_layout, conf->commithash_xattr_name,
+                          conf->vol_commit_hash);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Failed to set %s",
+               conf->commithash_xattr_name);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_setxattr(this, &loc, fix_layout, 0, NULL, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to set commit hash on %s. "
+               "Rebalance cannot proceed.",
+               loc.path);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
+
+    /* We now return to our regularly scheduled program. */
+
+    ret = dict_set_str(fix_layout, GF_XATTR_FIX_LAYOUT_KEY, "yes");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_START_FAILED,
+               "Failed to start rebalance:"
+               "Failed to set dictionary value: key = %s",
+               GF_XATTR_FIX_LAYOUT_KEY);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
 
-        old_THIS = THIS;
-        THIS = this;
+    defrag->new_commit_hash = conf->vol_commit_hash;
 
-        statfs_frame = create_frame (this, this->ctx->pool);
-        if (!statfs_frame) {
-                gf_msg (this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM,
-                        "Insufficient memory. Frame creation failed");
-                ret = -1;
-                goto out;
-        }
+    ret = syncop_setxattr(this, &loc, fix_layout, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_REBALANCE_FAILED,
+               "fix layout on %s failed", loc.path);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
 
-        /* async statfs update for honoring min-free-disk */
-        dht_get_du_info (statfs_frame, this, &loc);
-        THIS = old_THIS;
+    if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+        /* We need to migrate files */
 
-        fix_layout = dict_new ();
-        if (!fix_layout) {
-                ret = -1;
-                goto out;
+        migrate_data = dict_new();
+        if (!migrate_data) {
+            defrag->total_failures++;
+            ret = -1;
+            goto out;
         }
-
-        /*
-         * Unfortunately, we can't do special xattrs (like fix.layout) and
-         * real ones in the same call currently, and changing it seems
-         * riskier than just doing two calls.
-         */
-
-        gf_log (this->name, GF_LOG_INFO, "%s using commit hash %u",
-                __func__, conf->vol_commit_hash);
-
-        ret = dict_set_uint32 (fix_layout, conf->commithash_xattr_name,
-                               conf->vol_commit_hash);
+        ret = dict_set_str(
+            migrate_data, GF_XATTR_FILE_MIGRATE_KEY,
+            (defrag->cmd == GF_DEFRAG_CMD_START_FORCE) ? "force" : "non-force");
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set %s", conf->commithash_xattr_name);
-                defrag->total_failures++;
-                ret = -1;
-                goto out;
+            defrag->total_failures++;
+            ret = -1;
+            goto out;
         }
 
-        ret = syncop_setxattr (this, &loc, fix_layout, 0, NULL, NULL);
+        ret = dht_init_local_subvols_and_nodeuuids(this, conf, &loc);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set commit hash on %s. "
-                        "Rebalance cannot proceed.",
-                        loc.path);
-                defrag->total_failures++;
-                ret = -1;
-                goto out;
+            ret = -1;
+            goto out;
         }
 
-        /* We now return to our regularly scheduled program. */
-
-        ret = dict_set_str (fix_layout, GF_XATTR_FIX_LAYOUT_KEY, "yes");
+        /* Initialise the structures required for parallel migration */
+        ret = gf_defrag_parallel_migration_init(this, defrag, &tid,
+                                                &thread_index);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_REBALANCE_START_FAILED,
-                        "Failed to start rebalance:"
-                        "Failed to set dictionary value: key = %s",
-                        GF_XATTR_FIX_LAYOUT_KEY);
-                defrag->total_failures++;
-                ret = -1;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "Aborting rebalance.");
+            goto out;
         }
 
-        defrag->new_commit_hash = conf->vol_commit_hash;
-
-        ret = syncop_setxattr (this, &loc, fix_layout, 0, NULL, NULL);
+        ret = gf_defrag_estimates_init(this, &loc, &filecnt_thread);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_REBALANCE_FAILED,
-                        "fix layout on %s failed",
-                        loc.path);
-                defrag->total_failures++;
-                ret = -1;
-                goto out;
-        }
-
-        if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
-                migrate_data = dict_new ();
-                if (!migrate_data) {
-                        defrag->total_failures++;
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_str (migrate_data, GF_XATTR_FILE_MIGRATE_KEY,
-                        (defrag->cmd == GF_DEFRAG_CMD_START_FORCE)
-                        ?  "force" : "non-force");
-                if (ret) {
-                        defrag->total_failures++;
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dht_get_local_subvols_and_nodeuuids (this, conf, &loc);
-                if (ret) {
-                        ret = -1;
-                        goto out;
-                }
-
-                for (i = 0 ; i < conf->local_subvols_cnt; i++) {
-                        gf_msg (this->name, GF_LOG_INFO, 0, 0, "local subvols "
-                                "are %s", conf->local_subvols[i]->name);
-
-                        for (j = 0; j < conf->local_nodeuuids[i].count; j++) {
-                                uuid_ptr = &(conf->local_nodeuuids[i].elements[j].uuid);
-                                gf_msg (this->name, GF_LOG_INFO, 0, 0,
-                                        "node uuids are %s",
-                                        uuid_utoa(*uuid_ptr));
-                        }
-                }
-
-                g_totalsize = gf_defrag_total_file_size (this, &loc);
-                if (!g_totalsize) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0, "Failed to get "
-                                "the total data size. Unable to estimate "
-                                "time to complete rebalance.");
-                }
-
-                g_totalfiles = gf_defrag_total_file_cnt (this, &loc);
-                if (!g_totalfiles) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0, "Failed to get "
-                                "the total number of files. Unable to estimate "
-                                "time to complete rebalance.");
-                }
-
-                ret = gf_thread_create (&filecnt_thread, NULL,
-                                        &dht_file_counter_thread,
-                                        (void *)defrag, "dhtfcnt");
-
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, ret, 0, "Failed to "
-                                "create the file counter thread ");
-                        ret = 0;
-                } else {
-                        fc_thread_started = _gf_true;
-                }
-
-
-                /* Initialize global entry queue */
-                defrag->queue = GF_CALLOC (1, sizeof (struct dht_container),
-                                           gf_dht_mt_container_t);
-
-                if (!defrag->queue) {
-                        gf_log (this->name, GF_LOG_ERROR, "No memory for "
-                                "queue");
-                        ret = -1;
-                        goto out;
-                }
-
-                INIT_LIST_HEAD (&(defrag->queue[0].list));
-
-                thread_spawn_count = MAX (MAX_REBAL_THREADS, 4);
-
-                gf_msg_debug (this->name, 0, "thread_spawn_count: %d",
-                              thread_spawn_count);
-
-                tid = GF_CALLOC (thread_spawn_count, sizeof (pthread_t),
-                                 gf_common_mt_pthread_t);
-                if (!tid) {
-                        gf_log (this->name, GF_LOG_ERROR, "Insufficient memory "
-                                "for tid");
-                        ret = -1;
-                        goto out;
-                }
-
-                defrag->current_thread_count = thread_spawn_count;
-
-                /*Spawn Threads Here*/
-                while (thread_index < thread_spawn_count) {
-                        snprintf (thread_name, sizeof(thread_name),
-                                  "%s%d", "dhtdf", thread_index + 1);
-                        err = gf_thread_create (&(tid[thread_index]), NULL,
-                                                &gf_defrag_task, (void *)defrag,
-                                                thread_name);
-                        if (err != 0) {
-                                gf_log ("DHT", GF_LOG_ERROR,
-                                        "Thread[%d] creation failed. "
-                                        "Aborting Rebalance",
-                                         thread_index);
-                                ret = -1;
-                                goto out;
-                        } else {
-                                gf_log ("DHT", GF_LOG_INFO, "Thread[%d] "
-                                        "creation successful", thread_index);
-                        }
-                        thread_index++;
-                }
-        }
-
-        if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) {
-                /* Fix layout for attach tier */
-                ret = gf_tier_start_fix_layout (this, &loc, defrag, fix_layout);
-                if (ret) {
-                        goto out;
-                }
-
-                methods = &(conf->methods);
-
-                /* Calling tier_start of tier.c */
-                methods->migration_other(this, defrag);
-                if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER ||
-                    defrag->cmd == GF_DEFRAG_CMD_DETACH_START) {
-
-                        ret = dict_set_str (migrate_data,
-                                            GF_XATTR_FILE_MIGRATE_KEY,
-                                            "force");
-                        if (ret)
-                                goto out;
-
-                }
+            /* Not a fatal error. Allow the rebalance to proceed*/
+            ret = 0;
         } else {
-                ret = gf_defrag_fix_layout (this, defrag, &loc, fix_layout,
-                                    migrate_data);
-                if (ret && ret != 2) {
-                        defrag->total_failures++;
-                        ret = -1;
-                        goto out;
-                }
-
-                if (ret != 2 && gf_defrag_settle_hash
-                        (this, defrag, &loc, fix_layout) != 0) {
-                        defrag->total_failures++;
-                        ret = -1;
-                        goto out;
-                }
-
-                if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER ||
-                    defrag->cmd == GF_DEFRAG_CMD_DETACH_START)
-                        is_tier_detach = _gf_true;
-
+            fc_thread_started = _gf_true;
         }
+    }
 
-        gf_log ("DHT", GF_LOG_INFO, "crawling file-system completed");
-out:
-
-        /* We are here means crawling the entire file system is done
-           or something failed. Set defrag->crawl_done flag to intimate
-           the migrator threads to exhaust the defrag->queue and terminate*/
-
-        if (ret) {
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-        }
-
-        pthread_mutex_lock (&defrag->dfq_mutex);
-        {
-                defrag->crawl_done = 1;
-
-                pthread_cond_broadcast (
-                        &defrag->parallel_migration_cond);
-                pthread_cond_broadcast (
-                        &defrag->df_wakeup_thread);
-        }
-        pthread_mutex_unlock (&defrag->dfq_mutex);
-
-        /*Wait for all the threads to complete their task*/
-        for (i = 0; i < thread_index; i++) {
-                pthread_join (tid[i], NULL);
-        }
-
-        GF_FREE (tid);
+    ret = gf_defrag_fix_layout(this, defrag, &loc, fix_layout, migrate_data);
+    if (ret) {
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
 
-        if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) {
-                /* Wait for the tier fixlayout to
-                 * complete if its was started.*/
-                 gf_tier_wait_fix_lookup (defrag);
-        }
+    if (gf_defrag_settle_hash(this, defrag, &loc, fix_layout) != 0) {
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
 
-        if (is_tier_detach && ret == 0) {
-                /* If it was a detach remove the tier fix-layout
-                * xattr on root. Ignoring the failure, as nothing has to be
-                * done, logging is done in gf_tier_clear_fix_layout */
-                gf_tier_clear_fix_layout (this, &loc, defrag);
-        }
+    gf_log("DHT", GF_LOG_INFO, "crawling file-system completed");
+out:
 
-        if (defrag->queue) {
-                gf_dirent_free (defrag->queue[0].df_entry);
-                INIT_LIST_HEAD (&(defrag->queue[0].list));
-        }
+    /* We are here means crawling the entire file system is done
+       or something failed. Set defrag->crawl_done flag to intimate
+       the migrator threads to exhaust the defrag->queue and terminate*/
 
-        if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) &&
-            (defrag->defrag_status != GF_DEFRAG_STATUS_FAILED)) {
-                defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
-        }
+    if (ret) {
+        defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+    }
 
-        if (fc_thread_started) {
-                pthread_mutex_lock (&defrag->fc_mutex);
-                {
-                        pthread_cond_broadcast (&defrag->fc_wakeup_cond);
-                }
-                pthread_mutex_unlock (&defrag->fc_mutex);
+    gf_defrag_parallel_migration_cleanup(defrag, tid, thread_index);
 
-                pthread_join (filecnt_thread, NULL);
-        }
+    if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) &&
+        (defrag->defrag_status != GF_DEFRAG_STATUS_FAILED)) {
+        defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
+    }
 
-        dht_send_rebalance_event (this, defrag->cmd, defrag->defrag_status);
+    if (fc_thread_started) {
+        gf_defrag_estimates_cleanup(this, defrag, filecnt_thread);
+    }
 
-        LOCK (&defrag->lock);
-        {
-                status = dict_new ();
-                gf_defrag_status_get (conf, status);
-                if (ctx && ctx->notify)
-                        ctx->notify (GF_EN_DEFRAG_STATUS, status);
-                if (status)
-                        dict_unref (status);
-                defrag->is_exiting = 1;
-        }
-        UNLOCK (&defrag->lock);
+    dht_send_rebalance_event(this, defrag->cmd, defrag->defrag_status);
 
-        GF_FREE (defrag->queue);
+    status = dict_new();
+    LOCK(&defrag->lock);
+    {
+        gf_defrag_status_get(conf, status);
+        if (ctx && ctx->notify)
+            ctx->notify(GF_EN_DEFRAG_STATUS, status);
+        if (status)
+            dict_unref(status);
+        defrag->is_exiting = 1;
+    }
+    UNLOCK(&defrag->lock);
 
-        GF_FREE (defrag);
-        conf->defrag = NULL;
+    GF_FREE(defrag);
+    conf->defrag = NULL;
 
-        if (migrate_data)
-                dict_unref (migrate_data);
+    if (migrate_data)
+        dict_unref(migrate_data);
 
-        if (statfs_frame) {
-                STACK_DESTROY (statfs_frame->root);
-        }
+    if (statfs_frame) {
+        STACK_DESTROY(statfs_frame->root);
+    }
 exit:
-        return ret;
+    return ret;
 }
 
-
-
 static int
-gf_defrag_done  (int ret, call_frame_t *sync_frame, void *data)
+gf_defrag_done(int ret, call_frame_t *sync_frame, void *data)
 {
-        gf_listener_stop (sync_frame->this);
+    gf_listener_stop(sync_frame->this);
 
-        STACK_DESTROY (sync_frame->root);
-        kill (getpid(), SIGTERM);
-        return 0;
+    STACK_DESTROY(sync_frame->root);
+    kill(getpid(), SIGTERM);
+    return 0;
 }
 
 void *
-gf_defrag_start (void *data)
-{
-        int                      ret    = -1;
-        call_frame_t            *frame  = NULL;
-        dht_conf_t              *conf   = NULL;
-        gf_defrag_info_t        *defrag = NULL;
-        xlator_t                *this  = NULL;
-        xlator_t                *old_THIS = NULL;
-
-        this = data;
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        defrag = conf->defrag;
-        if (!defrag)
-                goto out;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        frame->root->pid = GF_CLIENT_PID_DEFRAG;
-
-        defrag->pid = frame->root->pid;
-
-        defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
-
-        old_THIS = THIS;
-        THIS = this;
-        ret = synctask_new (this->ctx->env, gf_defrag_start_crawl,
-                            gf_defrag_done, frame, this);
-
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_REBALANCE_START_FAILED,
-                        "Could not create task for rebalance");
-        THIS = old_THIS;
-out:
-        return NULL;
-}
-
-
-uint64_t
-gf_defrag_get_estimates_based_on_size (dht_conf_t *conf)
+gf_defrag_start(void *data)
 {
-        gf_defrag_info_t *defrag = NULL;
-        double            rate_processed = 0;
-        uint64_t          total_processed = 0;
-        uint64_t          tmp_count = 0;
-        uint64_t          time_to_complete = 0;
-        struct            timeval now = {0,};
-        double            elapsed = 0;
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    xlator_t *this = NULL;
+    xlator_t *old_THIS = NULL;
 
-        defrag = conf->defrag;
+    this = data;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        if (!g_totalsize)
-                goto out;
+    defrag = conf->defrag;
+    if (!defrag)
+        goto out;
 
-        gettimeofday (&now, NULL);
-        elapsed = now.tv_sec - defrag->start_time.tv_sec;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
 
-        /* Don't calculate the estimates for the first 10 minutes.
-         * It is unlikely to be accurate and estimates are not required
-         * if the process finishes in less than 10 mins.
-         */
+    frame->root->pid = GF_CLIENT_PID_DEFRAG;
 
-        if (elapsed < ESTIMATE_START_INTERVAL) {
-                gf_msg (THIS->name, GF_LOG_INFO, 0, 0,
-                        "Rebalance estimates will not be available for the "
-                        "first %d seconds.", ESTIMATE_START_INTERVAL);
-
-                goto out;
-        }
+    defrag->pid = frame->root->pid;
 
-        total_processed = defrag->size_processed;
-
-        /* rate at which files processed */
-        rate_processed = (total_processed)/elapsed;
-
-        tmp_count = g_totalsize;
-
-        if (rate_processed) {
-                time_to_complete = (tmp_count)/rate_processed;
-
-        } else {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0, 0,
-                        "Unable to calculate estimated time for rebalance");
-        }
+    defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
 
-        gf_log (THIS->name, GF_LOG_INFO,
-                "TIME: (size) total_processed=%"PRIu64" tmp_cnt = %"PRIu64","
-                "rate_processed=%f, elapsed = %f", total_processed, tmp_count,
-                rate_processed, elapsed);
+    old_THIS = THIS;
+    THIS = this;
+    ret = synctask_new(this->ctx->env, gf_defrag_start_crawl, gf_defrag_done,
+                       frame, this);
 
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_START_FAILED,
+               "Could not create task for rebalance");
+    THIS = old_THIS;
 out:
-        return time_to_complete;
+    return NULL;
 }
 
-
-
 uint64_t
-gf_defrag_get_estimates (dht_conf_t *conf)
+gf_defrag_get_estimates_based_on_size(dht_conf_t *conf)
 {
-        gf_defrag_info_t *defrag = NULL;
-        loc_t             loc = {0,};
-        double            rate_lookedup = 0;
-        uint64_t          dirs_processed = 0;
-        uint64_t          files_processed = 0;
-        uint64_t          total_processed = 0;
-        uint64_t          tmp_count = 0;
-        uint64_t          time_to_complete = 0;
-        struct            timeval now = {0,};
-        double            elapsed = 0;
+    gf_defrag_info_t *defrag = NULL;
+    double rate_processed = 0;
+    uint64_t total_processed = 0;
+    uint64_t tmp_count = 0;
+    uint64_t time_to_complete = 0;
+    double elapsed = 0;
 
+    defrag = conf->defrag;
 
-        defrag = conf->defrag;
+    if (!g_totalsize)
+        goto out;
 
-        if (!g_totalfiles)
-                goto out;
+    elapsed = gf_time() - defrag->start_time;
 
-        gettimeofday (&now, NULL);
-        elapsed = now.tv_sec - defrag->start_time.tv_sec;
+    /* Don't calculate the estimates for the first 10 minutes.
+     * It is unlikely to be accurate and estimates are not required
+     * if the process finishes in less than 10 mins.
+     */
 
-        /* I tried locking before accessing num_files_lookedup and
-         * num_dirs_processed but the status function
-         * never seemed to get the lock, causing the status cli to
-         * hang.
-         */
+    if (elapsed < ESTIMATE_START_INTERVAL) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, 0,
+               "Rebalance estimates will not be available for the "
+               "first %d seconds.",
+               ESTIMATE_START_INTERVAL);
 
-        dirs_processed = defrag->num_dirs_processed;
-        files_processed = defrag->num_files_lookedup;
+        goto out;
+    }
 
-        total_processed = files_processed + dirs_processed;
+    total_processed = defrag->size_processed;
 
-        if (total_processed > g_totalfiles) {
-                /* lookup the number of files again
-                 * The problem here is that not all the newly added files
-                 * might need to be processed. So this need not work
-                 * in some cases
-                 */
-                dht_build_root_loc (defrag->root_inode, &loc);
-                g_totalfiles = gf_defrag_total_file_cnt (defrag->this, &loc);
-                if (!g_totalfiles)
-                        goto out;
-        }
+    /* rate at which files processed */
+    rate_processed = (total_processed) / elapsed;
 
-        /* rate at which files looked up */
-        rate_lookedup = (total_processed)/elapsed;
+    tmp_count = g_totalsize;
 
-        /* We initially sum up dirs across all local subvols because we get the
-         * file count from the inodes on each subvol.
-         * The same directories will be counted for each subvol but
-         * we want them to be counted once.
-         */
-
-        tmp_count = g_totalfiles
-                     - (dirs_processed * (conf->local_subvols_cnt - 1));
-
-        if (rate_lookedup) {
-                time_to_complete = (tmp_count)/rate_lookedup;
+    if (rate_processed) {
+        time_to_complete = (tmp_count) / rate_processed;
 
-        } else {
-
-                gf_msg (THIS->name, GF_LOG_ERROR, 0, 0,
-                        "Unable to calculate estimated time for rebalance");
-        }
+    } else {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, 0,
+               "Unable to calculate estimated time for rebalance");
+    }
 
-        gf_log (THIS->name, GF_LOG_INFO,
-                "TIME: (count) total_processed=%"PRIu64" tmp_cnt = %"PRIu64","
-                "rate_lookedup=%f", total_processed, tmp_count,
-                rate_lookedup);
+    gf_log(THIS->name, GF_LOG_INFO,
+           "TIME: (size) total_processed=%" PRIu64 " tmp_cnt = %" PRIu64
+           ","
+           "rate_processed=%f, elapsed = %f",
+           total_processed, tmp_count, rate_processed, elapsed);
 
 out:
-        return time_to_complete;
+    return time_to_complete;
 }
 
-
 int
-gf_defrag_status_get (dht_conf_t *conf, dict_t *dict)
+gf_defrag_status_get(dht_conf_t *conf, dict_t *dict)
 {
-        int      ret    = 0;
-        uint64_t files  = 0;
-        uint64_t size   = 0;
-        uint64_t lookup = 0;
-        uint64_t failures = 0;
-        uint64_t skipped = 0;
-        uint64_t promoted = 0;
-        uint64_t demoted = 0;
-        char    *status = "";
-        double   elapsed = 0;
-        struct timeval end = {0,};
-        uint64_t time_to_complete = 0;
-        uint64_t time_left = 0;
-        gf_defrag_info_t *defrag = conf->defrag;
-
-        if (!defrag)
-                goto out;
-
-        ret = 0;
-        if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED)
-                goto out;
-
-        files  = defrag->total_files;
-        size   = defrag->total_data;
-        lookup = defrag->num_files_lookedup;
-        failures = defrag->total_failures;
-        skipped = defrag->skipped;
-        promoted = defrag->total_files_promoted;
-        demoted = defrag->total_files_demoted;
-
-        gettimeofday (&end, NULL);
-
-        elapsed = end.tv_sec - defrag->start_time.tv_sec;
-
-
-        /* The rebalance is still in progress */
-
-        if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER)
-            && (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED)) {
-
-/*
-                time_to_complete = gf_defrag_get_estimates (conf);
-
-                if (time_to_complete && (time_to_complete > elapsed))
-                        time_left = time_to_complete - elapsed;
-
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "TIME: Estimated total time to complete based on"
-                        " count = %"PRIu64 " seconds, seconds left = %"PRIu64"",
-                        time_to_complete, time_left);
-*/
-
-                time_to_complete = gf_defrag_get_estimates_based_on_size (conf);
-
-                if (time_to_complete && (time_to_complete > elapsed))
-                        time_left = time_to_complete - elapsed;
-
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "TIME: Estimated total time to complete (size)= %"PRIu64
-                        " seconds, seconds left = %"PRIu64"",
-                        time_to_complete, time_left);
-        }
-
-        if (!dict)
-                goto log;
-
-        ret = dict_set_uint64 (dict, "promoted", promoted);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set promoted count");
-
-        ret = dict_set_uint64 (dict, "demoted", demoted);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set demoted count");
-
-        ret = dict_set_uint64 (dict, "files", files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set file count");
-
-        ret = dict_set_uint64 (dict, "size", size);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set size of xfer");
-
-        ret = dict_set_uint64 (dict, "lookups", lookup);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set lookedup file count");
-
-
-        ret = dict_set_int32 (dict, "status", defrag->defrag_status);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set status");
-
-        ret = dict_set_double (dict, "run-time", elapsed);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set run-time");
-
-        ret = dict_set_uint64 (dict, "failures", failures);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set failure count");
-
-        ret = dict_set_uint64 (dict, "skipped", skipped);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set skipped file count");
-
-        ret = dict_set_uint64 (dict, "time-left", time_left);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set time-left");
+    int ret = 0;
+    uint64_t files = 0;
+    uint64_t size = 0;
+    uint64_t lookup = 0;
+    uint64_t failures = 0;
+    uint64_t skipped = 0;
+    char *status = "";
+    double elapsed = 0;
+    uint64_t time_to_complete = 0;
+    uint64_t time_left = 0;
+    gf_defrag_info_t *defrag = conf->defrag;
+
+    if (!defrag)
+        goto out;
+
+    ret = 0;
+    if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED)
+        goto out;
+
+    files = defrag->total_files;
+    size = defrag->total_data;
+    lookup = defrag->num_files_lookedup;
+    failures = defrag->total_failures;
+    skipped = defrag->skipped;
+
+    elapsed = gf_time() - defrag->start_time;
+
+    /* The rebalance is still in progress */
+
+    if (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED) {
+        time_to_complete = gf_defrag_get_estimates_based_on_size(conf);
+
+        if (time_to_complete && (time_to_complete > elapsed))
+            time_left = time_to_complete - elapsed;
+
+        gf_log(THIS->name, GF_LOG_INFO,
+               "TIME: Estimated total time to complete (size)= %" PRIu64
+               " seconds, seconds left = %" PRIu64 "",
+               time_to_complete, time_left);
+    }
+
+    if (!dict)
+        goto log;
+
+    ret = dict_set_uint64(dict, "files", files);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set file count");
+
+    ret = dict_set_uint64(dict, "size", size);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set size of xfer");
+
+    ret = dict_set_uint64(dict, "lookups", lookup);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set lookedup file count");
+
+    ret = dict_set_int32(dict, "status", defrag->defrag_status);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set status");
+
+    ret = dict_set_double(dict, "run-time", elapsed);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set run-time");
+
+    ret = dict_set_uint64(dict, "failures", failures);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set failure count");
+
+    ret = dict_set_uint64(dict, "skipped", skipped);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set skipped file count");
+
+    ret = dict_set_uint64(dict, "time-left", time_left);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set time-left");
 
 log:
-        switch (defrag->defrag_status) {
+    switch (defrag->defrag_status) {
         case GF_DEFRAG_STATUS_NOT_STARTED:
-                status = "not started";
-                break;
+            status = "not started";
+            break;
         case GF_DEFRAG_STATUS_STARTED:
-                status = "in progress";
-                break;
+            status = "in progress";
+            break;
         case GF_DEFRAG_STATUS_STOPPED:
-                status = "stopped";
-                break;
+            status = "stopped";
+            break;
         case GF_DEFRAG_STATUS_COMPLETE:
-                status = "completed";
-                break;
+            status = "completed";
+            break;
         case GF_DEFRAG_STATUS_FAILED:
-                status = "failed";
-                break;
+            status = "failed";
+            break;
         default:
-                break;
-        }
-
-        gf_msg (THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS,
-                "Rebalance is %s. Time taken is %.2f secs",
-                status, elapsed);
-        gf_msg (THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS,
-                "Files migrated: %"PRIu64", size: %"
-                PRIu64", lookups: %"PRIu64", failures: %"PRIu64", skipped: "
-                "%"PRIu64, files, size, lookup, failures, skipped);
-out:
-        return 0;
-}
-
-void
-gf_defrag_set_pause_state (gf_tier_conf_t *tier_conf, tier_pause_state_t state)
-{
-        pthread_mutex_lock (&tier_conf->pause_mutex);
-        tier_conf->pause_state = state;
-        pthread_mutex_unlock (&tier_conf->pause_mutex);
-}
-
-
-tier_pause_state_t
-gf_defrag_get_pause_state (gf_tier_conf_t *tier_conf)
-{
-        int state;
-
-        pthread_mutex_lock (&tier_conf->pause_mutex);
-        state = tier_conf->pause_state;
-        pthread_mutex_unlock (&tier_conf->pause_mutex);
-
-        return state;
-}
-
-tier_pause_state_t
-gf_defrag_check_pause_tier (gf_tier_conf_t *tier_conf)
-{
-        int woke = 0;
-        int state  = -1;
-
-        pthread_mutex_lock (&tier_conf->pause_mutex);
-
-        if (tier_conf->pause_state == TIER_RUNNING)
-                goto out;
-
-        if (tier_conf->pause_state == TIER_PAUSED)
-                goto out;
-
-        if (tier_conf->promote_in_progress ||
-            tier_conf->demote_in_progress)
-                goto out;
-
-        tier_conf->pause_state = TIER_PAUSED;
-
-        if (tier_conf->pause_synctask) {
-                synctask_wake (tier_conf->pause_synctask);
-                tier_conf->pause_synctask = 0;
-                woke = 1;
-        }
-
-        gf_msg ("tier", GF_LOG_DEBUG, 0,
-                DHT_MSG_TIER_PAUSED,
-                "woken %d", woke);
-
-        gf_event (EVENT_TIER_PAUSE, "vol=%s", tier_conf->volname);
-out:
-        state = tier_conf->pause_state;
-
-        pthread_mutex_unlock (&tier_conf->pause_mutex);
-
-        return state;
-}
-
-void
-gf_defrag_pause_tier_timeout (void *data)
-{
-        xlator_t         *this                  = NULL;
-        dht_conf_t       *conf                  = NULL;
-        gf_defrag_info_t *defrag                = NULL;
-
-        this   = (xlator_t *) data;
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
-
-        conf   = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-
-        defrag = conf->defrag;
-        GF_VALIDATE_OR_GOTO (this->name, defrag, out);
-
-        gf_msg (this->name, GF_LOG_DEBUG, 0,
-                DHT_MSG_TIER_PAUSED,
-                "Request pause timer timeout");
-
-        gf_defrag_check_pause_tier (&defrag->tier_conf);
-
+            break;
+    }
+
+    gf_msg(THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS,
+           "Rebalance is %s. Time taken is %.2f secs", status, elapsed);
+    gf_msg(THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS,
+           "Files migrated: %" PRIu64 ", size: %" PRIu64 ", lookups: %" PRIu64
+           ", failures: %" PRIu64
+           ", skipped: "
+           "%" PRIu64,
+           files, size, lookup, failures, skipped);
 out:
-        return;
+    return 0;
 }
 
 int
-gf_defrag_pause_tier (xlator_t *this, gf_defrag_info_t *defrag)
+gf_defrag_stop(dht_conf_t *conf, gf_defrag_status_t status, dict_t *output)
 {
-        int             ret         = 0;
-        struct timespec delta       = {0,};
-        int             delay       = 2;
+    /* TODO: set a variable 'stop_defrag' here, it should be checked
+       in defrag loop */
+    int ret = -1;
+    gf_defrag_info_t *defrag = conf->defrag;
 
-        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED)
-                goto out;
+    GF_ASSERT(defrag);
 
-        /*
-         * Set flag requesting to pause tiering. Wait 'delay' seconds for
-         * tiering to actually stop as indicated by the pause state
-         * before returning success or failure.
-         */
-        gf_defrag_set_pause_state (&defrag->tier_conf, TIER_REQUEST_PAUSE);
+    if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
+        goto out;
+    }
 
-        /*
-         * If migration is not underway, can pause immediately.
-         */
-        gf_defrag_check_pause_tier (&defrag->tier_conf);
-        if (gf_defrag_get_pause_state (&defrag->tier_conf) == TIER_PAUSED)
-                goto out;
-
-        gf_msg (this->name, GF_LOG_DEBUG, 0,
-                DHT_MSG_TIER_PAUSED,
-                "Request pause tier");
-
-        defrag->tier_conf.pause_synctask = synctask_get ();
-        delta.tv_sec  = delay;
-        delta.tv_nsec = 0;
-        defrag->tier_conf.pause_timer =
-                gf_timer_call_after (this->ctx, delta,
-                                     gf_defrag_pause_tier_timeout,
-                                     this);
-
-        synctask_yield (defrag->tier_conf.pause_synctask);
-
-        if (gf_defrag_get_pause_state (&defrag->tier_conf) == TIER_PAUSED)
-                goto out;
+    gf_msg("", GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STOPPED,
+           "Received stop command on rebalance");
+    defrag->defrag_status = status;
 
-        gf_defrag_set_pause_state (&defrag->tier_conf, TIER_RUNNING);
-
-        ret = -1;
+    if (output)
+        gf_defrag_status_get(conf, output);
+    ret = 0;
 out:
-
-        gf_msg (this->name, GF_LOG_DEBUG, 0,
-                DHT_MSG_TIER_PAUSED,
-                "Pause tiering ret=%d", ret);
-
-        return ret;
-}
-
-int
-gf_defrag_resume_tier (xlator_t *this, gf_defrag_info_t *defrag)
-{
-        gf_msg (this->name, GF_LOG_DEBUG, 0,
-                DHT_MSG_TIER_RESUME,
-                "Pause end. Resume tiering");
-
-        gf_defrag_set_pause_state (&defrag->tier_conf, TIER_RUNNING);
-
-        gf_event (EVENT_TIER_RESUME, "vol=%s", defrag->tier_conf.volname);
-
-        return 0;
-}
-
-int
-gf_defrag_start_detach_tier (gf_defrag_info_t *defrag)
-{
-        defrag->cmd = GF_DEFRAG_CMD_START_DETACH_TIER;
-
-        return 0;
-}
-
-int
-gf_defrag_stop (dht_conf_t *conf, gf_defrag_status_t status,
-                dict_t *output)
-{
-        /* TODO: set a variable 'stop_defrag' here, it should be checked
-           in defrag loop */
-        int     ret = -1;
-        gf_defrag_info_t *defrag = conf->defrag;
-
-        GF_ASSERT (defrag);
-
-        if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
-                goto out;
-        }
-
-        gf_msg ("", GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STOPPED,
-                "Received stop command on rebalance");
-        defrag->defrag_status = status;
-
-        if (output)
-                gf_defrag_status_get (conf, output);
-        ret = 0;
-out:
-        gf_msg_debug ("", 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("", 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c
index d311ac633e3..d9dbf50492f 100644
--- a/xlators/cluster/dht/src/dht-rename.c
+++ b/xlators/cluster/dht/src/dht-rename.c
@@ -11,489 +11,454 @@
 /* TODO: link(oldpath, newpath) fails if newpath already exists. DHT should
  *       delete the newpath if it gets EEXISTS from link() call.
  */
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
 #include "dht-lock.h"
-#include "defaults.h"
+#include <glusterfs/defaults.h>
 
-int dht_rename_unlock (call_frame_t *frame, xlator_t *this);
+int
+dht_rename_unlock(call_frame_t *frame, xlator_t *this);
 int32_t
-dht_rename_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata);
+dht_rename_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 int
-dht_rename_unlock_cbk (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       dict_t *xdata)
+dht_rename_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        dht_set_fixed_dir_stat (&local->preoldparent);
-        dht_set_fixed_dir_stat (&local->postoldparent);
-        dht_set_fixed_dir_stat (&local->preparent);
-        dht_set_fixed_dir_stat (&local->postparent);
+    dht_set_fixed_dir_stat(&local->preoldparent);
+    dht_set_fixed_dir_stat(&local->postoldparent);
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-        if (IA_ISREG (local->stbuf.ia_type))
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+    if (IA_ISREG(local->stbuf.ia_type))
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
 
-        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                          &local->stbuf, &local->preoldparent,
-                          &local->postoldparent, &local->preparent,
-                          &local->postparent, local->xattr);
-        return 0;
+    DHT_STACK_UNWIND(rename, frame, local->op_ret, local->op_errno,
+                     &local->stbuf, &local->preoldparent, &local->postoldparent,
+                     &local->preparent, &local->postparent, local->xattr);
+    return 0;
 }
 
 static void
-dht_rename_dir_unlock_src (call_frame_t *frame, xlator_t *this)
+dht_rename_dir_unlock_src(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t *local                      = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
-        dht_unlock_namespace (frame, &local->lock[0]);
-        return;
+    local = frame->local;
+    dht_unlock_namespace(frame, &local->lock[0]);
+    return;
 }
 
 static void
-dht_rename_dir_unlock_dst (call_frame_t *frame, xlator_t *this)
+dht_rename_dir_unlock_dst(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t *local                      = NULL;
-        int          op_ret                     = -1;
-        char         src_gfid[GF_UUID_BUF_SIZE] = {0};
-        char         dst_gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-
-        /* Unlock entrylk */
-        dht_unlock_entrylk_wrapper (frame, &local->lock[1].ns.directory_ns);
-
-        /* Unlock inodelk */
-        op_ret = dht_unlock_inodelk (frame,
-                                     local->lock[1].ns.parent_layout.locks,
-                                     local->lock[1].ns.parent_layout.lk_count,
-                                     dht_rename_unlock_cbk);
-        if (op_ret < 0) {
-                uuid_utoa_r (local->loc.inode->gfid, src_gfid);
-
-                if (local->loc2.inode)
-                        uuid_utoa_r (local->loc2.inode->gfid, dst_gfid);
-
-                if (IA_ISREG (local->stbuf.ia_type))
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_UNLOCKING_FAILED,
-                                "winding unlock inodelk failed "
-                                "rename (%s:%s:%s %s:%s:%s), "
-                                "stale locks left on bricks",
-                                local->loc.path, src_gfid,
-                                local->src_cached->name,
-                                local->loc2.path, dst_gfid,
-                                local->dst_cached ?
-                                local->dst_cached->name : NULL);
-                else
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_UNLOCKING_FAILED,
-                                "winding unlock inodelk failed "
-                                "rename (%s:%s %s:%s), "
-                                "stale locks left on bricks",
-                                local->loc.path, src_gfid,
-                                local->loc2.path, dst_gfid);
-
-                dht_rename_unlock_cbk (frame, NULL, this, 0, 0, NULL);
-        }
+    dht_local_t *local = NULL;
+    int op_ret = -1;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    /* Unlock entrylk */
+    dht_unlock_entrylk_wrapper(frame, &local->lock[1].ns.directory_ns);
+
+    /* Unlock inodelk */
+    op_ret = dht_unlock_inodelk(frame, local->lock[1].ns.parent_layout.locks,
+                                local->lock[1].ns.parent_layout.lk_count,
+                                dht_rename_unlock_cbk);
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        if (IA_ISREG(local->stbuf.ia_type))
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s:%s %s:%s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->src_cached->name,
+                   local->loc2.path, dst_gfid,
+                   local->dst_cached ? local->dst_cached->name : NULL);
+        else
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s %s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->loc2.path, dst_gfid);
 
-        return;
+        dht_rename_unlock_cbk(frame, NULL, this, 0, 0, NULL);
+    }
+
+    return;
 }
 
 static int
-dht_rename_dir_unlock (call_frame_t *frame, xlator_t *this)
+dht_rename_dir_unlock(call_frame_t *frame, xlator_t *this)
 {
-
-        dht_rename_dir_unlock_src (frame, this);
-        dht_rename_dir_unlock_dst (frame, this);
-        return 0;
+    dht_rename_dir_unlock_src(frame, this);
+    dht_rename_dir_unlock_dst(frame, this);
+    return 0;
 }
 int
-dht_rename_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
+dht_rename_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
 {
-        dht_conf_t  *conf                    = NULL;
-        dht_local_t *local                   = NULL;
-        int          this_call_cnt           = 0;
-        xlator_t    *prev                    = NULL;
-        int          i                       = 0;
-        char         gfid[GF_UUID_BUF_SIZE]  = {0};
-        int          subvol_cnt              = -1;
-
-        conf = this->private;
-        local = frame->local;
-        prev = cookie;
-        subvol_cnt = dht_subvol_cnt (this, prev);
-        local->ret_cache[subvol_cnt] = op_ret;
-
-        if (op_ret == -1) {
-                gf_uuid_unparse(local->loc.inode->gfid, gfid);
-
-                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                        DHT_MSG_RENAME_FAILED,
-                        "Rename %s -> %s on %s failed, (gfid = %s)",
-                        local->loc.path, local->loc2.path,
-                        prev->name, gfid);
-
-                local->op_ret   = op_ret;
-                local->op_errno = op_errno;
-                goto unwind;
-        }
-        /* TODO: construct proper stbuf for dir */
-        /*
-         * FIXME: is this the correct way to build stbuf and
-         * parent bufs?
-         */
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev);
-        dht_iatt_merge (this, &local->preoldparent, preoldparent, prev);
-        dht_iatt_merge (this, &local->postoldparent, postoldparent, prev);
-        dht_iatt_merge (this, &local->preparent, prenewparent, prev);
-        dht_iatt_merge (this, &local->postparent, postnewparent, prev);
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int subvol_cnt = -1;
+
+    conf = this->private;
+    local = frame->local;
+    prev = cookie;
+    subvol_cnt = dht_subvol_cnt(this, prev);
+    local->ret_cache[subvol_cnt] = op_ret;
+
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.inode->gfid, gfid);
+
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_RENAME_FAILED,
+               "Rename %s -> %s on %s failed, (gfid = %s)", local->loc.path,
+               local->loc2.path, prev->name, gfid);
+
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
+    /* TODO: construct proper stbuf for dir */
+    /*
+     * FIXME: is this the correct way to build stbuf and
+     * parent bufs?
+     */
+    dht_iatt_merge(this, &local->stbuf, stbuf);
+    dht_iatt_merge(this, &local->preoldparent, preoldparent);
+    dht_iatt_merge(this, &local->postoldparent, postoldparent);
+    dht_iatt_merge(this, &local->preparent, prenewparent);
+    dht_iatt_merge(this, &local->postparent, postnewparent);
 
 unwind:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                /* We get here with local->call_cnt == 0. Which means
-                 * we are the only one executing this code, there is
-                 * no contention. Therefore it's safe to manipulate or
-                 * deref local->call_cnt directly (without locking).
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        /* We get here with local->call_cnt == 0. Which means
+         * we are the only one executing this code, there is
+         * no contention. Therefore it's safe to manipulate or
+         * deref local->call_cnt directly (without locking).
+         */
+        if (local->ret_cache[conf->subvolume_cnt] == 0) {
+            /* count errant subvols in last field of ret_cache */
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (local->ret_cache[i] != 0)
+                    ++local->ret_cache[conf->subvolume_cnt];
+            }
+            if (local->ret_cache[conf->subvolume_cnt]) {
+                /* undoing the damage:
+                 * for all subvolumes, where rename
+                 * succeeded, we perform the reverse operation
                  */
-                if (local->ret_cache[conf->subvolume_cnt] == 0) {
-                        /* count errant subvols in last field of ret_cache */
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                if (local->ret_cache[i] != 0)
-                                        ++local->ret_cache[conf->subvolume_cnt];
-                        }
-                        if (local->ret_cache[conf->subvolume_cnt]) {
-                                /* undoing the damage:
-                                 * for all subvolumes, where rename
-                                 * succeeded, we perform the reverse operation
-                                 */
-                                for (i = 0; i < conf->subvolume_cnt; i++) {
-                                        if (local->ret_cache[i] == 0)
-                                                ++local->call_cnt;
-                                }
-                                for (i = 0; i < conf->subvolume_cnt; i++) {
-                                        if (local->ret_cache[i])
-                                                continue;
-
-                                        STACK_WIND (frame,
-                                                    dht_rename_dir_cbk,
-                                                    conf->subvolumes[i],
-                                                    conf->subvolumes[i]->fops->rename,
-                                                    &local->loc2, &local->loc,
-                                                    NULL);
-                                }
-
-                                return 0;
-                        }
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                    if (local->ret_cache[i] == 0)
+                        ++local->call_cnt;
                 }
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                    if (local->ret_cache[i])
+                        continue;
 
-                WIPE (&local->preoldparent);
-                WIPE (&local->postoldparent);
-                WIPE (&local->preparent);
-                WIPE (&local->postparent);
+                    STACK_WIND(frame, dht_rename_dir_cbk, conf->subvolumes[i],
+                               conf->subvolumes[i]->fops->rename, &local->loc2,
+                               &local->loc, NULL);
+                }
 
-                dht_rename_dir_unlock (frame, this);
+                return 0;
+            }
         }
 
-        return 0;
-}
+        WIPE(&local->preoldparent);
+        WIPE(&local->postoldparent);
+        WIPE(&local->preparent);
+        WIPE(&local->postparent);
+
+        dht_rename_dir_unlock(frame, this);
+    }
 
+    return 0;
+}
 
 int
-dht_rename_hashed_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                           struct iatt *preoldparent,
-                           struct iatt *postoldparent,
-                           struct iatt *prenewparent,
-                           struct iatt *postnewparent, dict_t *xdata)
+dht_rename_hashed_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                          struct iatt *preoldparent, struct iatt *postoldparent,
+                          struct iatt *prenewparent, struct iatt *postnewparent,
+                          dict_t *xdata)
 {
-        dht_conf_t   *conf = NULL;
-        dht_local_t  *local = NULL;
-        int           call_cnt = 0;
-        xlator_t     *prev = NULL;
-        int           i = 0;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
-
-        conf = this->private;
-        local = frame->local;
-        prev = cookie;
-
-
-        if (op_ret == -1) {
-                gf_uuid_unparse(local->loc.inode->gfid, gfid);
-
-                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                        DHT_MSG_RENAME_FAILED,
-                        "rename %s -> %s on %s failed, (gfid = %s) ",
-                        local->loc.path, local->loc2.path,
-                        prev->name, gfid);
-
-                local->op_ret   = op_ret;
-                local->op_errno = op_errno;
-                goto unwind;
-        }
-        /* TODO: construct proper stbuf for dir */
-        /*
-         * FIXME: is this the correct way to build stbuf and
-         * parent bufs?
-         */
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev);
-        dht_iatt_merge (this, &local->preoldparent, preoldparent, prev);
-        dht_iatt_merge (this, &local->postoldparent, postoldparent, prev);
-        dht_iatt_merge (this, &local->preparent, prenewparent, prev);
-        dht_iatt_merge (this, &local->postparent, postnewparent, prev);
-
-        call_cnt = local->call_cnt = conf->subvolume_cnt - 1;
-
-        if (!local->call_cnt)
-                goto unwind;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == local->dst_hashed)
-                        continue;
-                STACK_WIND_COOKIE (frame, dht_rename_dir_cbk,
-                                   conf->subvolumes[i],
-                                   conf->subvolumes[i],
-                                   conf->subvolumes[i]->fops->rename,
-                                   &local->loc, &local->loc2, NULL);
-                if (!--call_cnt)
-                        break;
-        }
-
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int call_cnt = 0;
+    xlator_t *prev = NULL;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    conf = this->private;
+    local = frame->local;
+    prev = cookie;
+
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.inode->gfid, gfid);
 
-        return 0;
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_RENAME_FAILED,
+               "rename %s -> %s on %s failed, (gfid = %s) ", local->loc.path,
+               local->loc2.path, prev->name, gfid);
+
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
+    /* TODO: construct proper stbuf for dir */
+    /*
+     * FIXME: is this the correct way to build stbuf and
+     * parent bufs?
+     */
+    dht_iatt_merge(this, &local->stbuf, stbuf);
+    dht_iatt_merge(this, &local->preoldparent, preoldparent);
+    dht_iatt_merge(this, &local->postoldparent, postoldparent);
+    dht_iatt_merge(this, &local->preparent, prenewparent);
+    dht_iatt_merge(this, &local->postparent, postnewparent);
+
+    call_cnt = local->call_cnt = conf->subvolume_cnt - 1;
+
+    if (!local->call_cnt)
+        goto unwind;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == local->dst_hashed)
+            continue;
+        STACK_WIND_COOKIE(
+            frame, dht_rename_dir_cbk, conf->subvolumes[i], conf->subvolumes[i],
+            conf->subvolumes[i]->fops->rename, &local->loc, &local->loc2, NULL);
+        if (!--call_cnt)
+            break;
+    }
+
+    return 0;
 unwind:
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
 
-        dht_rename_dir_unlock (frame, this);
-        return 0;
+    dht_rename_dir_unlock(frame, this);
+    return 0;
 }
 
-
 int
-dht_rename_dir_do (call_frame_t *frame, xlator_t *this)
+dht_rename_dir_do(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret == -1)
-                goto err;
+    if (local->op_ret == -1)
+        goto err;
 
-        local->op_ret = 0;
+    local->op_ret = 0;
 
-        STACK_WIND_COOKIE (frame, dht_rename_hashed_dir_cbk, local->dst_hashed,
-                           local->dst_hashed,
-                           local->dst_hashed->fops->rename,
-                           &local->loc, &local->loc2, NULL);
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_rename_hashed_dir_cbk, local->dst_hashed,
+                      local->dst_hashed, local->dst_hashed->fops->rename,
+                      &local->loc, &local->loc2, NULL);
+    return 0;
 
 err:
-        dht_rename_dir_unlock (frame, this);
-        return 0;
+    dht_rename_dir_unlock(frame, this);
+    return 0;
 }
 
-
 int
-dht_rename_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, gf_dirent_t *entries,
-                        dict_t *xdata)
+dht_rename_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, gf_dirent_t *entries,
+                       dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = -1;
-        xlator_t     *prev = NULL;
-
-        local = frame->local;
-        prev  = cookie;
-
-        if (op_ret > 2) {
-                gf_msg_trace (this->name, 0,
-                              "readdir on %s for %s returned %d entries",
-                              prev->name, local->loc.path, op_ret);
-                local->op_ret = -1;
-                local->op_errno = ENOTEMPTY;
-        }
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
+    xlator_t *prev = NULL;
 
-        this_call_cnt = dht_frame_return (frame);
+    local = frame->local;
+    prev = cookie;
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rename_dir_do (frame, this);
-        }
+    if (op_ret > 2) {
+        gf_msg_trace(this->name, 0, "readdir on %s for %s returned %d entries",
+                     prev->name, local->loc.path, op_ret);
+        local->op_ret = -1;
+        local->op_errno = ENOTEMPTY;
+    }
 
-        return 0;
-}
+    this_call_cnt = dht_frame_return(frame);
 
+    if (is_last_call(this_call_cnt)) {
+        dht_rename_dir_do(frame, this);
+    }
+
+    return 0;
+}
 
 int
-dht_rename_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+dht_rename_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = -1;
-        xlator_t     *prev = NULL;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
+    xlator_t *prev = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        local = frame->local;
-        prev  = cookie;
+    local = frame->local;
+    prev = cookie;
 
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.inode->gfid, gfid);
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_OPENDIR_FAILED,
+               "opendir on %s for %s failed,(gfid = %s) ", prev->name,
+               local->loc.path, gfid);
+        goto err;
+    }
 
-        if (op_ret == -1) {
-
-                gf_uuid_unparse(local->loc.inode->gfid, gfid);
-                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                        DHT_MSG_OPENDIR_FAILED,
-                        "opendir on %s for %s failed,(gfid = %s) ",
-                        prev->name, local->loc.path, gfid);
-                goto err;
-        }
-
-        fd_bind (fd);
-        STACK_WIND_COOKIE (frame, dht_rename_readdir_cbk, prev, prev,
-                           prev->fops->readdir, local->fd, 4096, 0, NULL);
+    fd_bind(fd);
+    STACK_WIND_COOKIE(frame, dht_rename_readdir_cbk, prev, prev,
+                      prev->fops->readdir, local->fd, 4096, 0, NULL);
 
-        return 0;
+    return 0;
 
 err:
-        this_call_cnt = dht_frame_return (frame);
+    this_call_cnt = dht_frame_return(frame);
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rename_dir_do (frame, this);
-        }
+    if (is_last_call(this_call_cnt)) {
+        dht_rename_dir_do(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-dht_rename_dir_lock2_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+dht_rename_dir_lock2_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local                      = NULL;
-        char         src_gfid[GF_UUID_BUF_SIZE] = {0};
-        char         dst_gfid[GF_UUID_BUF_SIZE] = {0};
-        dht_conf_t  *conf                       = NULL;
-        int          i                          = 0;
-
-        local = frame->local;
-        conf = this->private;
-
-        if (op_ret < 0) {
-                uuid_utoa_r (local->loc.inode->gfid, src_gfid);
-
-                if (local->loc2.inode)
-                        uuid_utoa_r (local->loc2.inode->gfid, dst_gfid);
-
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_INODE_LK_ERROR,
-                        "acquiring entrylk after inodelk failed"
-                        "rename (%s:%s:%s %s:%s:%s)",
-                        local->loc.path, src_gfid, local->src_cached->name,
-                        local->loc2.path, dst_gfid,
-                        local->dst_cached ? local->dst_cached->name : NULL);
-
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-                goto err;
-        }
-
-        local->fd = fd_create (local->loc.inode, frame->root->pid);
-        if (!local->fd) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->op_ret = 0;
-
-        if (!local->dst_cached) {
-                dht_rename_dir_do (frame, this);
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_conf_t *conf = NULL;
+    int i = 0;
+
+    local = frame->local;
+    conf = this->private;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring entrylk after inodelk failed"
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->loc.path, src_gfid, local->src_cached->name,
+               local->loc2.path, dst_gfid,
+               local->dst_cached ? local->dst_cached->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    local->fd = fd_create(local->loc.inode, frame->root->pid);
+    if (!local->fd) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->op_ret = 0;
+
+    if (!local->dst_cached) {
+        dht_rename_dir_do(frame, this);
+        return 0;
+    }
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND_COOKIE (frame, dht_rename_opendir_cbk,
-                                   conf->subvolumes[i],
-                                   conf->subvolumes[i],
-                                   conf->subvolumes[i]->fops->opendir,
-                                   &local->loc2, local->fd, NULL);
-        }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_rename_opendir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->opendir, &local->loc2,
+                          local->fd, NULL);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        /* No harm in calling an extra unlock */
-        dht_rename_dir_unlock (frame, this);
-        return 0;
+    /* No harm in calling an extra unlock */
+    dht_rename_dir_unlock(frame, this);
+    return 0;
 }
 
 int
-dht_rename_dir_lock1_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_rename_dir_lock1_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local                      = NULL;
-        char         src_gfid[GF_UUID_BUF_SIZE] = {0};
-        char         dst_gfid[GF_UUID_BUF_SIZE] = {0};
-        int          ret                        = 0;
-        loc_t       *loc                        = NULL;
-        xlator_t    *subvol                     = NULL;
-
-        local = frame->local;
-
-        if (op_ret < 0) {
-                uuid_utoa_r (local->loc.inode->gfid, src_gfid);
-
-                if (local->loc2.inode)
-                        uuid_utoa_r (local->loc2.inode->gfid, dst_gfid);
-
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_INODE_LK_ERROR,
-                        "acquiring entrylk after inodelk failed"
-                        "rename (%s:%s:%s %s:%s:%s)",
-                        local->loc.path, src_gfid, local->src_cached->name,
-                        local->loc2.path, dst_gfid,
-                        local->dst_cached ? local->dst_cached->name : NULL);
-
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-                goto err;
-        }
-
-        if (local->current == &local->lock[0]) {
-                loc = &local->loc2;
-                subvol = local->dst_hashed;
-                local->current = &local->lock[1];
-        } else {
-                loc = &local->loc;
-                subvol = local->src_hashed;
-                local->current = &local->lock[0];
-        }
-        ret = dht_protect_namespace (frame, loc, subvol, &local->current->ns,
-                                     dht_rename_dir_lock2_cbk);
-        if (ret < 0) {
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring entrylk after inodelk failed"
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->loc.path, src_gfid, local->src_cached->name,
+               local->loc2.path, dst_gfid,
+               local->dst_cached ? local->dst_cached->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    if (local->current == &local->lock[0]) {
+        loc = &local->loc2;
+        subvol = local->dst_hashed;
+        local->current = &local->lock[1];
+    } else {
+        loc = &local->loc;
+        subvol = local->src_hashed;
+        local->current = &local->lock[0];
+    }
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_dir_lock2_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
 err:
-        /* No harm in calling an extra unlock */
-        dht_rename_dir_unlock (frame, this);
-        return 0;
+    /* No harm in calling an extra unlock */
+    dht_rename_dir_unlock(frame, this);
+    return 0;
 }
 
-
 /*
  * If the hashed subvolumes of both source and dst are the different,
  * lock in dictionary order of hashed subvol->name. This is important
@@ -505,1548 +470,1528 @@ err:
  * dirs are the same.
  *
  */
-static void
-dht_order_rename_lock (call_frame_t *frame, loc_t **loc, xlator_t **subvol)
+static int
+dht_order_rename_lock(call_frame_t *frame, loc_t **loc, xlator_t **subvol)
 {
-        int ret                 = 0;
-        dht_local_t   *local    = NULL;
-        char           src[GF_UUID_BNAME_BUF_SIZE] = {0};
-        char           dst[GF_UUID_BNAME_BUF_SIZE] = {0};
-
+    int ret = 0;
+    int op_ret = 0;
+    dht_local_t *local = NULL;
+    char *src = NULL;
+    char *dst = NULL;
+
+    local = frame->local;
+
+    if (local->src_hashed->name == local->dst_hashed->name) {
+        ret = 0;
+    } else {
+        ret = strcmp(local->src_hashed->name, local->dst_hashed->name);
+    }
+
+    if (ret == 0) {
+        /* hashed subvols are the same for src and dst */
+        /* Entrylks need to be ordered*/
+
+        src = alloca(GF_UUID_BNAME_BUF_SIZE + strlen(local->loc.name) + 1);
+        if (!src) {
+            gf_msg(frame->this->name, GF_LOG_ERROR, ENOMEM, 0,
+                   "Insufficient memory for src");
+            op_ret = -1;
+            goto out;
+        }
+
+        if (!gf_uuid_is_null(local->loc.pargfid))
+            uuid_utoa_r(local->loc.pargfid, src);
+        else if (local->loc.parent)
+            uuid_utoa_r(local->loc.parent->gfid, src);
+        else
+            src[0] = '\0';
 
-        local = frame->local;
+        strcat(src, local->loc.name);
 
-        if (local->src_hashed->name == local->dst_hashed->name) {
-                ret = 0;
-        } else {
-                ret = strcmp (local->src_hashed->name, local->dst_hashed->name);
+        dst = alloca(GF_UUID_BNAME_BUF_SIZE + strlen(local->loc2.name) + 1);
+        if (!dst) {
+            gf_msg(frame->this->name, GF_LOG_ERROR, ENOMEM, 0,
+                   "Insufficient memory for dst");
+            op_ret = -1;
+            goto out;
         }
 
-        if (ret == 0) {
-
-                /* hashed subvols are the same for src and dst */
-                /* Entrylks need to be ordered*/
-                if (local->loc.pargfid)
-                        uuid_utoa_r (local->loc.pargfid, src);
-                else if (local->loc.parent)
-                        uuid_utoa_r (local->loc.parent->gfid, src);
-
-                strcat (src, local->loc.name);
+        if (!gf_uuid_is_null(local->loc2.pargfid))
+            uuid_utoa_r(local->loc2.pargfid, dst);
+        else if (local->loc2.parent)
+            uuid_utoa_r(local->loc2.parent->gfid, dst);
+        else
+            dst[0] = '\0';
 
-                if (local->loc2.pargfid)
-                        uuid_utoa_r (local->loc2.pargfid, dst);
-                else if (local->loc2.parent)
-                        uuid_utoa_r (local->loc2.parent->gfid, dst);
+        strcat(dst, local->loc2.name);
+        ret = strcmp(src, dst);
+    }
 
-                strcat (dst, local->loc2.name);
-                ret = strcmp (src, dst);
-        }
+    if (ret <= 0) {
+        /*inodelk in dictionary order of hashed subvol names*/
+        /*entrylk in dictionary order of gfid/basename */
+        local->current = &local->lock[0];
+        *loc = &local->loc;
+        *subvol = local->src_hashed;
 
-        if (ret <= 0) {
-                /*inodelk in dictionary order of hashed subvol names*/
-                /*entrylk in dictionary order of gfid/basename */
-                local->current = &local->lock[0];
-                *loc = &local->loc;
-                *subvol = local->src_hashed;
+    } else {
+        local->current = &local->lock[1];
+        *loc = &local->loc2;
+        *subvol = local->dst_hashed;
+    }
 
-        } else {
-                local->current = &local->lock[1];
-                *loc = &local->loc2;
-                *subvol = local->dst_hashed;
-        }
+    op_ret = 0;
 
-        return;
+out:
+    return op_ret;
 }
 
 int
-dht_rename_dir (call_frame_t *frame, xlator_t *this)
+dht_rename_dir(call_frame_t *frame, xlator_t *this)
 {
-        dht_conf_t    *conf         = NULL;
-        dht_local_t   *local        = NULL;
-        loc_t         *loc          = NULL;
-        xlator_t      *subvol       = NULL;
-        int            i            = 0;
-        int            ret          = 0;
-        int            op_errno     = -1;
-
-        conf = frame->this->private;
-        local = frame->local;
-
-        local->ret_cache = GF_CALLOC (conf->subvolume_cnt + 1, sizeof (int),
-                                      gf_dht_ret_cache_t);
-
-        if (local->ret_cache == NULL) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->call_cnt = conf->subvolume_cnt;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->subvolume_status[i]) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_RENAME_FAILED,
-                                "Rename dir failed: subvolume down (%s)",
-                                conf->subvolumes[i]->name);
-                        op_errno = ENOTCONN;
-                        goto err;
-                }
-        }
-
-
-        /* Locks on src and dst needs to ordered which otherwise might cause
-         * deadlocks when rename (src, dst) and rename (dst, src) is done from
-         * two different clients
-         */
-        dht_order_rename_lock (frame, &loc, &subvol);
-
-        /* Rename must take locks on src to avoid lookup selfheal from
-         * recreating src on those subvols where the rename was successful.
-         * The locks can't be issued parallel as two different clients might
-         * attempt same rename command and be in dead lock.
-         */
-        ret = dht_protect_namespace (frame, loc, subvol,
-                                     &local->current->ns,
-                                     dht_rename_dir_lock1_cbk);
-        if (ret < 0) {
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        return 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+    int i = 0;
+    int ret = 0;
+    int op_errno = -1;
+
+    conf = frame->this->private;
+    local = frame->local;
+
+    local->ret_cache = GF_CALLOC(conf->subvolume_cnt + 1, sizeof(int),
+                                 gf_dht_ret_cache_t);
+
+    if (local->ret_cache == NULL) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->call_cnt = conf->subvolume_cnt;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->subvolume_status[i]) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+                   "Rename dir failed: subvolume down (%s)",
+                   conf->subvolumes[i]->name);
+            op_errno = ENOTCONN;
+            goto err;
+        }
+    }
+
+    /* Locks on src and dst needs to ordered which otherwise might cause
+     * deadlocks when rename (src, dst) and rename (dst, src) is done from
+     * two different clients
+     */
+    ret = dht_order_rename_lock(frame, &loc, &subvol);
+    if (ret) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /* Rename must take locks on src to avoid lookup selfheal from
+     * recreating src on those subvols where the rename was successful.
+     * The locks can't be issued parallel as two different clients might
+     * attempt same rename command and be in dead lock.
+     */
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_dir_lock1_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-                          NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 static int
-dht_rename_track_for_changelog (xlator_t *this, dict_t *xattr,
-                                loc_t *oldloc, loc_t *newloc)
+dht_rename_track_for_changelog(xlator_t *this, dict_t *xattr, loc_t *oldloc,
+                               loc_t *newloc)
 {
-        int ret        = -1;
-        dht_changelog_rename_info_t *info = NULL;
-        char *name     = NULL;
-        int len1       = 0;
-        int len2       = 0;
-        int size       = 0;
-
-        if (!xattr || !oldloc || !newloc || !this)
-                return ret;
-
-        len1 = strlen (oldloc->name) + 1;
-        len2 = strlen (newloc->name) + 1;
-        size = sizeof (dht_changelog_rename_info_t) + len1 + len2;
-
-        info = GF_CALLOC (size, sizeof(char), gf_common_mt_char);
-        if (!info) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to calloc memory");
-                return ret;
-        }
+    int ret = -1;
+    dht_changelog_rename_info_t *info = NULL;
+    char *name = NULL;
+    int len1 = 0;
+    int len2 = 0;
+    int size = 0;
+
+    if (!xattr || !oldloc || !newloc || !this)
+        return ret;
 
-        gf_uuid_copy (info->old_pargfid, oldloc->pargfid);
-        gf_uuid_copy (info->new_pargfid, newloc->pargfid);
-
-        info->oldname_len = len1;
-        info->newname_len = len2;
-        strncpy (info->buffer, oldloc->name, len1);
-        name = info->buffer + len1;
-        strncpy (name, newloc->name, len2);
-
-        ret = dict_set_bin (xattr, DHT_CHANGELOG_RENAME_OP_KEY,
-                            info, size);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value: key = %s,"
-                        " path = %s", DHT_CHANGELOG_RENAME_OP_KEY,
-                        oldloc->name);
-                GF_FREE (info);
-        }
+    len1 = strlen(oldloc->name) + 1;
+    len2 = strlen(newloc->name) + 1;
+    size = sizeof(dht_changelog_rename_info_t) + len1 + len2;
 
+    info = GF_CALLOC(size, sizeof(char), gf_common_mt_char);
+    if (!info) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to calloc memory");
         return ret;
+    }
+
+    gf_uuid_copy(info->old_pargfid, oldloc->pargfid);
+    gf_uuid_copy(info->new_pargfid, newloc->pargfid);
+
+    info->oldname_len = len1;
+    info->newname_len = len2;
+    strncpy(info->buffer, oldloc->name, len1);
+    name = info->buffer + len1;
+    strncpy(name, newloc->name, len2);
+
+    ret = dict_set_bin(xattr, DHT_CHANGELOG_RENAME_OP_KEY, info, size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s,"
+               " path = %s",
+               DHT_CHANGELOG_RENAME_OP_KEY, oldloc->name);
+        GF_FREE(info);
+    }
+
+    return ret;
 }
 
-
-
-#define DHT_MARKER_DONT_ACCOUNT(xattr) do {                             \
-                int tmp = -1;                                                  \
-                if (!xattr) {                                                  \
-                        xattr = dict_new ();                                   \
-                        if (!xattr)                                            \
-                                break;                                         \
-                }                                                              \
-                tmp = dict_set_str (xattr, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY,  \
-                                    "yes");                                    \
-                if (tmp) {                                                     \
-                        gf_msg (this->name, GF_LOG_ERROR, 0,                    \
-                                DHT_MSG_DICT_SET_FAILED,                       \
-                                "Failed to set dictionary value: key = %s,"    \
-                                " path = %s",GLUSTERFS_MARKER_DONT_ACCOUNT_KEY, \
-                                local->loc.path);                             \
-                }                                                              \
-        }while (0)
-
-
-#define DHT_CHANGELOG_TRACK_AS_RENAME(xattr, oldloc, newloc) do {            \
-                int tmp = -1;                                                \
-                if (!xattr) {                                                \
-                        xattr = dict_new ();                                 \
-                        if (!xattr) {                                        \
-                                gf_msg (this->name, GF_LOG_ERROR, 0,         \
-                                        DHT_MSG_DICT_SET_FAILED,             \
-                                        "Failed to create dictionary to "    \
-                                        "track rename");                     \
-                                break;                                       \
-                        }                                                    \
-                }                                                            \
-                                                                             \
-                tmp = dht_rename_track_for_changelog (this, xattr,           \
-                                oldloc, newloc);                             \
-                                                                             \
-                if (tmp) {                                                   \
-                        gf_msg (this->name, GF_LOG_ERROR, 0,                 \
-                                DHT_MSG_DICT_SET_FAILED,                     \
-                                "Failed to set dictionary value: key = %s,"  \
-                                " path = %s", DHT_CHANGELOG_RENAME_OP_KEY,   \
-                                (oldloc)->path);                             \
-                }                                                            \
-        } while (0)
-
+#define DHT_MARKER_DONT_ACCOUNT(xattr)                                         \
+    do {                                                                       \
+        int tmp = -1;                                                          \
+        if (!xattr) {                                                          \
+            xattr = dict_new();                                                \
+            if (!xattr)                                                        \
+                break;                                                         \
+        }                                                                      \
+        tmp = dict_set_str(xattr, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY, "yes");   \
+        if (tmp) {                                                             \
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,       \
+                   "Failed to set dictionary value: key = %s,"                 \
+                   " path = %s",                                               \
+                   GLUSTERFS_MARKER_DONT_ACCOUNT_KEY, local->loc.path);        \
+        }                                                                      \
+    } while (0)
+
+#define DHT_CHANGELOG_TRACK_AS_RENAME(xattr, oldloc, newloc)                   \
+    do {                                                                       \
+        int tmp = -1;                                                          \
+        if (!xattr) {                                                          \
+            xattr = dict_new();                                                \
+            if (!xattr) {                                                      \
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,   \
+                       "Failed to create dictionary to "                       \
+                       "track rename");                                        \
+                break;                                                         \
+            }                                                                  \
+        }                                                                      \
+                                                                               \
+        tmp = dht_rename_track_for_changelog(this, xattr, oldloc, newloc);     \
+                                                                               \
+        if (tmp) {                                                             \
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,       \
+                   "Failed to set dictionary value: key = %s,"                 \
+                   " path = %s",                                               \
+                   DHT_CHANGELOG_RENAME_OP_KEY, (oldloc)->path);               \
+        }                                                                      \
+    } while (0)
 
 int
-dht_rename_unlock (call_frame_t *frame, xlator_t *this)
+dht_rename_unlock(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t *local                      = NULL;
-        int          op_ret                     = -1;
-        char         src_gfid[GF_UUID_BUF_SIZE] = {0};
-        char         dst_gfid[GF_UUID_BUF_SIZE] = {0};
-        dht_ilock_wrap_t inodelk_wrapper        = {0, };
-
-        local = frame->local;
-        inodelk_wrapper.locks = local->rename_inodelk_backward_compatible;
-        inodelk_wrapper.lk_count = local->rename_inodelk_bc_count;
-
-        op_ret = dht_unlock_inodelk_wrapper (frame, &inodelk_wrapper);
-        if (op_ret < 0) {
-                uuid_utoa_r (local->loc.inode->gfid, src_gfid);
-
-                if (local->loc2.inode)
-                        uuid_utoa_r (local->loc2.inode->gfid, dst_gfid);
-
-                if (IA_ISREG (local->stbuf.ia_type))
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_UNLOCKING_FAILED,
-                                "winding unlock inodelk failed "
-                                "rename (%s:%s:%s %s:%s:%s), "
-                                "stale locks left on bricks",
-                                local->loc.path, src_gfid,
-                                local->src_cached->name,
-                                local->loc2.path, dst_gfid,
-                                local->dst_cached ?
-                                local->dst_cached->name : NULL);
-                else
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_UNLOCKING_FAILED,
-                                "winding unlock inodelk failed "
-                                "rename (%s:%s %s:%s), "
-                                "stale locks left on bricks",
-                                local->loc.path, src_gfid,
-                                local->loc2.path, dst_gfid);
-        }
-
-        dht_unlock_namespace (frame, &local->lock[0]);
-        dht_unlock_namespace (frame, &local->lock[1]);
-
-        dht_rename_unlock_cbk (frame, NULL, this, local->op_ret,
-                               local->op_errno, NULL);
-        return 0;
+    dht_local_t *local = NULL;
+    int op_ret = -1;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_ilock_wrap_t inodelk_wrapper = {
+        0,
+    };
+
+    local = frame->local;
+    inodelk_wrapper.locks = local->rename_inodelk_backward_compatible;
+    inodelk_wrapper.lk_count = local->rename_inodelk_bc_count;
+
+    op_ret = dht_unlock_inodelk_wrapper(frame, &inodelk_wrapper);
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        if (IA_ISREG(local->stbuf.ia_type))
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s:%s %s:%s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->src_cached->name,
+                   local->loc2.path, dst_gfid,
+                   local->dst_cached ? local->dst_cached->name : NULL);
+        else
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s %s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->loc2.path, dst_gfid);
+    }
+
+    dht_unlock_namespace(frame, &local->lock[0]);
+    dht_unlock_namespace(frame, &local->lock[1]);
+
+    dht_rename_unlock_cbk(frame, NULL, this, local->op_ret, local->op_errno,
+                          NULL);
+    return 0;
 }
 
 int
-dht_rename_done (call_frame_t *frame, xlator_t *this)
+dht_rename_done(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t *local                      = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->linked == _gf_true) {
-                local->linked = _gf_false;
-                dht_linkfile_attr_heal (frame, this);
-        }
+    if (local->linked == _gf_true) {
+        local->linked = _gf_false;
+        dht_linkfile_attr_heal(frame, this);
+    }
 
-        dht_rename_unlock (frame, this);
-        return 0;
+    dht_rename_unlock(frame, this);
+    return 0;
 }
 
 int
-dht_rename_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                       struct iatt *postparent, dict_t *xdata)
+dht_rename_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *prev = NULL;
-        int           this_call_cnt = 0;
-
-        local = frame->local;
-        prev  = cookie;
-
-        FRAME_SU_UNDO (frame, dht_local_t);
-        if (!local) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_INVALID_VALUE,
-                        "!local, should not happen");
-                goto out;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
 
-        this_call_cnt = dht_frame_return (frame);
+    local = frame->local;
+    prev = cookie;
 
-        if (op_ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_UNLINK_FAILED,
-                        "%s: Rename: unlink on %s failed ",
-                        local->loc.path, prev->name);
-        }
+    FRAME_SU_UNDO(frame, dht_local_t);
+    if (!local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_VALUE,
+               "!local, should not happen");
+        goto out;
+    }
 
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
+    this_call_cnt = dht_frame_return(frame);
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rename_done (frame, this);
-        }
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLINK_FAILED,
+               "%s: Rename: unlink on %s failed ", local->loc.path, prev->name);
+    }
+
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
+
+    if (is_last_call(this_call_cnt)) {
+        dht_rename_done(frame, this);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-
 int
-dht_rename_cleanup (call_frame_t *frame)
+dht_rename_cleanup(call_frame_t *frame)
 {
-        dht_local_t *local                  = NULL;
-        xlator_t    *this                   = NULL;
-        xlator_t    *src_hashed             = NULL;
-        xlator_t    *src_cached             = NULL;
-        xlator_t    *dst_hashed             = NULL;
-        xlator_t    *dst_cached             = NULL;
-        int          call_cnt               = 0;
-        dict_t      *xattr                  = NULL;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        this  = frame->this;
-
-        src_hashed = local->src_hashed;
-        src_cached = local->src_cached;
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
-
-        if (src_cached == dst_cached)
-                goto nolinks;
-
-        if (local->linked && (dst_hashed != src_hashed )&&
-                (dst_hashed != src_cached)) {
-                call_cnt++;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    int call_cnt = 0;
+    dict_t *xattr = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        if (local->added_link && (src_cached != dst_hashed)) {
-                call_cnt++;
-        }
+    local = frame->local;
+    this = frame->this;
 
-        local->call_cnt = call_cnt;
+    src_hashed = local->src_hashed;
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
 
-        if (!call_cnt)
-                goto nolinks;
+    if (src_cached == dst_cached)
+        goto nolinks;
 
-        DHT_MARK_FOP_INTERNAL (xattr);
+    if (local->linked && (dst_hashed != src_hashed) &&
+        (dst_hashed != src_cached)) {
+        call_cnt++;
+    }
 
-        gf_uuid_unparse(local->loc.inode->gfid, gfid);
+    if (local->added_link && (src_cached != dst_hashed)) {
+        call_cnt++;
+    }
 
-        if (local->linked && (dst_hashed != src_hashed) &&
-                        (dst_hashed != src_cached)) {
-                dict_t *xattr_new = NULL;
+    local->call_cnt = call_cnt;
 
-                gf_msg_trace (this->name, 0,
-                              "unlinking linkfile %s @ %s => %s, (gfid = %s)",
-                              local->loc.path, dst_hashed->name,
-                              src_cached->name, gfid);
+    if (!call_cnt)
+        goto nolinks;
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+    DHT_MARK_FOP_INTERNAL(xattr);
 
+    gf_uuid_unparse(local->loc.inode->gfid, gfid);
 
-                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+    if (local->linked && (dst_hashed != src_hashed) &&
+        (dst_hashed != src_cached)) {
+        dict_t *xattr_new = NULL;
 
-                FRAME_SU_DO (frame, dht_local_t);
-                STACK_WIND_COOKIE (frame, dht_rename_unlink_cbk, dst_hashed,
-                                   dst_hashed, dst_hashed->fops->unlink,
-                                   &local->loc, 0, xattr_new);
+        gf_msg_trace(this->name, 0,
+                     "unlinking linkfile %s @ %s => %s, (gfid = %s)",
+                     local->loc.path, dst_hashed->name, src_cached->name, gfid);
 
-                dict_unref (xattr_new);
-                xattr_new = NULL;
-        }
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-        if (local->added_link && (src_cached != dst_hashed)) {
-                dict_t *xattr_new = NULL;
+        DHT_MARKER_DONT_ACCOUNT(xattr_new);
 
-                gf_msg_trace (this->name, 0,
-                              "unlinking link %s => %s (%s), (gfid = %s)",
-                              local->loc.path, local->loc2.path,
-                              src_cached->name, gfid);
+        FRAME_SU_DO(frame, dht_local_t);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, dst_hashed, dst_hashed,
+                          dst_hashed->fops->unlink, &local->loc, 0, xattr_new);
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-                if (gf_uuid_compare (local->loc.pargfid,
-                                  local->loc2.pargfid) == 0) {
-                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
-                }
-                /* *
-                 * The link to file is created using root permission.
-                 * Hence deletion should happen using root. Otherwise
-                 * it will fail.
-                 */
-                FRAME_SU_DO (frame, dht_local_t);
-                STACK_WIND_COOKIE (frame, dht_rename_unlink_cbk, src_cached,
-                                   src_cached, src_cached->fops->unlink,
-                                   &local->loc2, 0, xattr_new);
+    if (local->added_link && (src_cached != dst_hashed)) {
+        dict_t *xattr_new = NULL;
 
-                dict_unref (xattr_new);
-                xattr_new = NULL;
-        }
+        gf_msg_trace(this->name, 0, "unlinking link %s => %s (%s), (gfid = %s)",
+                     local->loc.path, local->loc2.path, src_cached->name, gfid);
 
-        if (xattr)
-                dict_unref (xattr);
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-        return 0;
-
-nolinks:
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
-
-        dht_rename_unlock (frame, this);
-        return 0;
-}
+        if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+            DHT_MARKER_DONT_ACCOUNT(xattr_new);
+        }
+        /* *
+         * The link to file is created using root permission.
+         * Hence deletion should happen using root. Otherwise
+         * it will fail.
+         */
+        FRAME_SU_DO(frame, dht_local_t);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, src_cached, src_cached,
+                          src_cached->fops->unlink, &local->loc2, 0, xattr_new);
 
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-int
-dht_rename_links_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             inode_t *inode, struct iatt *stbuf,
-                             struct iatt *preparent, struct iatt *postparent,
-                             dict_t *xdata)
-{
-        xlator_t *prev = NULL;
-        dht_local_t  *local = NULL;
+    if (xattr)
+        dict_unref(xattr);
 
-        prev = cookie;
-        local = frame->local;
+    return 0;
 
-        if (op_ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_CREATE_LINK_FAILED,
-                        "link/file %s on %s failed",
-                        local->loc.path, prev->name);
-        }
-
-        if (local->linked == _gf_true) {
-                local->linked = _gf_false;
-                dht_linkfile_attr_heal (frame, this);
-        }
-        DHT_STACK_DESTROY (frame);
+nolinks:
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
 
-        return 0;
+    dht_rename_unlock(frame, this);
+    return 0;
 }
 
-
 int
-dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                struct iatt *preoldparent, struct iatt *postoldparent,
-                struct iatt *prenewparent, struct iatt *postnewparent,
-                dict_t *xdata)
+dht_rename_unlink(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *prev = NULL;
-        xlator_t     *src_hashed = NULL;
-        xlator_t     *src_cached = NULL;
-        xlator_t     *dst_hashed = NULL;
-        xlator_t     *dst_cached = NULL;
-        xlator_t     *rename_subvol = NULL;
-        call_frame_t *link_frame = NULL;
-        dht_local_t *link_local = NULL;
-        dict_t       *xattr     = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        src_hashed = local->src_hashed;
-        src_cached = local->src_cached;
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
-
-        if (local->linked == _gf_true)
-                FRAME_SU_UNDO (frame, dht_local_t);
-
-        /* It is a critical failure iff we fail to rename the cached file
-         * if the rename of the linkto failed, it is not a critical failure,
-         * and we do not want to lose the created hard link for the new
-         * name as that could have been read by other clients.
-         *
-         * NOTE: If another client is attempting the same oldname -> newname
-         * rename, and finds both file names as existing, and are hard links
-         * to each other, then FUSE would send in an unlink for oldname. In
-         * this time duration if we treat the linkto as a critical error and
-         * unlink the newname we created, we would have effectively lost the
-         * file to rename operations.
-         *
-         * Repercussions of treating this as a non-critical error is that
-         * we could leave behind a stale linkto file and/or not create the new
-         * linkto file, the second case would be rectified by a subsequent
-         * lookup, the first case by a rebalance, like for all stale linkto
-         * files */
-
-        if (op_ret == -1) {
-                /* Critical failure: unable to rename the cached file */
-                if (prev == src_cached) {
-                        gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                                DHT_MSG_RENAME_FAILED,
-                                "%s: Rename on %s failed, (gfid = %s) ",
-                                local->loc.path, prev->name,
-                                local->loc.inode ?
-                                uuid_utoa(local->loc.inode->gfid):"");
-                        local->op_ret   = op_ret;
-                        local->op_errno = op_errno;
-                        goto cleanup;
-                } else {
-                        /* Non-critical failure, unable to rename the linkto
-                         * file
-                         */
-                        gf_msg (this->name, GF_LOG_INFO, op_errno,
-                                DHT_MSG_RENAME_FAILED,
-                                "%s: Rename (linkto file) on %s failed, "
-                                "(gfid = %s) ",
-                                local->loc.path, prev->name,
-                                local->loc.inode ?
-                                uuid_utoa(local->loc.inode->gfid):"");
-                }
-        }
-        if (xdata) {
-                if (!local->xattr)
-                        local->xattr = dict_ref (xdata);
-                else
-                        local->xattr = dict_copy_with_ref (xdata, local->xattr);
-        }
+    dht_local_t *local = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    xlator_t *rename_subvol = NULL;
+    dict_t *xattr = NULL;
 
-        if ((src_cached == dst_cached) && (dst_hashed != dst_cached)) {
-                link_frame = copy_frame (frame);
-                if (!link_frame) {
-                        goto err;
-                }
+    local = frame->local;
 
-                /* fop value sent as maxvalue because it is not used
-                   anywhere in this case */
-                link_local = dht_local_init (link_frame, &local->loc2, NULL,
-                                             GF_FOP_MAXVALUE);
-                if (!link_local) {
-                        goto err;
-                }
+    src_hashed = local->src_hashed;
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
 
-                if (link_local->loc.inode)
-                        inode_unref (link_local->loc.inode);
-                link_local->loc.inode = inode_ref (local->loc.inode);
-                gf_uuid_copy (link_local->gfid, local->loc.inode->gfid);
+    local->call_cnt = 0;
 
-                dht_linkfile_create (link_frame, dht_rename_links_create_cbk,
-                                     this, src_cached, dst_hashed,
-                                     &link_local->loc);
-        }
+    /* NOTE: rename_subvol is the same subvolume from which dht_rename_cbk
+     * is called. since rename has already happened on rename_subvol,
+     * unlink shouldn't be sent for oldpath (either linkfile or cached-file)
+     * on rename_subvol. */
+    if (src_cached == dst_cached)
+        rename_subvol = src_cached;
+    else
+        rename_subvol = dst_hashed;
 
-err:
-        /* Merge attrs only from src_cached. In case there of src_cached !=
-         * dst_hashed, this ignores linkfile attrs. */
-        if (prev == src_cached) {
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev);
-                dht_iatt_merge (this, &local->preoldparent, preoldparent,
-                                prev);
-                dht_iatt_merge (this, &local->postoldparent, postoldparent,
-                                prev);
-                dht_iatt_merge (this, &local->preparent, prenewparent, prev);
-                dht_iatt_merge (this, &local->postparent, postnewparent, prev);
-        }
-
-
-        /* NOTE: rename_subvol is the same subvolume from which dht_rename_cbk
-         *       is called. since rename has already happened on rename_subvol,
-         *       unlink should not be sent for oldpath (either linkfile or cached-file)
-         *       on rename_subvol. */
-        if (src_cached == dst_cached)
-                rename_subvol = src_cached;
-        else
-                rename_subvol = dst_hashed;
+    /* TODO: delete files in background */
 
-        /* TODO: delete files in background */
+    if (src_cached != dst_hashed && src_cached != dst_cached)
+        local->call_cnt++;
 
-        if (src_cached != dst_hashed && src_cached != dst_cached)
-                local->call_cnt++;
+    if (src_hashed != rename_subvol && src_hashed != src_cached)
+        local->call_cnt++;
 
-        if (src_hashed != rename_subvol && src_hashed != src_cached)
-                local->call_cnt++;
+    if (dst_cached && dst_cached != dst_hashed && dst_cached != src_cached)
+        local->call_cnt++;
 
-        if (dst_cached && dst_cached != dst_hashed && dst_cached != src_cached)
-                local->call_cnt++;
+    if (local->call_cnt == 0)
+        goto unwind;
 
-        if (local->call_cnt == 0)
-                goto unwind;
+    DHT_MARK_FOP_INTERNAL(xattr);
 
-        DHT_MARK_FOP_INTERNAL (xattr);
+    if (src_cached != dst_hashed && src_cached != dst_cached) {
+        dict_t *xattr_new = NULL;
 
-        if (src_cached != dst_hashed && src_cached != dst_cached) {
-                dict_t *xattr_new = NULL;
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+        gf_msg_trace(this->name, 0, "deleting old src datafile %s @ %s",
+                     local->loc.path, src_cached->name);
 
-                gf_msg_trace (this->name, 0,
-                              "deleting old src datafile %s @ %s",
-                              local->loc.path, src_cached->name);
-
-                if (gf_uuid_compare (local->loc.pargfid,
-                                  local->loc2.pargfid) == 0) {
-                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
-                }
-
-                DHT_CHANGELOG_TRACK_AS_RENAME(xattr_new, &local->loc,
-                                              &local->loc2);
-                STACK_WIND_COOKIE (frame, dht_rename_unlink_cbk, src_cached,
-                                   src_cached, src_cached->fops->unlink,
-                                   &local->loc, 0, xattr_new);
-
-                dict_unref (xattr_new);
-                xattr_new = NULL;
+        if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+            DHT_MARKER_DONT_ACCOUNT(xattr_new);
         }
 
-        if (src_hashed != rename_subvol && src_hashed != src_cached) {
-                dict_t *xattr_new = NULL;
+        DHT_CHANGELOG_TRACK_AS_RENAME(xattr_new, &local->loc, &local->loc2);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, src_cached, src_cached,
+                          src_cached->fops->unlink, &local->loc, 0, xattr_new);
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-                gf_msg_trace (this->name, 0,
-                              "deleting old src linkfile %s @ %s",
-                              local->loc.path, src_hashed->name);
+    if (src_hashed != rename_subvol && src_hashed != src_cached) {
+        dict_t *xattr_new = NULL;
 
-                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-                STACK_WIND_COOKIE (frame, dht_rename_unlink_cbk, src_hashed,
-                                   src_hashed, src_hashed->fops->unlink,
-                                   &local->loc, 0, xattr_new);
+        gf_msg_trace(this->name, 0, "deleting old src linkfile %s @ %s",
+                     local->loc.path, src_hashed->name);
 
-                dict_unref (xattr_new);
-                xattr_new = NULL;
-        }
+        DHT_MARKER_DONT_ACCOUNT(xattr_new);
 
-        if (dst_cached
-            && (dst_cached != dst_hashed)
-            && (dst_cached != src_cached)) {
-                gf_msg_trace (this->name, 0,
-                              "deleting old dst datafile %s @ %s",
-                              local->loc2.path, dst_cached->name);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, src_hashed, src_hashed,
+                          src_hashed->fops->unlink, &local->loc, 0, xattr_new);
 
-                STACK_WIND_COOKIE (frame, dht_rename_unlink_cbk, dst_cached,
-                                   dst_cached, dst_cached->fops->unlink,
-                                   &local->loc2, 0, xattr);
-        }
-        if (xattr)
-                dict_unref (xattr);
-        return 0;
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-unwind:
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
+    if (dst_cached && (dst_cached != dst_hashed) &&
+        (dst_cached != src_cached)) {
+        gf_msg_trace(this->name, 0, "deleting old dst datafile %s @ %s",
+                     local->loc2.path, dst_cached->name);
 
-        dht_rename_done (frame, this);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, dst_cached, dst_cached,
+                          dst_cached->fops->unlink, &local->loc2, 0, xattr);
+    }
+    if (xattr)
+        dict_unref(xattr);
+    return 0;
 
-        return 0;
+unwind:
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
 
-cleanup:
-        dht_rename_cleanup (frame);
+    dht_rename_done(frame, this);
 
-        return 0;
+    return 0;
 }
 
+int
+dht_rename_links_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, inode_t *inode,
+                            struct iatt *stbuf, struct iatt *preparent,
+                            struct iatt *postparent, dict_t *xdata)
+{
+    xlator_t *prev = NULL;
+    dht_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
+
+    prev = cookie;
+    local = frame->local;
+    main_frame = local->main_frame;
+
+    /* TODO: Handle this case in lookup-optimize */
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_CREATE_LINK_FAILED,
+               "link/file %s on %s failed", local->loc.path, prev->name);
+    }
+
+    if (local->linked == _gf_true) {
+        local->linked = _gf_false;
+        dht_linkfile_attr_heal(frame, this);
+    }
+
+    dht_rename_unlink(main_frame, this);
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
 int
-dht_do_rename (call_frame_t *frame)
+dht_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+               struct iatt *preoldparent, struct iatt *postoldparent,
+               struct iatt *prenewparent, struct iatt *postnewparent,
+               dict_t *xdata)
 {
-        dht_local_t *local         = NULL;
-        xlator_t    *dst_hashed    = NULL;
-        xlator_t    *src_cached    = NULL;
-        xlator_t    *dst_cached    = NULL;
-        xlator_t    *this          = NULL;
-        xlator_t    *rename_subvol = NULL;
-
-        local = frame->local;
-        this  = frame->this;
-
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
-        src_cached = local->src_cached;
-
-        if (src_cached == dst_cached)
-                rename_subvol = src_cached;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    call_frame_t *link_frame = NULL;
+    dht_local_t *link_local = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
+
+    if (local->linked == _gf_true)
+        FRAME_SU_UNDO(frame, dht_local_t);
+
+    /* It is a critical failure iff we fail to rename the cached file
+     * if the rename of the linkto failed, it is not a critical failure,
+     * and we do not want to lose the created hard link for the new
+     * name as that could have been read by other clients.
+     *
+     * NOTE: If another client is attempting the same oldname -> newname
+     * rename, and finds both file names as existing, and are hard links
+     * to each other, then FUSE would send in an unlink for oldname. In
+     * this time duration if we treat the linkto as a critical error and
+     * unlink the newname we created, we would have effectively lost the
+     * file to rename operations.
+     *
+     * Repercussions of treating this as a non-critical error is that
+     * we could leave behind a stale linkto file and/or not create the new
+     * linkto file, the second case would be rectified by a subsequent
+     * lookup, the first case by a rebalance, like for all stale linkto
+     * files */
+
+    if (op_ret == -1) {
+        /* Critical failure: unable to rename the cached file */
+        if (prev == src_cached) {
+            gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_RENAME_FAILED,
+                   "%s: Rename on %s failed, (gfid = %s) ", local->loc.path,
+                   prev->name,
+                   local->loc.inode ? uuid_utoa(local->loc.inode->gfid) : "");
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+            goto cleanup;
+        } else {
+            /* Non-critical failure, unable to rename the linkto
+             * file
+             */
+            gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_RENAME_FAILED,
+                   "%s: Rename (linkto file) on %s failed, "
+                   "(gfid = %s) ",
+                   local->loc.path, prev->name,
+                   local->loc.inode ? uuid_utoa(local->loc.inode->gfid) : "");
+        }
+    }
+    if (xdata) {
+        if (!local->xattr)
+            local->xattr = dict_ref(xdata);
         else
-                rename_subvol = dst_hashed;
+            local->xattr = dict_copy_with_ref(xdata, local->xattr);
+    }
+
+    /* Merge attrs only from src_cached. In case there of src_cached !=
+     * dst_hashed, this ignores linkfile attrs. */
+    if (prev == src_cached) {
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->preoldparent, preoldparent);
+        dht_iatt_merge(this, &local->postoldparent, postoldparent);
+        dht_iatt_merge(this, &local->preparent, prenewparent);
+        dht_iatt_merge(this, &local->postparent, postnewparent);
+    }
+
+    /* Create the linkto file for the dst file */
+    if ((src_cached == dst_cached) && (dst_hashed != dst_cached)) {
+        link_frame = copy_frame(frame);
+        if (!link_frame) {
+            goto unlink;
+        }
+
+        /* fop value sent as maxvalue because it is not used
+         * anywhere in this case */
+        link_local = dht_local_init(link_frame, &local->loc2, NULL,
+                                    GF_FOP_MAXVALUE);
+        if (!link_local) {
+            goto unlink;
+        }
+
+        if (link_local->loc.inode)
+            inode_unref(link_local->loc.inode);
+        link_local->loc.inode = inode_ref(local->loc.inode);
+        link_local->main_frame = frame;
+        link_local->stbuf = local->stbuf;
+        gf_uuid_copy(link_local->gfid, local->loc.inode->gfid);
+
+        dht_linkfile_create(link_frame, dht_rename_links_create_cbk, this,
+                            src_cached, dst_hashed, &link_local->loc);
+        return 0;
+    }
 
-        if ((src_cached != dst_hashed) && (rename_subvol == dst_hashed)) {
-                DHT_MARKER_DONT_ACCOUNT(local->xattr_req);
-        }
+unlink:
 
-        if (rename_subvol == src_cached) {
-                DHT_CHANGELOG_TRACK_AS_RENAME(local->xattr_req, &local->loc,
-                                              &local->loc2);
-        }
+    if (link_frame) {
+        DHT_STACK_DESTROY(link_frame);
+    }
+    dht_rename_unlink(frame, this);
+    return 0;
 
-        gf_msg_trace (this->name, 0,
-                      "renaming %s => %s (%s)",
-                      local->loc.path, local->loc2.path, rename_subvol->name);
+cleanup:
+    dht_rename_cleanup(frame);
 
-        if (local->linked == _gf_true)
-                FRAME_SU_DO (frame, dht_local_t);
-        STACK_WIND_COOKIE (frame, dht_rename_cbk, rename_subvol, rename_subvol,
-                           rename_subvol->fops->rename, &local->loc,
-                           &local->loc2, local->xattr_req);
-        return 0;
+    return 0;
 }
 
 int
-dht_rename_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
+dht_do_rename(call_frame_t *frame)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *prev = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_cached = NULL;
+    xlator_t *this = NULL;
+    xlator_t *rename_subvol = NULL;
+
+    local = frame->local;
+    this = frame->this;
+
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
+    src_cached = local->src_cached;
+
+    if (src_cached == dst_cached)
+        rename_subvol = src_cached;
+    else
+        rename_subvol = dst_hashed;
+
+    if ((src_cached != dst_hashed) && (rename_subvol == dst_hashed)) {
+        DHT_MARKER_DONT_ACCOUNT(local->xattr_req);
+    }
+
+    if (rename_subvol == src_cached) {
+        DHT_CHANGELOG_TRACK_AS_RENAME(local->xattr_req, &local->loc,
+                                      &local->loc2);
+    }
+
+    gf_msg_trace(this->name, 0, "renaming %s => %s (%s)", local->loc.path,
+                 local->loc2.path, rename_subvol->name);
+
+    if (local->linked == _gf_true)
+        FRAME_SU_DO(frame, dht_local_t);
+    STACK_WIND_COOKIE(frame, dht_rename_cbk, rename_subvol, rename_subvol,
+                      rename_subvol->fops->rename, &local->loc, &local->loc2,
+                      local->xattr_req);
+    return 0;
+}
 
-        local = frame->local;
-        prev = cookie;
+int
+dht_rename_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *stbuf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
 
-        if (op_ret == -1) {
-                gf_msg_debug (this->name, 0,
-                              "link/file on %s failed (%s)",
-                              prev->name, strerror (op_errno));
-                local->op_ret   = -1;
-                local->op_errno = op_errno;
-                local->added_link = _gf_false;
-        } else
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev);
+    local = frame->local;
+    prev = cookie;
 
-        if (local->op_ret == -1)
-                goto cleanup;
+    if (op_ret == -1) {
+        gf_msg_debug(this->name, 0, "link/file on %s failed (%s)", prev->name,
+                     strerror(op_errno));
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        local->added_link = _gf_false;
+    } else
+        dht_iatt_merge(this, &local->stbuf, stbuf);
 
-        dht_do_rename (frame);
+    if (local->op_ret == -1)
+        goto cleanup;
 
-        return 0;
+    dht_do_rename(frame);
+
+    return 0;
 
 cleanup:
-        dht_rename_cleanup (frame);
+    dht_rename_cleanup(frame);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_rename_linkto_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
+dht_rename_linkto_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
+                      struct iatt *stbuf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t     *local = NULL;
-        xlator_t        *prev = NULL;
-        xlator_t        *src_cached = NULL;
-        dict_t          *xattr = NULL;
-
-        local = frame->local;
-        DHT_MARK_FOP_INTERNAL (xattr);
-        prev = cookie;
-        src_cached = local->src_cached;
-
-        if (op_ret == -1) {
-                gf_msg_debug (this->name, 0,
-                              "link/file on %s failed (%s)",
-                              prev->name, strerror (op_errno));
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src_cached = NULL;
+    dict_t *xattr = NULL;
 
-        /* If linkto creation failed move to failure cleanup code,
-        * instead of continuing with creating the link file */
-        if (local->op_ret != 0) {
-                goto cleanup;
-        }
+    local = frame->local;
+    DHT_MARK_FOP_INTERNAL(xattr);
+    prev = cookie;
+    src_cached = local->src_cached;
 
-        gf_msg_trace (this->name, 0,
-                      "link %s => %s (%s)", local->loc.path,
-                      local->loc2.path, src_cached->name);
-        if (gf_uuid_compare (local->loc.pargfid,
-                                local->loc2.pargfid) == 0) {
-                DHT_MARKER_DONT_ACCOUNT(xattr);
-        }
+    if (op_ret == -1) {
+        gf_msg_debug(this->name, 0, "link/file on %s failed (%s)", prev->name,
+                     strerror(op_errno));
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    }
 
-        local->added_link = _gf_true;
+    /* If linkto creation failed move to failure cleanup code,
+     * instead of continuing with creating the link file */
+    if (local->op_ret != 0) {
+        goto cleanup;
+    }
 
-        STACK_WIND_COOKIE (frame, dht_rename_link_cbk, src_cached, src_cached,
-                           src_cached->fops->link, &local->loc, &local->loc2,
-                           xattr);
+    gf_msg_trace(this->name, 0, "link %s => %s (%s)", local->loc.path,
+                 local->loc2.path, src_cached->name);
+    if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+        DHT_MARKER_DONT_ACCOUNT(xattr);
+    }
 
-        if (xattr)
-                dict_unref (xattr);
+    local->added_link = _gf_true;
 
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_rename_link_cbk, src_cached, src_cached,
+                      src_cached->fops->link, &local->loc, &local->loc2, xattr);
+
+    if (xattr)
+        dict_unref(xattr);
+
+    return 0;
 
 cleanup:
-        dht_rename_cleanup (frame);
+    dht_rename_cleanup(frame);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_rename_unlink_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             struct iatt *preparent, struct iatt *postparent,
-                             dict_t *xdata)
+dht_rename_unlink_links_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            struct iatt *preparent, struct iatt *postparent,
+                            dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	xlator_t     *prev = NULL;
-
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
 
-	local = frame->local;
-	prev = cookie;
+    local = frame->local;
+    prev = cookie;
 
-	if ((op_ret == -1) && (op_errno != ENOENT)) {
-		gf_msg_debug (this->name, 0,
-		              "unlink of %s on %s failed (%s)",
-			      local->loc2.path, prev->name,
-                              strerror (op_errno));
-		local->op_ret   = -1;
-		local->op_errno = op_errno;
-	}
+    if ((op_ret == -1) && (op_errno != ENOENT)) {
+        gf_msg_debug(this->name, 0, "unlink of %s on %s failed (%s)",
+                     local->loc2.path, prev->name, strerror(op_errno));
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    }
 
-        if (local->op_ret == -1)
-                goto cleanup;
+    if (local->op_ret == -1)
+        goto cleanup;
 
-        dht_do_rename (frame);
+    dht_do_rename(frame);
 
-	return 0;
+    return 0;
 
 cleanup:
-        dht_rename_cleanup (frame);
+    dht_rename_cleanup(frame);
 
-	return 0;
+    return 0;
 }
 
-
 int
-dht_rename_create_links (call_frame_t *frame)
+dht_rename_create_links(call_frame_t *frame)
 {
-        dht_local_t *local = NULL;
-        xlator_t    *this = NULL;
-        xlator_t    *src_hashed = NULL;
-        xlator_t    *src_cached = NULL;
-        xlator_t    *dst_hashed = NULL;
-        xlator_t    *dst_cached = NULL;
-        int          call_cnt = 0;
-        dict_t      *xattr = NULL;
-
-
-        local = frame->local;
-        this  = frame->this;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    int call_cnt = 0;
+    dict_t *xattr = NULL;
 
-        src_hashed = local->src_hashed;
-        src_cached = local->src_cached;
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
+    local = frame->local;
+    this = frame->this;
 
-        DHT_MARK_FOP_INTERNAL (xattr);
+    src_hashed = local->src_hashed;
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
 
-        if (src_cached == dst_cached) {
-                dict_t *xattr_new = NULL;
+    DHT_MARK_FOP_INTERNAL(xattr);
 
-                if (dst_hashed == dst_cached)
-                        goto nolinks;
+    if (src_cached == dst_cached) {
+        dict_t *xattr_new = NULL;
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+        if (dst_hashed == dst_cached)
+            goto nolinks;
 
-		gf_msg_trace (this->name, 0,
-		              "unlinking dst linkfile %s @ %s",
-			      local->loc2.path, dst_hashed->name);
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+        gf_msg_trace(this->name, 0, "unlinking dst linkfile %s @ %s",
+                     local->loc2.path, dst_hashed->name);
 
-		STACK_WIND_COOKIE (frame, dht_rename_unlink_links_cbk,
-                                   dst_hashed, dst_hashed,
-			           dst_hashed->fops->unlink, &local->loc2, 0,
-			           xattr_new);
+        DHT_MARKER_DONT_ACCOUNT(xattr_new);
 
-                dict_unref (xattr_new);
-                if (xattr)
-                        dict_unref (xattr);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_links_cbk, dst_hashed,
+                          dst_hashed, dst_hashed->fops->unlink, &local->loc2, 0,
+                          xattr_new);
 
-                return 0;
-        }
+        dict_unref(xattr_new);
+        if (xattr)
+            dict_unref(xattr);
 
-        if (src_cached != dst_hashed) {
-                /* needed to create the link file */
-                call_cnt++;
-                if (dst_hashed != src_hashed)
-                        /* needed to create the linkto file */
-                        call_cnt ++;
+        return 0;
+    }
+
+    if (src_cached != dst_hashed) {
+        /* needed to create the link file */
+        call_cnt++;
+        if (dst_hashed != src_hashed)
+            /* needed to create the linkto file */
+            call_cnt++;
+    }
+
+    /* We should not have any failures post the link creation, as this
+     * introduces the newname into the namespace. Clients could have cached
+     * the existence of the newname and may start taking actions based on
+     * the same. Hence create the linkto first, and then attempt the link.
+     *
+     * NOTE: If another client is attempting the same oldname -> newname
+     * rename, and finds both file names as existing, and are hard links
+     * to each other, then FUSE would send in an unlink for oldname. In
+     * this time duration if we treat the linkto as a critical error and
+     * unlink the newname we created, we would have effectively lost the
+     * file to rename operations. */
+    if (dst_hashed != src_hashed && src_cached != dst_hashed) {
+        gf_msg_trace(this->name, 0, "linkfile %s @ %s => %s", local->loc.path,
+                     dst_hashed->name, src_cached->name);
+
+        memcpy(local->gfid, local->loc.inode->gfid, 16);
+        dht_linkfile_create(frame, dht_rename_linkto_cbk, this, src_cached,
+                            dst_hashed, &local->loc);
+    } else if (src_cached != dst_hashed) {
+        dict_t *xattr_new = NULL;
+
+        xattr_new = dict_copy_with_ref(xattr, NULL);
+
+        gf_msg_trace(this->name, 0, "link %s => %s (%s)", local->loc.path,
+                     local->loc2.path, src_cached->name);
+        if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+            DHT_MARKER_DONT_ACCOUNT(xattr_new);
         }
 
-        /* We should not have any failures post the link creation, as this
-         * introduces the newname into the namespace. Clients could have cached
-         * the existence of the newname and may start taking actions based on
-         * the same. Hence create the linkto first, and then attempt the link.
-         *
-         * NOTE: If another client is attempting the same oldname -> newname
-         * rename, and finds both file names as existing, and are hard links
-         * to each other, then FUSE would send in an unlink for oldname. In
-         * this time duration if we treat the linkto as a critical error and
-         * unlink the newname we created, we would have effectively lost the
-         * file to rename operations. */
-        if (dst_hashed != src_hashed && src_cached != dst_hashed) {
-                gf_msg_trace (this->name, 0,
-                              "linkfile %s @ %s => %s",
-                              local->loc.path, dst_hashed->name,
-                              src_cached->name);
-
-                memcpy (local->gfid, local->loc.inode->gfid, 16);
-                dht_linkfile_create (frame, dht_rename_linkto_cbk, this,
-                                     src_cached, dst_hashed, &local->loc);
-        } else if (src_cached != dst_hashed) {
-                dict_t *xattr_new = NULL;
-
-                xattr_new = dict_copy_with_ref (xattr, NULL);
-
-                gf_msg_trace (this->name, 0,
-                              "link %s => %s (%s)", local->loc.path,
-                              local->loc2.path, src_cached->name);
-                if (gf_uuid_compare (local->loc.pargfid,
-                                  local->loc2.pargfid) == 0) {
-                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
-                }
-
-                local->added_link = _gf_true;
+        local->added_link = _gf_true;
 
-                STACK_WIND_COOKIE (frame, dht_rename_link_cbk, src_cached,
-                                   src_cached, src_cached->fops->link,
-                                   &local->loc, &local->loc2, xattr_new);
+        STACK_WIND_COOKIE(frame, dht_rename_link_cbk, src_cached, src_cached,
+                          src_cached->fops->link, &local->loc, &local->loc2,
+                          xattr_new);
 
-                dict_unref (xattr_new);
-        }
+        dict_unref(xattr_new);
+    }
 
 nolinks:
-        if (!call_cnt) {
-                /* skip to next step */
-                dht_do_rename (frame);
-        }
-        if (xattr)
-                dict_unref (xattr);
-
-        return 0;
+    if (!call_cnt) {
+        /* skip to next step */
+        dht_do_rename(frame);
+    }
+    if (xattr)
+        dict_unref(xattr);
+
+    return 0;
 }
 
 int
-dht_rename_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                       struct iatt *postparent)
+dht_rename_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *stbuf, dict_t *xattr,
+                      struct iatt *postparent)
 {
-        dht_local_t *local                              = NULL;
-        int          call_cnt                           = 0;
-        dht_conf_t  *conf                               = NULL;
-        char         gfid_local[GF_UUID_BUF_SIZE]       = {0};
-        char         gfid_server[GF_UUID_BUF_SIZE]      = {0};
-        int          child_index                        = -1;
-        gf_boolean_t is_src                             = _gf_false;
-        loc_t       *loc                                = NULL;
-
-
-        child_index = (long)cookie;
-
-        local = frame->local;
-        conf = this->private;
-
-        is_src = (child_index == 0);
+    dht_local_t *local = NULL;
+    int call_cnt = 0;
+    dht_conf_t *conf = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char gfid_server[GF_UUID_BUF_SIZE] = {0};
+    int child_index = -1;
+    gf_boolean_t is_src = _gf_false;
+    loc_t *loc = NULL;
+
+    child_index = (long)cookie;
+
+    local = frame->local;
+    conf = this->private;
+
+    is_src = (child_index == 0);
+    if (is_src)
+        loc = &local->loc;
+    else
+        loc = &local->loc2;
+
+    if (op_ret >= 0) {
         if (is_src)
-                loc = &local->loc;
-        else
-                loc = &local->loc2;
-
-        if (op_ret >= 0) {
-                if (is_src)
-                        local->src_cached
-                                = dht_subvol_get_cached (this,
-                                                         local->loc.inode);
-                else {
-                        if (loc->inode)
-                                gf_uuid_unparse (loc->inode->gfid, gfid_local);
-
-                        gf_msg_debug (this->name, 0,
-                                      "dst_cached before lookup: %s, "
-                                      "(path:%s)(gfid:%s),",
-                                      local->loc2.path,
-                                      local->dst_cached
-                                      ? local->dst_cached->name :
-                                      NULL,
-                                      local->dst_cached ? gfid_local : NULL);
-
-                        local->dst_cached
-                                = dht_subvol_get_cached (this,
-                                                         local->loc2_copy.inode);
-
-                        gf_uuid_unparse (stbuf->ia_gfid, gfid_local);
-
-                        gf_msg_debug (this->name, GF_LOG_WARNING,
-                                      "dst_cached after lookup: %s, "
-                                      "(path:%s)(gfid:%s)",
-                                      local->loc2.path,
-                                      local->dst_cached
-                                      ? local->dst_cached->name :
-                                      NULL,
-                                      local->dst_cached ? gfid_local : NULL);
-
-
-                        if ((local->loc2.inode == NULL)
-                            || gf_uuid_compare (stbuf->ia_gfid,
-                                                local->loc2.inode->gfid)) {
-                                if (local->loc2.inode != NULL) {
-                                        inode_unlink (local->loc2.inode,
-                                                      local->loc2.parent,
-                                                      local->loc2.name);
-                                        inode_unref (local->loc2.inode);
-                                }
-
-                                local->loc2.inode
-                                        = inode_link (local->loc2_copy.inode,
-                                                      local->loc2_copy.parent,
-                                                      local->loc2_copy.name,
-                                                      stbuf);
-                                gf_uuid_copy (local->loc2.gfid,
-                                              stbuf->ia_gfid);
-                        }
-                }
-        }
-
-        if (op_ret < 0) {
-                if (is_src) {
-                        /* The meaning of is_linkfile is overloaded here. For locking
-                         * to work properly both rebalance and rename should acquire
-                         * lock on datafile. The reason for sending this lookup is to
-                         * find out whether we've acquired a lock on data file.
-                         * Between the lookup before rename and this rename, the
-                         * file could be migrated by a rebalance process and now this
-                         * file this might be a linkto file. We verify that by sending
-                         * this lookup. However, if this lookup fails we cannot really
-                         * say whether we've acquired lock on a datafile or linkto file.
-                         * So, we act conservatively and _assume_
-                         * that this is a linkfile and fail the rename operation.
-                         */
-                        local->is_linkfile = _gf_true;
-                        local->op_errno = op_errno;
-                } else {
-                        if (local->dst_cached)
-                                gf_msg_debug (this->name, op_errno,
-                                              "file %s (gfid:%s) was present "
-                                              "(hashed-subvol=%s, "
-                                              "cached-subvol=%s) before rename,"
-                                              " but lookup failed",
-                                              local->loc2.path,
-                                              uuid_utoa (local->loc2.inode->gfid),
-                                              local->dst_hashed->name,
-                                              local->dst_cached->name);
-                        if (dht_inode_missing (op_errno))
-                                local->dst_cached = NULL;
+            local->src_cached = dht_subvol_get_cached(this, local->loc.inode);
+        else {
+            if (loc->inode)
+                gf_uuid_unparse(loc->inode->gfid, gfid_local);
+
+            gf_msg_debug(this->name, 0,
+                         "dst_cached before lookup: %s, "
+                         "(path:%s)(gfid:%s),",
+                         local->loc2.path,
+                         local->dst_cached ? local->dst_cached->name : NULL,
+                         local->dst_cached ? gfid_local : NULL);
+
+            local->dst_cached = dht_subvol_get_cached(this,
+                                                      local->loc2_copy.inode);
+
+            gf_uuid_unparse(stbuf->ia_gfid, gfid_local);
+
+            gf_msg_debug(this->name, GF_LOG_WARNING,
+                         "dst_cached after lookup: %s, "
+                         "(path:%s)(gfid:%s)",
+                         local->loc2.path,
+                         local->dst_cached ? local->dst_cached->name : NULL,
+                         local->dst_cached ? gfid_local : NULL);
+
+            if ((local->loc2.inode == NULL) ||
+                gf_uuid_compare(stbuf->ia_gfid, local->loc2.inode->gfid)) {
+                if (local->loc2.inode != NULL) {
+                    inode_unlink(local->loc2.inode, local->loc2.parent,
+                                 local->loc2.name);
+                    inode_unref(local->loc2.inode);
                 }
-        } else if (is_src && xattr && check_is_linkfile (inode, stbuf, xattr,
-                                               conf->link_xattr_name)) {
-                local->is_linkfile = _gf_true;
-                /* Found linkto file instead of data file, passdown ENOENT
-                 * based on the above comment */
-                local->op_errno = ENOENT;
-        }
-
-        if (!local->is_linkfile && (op_ret >= 0) &&
-            gf_uuid_compare (loc->gfid, stbuf->ia_gfid)) {
-                gf_uuid_unparse (loc->gfid, gfid_local);
-                gf_uuid_unparse (stbuf->ia_gfid, gfid_server);
-
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_GFID_MISMATCH,
-                        "path:%s, received a different gfid, local_gfid= %s"
-                        " server_gfid: %s",
-                        local->loc.path, gfid_local, gfid_server);
-
-                /* Will passdown ENOENT anyway since the file we sent on
-                 * rename is replaced with a different file */
-                local->op_errno = ENOENT;
-                /* Since local->is_linkfile is used here to detect failure,
-                 * marking this to true */
-                local->is_linkfile = _gf_true;
-        }
-
-        call_cnt = dht_frame_return (frame);
-        if (is_last_call (call_cnt)) {
-                if (local->is_linkfile) {
-                        local->op_ret = -1;
-                        goto fail;
-                }
-
-                dht_rename_create_links (frame);
-        }
 
-        return 0;
+                local->loc2.inode = inode_link(local->loc2_copy.inode,
+                                               local->loc2_copy.parent,
+                                               local->loc2_copy.name, stbuf);
+                gf_uuid_copy(local->loc2.gfid, stbuf->ia_gfid);
+            }
+        }
+    }
+
+    if (op_ret < 0) {
+        if (is_src) {
+            /* The meaning of is_linkfile is overloaded here. For locking
+             * to work properly both rebalance and rename should acquire
+             * lock on datafile. The reason for sending this lookup is to
+             * find out whether we've acquired a lock on data file.
+             * Between the lookup before rename and this rename, the
+             * file could be migrated by a rebalance process and now this
+             * file this might be a linkto file. We verify that by sending
+             * this lookup. However, if this lookup fails we cannot really
+             * say whether we've acquired lock on a datafile or linkto file.
+             * So, we act conservatively and _assume_
+             * that this is a linkfile and fail the rename operation.
+             */
+            local->is_linkfile = _gf_true;
+            local->op_errno = op_errno;
+        } else {
+            if (local->dst_cached)
+                gf_msg_debug(this->name, op_errno,
+                             "file %s (gfid:%s) was present "
+                             "(hashed-subvol=%s, "
+                             "cached-subvol=%s) before rename,"
+                             " but lookup failed",
+                             local->loc2.path,
+                             uuid_utoa(local->loc2.inode->gfid),
+                             local->dst_hashed->name, local->dst_cached->name);
+            if (dht_inode_missing(op_errno))
+                local->dst_cached = NULL;
+        }
+    } else if (is_src && xattr &&
+               check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name)) {
+        local->is_linkfile = _gf_true;
+        /* Found linkto file instead of data file, passdown ENOENT
+         * based on the above comment */
+        local->op_errno = ENOENT;
+    }
+
+    if (!local->is_linkfile && (op_ret >= 0) &&
+        gf_uuid_compare(loc->gfid, stbuf->ia_gfid)) {
+        gf_uuid_unparse(loc->gfid, gfid_local);
+        gf_uuid_unparse(stbuf->ia_gfid, gfid_server);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "path:%s, received a different gfid, local_gfid= %s"
+               " server_gfid: %s",
+               local->loc.path, gfid_local, gfid_server);
+
+        /* Will passdown ENOENT anyway since the file we sent on
+         * rename is replaced with a different file */
+        local->op_errno = ENOENT;
+        /* Since local->is_linkfile is used here to detect failure,
+         * marking this to true */
+        local->is_linkfile = _gf_true;
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        if (local->is_linkfile) {
+            local->op_ret = -1;
+            goto fail;
+        }
+
+        dht_rename_create_links(frame);
+    }
+
+    return 0;
 fail:
-        dht_rename_unlock (frame, this);
-        return 0;
+    dht_rename_unlock(frame, this);
+    return 0;
 }
 
 int
-dht_rename_file_lock1_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_rename_file_lock1_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local                      = NULL;
-        char         src_gfid[GF_UUID_BUF_SIZE] = {0};
-        char         dst_gfid[GF_UUID_BUF_SIZE] = {0};
-        int          ret                        = 0;
-        loc_t       *loc                        = NULL;
-        xlator_t    *subvol                     = NULL;
-
-        local = frame->local;
-
-        if (op_ret < 0) {
-                uuid_utoa_r (local->loc.inode->gfid, src_gfid);
-
-                if (local->loc2.inode)
-                        uuid_utoa_r (local->loc2.inode->gfid, dst_gfid);
-
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_INODE_LK_ERROR,
-                        "protecting namespace of %s failed"
-                        "rename (%s:%s:%s %s:%s:%s)",
-                        local->current == &local->lock[0] ? local->loc.path
-                        : local->loc2.path,
-                        local->loc.path, src_gfid, local->src_hashed->name,
-                        local->loc2.path, dst_gfid,
-                        local->dst_hashed ? local->dst_hashed->name : NULL);
-
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-                goto err;
-        }
-
-        if (local->current == &local->lock[0]) {
-                loc = &local->loc2;
-                subvol = local->dst_hashed;
-                local->current = &local->lock[1];
-        } else {
-                loc = &local->loc;
-                subvol = local->src_hashed;
-                local->current = &local->lock[0];
-        }
-
-        ret = dht_protect_namespace (frame, loc, subvol, &local->current->ns,
-                                     dht_rename_lock_cbk);
-        if (ret < 0) {
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "protecting namespace of %s failed"
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->current == &local->lock[0] ? local->loc.path
+                                                 : local->loc2.path,
+               local->loc.path, src_gfid, local->src_hashed->name,
+               local->loc2.path, dst_gfid,
+               local->dst_hashed ? local->dst_hashed->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    if (local->current == &local->lock[0]) {
+        loc = &local->loc2;
+        subvol = local->dst_hashed;
+        local->current = &local->lock[1];
+    } else {
+        loc = &local->loc;
+        subvol = local->src_hashed;
+        local->current = &local->lock[0];
+    }
+
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_lock_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
 err:
-        /* No harm in calling an extra unlock */
-        dht_rename_unlock (frame, this);
-        return 0;
+    /* No harm in calling an extra unlock */
+    dht_rename_unlock(frame, this);
+    return 0;
 }
 
 int32_t
-dht_rename_file_protect_namespace (call_frame_t *frame, void *cookie,
-                                   xlator_t *this, int32_t op_ret,
-                                   int32_t op_errno, dict_t *xdata)
+dht_rename_file_protect_namespace(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int32_t op_ret,
+                                  int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local                     = NULL;
-        char         src_gfid[GF_UUID_BUF_SIZE] = {0};
-        char         dst_gfid[GF_UUID_BUF_SIZE] = {0};
-        int          ret                        = 0;
-        loc_t       *loc                        = NULL;
-        xlator_t    *subvol                     = NULL;
-
-        local = frame->local;
-
-        if (op_ret < 0) {
-                uuid_utoa_r (local->loc.inode->gfid, src_gfid);
-
-                if (local->loc2.inode)
-                        uuid_utoa_r (local->loc2.inode->gfid, dst_gfid);
-
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_INODE_LK_ERROR,
-                        "acquiring inodelk failed "
-                        "rename (%s:%s:%s %s:%s:%s)",
-                        local->loc.path, src_gfid, local->src_cached->name,
-                        local->loc2.path, dst_gfid,
-                        local->dst_cached ? local->dst_cached->name : NULL);
-
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-
-                goto err;
-        }
-
-        /* Locks on src and dst needs to ordered which otherwise might cause
-         * deadlocks when rename (src, dst) and rename (dst, src) is done from
-         * two different clients
-         */
-        dht_order_rename_lock (frame, &loc, &subvol);
-
-        ret = dht_protect_namespace (frame, loc, subvol,
-                                     &local->current->ns,
-                                     dht_rename_file_lock1_cbk);
-        if (ret < 0) {
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring inodelk failed "
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->loc.path, src_gfid, local->src_cached->name,
+               local->loc2.path, dst_gfid,
+               local->dst_cached ? local->dst_cached->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+
+        goto err;
+    }
+
+    /* Locks on src and dst needs to ordered which otherwise might cause
+     * deadlocks when rename (src, dst) and rename (dst, src) is done from
+     * two different clients
+     */
+    ret = dht_order_rename_lock(frame, &loc, &subvol);
+    if (ret) {
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_file_lock1_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
 
 err:
-        /* Its fine to call unlock even when no locks are acquired, as we check
-         * for lock->locked before winding a unlock call.
-         */
-        dht_rename_unlock (frame, this);
+    /* Its fine to call unlock even when no locks are acquired, as we check
+     * for lock->locked before winding a unlock call.
+     */
+    dht_rename_unlock(frame, this);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-dht_rename_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_rename_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local                      = NULL;
-        char         src_gfid[GF_UUID_BUF_SIZE] = {0};
-        char         dst_gfid[GF_UUID_BUF_SIZE] = {0};
-        dict_t      *xattr_req                  = NULL;
-        dht_conf_t  *conf                       = NULL;
-        int          i                          = 0;
-        xlator_t    *subvol                     = NULL;
-        dht_lock_t  *lock                       = NULL;
-
-        local = frame->local;
-        conf = this->private;
-
-        if (op_ret < 0) {
-                uuid_utoa_r (local->loc.inode->gfid, src_gfid);
-
-                if (local->loc2.inode)
-                        uuid_utoa_r (local->loc2.inode->gfid, dst_gfid);
-
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_INODE_LK_ERROR,
-                        "protecting namespace of %s failed. "
-                        "rename (%s:%s:%s %s:%s:%s)",
-                        local->current == &local->lock[0] ? local->loc.path
-                        : local->loc2.path,
-                        local->loc.path, src_gfid, local->src_hashed->name,
-                        local->loc2.path, dst_gfid,
-                        local->dst_hashed ? local->dst_hashed->name : NULL);
-
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-
-                goto done;
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    dict_t *xattr_req = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *subvol = NULL;
+    dht_lock_t *lock = NULL;
+
+    local = frame->local;
+    conf = this->private;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "protecting namespace of %s failed. "
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->current == &local->lock[0] ? local->loc.path
+                                                 : local->loc2.path,
+               local->loc.path, src_gfid, local->src_hashed->name,
+               local->loc2.path, dst_gfid,
+               local->dst_hashed ? local->dst_hashed->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+
+        goto done;
+    }
+
+    xattr_req = dict_new();
+    if (xattr_req == NULL) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto done;
+    }
+
+    op_ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = -op_ret;
+        goto done;
+    }
+
+    /* dst_cached might've changed. This normally happens for two reasons:
+     * 1. rebalance migrated dst
+     * 2. Another parallel rename was done overwriting dst
+     *
+     * Doing a lookup on local->loc2 when dst exists, but is associated
+     * with a different gfid will result in an ESTALE error. So, do a fresh
+     * lookup with a new inode on dst-path and handle change of dst-cached
+     * in the cbk. Also, to identify dst-cached changes we do a lookup on
+     * "this" rather than the subvol.
+     */
+    loc_copy(&local->loc2_copy, &local->loc2);
+    inode_unref(local->loc2_copy.inode);
+    local->loc2_copy.inode = inode_new(local->loc.inode->table);
+
+    /* Why not use local->lock.locks[?].loc for lookup post lock phase
+     * ---------------------------------------------------------------
+     * "layout.parent_layout.locks[?].loc" does not have the name and pargfid
+     * populated.
+     * Reason: If we had populated the name and pargfid, server might
+     * resolve to a successful lookup even if there is a file with same name
+     * with a different gfid(unlink & create) as server does name based
+     * resolution on first priority. And this can result in operating on a
+     * different inode entirely.
+     *
+     * Now consider a scenario where source file was renamed by some other
+     * client to a new name just before this lock was granted. So if a
+     * lookup would be done on local->lock[0].layout.parent_layout.locks[?].loc,
+     * server will send success even if the entry was renamed (since server will
+     * do a gfid based resolution). So once a lock is granted, make sure the
+     * file exists with the name that the client requested with.
+     * */
+
+    local->call_cnt = 2;
+    for (i = 0; i < 2; i++) {
+        if (i == 0) {
+            lock = local->rename_inodelk_backward_compatible[0];
+            if (gf_uuid_compare(local->loc.gfid, lock->loc.gfid) == 0)
+                subvol = lock->xl;
+            else {
+                lock = local->rename_inodelk_backward_compatible[1];
+                subvol = lock->xl;
+            }
+        } else {
+            subvol = this;
         }
 
-        xattr_req = dict_new ();
-        if (xattr_req == NULL) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto done;
-        }
+        STACK_WIND_COOKIE(frame, dht_rename_lookup_cbk, (void *)(long)i, subvol,
+                          subvol->fops->lookup,
+                          (i == 0) ? &local->loc : &local->loc2_copy,
+                          xattr_req);
+    }
 
-        op_ret = dict_set_uint32 (xattr_req,
-                                  conf->link_xattr_name, 256);
-        if (op_ret < 0) {
-                local->op_ret = -1;
-                local->op_errno = -op_ret;
-                goto done;
-        }
-
-        /* dst_cached might've changed. This normally happens for two reasons:
-         * 1. rebalance migrated dst
-         * 2. Another parallel rename was done overwriting dst
-         *
-         * Doing a lookup on local->loc2 when dst exists, but is associated
-         * with a different gfid will result in an ESTALE error. So, do a fresh
-         * lookup with a new inode on dst-path and handle change of dst-cached
-         * in the cbk. Also, to identify dst-cached changes we do a lookup on
-         * "this" rather than the subvol.
-         */
-        loc_copy (&local->loc2_copy, &local->loc2);
-        inode_unref (local->loc2_copy.inode);
-        local->loc2_copy.inode = inode_new (local->loc.inode->table);
-
-        /* Why not use local->lock.locks[?].loc for lookup post lock phase
-         * ---------------------------------------------------------------
-         * "layout.parent_layout.locks[?].loc" does not have the name and pargfid
-         * populated.
-         * Reason: If we had populated the name and pargfid, server might
-         * resolve to a successful lookup even if there is a file with same name
-         * with a different gfid(unlink & create) as server does name based
-         * resolution on first priority. And this can result in operating on a
-         * different inode entirely.
-         *
-         * Now consider a scenario where source file was renamed by some other
-         * client to a new name just before this lock was granted. So if a
-         * lookup would be done on local->lock[0].layout.parent_layout.locks[?].loc,
-         * server will send success even if the entry was renamed (since server will
-         * do a gfid based resolution). So once a lock is granted, make sure the file
-         * exists with the name that the client requested with.
-         * */
-
-        local->call_cnt = 2;
-        for (i = 0; i < 2; i++) {
-                if (i == 0) {
-                        lock = local->rename_inodelk_backward_compatible[0];
-                        if (gf_uuid_compare (local->loc.gfid,
-                                             lock->loc.gfid) == 0)
-                                subvol = lock->xl;
-                        else {
-                                lock = local->rename_inodelk_backward_compatible[1];
-                                subvol = lock->xl;
-                        }
-                } else {
-                        subvol = this;
-                }
-
-                STACK_WIND_COOKIE (frame, dht_rename_lookup_cbk,
-                                   (void *)(long)i, subvol,
-                                   subvol->fops->lookup,
-                                   (i == 0) ? &local->loc : &local->loc2_copy,
-                                   xattr_req);
-        }
-
-        dict_unref (xattr_req);
-        return 0;
+    dict_unref(xattr_req);
+    return 0;
 
 done:
-        /* Its fine to call unlock even when no locks are acquired, as we check
-         * for lock->locked before winding a unlock call.
-         */
-        dht_rename_unlock (frame, this);
+    /* Its fine to call unlock even when no locks are acquired, as we check
+     * for lock->locked before winding a unlock call.
+     */
+    dht_rename_unlock(frame, this);
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_rename_lock (call_frame_t *frame)
+dht_rename_lock(call_frame_t *frame)
 {
-        dht_local_t  *local    = NULL;
-        int           count    = 1, ret = -1;
-        dht_lock_t  **lk_array = NULL;
-
-        local = frame->local;
-
-        if (local->dst_cached)
-                count++;
-
-        lk_array = GF_CALLOC (count, sizeof (*lk_array),
-                              gf_common_mt_pointer);
-        if (lk_array == NULL)
-                goto err;
-
-        lk_array[0] = dht_lock_new (frame->this, local->src_cached, &local->loc,
-                                    F_WRLCK, DHT_FILE_MIGRATE_DOMAIN, NULL,
-                                    FAIL_ON_ANY_ERROR);
-        if (lk_array[0] == NULL)
-                goto err;
-
-        if (local->dst_cached) {
-                /* dst might be removed by the time inodelk reaches bricks,
-                 * which can result in ESTALE errors. POSIX imposes no
-                 * restriction for dst to be present for renames to be
-                 * successful. So, we'll ignore ESTALE errors. As far as
-                 * synchronization on dst goes, we'll achieve the same by
-                 * holding entrylk on parent directory of dst in the namespace
-                 * of basename(dst). Also, there might not be quorum in cluster
-                 * xlators like EC/disperse on errno, in which case they return
-                 * EIO. For eg., in a disperse (4 + 2), 3 might return success
-                 * and three might return ESTALE. Disperse, having no Quorum
-                 * unwinds inodelk with EIO. So, ignore EIO too.
-                 */
-                lk_array[1] = dht_lock_new (frame->this, local->dst_cached,
-                                            &local->loc2, F_WRLCK,
-                                            DHT_FILE_MIGRATE_DOMAIN, NULL,
-                                            IGNORE_ENOENT_ESTALE_EIO);
-                if (lk_array[1] == NULL)
-                        goto err;
-        }
-
-        local->rename_inodelk_backward_compatible = lk_array;
-        local->rename_inodelk_bc_count = count;
-
-        /* retaining inodelks for the sake of backward compatibility. Please
-         * make sure to remove this inodelk once all of 3.10, 3.12 and 3.13
-         * reach EOL. Better way of getting synchronization would be to acquire
-         * entrylks on src and dst parent directories in the namespace of
-         * basenames of src and dst
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1;
+    dht_lock_t **lk_array = NULL;
+
+    local = frame->local;
+
+    if (local->dst_cached)
+        count++;
+
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
+    if (lk_array == NULL)
+        goto err;
+
+    lk_array[0] = dht_lock_new(frame->this, local->src_cached, &local->loc,
+                               F_WRLCK, DHT_FILE_MIGRATE_DOMAIN, NULL,
+                               FAIL_ON_ANY_ERROR);
+    if (lk_array[0] == NULL)
+        goto err;
+
+    if (local->dst_cached) {
+        /* dst might be removed by the time inodelk reaches bricks,
+         * which can result in ESTALE errors. POSIX imposes no
+         * restriction for dst to be present for renames to be
+         * successful. So, we'll ignore ESTALE errors. As far as
+         * synchronization on dst goes, we'll achieve the same by
+         * holding entrylk on parent directory of dst in the namespace
+         * of basename(dst). Also, there might not be quorum in cluster
+         * xlators like EC/disperse on errno, in which case they return
+         * EIO. For eg., in a disperse (4 + 2), 3 might return success
+         * and three might return ESTALE. Disperse, having no Quorum
+         * unwinds inodelk with EIO. So, ignore EIO too.
          */
-        ret = dht_blocking_inodelk (frame, lk_array, count,
-                                    dht_rename_file_protect_namespace);
-        if (ret < 0) {
-                local->rename_inodelk_backward_compatible = NULL;
-                local->rename_inodelk_bc_count = 0;
-                goto err;
-        }
-
-        return 0;
+        lk_array[1] = dht_lock_new(frame->this, local->dst_cached, &local->loc2,
+                                   F_WRLCK, DHT_FILE_MIGRATE_DOMAIN, NULL,
+                                   IGNORE_ENOENT_ESTALE_EIO);
+        if (lk_array[1] == NULL)
+            goto err;
+    }
+
+    local->rename_inodelk_backward_compatible = lk_array;
+    local->rename_inodelk_bc_count = count;
+
+    /* retaining inodelks for the sake of backward compatibility. Please
+     * make sure to remove this inodelk once all of 3.10, 3.12 and 3.13
+     * reach EOL. Better way of getting synchronization would be to acquire
+     * entrylks on src and dst parent directories in the namespace of
+     * basenames of src and dst
+     */
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_rename_file_protect_namespace);
+    if (ret < 0) {
+        local->rename_inodelk_backward_compatible = NULL;
+        local->rename_inodelk_bc_count = 0;
+        goto err;
+    }
+
+    return 0;
 err:
-        if (lk_array != NULL) {
-                int tmp_count = 0, i = 0;
+    if (lk_array != NULL) {
+        int tmp_count = 0, i = 0;
 
-                for (i = 0; (i < count) && (lk_array[i]); i++, tmp_count++);
+        for (i = 0; (i < count) && (lk_array[i]); i++, tmp_count++)
+            ;
 
-                dht_lock_array_free (lk_array, tmp_count);
-                GF_FREE (lk_array);
-        }
+        dht_lock_array_free(lk_array, tmp_count);
+        GF_FREE(lk_array);
+    }
 
-        return -1;
+    return -1;
 }
 
 int
-dht_rename (call_frame_t *frame, xlator_t *this,
-            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+dht_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-        xlator_t    *src_cached             = NULL;
-        xlator_t    *src_hashed             = NULL;
-        xlator_t    *dst_cached             = NULL;
-        xlator_t    *dst_hashed             = NULL;
-        int          op_errno               = -1;
-        int          ret                    = -1;
-        dht_local_t *local                  = NULL;
-        dht_conf_t  *conf                   = NULL;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-        char         newgfid[GF_UUID_BUF_SIZE] = {0};
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (newloc, err);
-
-        conf = this->private;
-
-        if (conf->subvolume_cnt == 1) {
-                default_rename (frame, this, oldloc, newloc, xdata);
-                return 0;
-        }
-
-        gf_uuid_unparse(oldloc->inode->gfid, gfid);
-
-        src_hashed = dht_subvol_get_hashed (this, oldloc);
-        if (!src_hashed) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_RENAME_FAILED,
-                        "No hashed subvolume in layout for path=%s,"
-                        "(gfid = %s)", oldloc->path, gfid);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        src_cached = dht_subvol_get_cached (this, oldloc->inode);
-        if (!src_cached) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_RENAME_FAILED,
-                        "No cached subvolume for path = %s,"
-                        "(gfid = %s)", oldloc->path, gfid);
-
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        dst_hashed = dht_subvol_get_hashed (this, newloc);
-        if (!dst_hashed) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_RENAME_FAILED,
-                        "No hashed subvolume in layout for path=%s",
-                        newloc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (newloc->inode)
-                dst_cached = dht_subvol_get_cached (this, newloc->inode);
-
-        local = dht_local_init (frame, oldloc, NULL, GF_FOP_RENAME);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        /* cached_subvol will be set from dht_local_init, reset it to NULL,
-           as the logic of handling rename is different  */
-        local->cached_subvol = NULL;
-
-        ret = loc_copy (&local->loc2, newloc);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->src_hashed = src_hashed;
-        local->src_cached = src_cached;
-        local->dst_hashed = dst_hashed;
-        local->dst_cached = dst_cached;
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
-
-        if (newloc->inode)
-                gf_uuid_unparse(newloc->inode->gfid, newgfid);
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                DHT_MSG_RENAME_INFO,
-                "renaming %s (%s) (hash=%s/cache=%s) => %s (%s) "
-                "(hash=%s/cache=%s) ",
-                oldloc->path, gfid, src_hashed->name, src_cached->name,
-                newloc->path, newloc->inode ? newgfid : NULL, dst_hashed->name,
-                dst_cached ? dst_cached->name : "<nul>");
-
-        if (IA_ISDIR (oldloc->inode->ia_type)) {
-                dht_rename_dir (frame, this);
-        } else {
-                local->op_ret = 0;
-                ret = dht_rename_lock (frame);
-                if (ret < 0) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
+    xlator_t *src_cached = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    int op_errno = -1;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    char newgfid[GF_UUID_BUF_SIZE] = {0};
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(oldloc, err);
+    VALIDATE_OR_GOTO(newloc, err);
+
+    gf_uuid_unparse(oldloc->inode->gfid, gfid);
+
+    src_hashed = dht_subvol_get_hashed(this, oldloc);
+    if (!src_hashed) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+               "No hashed subvolume in layout for path=%s,"
+               "(gfid = %s)",
+               oldloc->path, gfid);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    src_cached = dht_subvol_get_cached(this, oldloc->inode);
+    if (!src_cached) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+               "No cached subvolume for path = %s,"
+               "(gfid = %s)",
+               oldloc->path, gfid);
+
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    dst_hashed = dht_subvol_get_hashed(this, newloc);
+    if (!dst_hashed) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+               "No hashed subvolume in layout for path=%s", newloc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (newloc->inode)
+        dst_cached = dht_subvol_get_cached(this, newloc->inode);
+
+    local = dht_local_init(frame, oldloc, NULL, GF_FOP_RENAME);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    /* cached_subvol will be set from dht_local_init, reset it to NULL,
+       as the logic of handling rename is different  */
+    local->cached_subvol = NULL;
+
+    ret = loc_copy(&local->loc2, newloc);
+    if (ret == -1) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->src_hashed = src_hashed;
+    local->src_cached = src_cached;
+    local->dst_hashed = dst_hashed;
+    local->dst_cached = dst_cached;
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (newloc->inode)
+        gf_uuid_unparse(newloc->inode->gfid, newgfid);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_INFO,
+           "renaming %s (%s) (hash=%s/cache=%s) => %s (%s) "
+           "(hash=%s/cache=%s) ",
+           oldloc->path, gfid, src_hashed->name, src_cached->name, newloc->path,
+           newloc->inode ? newgfid : NULL, dst_hashed->name,
+           dst_cached ? dst_cached->name : "<nul>");
+
+    if (IA_ISDIR(oldloc->inode->ia_type)) {
+        dht_rename_dir(frame, this);
+    } else {
+        local->op_ret = 0;
+        ret = dht_rename_lock(frame);
+        if (ret < 0) {
+            op_errno = ENOMEM;
+            goto err;
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-                          NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
+}
+
+int
+dht_pt_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
+{
+    gf_boolean_t free_xdata = _gf_false;
+
+    /* Just a pass through */
+    if (!IA_ISDIR(oldloc->inode->ia_type)) {
+        if (!xdata) {
+            free_xdata = _gf_true;
+        }
+        DHT_CHANGELOG_TRACK_AS_RENAME(xdata, oldloc, newloc);
+    }
+    default_rename(frame, this, oldloc, newloc, xdata);
+    if (free_xdata && xdata) {
+        dict_unref(xdata);
+        xdata = NULL;
+    }
+    return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index 6425bdc506e..3e24065227c 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -8,2746 +8,2531 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "dht-common.h"
-#include "dht-messages.h"
 #include "dht-lock.h"
-#include "glusterfs-acl.h"
-
-#define DHT_SET_LAYOUT_RANGE(layout,i,srt,chunk,path)    do {           \
-                layout->list[i].start = srt;                            \
-                layout->list[i].stop  = srt + chunk - 1;                \
-                layout->list[i].commit_hash = layout->commit_hash;      \
-                                                                        \
-                gf_msg_trace (this->name, 0,                            \
-                              "gave fix: %u - %u, with commit-hash %u"  \
-                              " on %s for %s",                          \
-                              layout->list[i].start,                    \
-                              layout->list[i].stop,                     \
-                              layout->list[i].commit_hash,              \
-                              layout->list[i].xlator->name, path);      \
-        } while (0)
-
-#define DHT_RESET_LAYOUT_RANGE(layout)    do {                          \
-                int cnt = 0;                                            \
-                for (cnt = 0; cnt < layout->cnt; cnt++ ) {              \
-                        layout->list[cnt].start = 0;                    \
-                        layout->list[cnt].stop  = 0;                    \
-                }                                                       \
-        } while (0)
 
-int
-dht_selfheal_layout_lock (call_frame_t *frame, dht_layout_t *layout,
-                          gf_boolean_t newdir,
-                          dht_selfheal_layout_t healer,
-                          dht_need_heal_t should_heal);
+#define DHT_SET_LAYOUT_RANGE(layout, i, srt, chunk, path)                      \
+    do {                                                                       \
+        layout->list[i].start = srt;                                           \
+        layout->list[i].stop = srt + chunk - 1;                                \
+        layout->list[i].commit_hash = layout->commit_hash;                     \
+                                                                               \
+        gf_msg_trace(this->name, 0,                                            \
+                     "gave fix: 0x%x - 0x%x, with commit-hash 0x%x"            \
+                     " on %s for %s",                                          \
+                     layout->list[i].start, layout->list[i].stop,              \
+                     layout->list[i].commit_hash,                              \
+                     layout->list[i].xlator->name, path);                      \
+    } while (0)
+
+#define DHT_RESET_LAYOUT_RANGE(layout)                                         \
+    do {                                                                       \
+        int cnt = 0;                                                           \
+        for (cnt = 0; cnt < layout->cnt; cnt++) {                              \
+            layout->list[cnt].start = 0;                                       \
+            layout->list[cnt].stop = 0;                                        \
+        }                                                                      \
+    } while (0)
+
+static int
+dht_selfheal_layout_lock(call_frame_t *frame, dht_layout_t *layout,
+                         gf_boolean_t newdir, dht_selfheal_layout_t healer,
+                         dht_need_heal_t should_heal);
 
 static uint32_t
-dht_overlap_calc (dht_layout_t *old, int o, dht_layout_t *new, int n)
+dht_overlap_calc(dht_layout_t *old, int o, dht_layout_t *new, int n)
 {
-	if (o >= old->cnt || n >= new->cnt)
-		return 0;
+    if (o >= old->cnt || n >= new->cnt)
+        return 0;
 
-	if (old->list[o].err > 0 || new->list[n].err > 0)
-		return 0;
+    if (old->list[o].err > 0 || new->list[n].err > 0)
+        return 0;
 
-        if (old->list[o].start == old->list[o].stop) {
-                return 0;
-        }
+    if (old->list[o].start == old->list[o].stop) {
+        return 0;
+    }
 
-        if (new->list[n].start == new->list[n].stop) {
-                return 0;
-        }
+    if (new->list[n].start == new->list[n].stop) {
+        return 0;
+    }
 
-	if ((old->list[o].start > new->list[n].stop) ||
-	    (old->list[o].stop < new->list[n].start))
-		return 0;
+    if ((old->list[o].start > new->list[n].stop) ||
+        (old->list[o].stop < new->list[n].start))
+        return 0;
 
-	return min (old->list[o].stop, new->list[n].stop) -
-	        max (old->list[o].start, new->list[n].start) + 1;
+    return min(old->list[o].stop, new->list[n].stop) -
+           max(old->list[o].start, new->list[n].start) + 1;
 }
 
 int
-dht_selfheal_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_selfheal_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
 int
-dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret,
-                         int invoke_cbk)
+dht_selfheal_dir_finish(call_frame_t *frame, xlator_t *this, int ret,
+                        int invoke_cbk)
 {
-        dht_local_t  *local      = NULL, *lock_local = NULL;
-        call_frame_t *lock_frame = NULL;
-        int           lock_count = 0;
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        /* Unlock entrylk */
-        dht_unlock_entrylk_wrapper (frame, &local->lock[0].ns.directory_ns);
+    /* Unlock entrylk */
+    dht_unlock_entrylk_wrapper(frame, &local->lock[0].ns.directory_ns);
 
-        /* Unlock inodelk */
-        lock_count = dht_lock_count (local->lock[0].ns.parent_layout.locks,
-                                     local->lock[0].ns.parent_layout.lk_count);
-        if (lock_count == 0)
-                goto done;
+    /* Unlock inodelk */
+    lock_count = dht_lock_count(local->lock[0].ns.parent_layout.locks,
+                                local->lock[0].ns.parent_layout.lk_count);
+    if (lock_count == 0)
+        goto done;
 
-        lock_frame = copy_frame (frame);
-        if (lock_frame == NULL) {
-                goto done;
-        }
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        goto done;
+    }
 
-        lock_local = dht_local_init (lock_frame, &local->loc, NULL,
-                                     lock_frame->root->op);
-        if (lock_local == NULL) {
-                goto done;
-        }
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL) {
+        goto done;
+    }
 
-        lock_local->lock[0].ns.parent_layout.locks = local->lock[0].ns.parent_layout.locks;
-        lock_local->lock[0].ns.parent_layout.lk_count = local->lock[0].ns.parent_layout.lk_count;
+    lock_local->lock[0].ns.parent_layout.locks = local->lock[0]
+                                                     .ns.parent_layout.locks;
+    lock_local->lock[0]
+        .ns.parent_layout.lk_count = local->lock[0].ns.parent_layout.lk_count;
 
-        local->lock[0].ns.parent_layout.locks = NULL;
-        local->lock[0].ns.parent_layout.lk_count = 0;
+    local->lock[0].ns.parent_layout.locks = NULL;
+    local->lock[0].ns.parent_layout.lk_count = 0;
 
-        dht_unlock_inodelk (lock_frame,
-                            lock_local->lock[0].ns.parent_layout.locks,
-                            lock_local->lock[0].ns.parent_layout.lk_count,
-                            dht_selfheal_unlock_cbk);
-        lock_frame = NULL;
+    dht_unlock_inodelk(lock_frame, lock_local->lock[0].ns.parent_layout.locks,
+                       lock_local->lock[0].ns.parent_layout.lk_count,
+                       dht_selfheal_unlock_cbk);
+    lock_frame = NULL;
 
 done:
-        if (invoke_cbk)
-                local->selfheal.dir_cbk (frame, NULL, frame->this, ret,
-                                         local->op_errno, NULL);
-        if (lock_frame != NULL) {
-                DHT_STACK_DESTROY (lock_frame);
-        }
+    if (invoke_cbk)
+        local->selfheal.dir_cbk(frame, NULL, frame->this, ret, local->op_errno,
+                                NULL);
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-dht_refresh_layout_done (call_frame_t *frame)
+dht_refresh_layout_done(call_frame_t *frame)
 {
-        int                    ret         = -1;
-        dht_layout_t          *refreshed   = NULL, *heal = NULL;
-        dht_local_t           *local       = NULL;
-        dht_need_heal_t        should_heal = NULL;
-        dht_selfheal_layout_t  healer      = NULL;
+    int ret = -1;
+    dht_layout_t *refreshed = NULL, *heal = NULL;
+    dht_local_t *local = NULL;
+    dht_need_heal_t should_heal = NULL;
+    dht_selfheal_layout_t healer = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        refreshed = local->selfheal.refreshed_layout;
-        heal = local->selfheal.layout;
+    refreshed = local->selfheal.refreshed_layout;
+    heal = local->selfheal.layout;
 
-        healer = local->selfheal.healer;
-        should_heal = local->selfheal.should_heal;
+    healer = local->selfheal.healer;
+    should_heal = local->selfheal.should_heal;
 
-        ret = dht_layout_sort (refreshed);
-        if (ret == -1) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LAYOUT_SORT_FAILED,
-                        "sorting the layout failed");
-                goto err;
-        }
+    ret = dht_layout_sort(refreshed);
+    if (ret == -1) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_LAYOUT_SORT_FAILED, NULL);
+        goto err;
+    }
 
-        if (should_heal (frame, &heal, &refreshed)) {
-                healer (frame, &local->loc, heal);
-        } else {
-                local->selfheal.layout = NULL;
-                local->selfheal.refreshed_layout = NULL;
-                local->selfheal.layout = refreshed;
+    if (should_heal(frame, &heal, &refreshed)) {
+        healer(frame, &local->loc, heal);
+    } else {
+        local->selfheal.layout = NULL;
+        local->selfheal.refreshed_layout = NULL;
+        local->selfheal.layout = refreshed;
 
-                dht_layout_unref (frame->this, heal);
+        dht_layout_unref(frame->this, heal);
 
-                dht_selfheal_dir_finish (frame, frame->this, 0, 1);
-        }
+        dht_selfheal_dir_finish(frame, frame->this, 0, 1);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        dht_selfheal_dir_finish (frame, frame->this, -1, 1);
-        return 0;
+    dht_selfheal_dir_finish(frame, frame->this, -1, 1);
+    return 0;
 }
 
 int
-dht_refresh_layout_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, inode_t *inode,
-                        struct iatt *stbuf, dict_t *xattr,
-                        struct iatt *postparent)
+dht_refresh_layout_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, inode_t *inode,
+                       struct iatt *stbuf, dict_t *xattr,
+                       struct iatt *postparent)
 {
-        dht_local_t  *local                     = NULL;
-        int           this_call_cnt             = 0;
-        xlator_t     *prev                      = NULL;
-        dht_layout_t *layout                    = NULL;
-        char          gfid[GF_UUID_BUF_SIZE]    = {0,};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, err);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, err);
-
-        local = frame->local;
-        prev  = cookie;
-
-        layout = local->selfheal.refreshed_layout;
-
-        LOCK (&frame->lock);
-        {
-                op_ret = dht_layout_merge (this, layout, prev,
-                                           op_ret, op_errno, xattr);
-
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev);
-
-                if (op_ret == -1) {
-                        gf_uuid_unparse (local->loc.gfid, gfid);
-                        local->op_errno = op_errno;
-                        gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                DHT_MSG_FILE_LOOKUP_FAILED,
-                                "lookup of %s on %s returned error, gfid: %s",
-                                local->loc.path, prev->name, gfid);
-
-                        goto unlock;
-                }
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, err);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, err);
+    GF_VALIDATE_OR_GOTO("dht", this->private, err);
 
-                local->op_ret = 0;
+    local = frame->local;
+    prev = cookie;
+
+    layout = local->selfheal.refreshed_layout;
+
+    LOCK(&frame->lock);
+    {
+        op_ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, xattr);
+
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+
+        if (op_ret == -1) {
+            gf_uuid_unparse(local->loc.gfid, gfid);
+            local->op_errno = op_errno;
+            gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                    DHT_MSG_FILE_LOOKUP_FAILED, "path=%s", local->loc.path,
+                    "name=%s", prev->name, "gfid=%s", gfid, NULL);
+
+            goto unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
+        local->op_ret = 0;
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (is_last_call (this_call_cnt)) {
-                if (local->op_ret == 0) {
-                        local->refresh_layout_done (frame);
-                } else {
-                        goto err;
-                }
+    this_call_cnt = dht_frame_return(frame);
 
+    if (is_last_call(this_call_cnt)) {
+        if (local->op_ret == 0) {
+            local->refresh_layout_done(frame);
+        } else {
+            goto err;
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        if (local) {
-                local->refresh_layout_unlock (frame, this, -1, 1);
-        }
-        return 0;
+    if (local) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    }
+    return 0;
 }
 
 int
-dht_refresh_layout (call_frame_t *frame)
+dht_refresh_layout(call_frame_t *frame)
 {
-        int          call_cnt                   = 0;
-        int          i                          = 0, ret = -1;
-        dht_conf_t  *conf                       = NULL;
-        dht_local_t *local                      = NULL;
-        xlator_t    *this                       = NULL;
-        char         gfid[GF_UUID_BUF_SIZE]     = {0,};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-
-        this = frame->this;
-        conf = this->private;
-        local = frame->local;
-
-        call_cnt = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
-        local->op_ret = -1;
-
-        if (local->selfheal.refreshed_layout) {
-                dht_layout_unref (this, local->selfheal.refreshed_layout);
-                local->selfheal.refreshed_layout = NULL;
-        }
-
-        local->selfheal.refreshed_layout = dht_layout_new (this,
-                                                           conf->subvolume_cnt);
-        if (!local->selfheal.refreshed_layout) {
-                gf_uuid_unparse (local->loc.gfid, gfid);
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
-                        "mem allocation for layout failed, path:%s gfid:%s",
-                        local->loc.path, gfid);
-                goto out;
-        }
+    int call_cnt = 0;
+    int i = 0, ret = -1;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+
+    this = frame->this;
+    conf = this->private;
+    local = frame->local;
+
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
+    local->op_ret = -1;
+
+    if (local->selfheal.refreshed_layout) {
+        dht_layout_unref(this, local->selfheal.refreshed_layout);
+        local->selfheal.refreshed_layout = NULL;
+    }
+
+    local->selfheal.refreshed_layout = dht_layout_new(this,
+                                                      conf->subvolume_cnt);
+    if (!local->selfheal.refreshed_layout) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                "path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
 
-        if (local->xattr != NULL) {
-                dict_del (local->xattr, conf->xattr_name);
-        }
+    if (local->xattr != NULL) {
+        dict_del(local->xattr, conf->xattr_name);
+    }
 
+    if (local->xattr_req == NULL) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        local->xattr_req = dict_new();
         if (local->xattr_req == NULL) {
-                gf_uuid_unparse (local->loc.gfid, gfid);
-                local->xattr_req = dict_new ();
-                if (local->xattr_req == NULL) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                DHT_MSG_NO_MEMORY,
-                                "dict mem allocation failed, path:%s gfid:%s",
-                                local->loc.path, gfid);
-                        goto out;
-                }
+            gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                    "path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+            goto out;
         }
+    }
 
-        if (dict_get (local->xattr_req, conf->xattr_name) == 0) {
-                ret = dict_set_uint32 (local->xattr_req, conf->xattr_name,
-                                       4 * 4);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "%s: Failed to set dictionary value:key = %s",
-                                local->loc.path, conf->xattr_name);
-        }
+    if (dict_get(local->xattr_req, conf->xattr_name) == 0) {
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                    "path=%s", local->loc.path, "key=%s", conf->xattr_name,
+                    NULL);
+    }
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND_COOKIE (frame, dht_refresh_layout_cbk,
-                                   conf->subvolumes[i], conf->subvolumes[i],
-                                   conf->subvolumes[i]->fops->lookup,
-                                   &local->loc, local->xattr_req);
-        }
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_refresh_layout_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->lookup, &local->loc,
+                          local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 out:
-        if (local) {
-                local->refresh_layout_unlock (frame, this, -1, 1);
-        }
-        return 0;
+    if (local) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    }
+    return 0;
 }
 
-
 int32_t
-dht_selfheal_layout_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_selfheal_layout_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t     *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (!local) {
-                goto err;
-        }
+    if (!local) {
+        goto err;
+    }
 
-        if (op_ret < 0) {
-                local->op_errno = op_errno;
-                goto err;
-        }
+    if (op_ret < 0) {
+        local->op_errno = op_errno;
+        goto err;
+    }
 
-        local->refresh_layout_unlock = dht_selfheal_dir_finish;
-        local->refresh_layout_done = dht_refresh_layout_done;
+    local->refresh_layout_unlock = dht_selfheal_dir_finish;
+    local->refresh_layout_done = dht_refresh_layout_done;
 
-        dht_refresh_layout (frame);
-        return 0;
+    dht_refresh_layout(frame);
+    return 0;
 
 err:
-        dht_selfheal_dir_finish (frame, this, -1, 1);
-        return 0;
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
 }
 
-
 gf_boolean_t
-dht_should_heal_layout (call_frame_t *frame, dht_layout_t **heal,
-                        dht_layout_t **ondisk)
+dht_should_heal_layout(call_frame_t *frame, dht_layout_t **heal,
+                       dht_layout_t **ondisk)
 {
-        gf_boolean_t  fixit = _gf_true;
-        dht_local_t  *local = NULL;
-        int           ret   = -1, heal_missing_dirs = 0;
-
-        local = frame->local;
-
-        if ((heal == NULL) || (*heal == NULL) || (ondisk == NULL)
-            || (*ondisk == NULL))
-                goto out;
-
-        ret = dht_layout_anomalies (frame->this, &local->loc, *ondisk,
-                                    &local->selfheal.hole_cnt,
-                                    &local->selfheal.overlaps_cnt,
-                                    &local->selfheal.missing_cnt,
-                                    &local->selfheal.down,
-                                    &local->selfheal.misc, NULL);
-
-        if (ret < 0)
-                goto out;
-
-        /* Directories might've been created as part of this self-heal. We've to
-         * sync non-layout xattrs and set range 0-0 on new directories
+    gf_boolean_t fixit = _gf_true;
+    dht_local_t *local = NULL;
+    int heal_missing_dirs = 0;
+
+    local = frame->local;
+
+    if ((heal == NULL) || (*heal == NULL) || (ondisk == NULL) ||
+        (*ondisk == NULL))
+        goto out;
+
+    dht_layout_anomalies(
+        frame->this, &local->loc, *ondisk, &local->selfheal.hole_cnt,
+        &local->selfheal.overlaps_cnt, &local->selfheal.missing_cnt,
+        &local->selfheal.down, &local->selfheal.misc, NULL);
+
+    /* Directories might've been created as part of this self-heal. We've to
+     * sync non-layout xattrs and set range 0-0 on new directories
+     */
+    heal_missing_dirs = local->selfheal.force_mkdir
+                            ? local->selfheal.force_mkdir
+                            : dht_layout_missing_dirs(*heal);
+
+    if ((local->selfheal.hole_cnt == 0) &&
+        (local->selfheal.overlaps_cnt == 0) && heal_missing_dirs) {
+        dht_layout_t *tmp = NULL;
+
+        /* Just added a brick and need to set 0-0 range on this brick.
+         * But ondisk layout is well-formed. So, swap layouts "heal" and
+         * "ondisk". Now "ondisk" layout will be used for healing
+         * xattrs. If there are any non-participating subvols in
+         * "ondisk" layout, dht_selfheal_dir_xattr_persubvol will set
+         * 0-0 and non-layout xattrs. This way we won't end up in
+         * "corrupting" already set and well-formed "ondisk" layout.
          */
-        heal_missing_dirs = local->selfheal.force_mkdir
-                ? local->selfheal.force_mkdir : dht_layout_missing_dirs (*heal);
-
-        if ((local->selfheal.hole_cnt == 0)
-            && (local->selfheal.overlaps_cnt == 0) && heal_missing_dirs) {
-                dht_layout_t *tmp = NULL;
-
-                /* Just added a brick and need to set 0-0 range on this brick.
-                 * But ondisk layout is well-formed. So, swap layouts "heal" and
-                 * "ondisk". Now "ondisk" layout will be used for healing
-                 * xattrs. If there are any non-participating subvols in
-                 * "ondisk" layout, dht_selfheal_dir_xattr_persubvol will set
-                 * 0-0 and non-layout xattrs. This way we won't end up in
-                 * "corrupting" already set and well-formed "ondisk" layout.
-                 */
-                tmp = *heal;
-                *heal = *ondisk;
-                *ondisk = tmp;
-
-                /* Current selfheal code, heals non-layout xattrs only after
-                 * an add-brick. In fact non-layout xattrs are considered as
-                 * secondary citizens which are healed only if layout xattrs
-                 * need to be healed. This is wrong, since for eg., quota can be
-                 * set when layout is well-formed, but a node is down. Also,
-                 * just for healing non-layout xattrs, we don't need locking.
-                 * This issue is _NOT FIXED_ by this patch.
-                 */
-        }
+        tmp = *heal;
+        *heal = *ondisk;
+        *ondisk = tmp;
+
+        /* Current selfheal code, heals non-layout xattrs only after
+         * an add-brick. In fact non-layout xattrs are considered as
+         * secondary citizens which are healed only if layout xattrs
+         * need to be healed. This is wrong, since for eg., quota can be
+         * set when layout is well-formed, but a node is down. Also,
+         * just for healing non-layout xattrs, we don't need locking.
+         * This issue is _NOT FIXED_ by this patch.
+         */
+    }
 
-        fixit = (local->selfheal.hole_cnt || local->selfheal.overlaps_cnt
-                 || heal_missing_dirs);
+    fixit = (local->selfheal.hole_cnt || local->selfheal.overlaps_cnt ||
+             heal_missing_dirs);
 
 out:
-        return fixit;
+    return fixit;
 }
 
 int
-dht_layout_span (dht_layout_t *layout)
+dht_layout_span(dht_layout_t *layout)
 {
-        int i = 0, count = 0;
+    int i = 0, count = 0;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err)
-                        continue;
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err)
+            continue;
 
-                if (layout->list[i].start != layout->list[i].stop)
-                        count++;
-        }
+        if (layout->list[i].start != layout->list[i].stop)
+            count++;
+    }
 
-        return count;
+    return count;
 }
 
 int
-dht_decommissioned_bricks_in_layout (xlator_t *this, dht_layout_t *layout)
+dht_decommissioned_bricks_in_layout(xlator_t *this, dht_layout_t *layout)
 {
-        dht_conf_t *conf  = NULL;
-        int         count = 0, i = 0, j = 0;
+    dht_conf_t *conf = NULL;
+    int count = 0, i = 0, j = 0;
 
-        if ((this == NULL) || (layout == NULL))
-                goto out;
+    if ((this == NULL) || (layout == NULL))
+        goto out;
 
-        conf = this->private;
+    conf = this->private;
 
-        for (i = 0; i < layout->cnt; i++) {
-                for (j = 0; j < conf->subvolume_cnt; j++) {
-                        if (conf->decommissioned_bricks[j] &&
-                            conf->decommissioned_bricks[j]
-                            == layout->list[i].xlator) {
-                                count++;
-                        }
-                }
+    for (i = 0; i < layout->cnt; i++) {
+        for (j = 0; j < conf->subvolume_cnt; j++) {
+            if (conf->decommissioned_bricks[j] &&
+                conf->decommissioned_bricks[j] == layout->list[i].xlator) {
+                count++;
+            }
         }
+    }
 
 out:
-        return count;
+    return count;
 }
 
 dht_distribution_type_t
-dht_distribution_type (xlator_t *this, dht_layout_t *layout)
+dht_distribution_type(xlator_t *this, dht_layout_t *layout)
 {
-        dht_distribution_type_t type        = GF_DHT_EQUAL_DISTRIBUTION;
-        int                     i           = 0;
-        uint32_t                start_range = 0, range = 0, diff = 0;
+    dht_distribution_type_t type = GF_DHT_EQUAL_DISTRIBUTION;
+    int i = 0;
+    uint32_t start_range = 0, range = 0, diff = 0;
 
-        if ((this == NULL) || (layout == NULL) || (layout->cnt < 1)) {
-                goto out;
-        }
+    if ((this == NULL) || (layout == NULL) || (layout->cnt < 1)) {
+        goto out;
+    }
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (start_range == 0) {
-                        start_range = layout->list[i].stop
-                                - layout->list[i].start;
-                        continue;
-                }
+    for (i = 0; i < layout->cnt; i++) {
+        if (start_range == 0) {
+            start_range = layout->list[i].stop - layout->list[i].start;
+            continue;
+        }
 
-                range = layout->list[i].stop - layout->list[i].start;
-                diff = (range >= start_range)
-                        ? range - start_range
-                        : start_range - range;
+        range = layout->list[i].stop - layout->list[i].start;
+        diff = (range >= start_range) ? range - start_range
+                                      : start_range - range;
 
-                if ((range != 0) && (diff > layout->cnt)) {
-                        type = GF_DHT_WEIGHTED_DISTRIBUTION;
-                        break;
-                }
+        if ((range != 0) && (diff > layout->cnt)) {
+            type = GF_DHT_WEIGHTED_DISTRIBUTION;
+            break;
         }
+    }
 
 out:
-        return type;
+    return type;
 }
 
 gf_boolean_t
-dht_should_fix_layout (call_frame_t *frame, dht_layout_t **inmem,
-                       dht_layout_t **ondisk)
+dht_should_fix_layout(call_frame_t *frame, dht_layout_t **inmem,
+                      dht_layout_t **ondisk)
 {
-        gf_boolean_t             fixit                 = _gf_true;
+    gf_boolean_t fixit = _gf_true;
 
-        dht_local_t             *local                 = NULL;
-        int                      layout_span           = 0;
-        int                      decommissioned_bricks = 0;
-        int                      ret                   = 0;
-        dht_conf_t              *conf                  = NULL;
-        dht_distribution_type_t  inmem_dist_type       = 0;
-        dht_distribution_type_t  ondisk_dist_type      = 0;
+    dht_local_t *local = NULL;
+    int layout_span = 0;
+    int decommissioned_bricks = 0;
+    dht_conf_t *conf = NULL;
+    dht_distribution_type_t inmem_dist_type = 0;
+    dht_distribution_type_t ondisk_dist_type = 0;
 
-        conf = frame->this->private;
+    conf = frame->this->private;
 
-        local = frame->local;
+    local = frame->local;
 
-        if ((inmem == NULL) || (*inmem == NULL) || (ondisk == NULL)
-            || (*ondisk == NULL))
-                goto out;
+    if ((inmem == NULL) || (*inmem == NULL) || (ondisk == NULL) ||
+        (*ondisk == NULL))
+        goto out;
 
-        ret = dht_layout_anomalies (frame->this, &local->loc, *ondisk,
-                                    &local->selfheal.hole_cnt,
-                                    &local->selfheal.overlaps_cnt, NULL,
-                                    &local->selfheal.down,
-                                    &local->selfheal.misc, NULL);
-        if (ret < 0) {
-                fixit = _gf_false;
-                goto out;
-        }
+    dht_layout_anomalies(frame->this, &local->loc, *ondisk,
+                         &local->selfheal.hole_cnt,
+                         &local->selfheal.overlaps_cnt, NULL,
+                         &local->selfheal.down, &local->selfheal.misc, NULL);
 
-        if (local->selfheal.down || local->selfheal.misc) {
-                fixit = _gf_false;
-                goto out;
-        }
+    if (local->selfheal.down || local->selfheal.misc) {
+        fixit = _gf_false;
+        goto out;
+    }
 
-        if (local->selfheal.hole_cnt || local->selfheal.overlaps_cnt)
-                goto out;
+    if (local->selfheal.hole_cnt || local->selfheal.overlaps_cnt)
+        goto out;
 
-        /* If commit hashes are being updated, let it through */
-        if ((*inmem)->commit_hash != (*ondisk)->commit_hash)
-                goto out;
+    /* If commit hashes are being updated, let it through */
+    if ((*inmem)->commit_hash != (*ondisk)->commit_hash)
+        goto out;
 
-        layout_span = dht_layout_span (*ondisk);
+    layout_span = dht_layout_span(*ondisk);
 
-        decommissioned_bricks
-                = dht_decommissioned_bricks_in_layout (frame->this,
-                                                       *ondisk);
-        inmem_dist_type = dht_distribution_type (frame->this, *inmem);
-        ondisk_dist_type = dht_distribution_type (frame->this, *ondisk);
+    decommissioned_bricks = dht_decommissioned_bricks_in_layout(frame->this,
+                                                                *ondisk);
+    inmem_dist_type = dht_distribution_type(frame->this, *inmem);
+    ondisk_dist_type = dht_distribution_type(frame->this, *ondisk);
 
-        if ((decommissioned_bricks == 0)
-            && (layout_span == (conf->subvolume_cnt
-                                - conf->decommission_subvols_cnt))
-            && (inmem_dist_type == ondisk_dist_type))
-                fixit = _gf_false;
+    if ((decommissioned_bricks == 0) &&
+        (layout_span ==
+         (conf->subvolume_cnt - conf->decommission_subvols_cnt)) &&
+        (inmem_dist_type == ondisk_dist_type))
+        fixit = _gf_false;
 
 out:
 
-        return fixit;
+    return fixit;
 }
 
-int
-dht_selfheal_layout_lock (call_frame_t *frame, dht_layout_t *layout,
-                          gf_boolean_t newdir,
-                          dht_selfheal_layout_t healer,
-                          dht_need_heal_t should_heal)
+static int
+dht_selfheal_layout_lock(call_frame_t *frame, dht_layout_t *layout,
+                         gf_boolean_t newdir, dht_selfheal_layout_t healer,
+                         dht_need_heal_t should_heal)
 {
-        dht_local_t   *local    = NULL;
-        int            count    = 1, ret = -1, i = 0;
-        dht_lock_t   **lk_array = NULL;
-        dht_conf_t    *conf     = NULL;
-        dht_layout_t  *tmp      = NULL;
-        char           gfid[GF_UUID_BUF_SIZE] = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO (frame->this->name, frame->local, err);
-
-        local = frame->local;
-
-        conf = frame->this->private;
-
-        local->selfheal.healer = healer;
-        local->selfheal.should_heal = should_heal;
-
-        tmp = local->selfheal.layout;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
-        dht_layout_unref (frame->this, tmp);
-
-        if (!newdir) {
-                count = conf->subvolume_cnt;
-
-                lk_array = GF_CALLOC (count, sizeof (*lk_array),
-                                      gf_common_mt_char);
-                if (lk_array == NULL) {
-                        gf_uuid_unparse (local->stbuf.ia_gfid, gfid);
-                        gf_msg ("dht", GF_LOG_ERROR, ENOMEM,
-                                DHT_MSG_NO_MEMORY, "mem allocation failed for "
-                                "lk_array, gfid:%s path: %s", gfid,
-                                local->loc.path);
-                        goto err;
-                }
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1, i = 0;
+    dht_lock_t **lk_array = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *tmp = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-                for (i = 0; i < count; i++) {
-                        lk_array[i] = dht_lock_new (frame->this,
-                                                    conf->subvolumes[i],
-                                                    &local->loc, F_WRLCK,
-                                                    DHT_LAYOUT_HEAL_DOMAIN,
-                                                    NULL,
-                                                    FAIL_ON_ANY_ERROR);
-                        if (lk_array[i] == NULL) {
-                                gf_uuid_unparse (local->stbuf.ia_gfid, gfid);
-                                gf_msg (THIS->name, GF_LOG_ERROR, ENOMEM,
-                                        DHT_MSG_NO_MEMORY, "mem allocation "
-                                        "failed for lk_array, gfid:%s path:%s",
-                                        gfid, local->loc.path);
-                                goto err;
-                        }
-                }
-        } else {
-                count = 1;
-                lk_array = GF_CALLOC (count, sizeof (*lk_array),
-                                      gf_common_mt_char);
-                if (lk_array == NULL) {
-                        gf_uuid_unparse (local->stbuf.ia_gfid, gfid);
-                        gf_msg (THIS->name, GF_LOG_ERROR, ENOMEM,
-                                DHT_MSG_NO_MEMORY, "mem allocation failed for "
-                                "lk_array, gfid:%s path:%s",
-                                gfid, local->loc.path);
-                        goto err;
-                }
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
 
-                lk_array[0] = dht_lock_new (frame->this, local->hashed_subvol,
-                                            &local->loc, F_WRLCK,
-                                            DHT_LAYOUT_HEAL_DOMAIN, NULL,
-                                            FAIL_ON_ANY_ERROR);
-                if (lk_array[0] == NULL) {
-                        gf_uuid_unparse (local->stbuf.ia_gfid, gfid);
-                        gf_msg (THIS->name, GF_LOG_ERROR, ENOMEM,
-                                DHT_MSG_NO_MEMORY, "mem allocation failed for "
-                                "lk_array, gfid:%s path:%s", gfid,
-                                local->loc.path);
-                        goto err;
-                }
-        }
+    local = frame->local;
 
-        local->lock[0].layout.my_layout.locks = lk_array;
-        local->lock[0].layout.my_layout.lk_count = count;
+    conf = frame->this->private;
 
-        ret = dht_blocking_inodelk (frame, lk_array, count,
-                                    dht_selfheal_layout_lock_cbk);
-        if (ret < 0) {
-                local->lock[0].layout.my_layout.locks = NULL;
-                local->lock[0].layout.my_layout.lk_count = 0;
-                goto err;
+    local->selfheal.healer = healer;
+    local->selfheal.should_heal = should_heal;
+
+    tmp = local->selfheal.layout;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
+    dht_layout_unref(frame->this, tmp);
+
+    if (!newdir) {
+        count = conf->subvolume_cnt;
+
+        lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_char);
+        if (lk_array == NULL) {
+            gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+            gf_smsg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                    "lk_array-gfid=%s", gfid, "path=%s", local->loc.path, NULL);
+            goto err;
         }
 
-        return 0;
+        for (i = 0; i < count; i++) {
+            lk_array[i] = dht_lock_new(
+                frame->this, conf->subvolumes[i], &local->loc, F_WRLCK,
+                DHT_LAYOUT_HEAL_DOMAIN, NULL, FAIL_ON_ANY_ERROR);
+            if (lk_array[i] == NULL) {
+                gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+                gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM,
+                        DHT_MSG_MEM_ALLOC_FAILED, "lk_array-gfid=%s", gfid,
+                        "path=%s", local->loc.path, NULL);
+                goto err;
+            }
+        }
+    } else {
+        count = 1;
+        lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_char);
+        if (lk_array == NULL) {
+            gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+            gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                    "lk_array-gfid=%s", gfid, "path=%s", local->loc.path, NULL);
+            goto err;
+        }
+
+        lk_array[0] = dht_lock_new(frame->this, local->hashed_subvol,
+                                   &local->loc, F_WRLCK, DHT_LAYOUT_HEAL_DOMAIN,
+                                   NULL, FAIL_ON_ANY_ERROR);
+        if (lk_array[0] == NULL) {
+            gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+            gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                    "lk_array-gfid=%s", gfid, "path=%s", local->loc.path, NULL);
+            goto err;
+        }
+    }
+
+    local->lock[0].layout.my_layout.locks = lk_array;
+    local->lock[0].layout.my_layout.lk_count = count;
+
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_selfheal_layout_lock_cbk);
+    if (ret < 0) {
+        local->lock[0].layout.my_layout.locks = NULL;
+        local->lock[0].layout.my_layout.lk_count = 0;
+        goto err;
+    }
+
+    return 0;
 err:
-        if (lk_array != NULL) {
-                dht_lock_array_free (lk_array, count);
-                GF_FREE (lk_array);
-        }
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
 
-        return -1;
+    return -1;
 }
 
-int
-dht_selfheal_dir_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_selfheal_dir_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local                     = NULL;
-        xlator_t     *subvol                    = NULL;
-        struct iatt  *stbuf                     = NULL;
-        int           i                         = 0;
-        int           ret                       = 0;
-        dht_layout_t *layout                    = NULL;
-        int           err                       = 0;
-        int           this_call_cnt             = 0;
-        char          gfid[GF_UUID_BUF_SIZE]    = {0};
-
-        local = frame->local;
-        layout = local->selfheal.layout;
-        subvol = cookie;
-
-        if (op_ret == 0) {
-                err = 0;
-        } else {
-                gf_uuid_unparse (local->loc.gfid, gfid);
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                        "layout setxattr failed on %s, path:%s gfid:%s",
-                        subvol->name, local->loc.path, gfid);
-                err = op_errno;
-        }
-
-        ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf);
-        if (ret < 0) {
-                gf_uuid_unparse (local->loc.gfid, gfid);
-                gf_msg_debug (this->name, 0, "key = %s not present in dict"
-                              ", path:%s gfid:%s", DHT_IATT_IN_XDATA_KEY,
-                              local->loc.path, gfid);
-        }
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    struct iatt *stbuf = NULL;
+    int i = 0;
+    int ret = 0;
+    dht_layout_t *layout = NULL;
+    int err = 0;
+    int this_call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    layout = local->selfheal.layout;
+    subvol = cookie;
+
+    if (op_ret == 0) {
+        err = 0;
+    } else {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED, "name=%s", subvol->name,
+                "path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+        err = op_errno;
+    }
+
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
+    if (ret < 0) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_msg_debug(this->name, 0,
+                     "key = %s not present in dict"
+                     ", path:%s gfid:%s",
+                     DHT_IATT_IN_XDATA_KEY, local->loc.path, gfid);
+    }
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].xlator == subvol) {
-                        layout->list[i].err = err;
-                        break;
-                }
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == subvol) {
+            layout->list[i].err = err;
+            break;
         }
+    }
 
-        LOCK (&frame->lock);
-        {
-                dht_iatt_merge (this, &local->stbuf, stbuf, subvol);
-        }
-        UNLOCK (&frame->lock);
+    LOCK(&frame->lock);
+    {
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+    }
+    UNLOCK(&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
+    this_call_cnt = dht_frame_return(frame);
 
-        if (is_last_call (this_call_cnt)) {
-                dht_selfheal_dir_finish (frame, this, 0, 1);
-        }
+    if (is_last_call(this_call_cnt)) {
+        dht_selfheal_dir_finish(frame, this, 0, 1);
+    }
 
-        return 0;
+    return 0;
 }
 
 /* Code is required to set user xattr to local->xattr
-*/
+ */
 int
-dht_set_user_xattr (dict_t *dict, char *k, data_t *v, void *data)
+dht_set_user_xattr(dict_t *dict, char *k, data_t *v, void *data)
 {
-        dict_t          *set_xattr          = data;
-        int              ret                = -1;
+    dict_t *set_xattr = data;
+    int ret = -1;
 
-        ret = dict_set (set_xattr, k, v);
-        return ret;
+    ret = dict_set(set_xattr, k, v);
+    return ret;
 }
 
-
-int
-dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
-                                  dht_layout_t *layout, int i,
-                                  xlator_t *req_subvol)
+static int
+dht_selfheal_dir_xattr_persubvol(call_frame_t *frame, loc_t *loc,
+                                 dht_layout_t *layout, int i,
+                                 xlator_t *req_subvol)
 {
-        xlator_t          *subvol = NULL;
-        dict_t            *xattr = NULL;
-        dict_t            *xdata = NULL;
-        int                ret = 0;
-        xlator_t          *this = NULL;
-        int32_t           *disk_layout = NULL;
-        dht_local_t       *local = NULL;
-        dht_conf_t        *conf = NULL;
-        data_t            *data = NULL;
-        char              gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        if (req_subvol)
-                subvol = req_subvol;
-        else
-                subvol = layout->list[i].xlator;
-        this = frame->this;
-
-        GF_VALIDATE_OR_GOTO ("", this, err);
-        GF_VALIDATE_OR_GOTO (this->name, layout, err);
-        GF_VALIDATE_OR_GOTO (this->name, local, err);
-        GF_VALIDATE_OR_GOTO (this->name, subvol, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        xattr = dict_new ();
-        if (!xattr) {
-                goto err;
-        }
-
-        xdata = dict_new ();
-        if (!xdata)
-                goto err;
-
-        ret = dict_set_str (xdata, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value: key = %s,"
-                        " gfid = %s", loc->path,
-                        GLUSTERFS_INTERNAL_FOP_KEY, gfid);
-                goto err;
-        }
-
-        ret = dict_set_int8 (xdata, DHT_IATT_IN_XDATA_KEY, 1);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value: key = %s,"
-                        " gfid = %s", loc->path,
-                        DHT_IATT_IN_XDATA_KEY, gfid);
-                goto err;
-        }
-
-        gf_uuid_unparse(loc->inode->gfid, gfid);
-
-        ret = dht_disk_layout_extract (this, layout, i, &disk_layout);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                        "Directory self heal xattr failed:"
-                        " %s: (subvol %s) Failed to extract disk layout,"
-                        " gfid = %s", loc->path, subvol->name, gfid);
-                goto err;
-        }
-
-        ret = dict_set_bin (xattr, conf->xattr_name, disk_layout, 4 * 4);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                        "Directory self heal xattr failed:"
-                        "%s: (subvol %s) Failed to set xattr dictionary,"
-                        " gfid = %s", loc->path, subvol->name, gfid);
-                goto err;
-        }
-        disk_layout = NULL;
-
-        gf_msg_trace (this->name, 0,
-                      "setting hash range %u - %u (type %d) on subvolume %s"
-                      " for %s", layout->list[i].start, layout->list[i].stop,
-                      layout->type, subvol->name, loc->path);
-
-        if (local->xattr) {
-                data = dict_get (local->xattr, QUOTA_LIMIT_KEY);
-                if (data) {
-                        ret = dict_add (xattr, QUOTA_LIMIT_KEY, data);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "%s: Failed to set dictionary value:"
-                                        " key = %s",
-                                        loc->path, QUOTA_LIMIT_KEY);
-                        }
-                }
-                data = dict_get (local->xattr, QUOTA_LIMIT_OBJECTS_KEY);
-                if (data) {
-                        ret = dict_add (xattr, QUOTA_LIMIT_OBJECTS_KEY, data);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "%s: Failed to set dictionary value:"
-                                        " key = %s",
-                                        loc->path, QUOTA_LIMIT_OBJECTS_KEY);
-                        }
-                }
-        }
-
-        if (!gf_uuid_is_null (local->gfid))
-                gf_uuid_copy (loc->gfid, local->gfid);
-
-        STACK_WIND_COOKIE (frame, dht_selfheal_dir_xattr_cbk,
-                    (void *) subvol, subvol, subvol->fops->setxattr,
-                    loc, xattr, 0, xdata);
-
-        dict_unref (xattr);
-        dict_unref (xdata);
-
-        return 0;
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xdata = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+    int32_t *disk_layout = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    data_t *data = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    if (req_subvol)
+        subvol = req_subvol;
+    else
+        subvol = layout->list[i].xlator;
+    this = frame->this;
+
+    GF_VALIDATE_OR_GOTO("", this, err);
+    GF_VALIDATE_OR_GOTO(this->name, layout, err);
+    GF_VALIDATE_OR_GOTO(this->name, local, err);
+    GF_VALIDATE_OR_GOTO(this->name, subvol, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    xattr = dict_new();
+    if (!xattr) {
+        goto err;
+    }
+
+    xdata = dict_new();
+    if (!xdata)
+        goto err;
+
+    ret = dict_set_str(xdata, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                "path=%s", loc->path, "key=%s", GLUSTERFS_INTERNAL_FOP_KEY,
+                "gfid=%s", gfid, NULL);
+        goto err;
+    }
+
+    ret = dict_set_int8(xdata, DHT_IATT_IN_XDATA_KEY, 1);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                "path=%s", loc->path, "key=%s", DHT_IATT_IN_XDATA_KEY,
+                "gfid=%s", gfid, NULL);
+        goto err;
+    }
+
+    gf_uuid_unparse(loc->inode->gfid, gfid);
+
+    ret = dht_disk_layout_extract(this, layout, i, &disk_layout);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
+                "extract-disk-layout-failed, path=%s", loc->path, "subvol=%s",
+                subvol->name, "gfid=%s", gfid, NULL);
+        goto err;
+    }
+
+    ret = dict_set_bin(xattr, conf->xattr_name, disk_layout, 4 * 4);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED, "path=%s", loc->path,
+                "subvol=%s", subvol->name,
+                "set-xattr-dictionary-failed"
+                "gfid=%s",
+                gfid, NULL);
+        goto err;
+    }
+    disk_layout = NULL;
+
+    gf_msg_trace(this->name, 0,
+                 "setting hash range 0x%x - 0x%x (type %d) on subvolume %s"
+                 " for %s",
+                 layout->list[i].start, layout->list[i].stop, layout->type,
+                 subvol->name, loc->path);
+
+    if (local->xattr) {
+        data = dict_get(local->xattr, QUOTA_LIMIT_KEY);
+        if (data) {
+            ret = dict_add(xattr, QUOTA_LIMIT_KEY, data);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                        "path=%s", loc->path, "key=%s", QUOTA_LIMIT_KEY, NULL);
+            }
+        }
+        data = dict_get(local->xattr, QUOTA_LIMIT_OBJECTS_KEY);
+        if (data) {
+            ret = dict_add(xattr, QUOTA_LIMIT_OBJECTS_KEY, data);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                        "path=%s", loc->path, "key=%s", QUOTA_LIMIT_OBJECTS_KEY,
+                        NULL);
+            }
+        }
+    }
+
+    if (!gf_uuid_is_null(local->gfid))
+        gf_uuid_copy(loc->gfid, local->gfid);
+
+    STACK_WIND_COOKIE(frame, dht_selfheal_dir_xattr_cbk, (void *)subvol, subvol,
+                      subvol->fops->setxattr, loc, xattr, 0, xdata);
+
+    dict_unref(xattr);
+    dict_unref(xdata);
+
+    return 0;
 
 err:
-        if (xattr)
-                dict_unref (xattr);
-        if (xdata)
-                dict_unref (xdata);
+    if (xattr)
+        dict_unref(xattr);
+    if (xdata)
+        dict_unref(xdata);
 
-        GF_FREE (disk_layout);
+    GF_FREE(disk_layout);
 
-        dht_selfheal_dir_xattr_cbk (frame, (void *) subvol, frame->this,
-                                    -1, ENOMEM, NULL);
-        return 0;
+    dht_selfheal_dir_xattr_cbk(frame, (void *)subvol, frame->this, -1, ENOMEM,
+                               NULL);
+    return 0;
 }
 
-int
-dht_fix_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+static int
+dht_fix_dir_xattr(call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
 {
-        dht_local_t *local = NULL;
-        int          i = 0;
-        int          count = 0;
-        xlator_t    *this = NULL;
-        dht_conf_t  *conf = NULL;
-        dht_layout_t *dummy = NULL;
-
-        local = frame->local;
-        this = frame->this;
-        conf = this->private;
-
-        gf_msg_debug (this->name, 0,
-                      "%s: Writing the new range for all subvolumes",
-                      loc->path);
-
-        local->call_cnt = count = conf->subvolume_cnt;
-
-        if (gf_log_get_loglevel () >= GF_LOG_DEBUG)
-                dht_log_new_layout_for_dir_selfheal (this, loc, layout);
-
-        for (i = 0; i < layout->cnt; i++) {
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
-
-                if (--count == 0)
-                        goto out;
-        }
-        /* if we are here, subvolcount > layout_count. subvols-per-directory
-         * option might be set here. We need to clear out layout from the
-         * non-participating subvolumes, else it will result in overlaps */
-        dummy = dht_layout_new (this, 1);
-        if (!dummy)
-                goto out;
-        dummy->commit_hash = layout->commit_hash;
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (_gf_false ==
-                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
-                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
-                                                          conf->subvolumes[i]);
-                        if (--count == 0)
-                                break;
-                }
-        }
-
-        dht_layout_unref (this, dummy);
+    dht_local_t *local = NULL;
+    int i = 0;
+    int count = 0;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *dummy = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    gf_msg_debug(this->name, 0, "%s: Writing the new range for all subvolumes",
+                 loc->path);
+
+    local->call_cnt = count = conf->subvolume_cnt;
+
+    if (gf_log_get_loglevel() >= GF_LOG_DEBUG)
+        dht_log_new_layout_for_dir_selfheal(this, loc, layout);
+
+    for (i = 0; i < layout->cnt; i++) {
+        dht_selfheal_dir_xattr_persubvol(frame, loc, layout, i, NULL);
+
+        if (--count == 0)
+            goto out;
+    }
+    /* if we are here, subvolcount > layout_count. subvols-per-directory
+     * option might be set here. We need to clear out layout from the
+     * non-participating subvolumes, else it will result in overlaps */
+    dummy = dht_layout_new(this, 1);
+    if (!dummy)
+        goto out;
+    dummy->commit_hash = layout->commit_hash;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (_gf_false == dht_is_subvol_in_layout(layout, conf->subvolumes[i])) {
+            dht_selfheal_dir_xattr_persubvol(frame, loc, dummy, 0,
+                                             conf->subvolumes[i]);
+            if (--count == 0)
+                break;
+        }
+    }
+
+    dht_layout_unref(this, dummy);
 out:
-        return 0;
+    return 0;
 }
 
-int
-dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+static int
+dht_selfheal_dir_xattr(call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
 {
-        dht_local_t *local = NULL;
-        int          missing_xattr = 0;
-        int          i = 0;
-        xlator_t    *this = NULL;
-        dht_conf_t   *conf = NULL;
-        dht_layout_t *dummy = NULL;
-        char          gfid[GF_UUID_BUF_SIZE] = {0,};
-
-        local = frame->local;
-        this = frame->this;
-        conf = this->private;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err != -1 || !layout->list[i].stop) {
-                        /* err != -1 would mean xattr present on the directory
-                         * or the directory is non existent.
-                         * !layout->list[i].stop would mean layout absent
-                         */
-
-                        continue;
-                }
-                missing_xattr++;
-        }
-        /* Also account for subvolumes with no-layout. Used for zero'ing out
-         * the layouts and for setting quota key's if present */
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (_gf_false ==
-                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
-                        missing_xattr++;
-                }
-        }
-        gf_msg_trace (this->name, 0,
-                      "%d subvolumes missing xattr for %s",
-                      missing_xattr, loc->path);
-
-        if (missing_xattr == 0) {
-                dht_selfheal_dir_finish (frame, this, 0, 1);
-                return 0;
-        }
-
-        local->call_cnt = missing_xattr;
-
-        if (gf_log_get_loglevel () >= GF_LOG_DEBUG)
-                dht_log_new_layout_for_dir_selfheal (this, loc, layout);
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err != -1 || !layout->list[i].stop)
-                        continue;
-
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
-
-                if (--missing_xattr == 0)
-                        break;
-        }
-        dummy = dht_layout_new (this, 1);
-        if (!dummy) {
-                gf_uuid_unparse (loc->gfid, gfid);
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
-                        "failed to allocate dummy layout, path:%s gfid:%s",
-                        loc->path, gfid);
-                goto out;
-        }
-        for (i = 0; i < conf->subvolume_cnt && missing_xattr; i++) {
-                if (_gf_false ==
-                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
-                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
-                                                          conf->subvolumes[i]);
-                        missing_xattr--;
-                }
-        }
-
-        dht_layout_unref (this, dummy);
-out:
+    dht_local_t *local = NULL;
+    int missing_xattr = 0;
+    int i = 0;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *dummy = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err != -1 || !layout->list[i].stop) {
+            /* err != -1 would mean xattr present on the directory
+             * or the directory is non existent.
+             * !layout->list[i].stop would mean layout absent
+             */
+
+            continue;
+        }
+        missing_xattr++;
+    }
+    /* Also account for subvolumes with no-layout. Used for zero'ing out
+     * the layouts and for setting quota key's if present */
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (_gf_false == dht_is_subvol_in_layout(layout, conf->subvolumes[i])) {
+            missing_xattr++;
+        }
+    }
+    gf_msg_trace(this->name, 0, "%d subvolumes missing xattr for %s",
+                 missing_xattr, loc->path);
+
+    if (missing_xattr == 0) {
+        dht_selfheal_dir_finish(frame, this, 0, 1);
         return 0;
-}
-
-gf_boolean_t
-dht_is_subvol_part_of_layout (dht_layout_t *layout, xlator_t *xlator)
-{
-        int               i = 0;
-        gf_boolean_t    ret = _gf_false;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (!strcmp (layout->list[i].xlator->name, xlator->name)) {
-                        ret = _gf_true;
-                        break;
-
-                }
-        }
-
-        return ret;
-}
-
-int
-dht_layout_index_from_conf (dht_layout_t *layout, xlator_t *xlator)
-{
-        int i = -1;
-        int j = 0;
-
-        for (j = 0; j < layout->cnt; j++) {
-                if (!strcmp (layout->list[j].xlator->name, xlator->name)) {
-                        i = j;
-                        break;
-                }
-        }
-
-        return i;
+    }
+
+    local->call_cnt = missing_xattr;
+
+    if (gf_log_get_loglevel() >= GF_LOG_DEBUG)
+        dht_log_new_layout_for_dir_selfheal(this, loc, layout);
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err != -1 || !layout->list[i].stop)
+            continue;
+
+        dht_selfheal_dir_xattr_persubvol(frame, loc, layout, i, NULL);
+
+        if (--missing_xattr == 0)
+            break;
+    }
+    dummy = dht_layout_new(this, 1);
+    if (!dummy) {
+        gf_uuid_unparse(loc->gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_DUMMY_ALLOC_FAILED,
+                "path=%s", loc->path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+    for (i = 0; i < conf->subvolume_cnt && missing_xattr; i++) {
+        if (_gf_false == dht_is_subvol_in_layout(layout, conf->subvolumes[i])) {
+            dht_selfheal_dir_xattr_persubvol(frame, loc, dummy, 0,
+                                             conf->subvolumes[i]);
+            missing_xattr--;
+        }
+    }
+
+    dht_layout_unref(this, dummy);
+out:
+    return 0;
 }
 
 int
-dht_selfheal_dir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int op_ret, int op_errno, struct iatt *statpre,
-                              struct iatt *statpost, dict_t *xdata)
+dht_selfheal_dir_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, struct iatt *statpre,
+                             struct iatt *statpost, dict_t *xdata)
 {
-        dht_local_t   *local = NULL;
-        dht_layout_t  *layout = NULL;
-        int            this_call_cnt = 0, ret = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int this_call_cnt = 0, ret = -1;
 
-        local  = frame->local;
-        layout = local->selfheal.layout;
+    local = frame->local;
+    layout = local->selfheal.layout;
 
-        this_call_cnt = dht_frame_return (frame);
+    this_call_cnt = dht_frame_return(frame);
 
-        if (is_last_call (this_call_cnt)) {
-                if (!local->heal_layout) {
-                        gf_msg_trace (this->name, 0,
-                                      "Skip heal layout for %s gfid = %s ",
-                                      local->loc.path, uuid_utoa(local->gfid));
+    if (is_last_call(this_call_cnt)) {
+        if (!local->heal_layout) {
+            gf_msg_trace(this->name, 0, "Skip heal layout for %s gfid = %s ",
+                         local->loc.path, uuid_utoa(local->gfid));
 
-                        dht_selfheal_dir_finish (frame, this, 0, 1);
-                        return 0;
-                }
-                ret = dht_selfheal_layout_lock (frame, layout, _gf_false,
-                                                dht_selfheal_dir_xattr,
-                                                dht_should_heal_layout);
+            dht_selfheal_dir_finish(frame, this, 0, 1);
+            return 0;
+        }
+        ret = dht_selfheal_layout_lock(frame, layout, _gf_false,
+                                       dht_selfheal_dir_xattr,
+                                       dht_should_heal_layout);
 
-                if (ret < 0) {
-                        dht_selfheal_dir_finish (frame, this, -1, 1);
-                }
+        if (ret < 0) {
+            dht_selfheal_dir_finish(frame, this, -1, 1);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_selfheal_dir_setattr (call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
-                          int32_t valid, dht_layout_t *layout)
+dht_selfheal_dir_setattr(call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
+                         int32_t valid, dht_layout_t *layout)
 {
-        int           missing_attr = 0;
-        int           i     = 0, ret = -1;
-        dht_local_t  *local = NULL;
-        xlator_t     *this = NULL;
-        int           cnt  = 0;
-
-        local = frame->local;
-        this = frame->this;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err == -1)
-                        missing_attr++;
-        }
-
-        if (missing_attr == 0) {
-                if (!local->heal_layout) {
-                        gf_msg_trace (this->name, 0,
-                                      "Skip heal layout for %s gfid = %s ",
-                                      loc->path, uuid_utoa(loc->gfid));
-                        dht_selfheal_dir_finish (frame, this, 0, 1);
-                        return 0;
-                }
-                ret = dht_selfheal_layout_lock (frame, layout, _gf_false,
-                                                dht_selfheal_dir_xattr,
-                                                dht_should_heal_layout);
+    int missing_attr = 0;
+    int i = 0, ret = -1;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    int cnt = 0;
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    /* We need to heal the attrs if:
+     * 1. Any directories were missing - the newly created dirs will need
+     *    to have the correct attrs set
+     * 2. An existing dir does not have the correct permissions -they may
+     *    have been changed when a brick was down.
+     */
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err == -1)
+            missing_attr++;
+    }
+
+    if ((missing_attr == 0) && (local->need_attrheal == 0)) {
+        if (!local->heal_layout) {
+            gf_msg_trace(this->name, 0, "Skip heal layout for %s gfid = %s ",
+                         loc->path, uuid_utoa(loc->gfid));
+            dht_selfheal_dir_finish(frame, this, 0, 1);
+            return 0;
+        }
+        ret = dht_selfheal_layout_lock(frame, layout, _gf_false,
+                                       dht_selfheal_dir_xattr,
+                                       dht_should_heal_layout);
 
-                if (ret < 0) {
-                        dht_selfheal_dir_finish (frame, this, -1, 1);
-                }
-
-                return 0;
+        if (ret < 0) {
+            dht_selfheal_dir_finish(frame, this, -1, 1);
         }
 
-        local->call_cnt = missing_attr;
-        cnt = layout->cnt;
+        return 0;
+    }
 
-        for (i = 0; i < cnt; i++) {
-                if (layout->list[i].err == -1) {
-                        gf_msg_trace (this->name, 0,
-                                      "%s: setattr on subvol %s, gfid = %s",
-                                      loc->path, layout->list[i].xlator->name,
-                                      uuid_utoa(loc->gfid));
+    cnt = local->call_cnt = conf->subvolume_cnt;
 
-                        STACK_WIND (frame, dht_selfheal_dir_setattr_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->setattr,
-                                    loc, stbuf, valid, NULL);
-                }
-        }
+    for (i = 0; i < cnt; i++) {
+        STACK_WIND(frame, dht_selfheal_dir_setattr_cbk, layout->list[i].xlator,
+                   layout->list[i].xlator->fops->setattr, loc, stbuf, valid,
+                   NULL);
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_selfheal_dir_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int op_ret, int op_errno,
-                            inode_t *inode, struct iatt *stbuf,
-                            struct iatt *preparent, struct iatt *postparent,
-                            dict_t *xdata)
+static int
+dht_selfheal_dir_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int op_ret, int op_errno, inode_t *inode,
+                           struct iatt *stbuf, struct iatt *preparent,
+                           struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t   *local = NULL;
-        dht_layout_t  *layout = NULL;
-        xlator_t      *prev = NULL;
-        xlator_t      *subvol = NULL;
-        int            i = 0, ret = -1;
-        int            this_call_cnt = 0;
-        char           gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local  = frame->local;
-        layout = local->selfheal.layout;
-        prev   = cookie;
-        subvol = prev;
-
-        if ((op_ret == 0) || ((op_ret == -1) && (op_errno == EEXIST))) {
-                for (i = 0; i < layout->cnt; i++) {
-                        if (layout->list[i].xlator == subvol) {
-                                layout->list[i].err = -1;
-                                break;
-                        }
-                }
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *subvol = NULL;
+    int i = 0, ret = -1;
+    int this_call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    layout = local->selfheal.layout;
+    prev = cookie;
+    subvol = prev;
+
+    if ((op_ret == 0) || ((op_ret == -1) && (op_errno == EEXIST))) {
+        for (i = 0; i < layout->cnt; i++) {
+            if (layout->list[i].xlator == subvol) {
+                layout->list[i].err = -1;
+                break;
+            }
         }
+    }
 
-        if (op_ret) {
-                gf_uuid_unparse(local->loc.gfid, gfid);
-                gf_msg (this->name, ((op_errno == EEXIST) ? GF_LOG_DEBUG :
-                                     GF_LOG_WARNING),
-                        op_errno, DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "Directory selfheal failed: path = %s, gfid = %s",
-                        local->loc.path, gfid );
-                goto out;
-        }
-        dht_iatt_merge (this, &local->preparent, preparent, prev);
-        dht_iatt_merge (this, &local->postparent, postparent, prev);
-        ret = 0;
+    if (op_ret) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name,
+                ((op_errno == EEXIST) ? GF_LOG_DEBUG : GF_LOG_WARNING),
+                op_errno, DHT_MSG_DIR_SELFHEAL_FAILED, "path=%s",
+                local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+    dht_iatt_merge(this, &local->preparent, preparent);
+    dht_iatt_merge(this, &local->postparent, postparent);
+    ret = 0;
 
 out:
-        this_call_cnt = dht_frame_return (frame);
+    this_call_cnt = dht_frame_return(frame);
 
-        if (is_last_call (this_call_cnt)) {
-                dht_selfheal_dir_finish (frame, this, ret, 0);
-                dht_selfheal_dir_setattr (frame, &local->loc, &local->stbuf, 0xffffff, layout);
-        }
+    if (is_last_call(this_call_cnt)) {
+        dht_selfheal_dir_finish(frame, this, ret, 0);
+        dht_selfheal_dir_setattr(frame, &local->loc, &local->stbuf, 0xffffff,
+                                 layout);
+    }
 
-        return 0;
+    return 0;
 }
 
-void
-dht_selfheal_dir_mkdir_setacl (dict_t *xattr, dict_t *dict)
+static int
+dht_selfheal_dir_mkdir_lookup_done(call_frame_t *frame, xlator_t *this)
 {
-        data_t          *acl_default = NULL;
-        data_t          *acl_access = NULL;
-        xlator_t        *this = NULL;
-        int     ret = -1;
-
-        GF_ASSERT (xattr);
-        GF_ASSERT (dict);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        acl_default = dict_get (xattr, POSIX_ACL_DEFAULT_XATTR);
-
-        if (!acl_default) {
-                gf_msg_debug (this->name, 0,
-                              "ACL_DEFAULT xattr not present");
-                goto cont;
-        }
-        ret = dict_set (dict, POSIX_ACL_DEFAULT_XATTR, acl_default);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value.key = %s",
-                        POSIX_ACL_DEFAULT_XATTR);
-cont:
-        acl_access = dict_get (xattr, POSIX_ACL_ACCESS_XATTR);
-        if (!acl_access) {
-                gf_msg_debug (this->name, 0,
-                              "ACL_ACCESS xattr not present");
-                goto out;
-        }
-        ret = dict_set (dict, POSIX_ACL_ACCESS_XATTR, acl_access);
+    dht_local_t *local = NULL;
+    int i = 0;
+    dict_t *dict = NULL;
+    dht_layout_t *layout = NULL;
+    loc_t *loc = NULL;
+    int cnt = 0;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(this->private, err);
+
+    local = frame->local;
+    layout = local->layout;
+    loc = &local->loc;
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        dict = dict_new();
+        if (!dict)
+            return -1;
+
+        ret = dict_set_gfuuid(dict, "gfid-req", local->gfid, true);
         if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value.key = %s",
-                        POSIX_ACL_ACCESS_XATTR);
-
-out:
-        return;
-}
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                    "path=%s", loc->path, "key=gfid-req", NULL);
+    } else if (local->params) {
+        /* Send the dictionary from higher layers directly */
+
+        dict = dict_ref(local->params);
+    }
+    /* Code to update all extended attributed from local->xattr
+       to dict
+    */
+    dht_dir_set_heal_xattr(this, local, dict, local->xattr, NULL, NULL);
+
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_IS_NULL, NULL);
+        dict = dict_new();
+        if (!dict)
+            return -1;
+    }
+    ret = dict_set_flag(dict, GF_INTERNAL_CTX_KEY, GF_DHT_HEAL_DIR);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED, "key=%s",
+                GF_INTERNAL_CTX_KEY, "path=%s", loc->path, NULL);
+        /* We can still continue. As heal can still happen
+         * unless quota limits have reached for the dir.
+         */
+    }
 
+    cnt = layout->cnt;
+    for (i = 0; i < cnt; i++) {
+        if (layout->list[i].err == ESTALE || layout->list[i].err == ENOENT ||
+            local->selfheal.force_mkdir) {
+            gf_msg_debug(this->name, 0, "Creating directory %s on subvol %s",
+                         loc->path, layout->list[i].xlator->name);
 
-void
-dht_selfheal_dir_mkdir_setquota (dict_t *src, dict_t *dst)
-{
-        data_t           *quota_limit_key = NULL;
-        data_t           *quota_limit_obj_key = NULL;
-        xlator_t        *this = NULL;
-        int     ret = -1;
-
-        GF_ASSERT (src);
-        GF_ASSERT (dst);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        quota_limit_key = dict_get (src, QUOTA_LIMIT_KEY);
-        if (!quota_limit_key) {
-                gf_msg_debug (this->name, 0,
-                              "QUOTA_LIMIT_KEY xattr not present");
-                goto cont;
-        }
-        ret = dict_set(dst, QUOTA_LIMIT_KEY, quota_limit_key);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value.key = %s",
-                        QUOTA_LIMIT_KEY);
-
-cont:
-        quota_limit_obj_key = dict_get (src, QUOTA_LIMIT_OBJECTS_KEY);
-        if (!quota_limit_obj_key) {
-                gf_msg_debug (this->name, 0,
-                              "QUOTA_LIMIT_OBJECTS_KEY xattr not present");
-                goto out;
+            STACK_WIND_COOKIE(
+                frame, dht_selfheal_dir_mkdir_cbk, layout->list[i].xlator,
+                layout->list[i].xlator, layout->list[i].xlator->fops->mkdir,
+                loc,
+                st_mode_from_ia(local->stbuf.ia_prot, local->stbuf.ia_type), 0,
+                dict);
         }
-        ret = dict_set (dst, QUOTA_LIMIT_OBJECTS_KEY, quota_limit_obj_key);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value.key = %s",
-                        QUOTA_LIMIT_OBJECTS_KEY);
-
-out:
-        return;
-}
-
+    }
 
+    if (dict)
+        dict_unref(dict);
 
+    return 0;
 
+err:
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
+}
 
-int
-dht_selfheal_dir_mkdir_lookup_done (call_frame_t *frame, xlator_t *this)
+static int
+dht_selfheal_dir_mkdir_lookup_cbk(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int op_ret, int op_errno,
+                                  inode_t *inode, struct iatt *stbuf,
+                                  dict_t *xattr, struct iatt *postparent)
 {
-        dht_local_t  *local = NULL;
-        int           i     = 0;
-        dict_t       *dict = NULL;
-        dht_layout_t  *layout = NULL;
-        loc_t        *loc   = NULL;
-        int           cnt   = 0;
-        int          ret    = -1;
-
-        VALIDATE_OR_GOTO (this->private, err);
-
-        local = frame->local;
-        layout = local->layout;
-        loc    = &local->loc;
-
-        if (!gf_uuid_is_null (local->gfid)) {
-                dict = dict_new ();
-                if (!dict)
-                        return -1;
-
-                ret = dict_set_gfuuid (dict, "gfid-req", local->gfid, true);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "%s: Failed to set dictionary value:"
-                                " key = gfid-req", loc->path);
-        } else if (local->params) {
-                /* Send the dictionary from higher layers directly */
-
-                dict = dict_ref (local->params);
-        }
-        /* Code to update all extended attributed from local->xattr
-           to dict
-        */
-        dht_dir_set_heal_xattr (this, local, dict, local->xattr, NULL,
-                                NULL);
-
-        if (!dict) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "dict is NULL, need to make sure gfids are same");
-                dict = dict_new ();
-                if (!dict)
-                        return -1;
-        }
-        ret = dict_set_flag (dict, GF_INTERNAL_CTX_KEY, GF_DHT_HEAL_DIR);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value for"
-                        " key = %s at path: %s",
-                        GF_INTERNAL_CTX_KEY, loc->path);
-                /* We can still continue. As heal can still happen
-                 * unless quota limits have reached for the dir.
-                 */
-        }
+    dht_local_t *local = NULL;
+    int i = 0;
+    int this_call_cnt = 0;
+    int missing_dirs = 0;
+    dht_layout_t *layout = NULL;
+    xlator_t *prev = 0;
+    loc_t *loc = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    int index = -1;
+
+    VALIDATE_OR_GOTO(this->private, err);
+
+    local = frame->local;
+    layout = local->layout;
+    loc = &local->loc;
+    prev = cookie;
+
+    if (!gf_uuid_is_null(local->gfid))
+        gf_uuid_unparse(local->gfid, gfid_local);
+
+    LOCK(&frame->lock);
+    {
+        index = dht_layout_index_for_subvol(layout, prev);
+        if ((op_ret < 0) && (op_errno == ENOENT || op_errno == ESTALE)) {
+            local->selfheal.hole_cnt = !local->selfheal.hole_cnt
+                                           ? 1
+                                           : local->selfheal.hole_cnt + 1;
+            /* the status might have changed. Update the layout with the
+             * new status
+             */
+            if (index >= 0) {
+                layout->list[index].err = op_errno;
+            }
+        }
+
+        if (!op_ret) {
+            dht_iatt_merge(this, &local->stbuf, stbuf);
+            if (prev == local->mds_subvol) {
+                dict_unref(local->xattr);
+                local->xattr = dict_ref(xattr);
+            }
+            /* the status might have changed. Update the layout with the
+             * new status
+             */
+            if (index >= 0) {
+                layout->list[index].err = -1;
+            }
+        }
+    }
+    UNLOCK(&frame->lock);
+
+    this_call_cnt = dht_frame_return(frame);
+
+    if (is_last_call(this_call_cnt)) {
+        if (local->selfheal.hole_cnt == layout->cnt) {
+            gf_msg_debug(this->name, op_errno,
+                         "Lookup failed, an rmdir could have "
+                         "deleted this entry %s",
+                         loc->name);
+            local->op_errno = op_errno;
+            goto err;
+        } else {
+            for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err == ENOENT ||
+                    layout->list[i].err == ESTALE ||
+                    local->selfheal.force_mkdir)
+                    missing_dirs++;
+            }
+
+            if (missing_dirs == 0) {
+                dht_selfheal_dir_finish(frame, this, 0, 0);
+                dht_selfheal_dir_setattr(frame, loc, &local->stbuf, 0xffffffff,
+                                         layout);
+                return 0;
+            }
 
-        cnt = layout->cnt;
-        for (i = 0; i < cnt; i++) {
-                if (layout->list[i].err == ESTALE ||
-                    layout->list[i].err == ENOENT ||
-                    local->selfheal.force_mkdir) {
-                        gf_msg_debug (this->name, 0,
-                                      "Creating directory %s on subvol %s",
-                                      loc->path, layout->list[i].xlator->name);
-
-                        STACK_WIND_COOKIE (frame, dht_selfheal_dir_mkdir_cbk,
-                                           layout->list[i].xlator,
-                                           layout->list[i].xlator,
-                                           layout->list[i].xlator->fops->mkdir,
-                                           loc,
-                                           st_mode_from_ia (local->stbuf.ia_prot,
-                                                            local->stbuf.ia_type),
-                                           0, dict);
-                }
+            local->call_cnt = missing_dirs;
+            dht_selfheal_dir_mkdir_lookup_done(frame, this);
         }
+    }
 
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
+    return 0;
 
 err:
-        dht_selfheal_dir_finish (frame, this, -1, 1);
-        return 0;
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
 }
 
-int
-dht_selfheal_dir_mkdir_lookup_cbk (call_frame_t *frame, void *cookie,
-                                   xlator_t *this, int op_ret, int op_errno,
-                                   inode_t *inode, struct iatt *stbuf,
-                                   dict_t *xattr, struct iatt *postparent)
+static int
+dht_selfheal_dir_mkdir_lock_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           i     = 0;
-        int           this_call_cnt = 0;
-        int           missing_dirs = 0;
-        dht_layout_t  *layout = NULL;
-        dht_conf_t    *conf   = 0;
-        loc_t         *loc    = NULL;
-        xlator_t      *prev    = NULL;
-        int           check_mds = 0;
-        int           errst     = 0;
-        int32_t       mds_xattr_val[1] = {0};
-        char          gfid_local[GF_UUID_BUF_SIZE] = {0};
-
-        VALIDATE_OR_GOTO (this->private, err);
-
-        local = frame->local;
-        layout = local->layout;
-        loc = &local->loc;
-        prev  = cookie;
-        conf = this->private;
-
-        if (local->gfid)
-                gf_uuid_unparse(local->gfid, gfid_local);
-
-        this_call_cnt = dht_frame_return (frame);
-
-        LOCK (&frame->lock);
-        {
-                if ((op_ret < 0) &&
-                    (op_errno == ENOENT || op_errno == ESTALE)) {
-                        local->selfheal.hole_cnt = !local->selfheal.hole_cnt ? 1
-                                                : local->selfheal.hole_cnt + 1;
-                }
-
-                if (!op_ret) {
-                        dht_iatt_merge (this, &local->stbuf, stbuf, prev);
-                }
-                check_mds = dht_dict_get_array (xattr, conf->mds_xattr_key,
-                                                mds_xattr_val, 1, &errst);
-                if (dict_get (xattr, conf->mds_xattr_key) && check_mds && !errst) {
-                        dict_unref (local->xattr);
-                        local->xattr = dict_ref (xattr);
-                }
-
-        }
-        UNLOCK (&frame->lock);
-
-        if (is_last_call (this_call_cnt)) {
-                if (local->selfheal.hole_cnt == layout->cnt) {
-                        gf_msg_debug (this->name, op_errno,
-                                      "Lookup failed, an rmdir could have "
-                                      "deleted this entry %s", loc->name);
-                        local->op_errno = op_errno;
-                        goto err;
-                } else {
-                        for (i = 0; i < layout->cnt; i++) {
-                                if (layout->list[i].err == ENOENT ||
-                                    layout->list[i].err == ESTALE ||
-                                    local->selfheal.force_mkdir)
-                                        missing_dirs++;
-                        }
-
-                        if (missing_dirs == 0) {
-                                dht_selfheal_dir_finish (frame, this, 0, 0);
-                                dht_selfheal_dir_setattr (frame, loc,
-                                                          &local->stbuf,
-                                                          0xffffffff, layout);
-                                return 0;
-                        }
-
-                        local->call_cnt = missing_dirs;
-                        dht_selfheal_dir_mkdir_lookup_done (frame, this);
-                }
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    int ret = -1;
+    xlator_t *mds_subvol = NULL;
+
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+    local = frame->local;
+    mds_subvol = local->mds_subvol;
+
+    local->call_cnt = conf->subvolume_cnt;
+
+    if (op_ret < 0) {
+        if (op_errno == EINVAL) {
+            local->call_cnt = 1;
+            dht_selfheal_dir_mkdir_lookup_done(frame, this);
+            return 0;
+        }
+
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_ENTRYLK_ERROR,
+                "path=%s", local->loc.path, NULL);
+
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    /* After getting locks, perform lookup again to ensure that the
+       directory was not deleted by a racing rmdir
+    */
+    if (!local->xattr_req)
+        local->xattr_req = dict_new();
+
+    ret = dict_set_int32(local->xattr_req, "list-xattr", 1);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED, "path=%s",
+                local->loc.path, NULL);
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (mds_subvol && conf->subvolumes[i] == mds_subvol) {
+            STACK_WIND_COOKIE(frame, dht_selfheal_dir_mkdir_lookup_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
+        } else {
+            STACK_WIND_COOKIE(frame, dht_selfheal_dir_mkdir_lookup_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              NULL);
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        dht_selfheal_dir_finish (frame, this, -1, 1);
-        return 0;
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
 }
 
-
-int
-dht_selfheal_dir_mkdir_lock_cbk (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *xdata)
+static int
+dht_selfheal_dir_mkdir(call_frame_t *frame, loc_t *loc, dht_layout_t *layout,
+                       int force)
 {
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf  = NULL;
-        int           i     = 0;
-        int           ret   = -1;
-        xlator_t     *mds_subvol = NULL;
-
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-        local = frame->local;
-        mds_subvol = local->mds_subvol;
-
-        local->call_cnt = conf->subvolume_cnt;
-
-        if (op_ret < 0) {
-
-                /* We get this error when the directory entry was not created
-                 * on a newky attatched tier subvol. Hence proceed and do mkdir
-                 * on the tier subvol.
-                 */
-                if (op_errno == EINVAL) {
-                        local->call_cnt = 1;
-                        dht_selfheal_dir_mkdir_lookup_done (frame, this);
-                        return 0;
+    int missing_dirs = 0;
+    int i = 0;
+    int op_errno = 0;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    local->selfheal.force_mkdir = force;
+    local->selfheal.hole_cnt = 0;
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err == ENOENT || force)
+            missing_dirs++;
+    }
+
+    if (missing_dirs == 0) {
+        /* We don't need to create any directories. Proceed to heal the
+         * attrs and xattrs
+         */
+        if (!__is_root_gfid(local->stbuf.ia_gfid)) {
+            if (local->need_xattr_heal) {
+                local->need_xattr_heal = 0;
+                ret = dht_dir_xattr_heal(this, local, &op_errno);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                            DHT_MSG_DIR_XATTR_HEAL_FAILED, "path=%s",
+                            local->loc.path, "gfid=%s", local->gfid, NULL);
                 }
+            } else {
+                if (!gf_uuid_is_null(local->gfid))
+                    gf_uuid_copy(loc->gfid, local->gfid);
 
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_ENTRYLK_ERROR,
-                        "acquiring entrylk after inodelk failed for %s",
-                        local->loc.path);
-
-                local->op_errno = op_errno;
-                goto err;
-        }
-
-        /* After getting locks, perform lookup again to ensure that the
-           directory was not deleted by a racing rmdir
-        */
-        if (!local->xattr_req)
-                local->xattr_req = dict_new ();
-
-        ret = dict_set_int32 (local->xattr_req, "list-xattr", 1);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary key list-xattr value "
-                        " for path %s ", local->loc.path);
+                ret = dht_common_mark_mdsxattr(frame, NULL, 0);
+                if (!ret)
+                    return 0;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (mds_subvol && conf->subvolumes[i] == mds_subvol) {
-                        STACK_WIND_COOKIE (frame,
-                                           dht_selfheal_dir_mkdir_lookup_cbk,
-                                           conf->subvolumes[i],
-                                           conf->subvolumes[i],
-                                           conf->subvolumes[i]->fops->lookup,
-                                           &local->loc, local->xattr_req);
-                } else {
-                       STACK_WIND_COOKIE (frame,
-                                          dht_selfheal_dir_mkdir_lookup_cbk,
-                                          conf->subvolumes[i],
-                                          conf->subvolumes[i],
-                                          conf->subvolumes[i]->fops->lookup,
-                                          &local->loc, NULL);
-                }
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_SET_XATTR_FAILED,
+                        "path=%s", local->loc.path, "gfid=%s", local->gfid,
+                        NULL);
+            }
         }
-
+        dht_selfheal_dir_setattr(frame, loc, &local->stbuf, 0xffffffff, layout);
         return 0;
-
-err:
-        dht_selfheal_dir_finish (frame, this, -1, 1);
-        return 0;
-}
-
-int
-dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
-                        dht_layout_t *layout, int force)
-{
-        int           missing_dirs = 0;
-        int           i     = 0;
-        int           ret   = -1;
-        dht_local_t  *local = NULL;
-        xlator_t     *this  = NULL;
-
-        local = frame->local;
-        this = frame->this;
-
-        local->selfheal.force_mkdir = force;
-        local->selfheal.hole_cnt = 0;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err == ENOENT || force)
-                        missing_dirs++;
+    }
+
+    /* MDS xattr is populated only while DHT is having more than one
+     subvol.In case of graph switch while adding more dht subvols need to
+     consider hash subvol as a MDS to avoid MDS check failure at the time
+     of running fop on directory
+    */
+    if (!dict_get(local->xattr, conf->mds_xattr_key) &&
+        (conf->subvolume_cnt > 1)) {
+        if (local->hashed_subvol == NULL) {
+            local->hashed_subvol = dht_subvol_get_hashed(this, loc);
+            if (local->hashed_subvol == NULL) {
+                local->op_errno = EINVAL;
+                gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                        DHT_MSG_HASHED_SUBVOL_GET_FAILED, "gfid=%s",
+                        loc->pargfid, "name=%s", loc->name, "path=%s",
+                        loc->path, NULL);
+                goto err;
+            }
         }
-
-        if (missing_dirs == 0) {
-                if (!__is_root_gfid (local->stbuf.ia_gfid)) {
-                        if (local->need_xattr_heal) {
-                                local->need_xattr_heal = 0;
-                                ret =  dht_dir_xattr_heal (this, local);
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_ERROR,
-                                                ret,
-                                                DHT_MSG_DIR_XATTR_HEAL_FAILED,
-                                                "xattr heal failed for "
-                                                "directory  %s gfid %s ",
-                                                local->loc.path,
-                                                local->gfid);
-                        } else {
-                                if (!gf_uuid_is_null (local->gfid))
-                                        gf_uuid_copy (loc->gfid, local->gfid);
-
-                                ret = dht_common_mark_mdsxattr (frame, NULL, 0);
-                                if (!ret)
-                                        return 0;
-
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_DIR_XATTR_HEAL_FAILED,
-                                        "Failed to set mds xattr "
-                                        "for directory  %s gfid %s ",
-                                        local->loc.path, local->gfid);
-                        }
-                }
-                dht_selfheal_dir_setattr (frame, loc, &local->stbuf,
-                                          0xffffffff, layout);
-                return 0;
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this,
+                                       local->hashed_subvol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "Failed to set hashed subvol for %s on inode vol is %s",
+                   local->loc.path,
+                   local->hashed_subvol ? local->hashed_subvol->name : "NULL");
+            goto err;
         }
+    }
 
-        if (local->hashed_subvol == NULL)
-                local->hashed_subvol = dht_subvol_get_hashed (this, loc);
-
+    if (local->hashed_subvol == NULL) {
+        local->hashed_subvol = dht_subvol_get_hashed(this, loc);
         if (local->hashed_subvol == NULL) {
-                local->op_errno = EINVAL;
-                gf_msg (this->name, GF_LOG_WARNING, local->op_errno,
-                        DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                        "(%s/%s) (path: %s): "
-                        "hashed subvolume not found", loc->pargfid, loc->name,
-                        loc->path);
-                goto err;
+            local->op_errno = EINVAL;
+            gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                    DHT_MSG_HASHED_SUBVOL_GET_FAILED, "gfid=%s", loc->pargfid,
+                    "name=%s", loc->name, "path=%s", loc->path, NULL);
+            goto err;
         }
+    }
 
-        local->current = &local->lock[0];
-        ret = dht_protect_namespace (frame, loc, local->hashed_subvol,
-                                     &local->current->ns,
-                                     dht_selfheal_dir_mkdir_lock_cbk);
+    local->current = &local->lock[0];
+    ret = dht_protect_namespace(frame, loc, local->hashed_subvol,
+                                &local->current->ns,
+                                dht_selfheal_dir_mkdir_lock_cbk);
 
-        if (ret < 0)
-                goto err;
+    if (ret < 0)
+        goto err;
 
-        return 0;
+    return 0;
 err:
-        return -1;
+    return -1;
 }
 
-int
-dht_selfheal_layout_alloc_start (xlator_t *this, loc_t *loc,
-                                 dht_layout_t *layout)
+static int
+dht_selfheal_layout_alloc_start(xlator_t *this, loc_t *loc,
+                                dht_layout_t *layout)
 {
-        int         start                               = 0;
-        uint32_t    hashval                             = 0;
-        int         ret                                 = 0;
-        const char *str                                 = NULL;
-        dht_conf_t *conf                                = NULL;
-        char           buf[UUID_CANONICAL_FORM_LEN + 1] = {0, };
-
-        conf = this->private;
-
-        if (conf->randomize_by_gfid) {
-                str = uuid_utoa_r (loc->gfid, buf);
-        } else {
-                str = loc->path;
-        }
-
-        ret = dht_hash_compute (this, layout->type, str, &hashval);
-        if (ret == 0) {
-                start = (hashval % layout->cnt);
-        }
-
-        return start;
+    int start = 0;
+    uint32_t hashval = 0;
+    int ret = 0;
+    const char *str = NULL;
+    dht_conf_t *conf = NULL;
+    char buf[UUID_CANONICAL_FORM_LEN + 1] = {
+        0,
+    };
+
+    conf = this->private;
+
+    if (conf->randomize_by_gfid) {
+        str = uuid_utoa_r(loc->gfid, buf);
+    } else {
+        str = loc->path;
+    }
+
+    ret = dht_hash_compute(this, layout->type, str, &hashval);
+    if (ret == 0) {
+        start = (hashval % layout->cnt);
+    }
+
+    return start;
 }
 
 static int
-dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
+dht_get_layout_count(xlator_t *this, dht_layout_t *layout, int new_layout)
 {
-        int i = 0;
-        int j = 0;
-        int err = 0;
-        int count = 0;
-        dht_conf_t *conf = NULL;
-
-        /* Gets in use only for replace-brick, remove-brick */
-        conf = this->private;
+    int i = 0;
+    int j = 0;
+    int err = 0;
+    int count = 0;
+    dht_conf_t *conf = NULL;
+
+    /* Gets in use only for replace-brick, remove-brick */
+    conf = this->private;
+    for (i = 0; i < layout->cnt; i++) {
+        for (j = 0; j < conf->subvolume_cnt; j++) {
+            if (conf->decommissioned_bricks[j] &&
+                conf->decommissioned_bricks[j] == layout->list[i].xlator) {
+                layout->list[i].err = EINVAL;
+                break;
+            }
+        }
+    }
+
+    for (i = 0; i < layout->cnt; i++) {
+        err = layout->list[i].err;
+        if (err == -1 || err == 0 || err == ENOENT) {
+            /* Take this with a pinch of salt. The behaviour seems
+             * to be slightly different when this function is
+             * invoked from mkdir codepath. For eg., err == 0 in
+             * mkdir codepath means directory created but xattr
+             * is not set yet.
+             */
+
+            /* Setting list[i].err = -1 is an indication for
+               dht_selfheal_layout_new_directory() to assign
+               a range. We set it to -1 based on any one of
+               the three criteria:
+
+               - err == -1 already, which means directory
+                 existed but layout was not set on it.
+
+               - err == 0, which means directory exists and
+                 has an old layout piece which will be
+                 overwritten now.
+
+               - err == ENOENT, which means directory does
+                 not exist (possibly racing with mkdir or
+                 finishing half done mkdir). The missing
+                 directory will be attempted to be recreated.
+            */
+            count++;
+            if (!err)
+                layout->list[i].err = -1;
+        }
+    }
+
+    /* no subvolume has enough space, but can't stop directory creation */
+    if (!count || !new_layout) {
         for (i = 0; i < layout->cnt; i++) {
-                for (j = 0; j < conf->subvolume_cnt; j++) {
-                        if (conf->decommissioned_bricks[j] &&
-                            conf->decommissioned_bricks[j] == layout->list[i].xlator) {
-                                layout->list[i].err = EINVAL;
-                                break;
-                        }
-                }
-        }
-
-        for (i = 0; i < layout->cnt; i++) {
-                err = layout->list[i].err;
-                if (err == -1 || err == 0 || err == ENOENT) {
-                        /* Take this with a pinch of salt. The behaviour seems
-                         * to be slightly different when this function is
-                         * invoked from mkdir codepath. For eg., err == 0 in
-                         * mkdir codepath means directory created but xattr
-                         * is not set yet.
-                         */
-
-			/* Setting list[i].err = -1 is an indication for
-			   dht_selfheal_layout_new_directory() to assign
-			   a range. We set it to -1 based on any one of
-			   the three criteria:
-
-			   - err == -1 already, which means directory
-			     existed but layout was not set on it.
-
-			   - err == 0, which means directory exists and
-			     has an old layout piece which will be
-			     overwritten now.
-
-			   - err == ENOENT, which means directory does
-			     not exist (possibly racing with mkdir or
-			     finishing half done mkdir). The missing
-			     directory will be attempted to be recreated.
-			*/
-                        count++;
-			if (!err)
-				layout->list[i].err = -1;
-                }
-        }
-
-        /* no subvolume has enough space, but can't stop directory creation */
-        if (!count || !new_layout) {
-                for (i = 0; i < layout->cnt; i++) {
-                        err = layout->list[i].err;
-                        if (err == ENOSPC) {
-                                layout->list[i].err = -1;
-                                count++;
-                        }
-                }
-        }
-
-        /* if layout->spread_cnt is set, check if it is <= available
-         * subvolumes (down brick and decommissioned bricks are considered
-         * un-availbale). Else return count (available up bricks) */
-        count = ((layout->spread_cnt &&
-                 (layout->spread_cnt <= count)) ?
-                 layout->spread_cnt : ((count) ? count : 1));
-
-        return count;
+            err = layout->list[i].err;
+            if (err == ENOSPC) {
+                layout->list[i].err = -1;
+                count++;
+            }
+        }
+    }
+
+    /* if layout->spread_cnt is set, check if it is <= available
+     * subvolumes (down brick and decommissioned bricks are considered
+     * un-available). Else return count (available up bricks) */
+    count = ((layout->spread_cnt && (layout->spread_cnt <= count))
+                 ? layout->spread_cnt
+                 : ((count) ? count : 1));
+
+    return count;
 }
 
+void
+dht_selfheal_layout_new_directory(call_frame_t *frame, loc_t *loc,
+                                  dht_layout_t *new_layout);
 
-void dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
-					dht_layout_t *new_layout);
-
-void dht_layout_entry_swap (dht_layout_t *layout, int i, int j);
-void dht_layout_range_swap (dht_layout_t *layout, int i, int j);
+void
+dht_layout_range_swap(dht_layout_t *layout, int i, int j);
 
 /*
  * It's a bit icky using local variables in a macro, but it makes the rest
  * of the code a lot clearer.
  */
-#define OV_ENTRY(x,y)      table[x*new->cnt+y]
+#define OV_ENTRY(x, y) table[x * new->cnt + y]
 
-void
-dht_selfheal_layout_maximize_overlap (call_frame_t *frame, loc_t *loc,
-				      dht_layout_t *new, dht_layout_t *old)
+static void
+dht_selfheal_layout_maximize_overlap(call_frame_t *frame, loc_t *loc,
+                                     dht_layout_t *new, dht_layout_t *old)
 {
-	int           i            = 0;
-	int           j            = 0;
-	uint32_t      curr_overlap = 0;
-	uint32_t      max_overlap  = 0;
-	int           max_overlap_idx = -1;
-	uint32_t      overlap      = 0;
-        uint32_t     *table = NULL;
-
-	dht_layout_sort_volname (old);
-	/* Now both old_layout->list[] and new_layout->list[]
-	   are match the same xlators/subvolumes. i.e,
-	   old_layout->[i] and new_layout->[i] are referring
-	   to the same subvolumes
-	*/
-
-        /* Build a table of overlaps between new[i] and old[j]. */
-        table = alloca(sizeof(overlap)*old->cnt*new->cnt);
-        if (!table) {
-                return;
-        }
-        memset(table,0,sizeof(overlap)*old->cnt*new->cnt);
-        for (i = 0; i < new->cnt; ++i) {
-                for (j = 0; j < old->cnt; ++j) {
-                        OV_ENTRY(i,j) = dht_overlap_calc(old,j,new,i);
-                }
-        }
-
-	for (i = 0; i < new->cnt; i++) {
-		if (new->list[i].err > 0) {
-			/* Subvol might be marked for decommission
-			   with EINVAL, or some other serious error
-			   marked with positive errno.
-			*/
-			continue;
-                }
-
-                max_overlap = 0;
-                max_overlap_idx = i;
-                for (j = (i + 1); j < new->cnt; ++j) {
-                        if (new->list[j].err > 0) {
-			        /* Subvol might be marked for decommission
-			        with EINVAL, or some other serious error
-			        marked with positive errno.
-			        */
-			        continue;
-                        }
-                        /* Calculate the overlap now. */
-                        curr_overlap = OV_ENTRY(i,i) + OV_ENTRY(j,j);
-                        /* Calculate the overlap after the proposed swap. */
-                        overlap = OV_ENTRY(i,j) + OV_ENTRY(j,i);
-                        /* Are we better than status quo? */
-                        if (overlap > curr_overlap) {
-                                overlap -= curr_overlap;
-                                /* Are we better than the previous choice? */
-                                if (overlap > max_overlap) {
-                                        max_overlap = overlap;
-                                        max_overlap_idx = j;
-                                }
-                        }
-                }
-
-		if (max_overlap_idx != i) {
- 			dht_layout_range_swap (new, i, max_overlap_idx);
-                        /* Need to swap the table values too. */
-                        for (j = 0; j < old->cnt; ++j) {
-                                overlap = OV_ENTRY(i,j);
-                                OV_ENTRY(i,j) = OV_ENTRY(max_overlap_idx,j);
-                                OV_ENTRY(max_overlap_idx,j) = overlap;
-                        }
-                }
-	}
+    int i = 0;
+    int j = 0;
+    uint32_t curr_overlap = 0;
+    uint32_t max_overlap = 0;
+    int max_overlap_idx = -1;
+    uint32_t overlap = 0;
+    uint32_t *table = NULL;
+
+    dht_layout_sort_volname(old);
+    /* Now both old_layout->list[] and new_layout->list[]
+       are match the same xlators/subvolumes. i.e,
+       old_layout->[i] and new_layout->[i] are referring
+       to the same subvolumes
+    */
+
+    /* Build a table of overlaps between new[i] and old[j]. */
+    table = alloca(sizeof(overlap) * old->cnt * new->cnt);
+    if (!table) {
+        return;
+    }
+    memset(table, 0, sizeof(overlap) * old->cnt * new->cnt);
+    for (i = 0; i < new->cnt; ++i) {
+        for (j = 0; j < old->cnt; ++j) {
+            OV_ENTRY(i, j) = dht_overlap_calc(old, j, new, i);
+        }
+    }
+
+    for (i = 0; i < new->cnt; i++) {
+        if (new->list[i].err > 0) {
+            /* Subvol might be marked for decommission
+               with EINVAL, or some other serious error
+               marked with positive errno.
+            */
+            continue;
+        }
+
+        max_overlap = 0;
+        max_overlap_idx = i;
+        for (j = (i + 1); j < new->cnt; ++j) {
+            if (new->list[j].err > 0) {
+                /* Subvol might be marked for decommission
+                with EINVAL, or some other serious error
+                marked with positive errno.
+                */
+                continue;
+            }
+            /* Calculate the overlap now. */
+            curr_overlap = OV_ENTRY(i, i) + OV_ENTRY(j, j);
+            /* Calculate the overlap after the proposed swap. */
+            overlap = OV_ENTRY(i, j) + OV_ENTRY(j, i);
+            /* Are we better than status quo? */
+            if (overlap > curr_overlap) {
+                overlap -= curr_overlap;
+                /* Are we better than the previous choice? */
+                if (overlap > max_overlap) {
+                    max_overlap = overlap;
+                    max_overlap_idx = j;
+                }
+            }
+        }
+
+        if (max_overlap_idx != i) {
+            dht_layout_range_swap(new, i, max_overlap_idx);
+            /* Need to swap the table values too. */
+            for (j = 0; j < old->cnt; ++j) {
+                overlap = OV_ENTRY(i, j);
+                OV_ENTRY(i, j) = OV_ENTRY(max_overlap_idx, j);
+                OV_ENTRY(max_overlap_idx, j) = overlap;
+            }
+        }
+    }
 }
 
-
-dht_layout_t *
-dht_fix_layout_of_directory (call_frame_t *frame, loc_t *loc,
-                             dht_layout_t *layout)
+static dht_layout_t *
+dht_fix_layout_of_directory(call_frame_t *frame, loc_t *loc,
+                            dht_layout_t *layout)
 {
-        int           i                         = 0;
-        xlator_t     *this                      = NULL;
-        dht_layout_t *new_layout                = NULL;
-        dht_conf_t   *priv                      = NULL;
-        dht_local_t  *local                     = NULL;
-        uint32_t      subvol_down               = 0;
-        int           ret                       = 0;
-        gf_boolean_t  maximize_overlap          = _gf_true;
-        char          gfid[GF_UUID_BUF_SIZE]    = {0};
-
-        this  = frame->this;
-        priv  = this->private;
-        local = frame->local;
-
-        if (layout->type == DHT_HASH_TYPE_DM_USER) {
-                gf_msg_debug (THIS->name, 0, "leaving %s alone",
-                              loc->path);
-                goto done;
-        }
-
-        new_layout = dht_layout_new (this, priv->subvolume_cnt);
-        if (!new_layout) {
-                gf_uuid_unparse (loc->gfid, gfid);
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
-                        "mem allocation failed for new_layout, path:%s gfid:%s",
-                        loc->path, gfid);
-                goto done;
-        }
-
-        /* If a subvolume is down, do not re-write the layout. */
-        ret = dht_layout_anomalies (this, loc, layout, NULL, NULL, NULL,
-                                    &subvol_down, NULL, NULL);
-
-        if (subvol_down || (ret == -1)) {
-                gf_uuid_unparse (loc->gfid, gfid);
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LAYOUT_FIX_FAILED,
-                        "Layout fix failed: %u subvolume(s) are down"
-                        ". Skipping fix layout. path:%s gfid:%s", subvol_down,
-                        loc->path, gfid);
-                GF_FREE (new_layout);
-                return NULL;
-        }
-
-        for (i = 0; i < new_layout->cnt; i++) {
-		if (layout->list[i].err != ENOSPC)
-			new_layout->list[i].err = layout->list[i].err;
-		else
-			new_layout->list[i].err = -1;
-
-		new_layout->list[i].xlator = layout->list[i].xlator;
-        }
-
-        new_layout->commit_hash = layout->commit_hash;
-
-        if (priv->du_stats) {
-                for (i = 0; i < priv->subvolume_cnt; ++i) {
-                        gf_msg (this->name, GF_LOG_DEBUG, 0,
-                                DHT_MSG_SUBVOL_INFO,
-                                "subvolume %d (%s): %u chunks, path:%s", i,
-                                priv->subvolumes[i]->name,
-                                priv->du_stats[i].chunks, loc->path);
-
-                        /* Maximize overlap if the bricks are all the same
-                         *  size.
-                         * This is probably not going to be very common on
-                         * live setups but will benefit our regression tests
-                         */
-                        if (i && (priv->du_stats[i].chunks
-                                  != priv->du_stats[0].chunks)) {
-                                maximize_overlap = _gf_false;
-                        }
-                }
-        } else {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_NO_DISK_USAGE_STATUS, "no du stats ?!?");
-        }
-
-	/* First give it a layout as though it is a new directory. This
-	   ensures rotation to kick in */
-        dht_layout_sort_volname (new_layout);
-	dht_selfheal_layout_new_directory (frame, loc, new_layout);
-
-
-        /* Maximize overlap if weighted-rebalance is disabled */
-        if (!priv->do_weighting)
-                maximize_overlap = _gf_true;
-
-	/* Now selectively re-assign ranges only when it helps */
-        if (maximize_overlap) {
-                dht_selfheal_layout_maximize_overlap (frame, loc, new_layout,
-                                                      layout);
-        }
+    int i = 0;
+    xlator_t *this = NULL;
+    dht_layout_t *new_layout = NULL;
+    dht_conf_t *priv = NULL;
+    dht_local_t *local = NULL;
+    uint32_t subvol_down = 0;
+    gf_boolean_t maximize_overlap = _gf_true;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    this = frame->this;
+    priv = this->private;
+    local = frame->local;
+
+    if (layout->type == DHT_HASH_TYPE_DM_USER) {
+        gf_msg_debug(THIS->name, 0, "leaving %s alone", loc->path);
+        goto done;
+    }
+
+    new_layout = dht_layout_new(this, priv->subvolume_cnt);
+    if (!new_layout) {
+        gf_uuid_unparse(loc->gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                "new_layout, path=%s", loc->path, "gfid=%s", gfid, NULL);
+        goto done;
+    }
+
+    /* If a subvolume is down, do not re-write the layout. */
+    dht_layout_anomalies(this, loc, layout, NULL, NULL, NULL, &subvol_down,
+                         NULL, NULL);
+
+    if (subvol_down) {
+        gf_uuid_unparse(loc->gfid, gfid);
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_FIX_FAILED,
+                "subvol-down=%u", subvol_down, "Skipping-fix-layout", "path=%s",
+                loc->path, "gfid=%s", gfid, NULL);
+        GF_FREE(new_layout);
+        return NULL;
+    }
+
+    for (i = 0; i < new_layout->cnt; i++) {
+        if (layout->list[i].err != ENOSPC)
+            new_layout->list[i].err = layout->list[i].err;
+        else
+            new_layout->list[i].err = -1;
+
+        new_layout->list[i].xlator = layout->list[i].xlator;
+    }
+
+    new_layout->commit_hash = layout->commit_hash;
+
+    if (priv->du_stats) {
+        for (i = 0; i < priv->subvolume_cnt; ++i) {
+            gf_smsg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_SUBVOL_INFO,
+                    "index=%d", i, "name=%s", priv->subvolumes[i]->name,
+                    "chunks=%u", priv->du_stats[i].chunks, "path=%s", loc->path,
+                    NULL);
+
+            /* Maximize overlap if the bricks are all the same
+             *  size.
+             * This is probably not going to be very common on
+             * live setups but will benefit our regression tests
+             */
+            if (i && (priv->du_stats[i].chunks != priv->du_stats[0].chunks)) {
+                maximize_overlap = _gf_false;
+            }
+        }
+    } else {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NO_DISK_USAGE_STATUS,
+                NULL);
+    }
+
+    /* First give it a layout as though it is a new directory. This
+       ensures rotation to kick in */
+    dht_layout_sort_volname(new_layout);
+    dht_selfheal_layout_new_directory(frame, loc, new_layout);
+
+    /* Maximize overlap if weighted-rebalance is disabled */
+    if (!priv->do_weighting)
+        maximize_overlap = _gf_true;
+
+    /* Now selectively re-assign ranges only when it helps */
+    if (maximize_overlap) {
+        dht_selfheal_layout_maximize_overlap(frame, loc, new_layout, layout);
+    }
 done:
-        if (new_layout) {
+    if (new_layout) {
+        /* Make sure the extra 'ref' for existing layout is removed */
+        dht_layout_unref(this, local->layout);
 
-                /* Make sure the extra 'ref' for existing layout is removed */
-                dht_layout_unref (this, local->layout);
+        local->layout = new_layout;
+    }
 
-                local->layout = new_layout;
-        }
-
-        return local->layout;
+    return local->layout;
 }
 
-
 /*
  * Having to call this 2x for each entry in the layout is pretty horrible, but
  * that's what all of this layout-sorting nonsense gets us.
  */
-uint32_t
-dht_get_chunks_from_xl (xlator_t *parent, xlator_t *child)
+static uint32_t
+dht_get_chunks_from_xl(xlator_t *parent, xlator_t *child)
 {
-        dht_conf_t      *priv   = parent->private;
-        xlator_list_t   *trav;
-        uint32_t        index   = 0;
+    dht_conf_t *priv = parent->private;
+    xlator_list_t *trav;
+    uint32_t index = 0;
 
-        if (!priv->du_stats) {
-                return 0;
-        }
+    if (!priv->du_stats) {
+        return 0;
+    }
 
-        for (trav = parent->children; trav; trav = trav->next) {
-                if (trav->xlator == child) {
-                        return priv->du_stats[index].chunks;
-                }
-                ++index;
+    for (trav = parent->children; trav; trav = trav->next) {
+        if (trav->xlator == child) {
+            return priv->du_stats[index].chunks;
         }
+        ++index;
+    }
 
-        return 0;
+    return 0;
 }
 
-
 void
-dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
-                                   dht_layout_t *layout)
+dht_selfheal_layout_new_directory(call_frame_t *frame, loc_t *loc,
+                                  dht_layout_t *layout)
 {
-        xlator_t    *this = NULL;
-        double       chunk = 0;
-        int          i = 0;
-        uint32_t     start = 0;
-        int          bricks_to_use = 0;
-        int          err = 0;
-        int          start_subvol = 0;
-        uint32_t     curr_size;
-        uint32_t     range_size;
-        uint64_t     total_size = 0;
-        int          real_i;
-        dht_conf_t   *priv;
-        gf_boolean_t weight_by_size;
-        int          bricks_used = 0;
-
-        this = frame->this;
-        priv = this->private;
-        weight_by_size = priv->do_weighting;
-
-        bricks_to_use = dht_get_layout_count (this, layout, 1);
-        GF_ASSERT (bricks_to_use > 0);
-
-        bricks_used = 0;
-        for (i = 0; i < layout->cnt; ++i) {
-                err = layout->list[i].err;
-                if ((err != -1) && (err != ENOENT)) {
-                        continue;
-                }
-                curr_size = dht_get_chunks_from_xl (this,
-                                                    layout->list[i].xlator);
-                if (!curr_size) {
-                        weight_by_size = _gf_false;
-                        break;
-                }
-                total_size += curr_size;
-                if (++bricks_used >= bricks_to_use) {
-                        break;
-                }
-        }
-
-        if (weight_by_size && total_size) {
-                /* We know total_size is not zero. */
-                chunk = ((double) 0xffffffff) / ((double) total_size);
-                gf_msg_debug (this->name, 0,
-                              "chunk size = 0xffffffff / %lu = %f",
-                              total_size, chunk);
-        }
-        else {
-                weight_by_size = _gf_false;
-                chunk = ((unsigned long) 0xffffffff) / bricks_to_use;
+    xlator_t *this = NULL;
+    double chunk = 0;
+    int i = 0;
+    uint32_t start = 0;
+    int bricks_to_use = 0;
+    int err = 0;
+    int start_subvol = 0;
+    uint32_t curr_size;
+    uint32_t range_size;
+    uint64_t total_size = 0;
+    int real_i;
+    dht_conf_t *priv;
+    gf_boolean_t weight_by_size;
+    int bricks_used = 0;
+
+    this = frame->this;
+    priv = this->private;
+    weight_by_size = priv->do_weighting;
+
+    bricks_to_use = dht_get_layout_count(this, layout, 1);
+    GF_ASSERT(bricks_to_use > 0);
+
+    bricks_used = 0;
+    for (i = 0; i < layout->cnt; ++i) {
+        err = layout->list[i].err;
+        if ((err != -1) && (err != ENOENT)) {
+            continue;
+        }
+        curr_size = dht_get_chunks_from_xl(this, layout->list[i].xlator);
+        if (!curr_size) {
+            weight_by_size = _gf_false;
+            break;
+        }
+        total_size += curr_size;
+        if (++bricks_used >= bricks_to_use) {
+            break;
+        }
+    }
+
+    if (weight_by_size && total_size) {
+        /* We know total_size is not zero. */
+        chunk = ((double)0xffffffff) / ((double)total_size);
+        gf_msg_debug(this->name, 0,
+                     "chunk size = 0xffffffff / %" PRIu64 " = %f", total_size,
+                     chunk);
+    } else {
+        weight_by_size = _gf_false;
+        chunk = ((unsigned long)0xffffffff) / bricks_to_use;
+    }
+
+    start_subvol = dht_selfheal_layout_alloc_start(this, loc, layout);
+
+    /* clear out the range, as we are re-computing here */
+    DHT_RESET_LAYOUT_RANGE(layout);
+
+    /*
+     * OK, what's this "real_i" stuff about?  This used to be two loops -
+     * from start_subvol to layout->cnt-1, then from 0 to start_subvol-1.
+     * That way is practically an open invitation to bugs when only one
+     * of the loops is updated.  Using real_i and modulo operators to make
+     * it one loop avoids this problem.  Remember, folks: it's everyone's
+     * responsibility to help stamp out copy/paste abuse.
+     */
+    bricks_used = 0;
+    for (real_i = 0; real_i < layout->cnt; real_i++) {
+        i = (real_i + start_subvol) % layout->cnt;
+        err = layout->list[i].err;
+        if ((err != -1) && (err != ENOENT)) {
+            continue;
+        }
+        if (weight_by_size) {
+            curr_size = dht_get_chunks_from_xl(this, layout->list[i].xlator);
+            if (!curr_size) {
+                continue;
+            }
+        } else {
+            curr_size = 1;
         }
-
-        start_subvol = dht_selfheal_layout_alloc_start (this, loc, layout);
-
-        /* clear out the range, as we are re-computing here */
-        DHT_RESET_LAYOUT_RANGE (layout);
-
-        /*
-         * OK, what's this "real_i" stuff about?  This used to be two loops -
-         * from start_subvol to layout->cnt-1, then from 0 to start_subvol-1.
-         * That way is practically an open invitation to bugs when only one
-         * of the loops is updated.  Using real_i and modulo operators to make
-         * it one loop avoids this problem.  Remember, folks: it's everyone's
-         * responsibility to help stamp out copy/paste abuse.
-         */
-        bricks_used = 0;
-        for (real_i = 0; real_i < layout->cnt; real_i++) {
-                i = (real_i + start_subvol) % layout->cnt;
-                err = layout->list[i].err;
-                if ((err != -1) && (err != ENOENT)) {
-                        continue;
-                }
-                if (weight_by_size) {
-                        curr_size = dht_get_chunks_from_xl (this,
-                                layout->list[i].xlator);
-                        if (!curr_size) {
-                                continue;
-                        }
-                }
-                else {
-                        curr_size = 1;
-                }
-                range_size = chunk * curr_size;
-                gf_msg_debug (this->name, 0,
-                              "assigning range size 0x%x to %s",
-                              range_size,
-                              layout->list[i].xlator->name);
-                DHT_SET_LAYOUT_RANGE(layout, i, start, range_size,
-                                     loc->path);
-                if (++bricks_used >= bricks_to_use) {
-                        layout->list[i].stop = 0xffffffff;
-                        goto done;
-                }
-                start += range_size;
+        range_size = chunk * curr_size;
+        gf_msg_debug(this->name, 0, "assigning range size 0x%x to %s",
+                     range_size, layout->list[i].xlator->name);
+        DHT_SET_LAYOUT_RANGE(layout, i, start, range_size, loc->path);
+        if (++bricks_used >= bricks_to_use) {
+            layout->list[i].stop = 0xffffffff;
+            goto done;
         }
+        start += range_size;
+    }
 
 done:
-        return;
+    return;
 }
 
-int
-dht_selfheal_dir_getafix (call_frame_t *frame, loc_t *loc,
-                          dht_layout_t *layout)
+static int
+dht_selfheal_dir_getafix(call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
 {
-        dht_local_t *local = NULL;
-        uint32_t     holes = 0;
-        int          ret = -1;
-        int          i = -1;
-        uint32_t     overlaps = 0;
-
-        local = frame->local;
-
-        holes = local->selfheal.hole_cnt;
-        overlaps = local->selfheal.overlaps_cnt;
-
-        if (holes || overlaps) {
-                /* If the layout has anomolies which would change the hash
-                 * ranges, then we need to reset the commit_hash for this
-                 * directory, as the layout would change and things may not
-                 * be in place as expected */
-                layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
-                dht_selfheal_layout_new_directory (frame, loc, layout);
-                ret = 0;
-        }
+    dht_local_t *local = NULL;
+    uint32_t holes = 0;
+    int ret = -1;
+    int i = -1;
+    uint32_t overlaps = 0;
+
+    local = frame->local;
+
+    holes = local->selfheal.hole_cnt;
+    overlaps = local->selfheal.overlaps_cnt;
+
+    if (holes || overlaps) {
+        /* If the layout has anomalies which would change the hash
+         * ranges, then we need to reset the commit_hash for this
+         * directory, as the layout would change and things may not
+         * be in place as expected */
+        layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
+        dht_selfheal_layout_new_directory(frame, loc, layout);
+        ret = 0;
+    }
 
-        for (i = 0; i < layout->cnt; i++) {
-                /* directory not present */
-                if (layout->list[i].err == ENOENT) {
-                        ret = 0;
-                        break;
-                }
+    for (i = 0; i < layout->cnt; i++) {
+        /* directory not present */
+        if (layout->list[i].err == ENOENT) {
+            ret = 0;
+            break;
         }
+    }
 
-        /* TODO: give a fix to these non-virgins */
+    /* TODO: give a fix to these non-virgins */
 
-        return ret;
+    return ret;
 }
 
 int
-dht_selfheal_new_directory (call_frame_t *frame,
-                            dht_selfheal_dir_cbk_t dir_cbk,
-                            dht_layout_t *layout)
+dht_selfheal_new_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                           dht_layout_t *layout)
 {
-        dht_local_t *local                   = NULL;
-        int          ret                     = 0;
-        inode_t     *linked_inode            = NULL, *inode = NULL;
-        loc_t       *loc                     = NULL;
-        char         pgfid[GF_UUID_BUF_SIZE] = {0};
-        char         gfid[GF_UUID_BUF_SIZE]  = {0};
-        int32_t      op_errno                = EIO;
+    dht_local_t *local = NULL;
+    int ret = 0;
+    inode_t *linked_inode = NULL, *inode = NULL;
+    loc_t *loc = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t op_errno = EIO;
 
-        local = frame->local;
+    local = frame->local;
 
-        loc = &local->loc;
+    loc = &local->loc;
 
-        gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
-        gf_uuid_unparse(loc->parent->gfid, pgfid);
+    gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
 
-        linked_inode = inode_link (loc->inode, loc->parent, loc->name,
-                                   &local->stbuf);
-        if (!linked_inode) {
-                gf_msg (frame->this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "linking inode failed (%s/%s) => %s",
-                        pgfid, loc->name, gfid);
-                ret = -1;
-                goto out;
-        }
+    linked_inode = inode_link(loc->inode, loc->parent, loc->name,
+                              &local->stbuf);
+    if (!linked_inode) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0, DHT_MSG_LINK_INODE_FAILED,
+                "pgfid=%s", pgfid, "name=%s", loc->name, "gfid=%s", gfid, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        inode = loc->inode;
-        loc->inode = linked_inode;
-        inode_unref (inode);
+    inode = loc->inode;
+    loc->inode = linked_inode;
+    inode_unref(inode);
 
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
 
-        dht_layout_sort_volname (layout);
-        dht_selfheal_layout_new_directory (frame, &local->loc, layout);
+    dht_layout_sort_volname(layout);
+    dht_selfheal_layout_new_directory(frame, &local->loc, layout);
 
-        op_errno = ENOMEM;
-        ret = dht_selfheal_layout_lock (frame, layout, _gf_true,
-                                        dht_selfheal_dir_xattr,
-                                        dht_should_heal_layout);
+    op_errno = ENOMEM;
+    ret = dht_selfheal_layout_lock(frame, layout, _gf_true,
+                                   dht_selfheal_dir_xattr,
+                                   dht_should_heal_layout);
 
 out:
-        if (ret < 0) {
-                dir_cbk (frame, NULL, frame->this, -1, op_errno, NULL);
-        }
+    if (ret < 0) {
+        dir_cbk(frame, NULL, frame->this, -1, op_errno, NULL);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-dht_fix_directory_layout (call_frame_t *frame,
-                          dht_selfheal_dir_cbk_t dir_cbk,
-                          dht_layout_t *layout)
+dht_fix_directory_layout(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                         dht_layout_t *layout)
 {
-        dht_local_t  *local      = NULL;
-        dht_layout_t *tmp_layout = NULL;
-        int           ret        = 0;
+    dht_local_t *local = NULL;
+    dht_layout_t *tmp_layout = NULL;
+    int ret = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
 
-        /* No layout sorting required here */
-        tmp_layout = dht_fix_layout_of_directory (frame, &local->loc, layout);
-        if (!tmp_layout) {
-                return -1;
-        }
+    /* No layout sorting required here */
+    tmp_layout = dht_fix_layout_of_directory(frame, &local->loc, layout);
+    if (!tmp_layout) {
+        return -1;
+    }
 
-        ret = dht_selfheal_layout_lock (frame, tmp_layout, _gf_false,
-                                        dht_fix_dir_xattr,
-                                        dht_should_fix_layout);
+    ret = dht_selfheal_layout_lock(frame, tmp_layout, _gf_false,
+                                   dht_fix_dir_xattr, dht_should_fix_layout);
 
-        return ret;
+    return ret;
 }
 
-
 int
-dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
-                        loc_t *loc, dht_layout_t *layout)
+dht_selfheal_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                       loc_t *loc, dht_layout_t *layout)
 {
-        dht_local_t *local                   = NULL;
-        uint32_t     down                    = 0;
-        uint32_t     misc                    = 0;
-        int          ret                     = 0;
-        xlator_t    *this                    = NULL;
-        char         pgfid[GF_UUID_BUF_SIZE] = {0};
-        char         gfid[GF_UUID_BUF_SIZE]  = {0};
-        inode_t     *linked_inode            = NULL, *inode = NULL;
-
-        local = frame->local;
-        this = frame->this;
-
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (this, layout);
-
-        if (!__is_root_gfid (local->stbuf.ia_gfid)) {
-                gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
-                gf_uuid_unparse(loc->parent->gfid, pgfid);
-
-                linked_inode = inode_link (loc->inode, loc->parent, loc->name,
-                                           &local->stbuf);
-                if (!linked_inode) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DIR_SELFHEAL_FAILED,
-                                "linking inode failed (%s/%s) => %s",
-                                pgfid, loc->name, gfid);
-                        ret = 0;
-                        goto sorry_no_fix;
-                }
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    uint32_t down = 0;
+    uint32_t misc = 0;
+    int ret = 0;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    inode_t *linked_inode = NULL, *inode = NULL;
 
-                inode = loc->inode;
-                loc->inode = linked_inode;
-                inode_unref (inode);
-        }
+    local = frame->local;
+    this = frame->this;
 
-        dht_layout_anomalies (this, loc, layout,
-                              &local->selfheal.hole_cnt,
-                              &local->selfheal.overlaps_cnt,
-                              &local->selfheal.missing_cnt,
-                              &local->selfheal.down,
-                              &local->selfheal.misc, NULL);
-
-        down     = local->selfheal.down;
-        misc     = local->selfheal.misc;
-
-        if (down) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "Directory selfheal failed: %d subvolumes down."
-                        "Not fixing. path = %s, gfid = %s",
-                        down, loc->path, gfid);
-                ret = 0;
-                goto sorry_no_fix;
-        }
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(this, layout);
 
-        if (misc) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "Directory selfheal failed : %d subvolumes "
-                        "have unrecoverable errors. path = %s, gfid = %s",
-                        misc, loc->path, gfid);
+    if (local->need_attrheal) {
+        if (__is_root_gfid(local->stbuf.ia_gfid)) {
+            local->stbuf.ia_gid = local->prebuf.ia_gid;
+            local->stbuf.ia_uid = local->prebuf.ia_uid;
 
-                ret = 0;
-                goto sorry_no_fix;
-        }
+            local->stbuf.ia_ctime = local->prebuf.ia_ctime;
+            local->stbuf.ia_ctime_nsec = local->prebuf.ia_ctime_nsec;
+            local->stbuf.ia_prot = local->prebuf.ia_prot;
 
-        dht_layout_sort_volname (layout);
-        local->heal_layout = _gf_true;
+        } else if (!IA_ISINVAL(local->mds_stbuf.ia_type)) {
+            local->stbuf = local->mds_stbuf;
+        }
+    }
 
-        /* Ignore return value as it can be inferred from result of
-         * dht_layout_anomalies
-         */
-        dht_selfheal_dir_getafix (frame, loc, layout);
+    if (!__is_root_gfid(local->stbuf.ia_gfid)) {
+        gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+        gf_uuid_unparse(loc->parent->gfid, pgfid);
 
-        if (!(local->selfheal.hole_cnt || local->selfheal.overlaps_cnt ||
-              local->selfheal.missing_cnt)) {
-                local->heal_layout = _gf_false;
+        linked_inode = inode_link(loc->inode, loc->parent, loc->name,
+                                  &local->stbuf);
+        if (!linked_inode) {
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LINK_INODE_FAILED,
+                    "pgfid=%s", pgfid, "name=%s", loc->name, "gfid=%s", gfid,
+                    NULL);
+            ret = 0;
+            goto sorry_no_fix;
         }
 
-        ret = dht_selfheal_dir_mkdir (frame, loc, layout, 0);
-        if (ret < 0) {
-                ret = 0;
-                goto sorry_no_fix;
-        }
+        inode = loc->inode;
+        loc->inode = linked_inode;
+        inode_unref(inode);
+    }
+
+    if (local->need_xattr_heal && (local->mds_xattr)) {
+        dht_dir_set_heal_xattr(this, local, local->xattr, local->mds_xattr,
+                               NULL, NULL);
+        dict_unref(local->mds_xattr);
+        local->mds_xattr = NULL;
+    }
+
+    dht_layout_anomalies(this, loc, layout, &local->selfheal.hole_cnt,
+                         &local->selfheal.overlaps_cnt,
+                         &local->selfheal.missing_cnt, &local->selfheal.down,
+                         &local->selfheal.misc, NULL);
+
+    down = local->selfheal.down;
+    misc = local->selfheal.misc;
+
+    if (down) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SELFHEAL_FAILED,
+                "path=%s", loc->path, "subvol-down=%d", down, "Not-fixing",
+                "gfid=%s", gfid, NULL);
+        ret = 0;
+        goto sorry_no_fix;
+    }
 
-        return 0;
+    if (misc) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SELFHEAL_FAILED,
+                "path=%s", loc->path, "misc=%d", misc, "unrecoverable-errors",
+                "gfid=%s", gfid, NULL);
 
-sorry_no_fix:
-        /* TODO: need to put appropriate local->op_errno */
-        dht_selfheal_dir_finish (frame, this, ret, 1);
+        ret = 0;
+        goto sorry_no_fix;
+    }
 
-        return 0;
-}
+    dht_layout_sort_volname(layout);
+    local->heal_layout = _gf_true;
 
-int
-dht_selfheal_restore (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
-                      loc_t *loc, dht_layout_t *layout)
-{
-        int          ret = 0;
-        dht_local_t *local    = NULL;
+    /* Ignore return value as it can be inferred from result of
+     * dht_layout_anomalies
+     */
+    dht_selfheal_dir_getafix(frame, loc, layout);
 
-        local = frame->local;
+    if (!(local->selfheal.hole_cnt || local->selfheal.overlaps_cnt ||
+          local->selfheal.missing_cnt)) {
+        local->heal_layout = _gf_false;
+    }
 
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
+    ret = dht_selfheal_dir_mkdir(frame, loc, layout, 0);
+    if (ret < 0) {
+        ret = 0;
+        goto sorry_no_fix;
+    }
 
-        ret = dht_selfheal_dir_mkdir (frame, loc, layout, 1);
+    return 0;
 
-        return ret;
+sorry_no_fix:
+    /* TODO: need to put appropriate local->op_errno */
+    dht_selfheal_dir_finish(frame, this, ret, 1);
+
+    return 0;
 }
 
 int
-dht_dir_heal_xattrs (void *data)
+dht_selfheal_restore(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                     loc_t *loc, dht_layout_t *layout)
 {
-        call_frame_t    *frame          = NULL;
-        dht_local_t     *local          = NULL;
-        xlator_t        *subvol         = NULL;
-        xlator_t        *mds_subvol     = NULL;
-        xlator_t        *this           = NULL;
-        dht_conf_t      *conf           = NULL;
-        dict_t          *user_xattr     = NULL;
-        dict_t          *internal_xattr = NULL;
-        dict_t          *mds_xattr   = NULL;
-        dict_t          *xdata          = NULL;
-        int              call_cnt       = 0;
-        int              ret            = -1;
-        int              uret           = 0;
-        int              uflag          = 0;
-        int              i              = 0;
-        int              xattr_hashed   = 0;
-        char     gfid[GF_UUID_BUF_SIZE] = {0};
-        int32_t    allzero[1]           = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", data, out);
-
-        frame = data;
-        local = frame->local;
-        this = frame->this;
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-        mds_subvol = local->mds_subvol;
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-        gf_uuid_unparse(local->loc.gfid, gfid);
+    int ret = 0;
+    dht_local_t *local = NULL;
 
-        if (!mds_subvol) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_XATTR_HEAL_FAILED,
-                        "No mds subvol for %s gfid = %s",
-                        local->loc.path, gfid);
-                goto out;
-        }
+    local = frame->local;
 
-        if ((local->loc.inode && gf_uuid_is_null (local->loc.inode->gfid)) ||
-            gf_uuid_is_null (local->loc.gfid)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_XATTR_HEAL_FAILED,
-                        "No gfid present so skip heal for path %s gfid = %s",
-                        local->loc.path, gfid);
-                goto out;
-        }
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
 
-        internal_xattr = dict_new ();
-        if (!internal_xattr) {
-                gf_msg (this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, 0,
-                        "dictionary creation failed");
-                goto out;
-        }
-        xdata = dict_new ();
-        if (!xdata) {
-                gf_msg (this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, 0,
-                        "dictionary creation failed");
-                goto out;
-        }
+    ret = dht_selfheal_dir_mkdir(frame, loc, layout, 1);
 
-        call_cnt = conf->subvolume_cnt;
+    return ret;
+}
 
-        user_xattr = dict_new ();
-        if (!user_xattr) {
-                gf_msg (this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, 0,
-                        "dictionary creation failed");
+int
+dht_dir_heal_xattrs(void *data)
+{
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dict_t *user_xattr = NULL;
+    dict_t *internal_xattr = NULL;
+    dict_t *mds_xattr = NULL;
+    dict_t *xdata = NULL;
+    int call_cnt = 0;
+    int ret = -1;
+    int uret = 0;
+    int uflag = 0;
+    int i = 0;
+    int xattr_hashed = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t allzero[1] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", data, out);
+
+    frame = data;
+    local = frame->local;
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, local, out);
+    mds_subvol = local->mds_subvol;
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    if (!mds_subvol) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NO_MDS_SUBVOL, "path=%s",
+                local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    if ((local->loc.inode && gf_uuid_is_null(local->loc.inode->gfid)) ||
+        gf_uuid_is_null(local->loc.gfid)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_NOT_PRESENT,
+                "skip-heal path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    internal_xattr = dict_new();
+    if (!internal_xattr) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CREATE_FAILED,
+                "dictionary", NULL);
+        goto out;
+    }
+    xdata = dict_new();
+    if (!xdata) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CREATE_FAILED,
+                "dictionary", NULL);
+        goto out;
+    }
+
+    call_cnt = conf->subvolume_cnt;
+
+    user_xattr = dict_new();
+    if (!user_xattr) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CREATE_FAILED,
+                "dictionary", NULL);
+        goto out;
+    }
+
+    ret = syncop_listxattr(local->mds_subvol, &local->loc, &mds_xattr, NULL,
+                           NULL);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LIST_XATTRS_FAILED,
+                "path=%s", local->loc.path, "name=%s", local->mds_subvol->name,
+                NULL);
+    }
+
+    if (!mds_xattr)
+        goto out;
+
+    dht_dir_set_heal_xattr(this, local, user_xattr, mds_xattr, &uret, &uflag);
+
+    /* To set quota related xattr need to set GLUSTERFS_INTERNAL_FOP_KEY
+     * key value to 1
+     */
+    if (dict_get(user_xattr, QUOTA_LIMIT_KEY) ||
+        dict_get(user_xattr, QUOTA_LIMIT_OBJECTS_KEY)) {
+        ret = dict_set_int32(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                    "key=%s", GLUSTERFS_INTERNAL_FOP_KEY, "path=%s",
+                    local->loc.path, NULL);
+            goto out;
+        }
+    }
+    if (uret <= 0 && !uflag)
+        goto out;
+
+    for (i = 0; i < call_cnt; i++) {
+        subvol = conf->subvolumes[i];
+        if (subvol == mds_subvol)
+            continue;
+        if (uret || uflag) {
+            /* Custom xattr heal is required - let posix handle it */
+            ret = dict_set_int8(xdata, "sync_backend_xattrs", _gf_true);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                        "path=%s", local->loc.path, "key=%s",
+                        "sync_backend_xattrs", NULL);
                 goto out;
-        }
+            }
 
-        ret = syncop_listxattr (local->mds_subvol, &local->loc,
-                                &mds_xattr, NULL, NULL);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
+            ret = syncop_setxattr(subvol, &local->loc, user_xattr, 0, xdata,
+                                  NULL);
+            if (ret) {
+                xattr_hashed = 1;
+                gf_smsg(this->name, GF_LOG_ERROR, -ret,
                         DHT_MSG_DIR_XATTR_HEAL_FAILED,
-                        "failed to list xattrs for "
-                        "%s: on %s ",
-                        local->loc.path, local->mds_subvol->name);
-        }
-
-        if (!mds_xattr)
-                goto out;
-
-        dht_dir_set_heal_xattr (this, local, user_xattr, mds_xattr,
-                                &uret, &uflag);
-
-        /* To set quota related xattr need to set GLUSTERFS_INTERNAL_FOP_KEY
-         * key value to 1
-         */
-        if (dict_get (user_xattr, QUOTA_LIMIT_KEY) ||
-            dict_get (user_xattr, QUOTA_LIMIT_OBJECTS_KEY)) {
-                ret = dict_set_int32 (xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value: key = %s,"
-                                " path = %s", GLUSTERFS_INTERNAL_FOP_KEY,
-                                 local->loc.path);
-                        goto out;
-                }
-        }
-        if (uret <= 0 && !uflag)
-                goto out;
-
-        for (i = 0; i < call_cnt; i++) {
-                subvol = conf->subvolumes[i];
-                if (subvol == mds_subvol)
-                        continue;
-                if (uret || uflag) {
-                        ret = syncop_setxattr (subvol, &local->loc, user_xattr,
-                                               0, xdata, NULL);
-                        if (ret) {
-                                xattr_hashed = 1;
-                                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                        DHT_MSG_DIR_XATTR_HEAL_FAILED,
-                                        "Directory xattr heal failed. Failed to set"
-                                        "user xattr on path %s on "
-                                        "subvol %s, gfid = %s ",
-                                        local->loc.path, subvol->name, gfid);
-                        }
-                }
+                        "set-user-xattr-failed path=%s", local->loc.path,
+                        "subvol=%s", subvol->name, "gfid=%s", gfid, NULL);
+            } else {
+                dict_del(xdata, "sync_backend_xattrs");
+            }
+        }
+    }
+    /* After heal all custom xattr reset internal MDS xattr to 0 */
+    if (!xattr_hashed) {
+        ret = dht_dict_set_array(internal_xattr, conf->mds_xattr_key, allzero,
+                                 1);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                    "key=%s", conf->mds_xattr_key, "path=%s", local->loc.path,
+                    NULL);
+            goto out;
         }
-        /* After heal all custom xattr reset internal MDS xattr to 0 */
-        if (!xattr_hashed) {
-                ret = dht_dict_set_array (internal_xattr,
-                                          conf->mds_xattr_key,
-                                          allzero, 1);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", conf->mds_xattr_key,
-                                local->loc.path);
-                        goto out;
-                }
-                ret = syncop_setxattr (mds_subvol, &local->loc, internal_xattr,
-                                       0, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_DIR_XATTR_HEAL_FAILED,
-                                "Failed to reset internal xattr "
-                                "on path %s on subvol %s"
-                                "gfid = %s ", local->loc.path,
-                                mds_subvol->name, gfid);
-                }
+        ret = syncop_setxattr(mds_subvol, &local->loc, internal_xattr, 0, NULL,
+                              NULL);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_DIR_XATTR_HEAL_FAILED, "path=%s", local->loc.path,
+                    "subvol=%s", mds_subvol->name, "gfid=%s", gfid, NULL);
         }
+    }
 
 out:
-        if (user_xattr)
-                dict_unref (user_xattr);
-        if (mds_xattr)
-                dict_unref (mds_xattr);
-        if (internal_xattr)
-                dict_unref (internal_xattr);
-        if (xdata)
-                dict_unref (xdata);
-        return 0;
+    if (user_xattr)
+        dict_unref(user_xattr);
+    if (mds_xattr)
+        dict_unref(mds_xattr);
+    if (internal_xattr)
+        dict_unref(internal_xattr);
+    if (xdata)
+        dict_unref(xdata);
+    return 0;
 }
 
-
 int
-dht_dir_heal_xattrs_done (int ret, call_frame_t *sync_frame, void *data)
+dht_dir_heal_xattrs_done(int ret, call_frame_t *sync_frame, void *data)
 {
-        DHT_STACK_DESTROY (sync_frame);
-        return 0;
+    DHT_STACK_DESTROY(sync_frame);
+    return 0;
 }
 
-
 int
-dht_dir_attr_heal (void *data)
+dht_dir_attr_heal(void *data)
 {
-        call_frame_t    *frame      = NULL;
-        dht_local_t     *local      = NULL;
-        xlator_t        *subvol     = NULL;
-        xlator_t        *mds_subvol = NULL;
-        xlator_t        *this       = NULL;
-        dht_conf_t      *conf  = NULL;
-        int              call_cnt = 0;
-        int              ret   = -1;
-        int              i     = 0;
-        char             gfid[GF_UUID_BUF_SIZE] = {0};
-
-
-        GF_VALIDATE_OR_GOTO ("dht", data, out);
-
-        frame = data;
-        local = frame->local;
-        mds_subvol = local->mds_subvol;
-        this = frame->this;
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", local, out);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO ("dht", conf, out);
-
-        call_cnt = conf->subvolume_cnt;
-
-        if (!__is_root_gfid (local->stbuf.ia_gfid) && (!mds_subvol)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_ATTR_HEAL_FAILED,
-                        "No mds subvol for %s gfid = %s",
-                        local->loc.path, gfid);
-                goto out;
-        }
-
-        if (!__is_root_gfid (local->stbuf.ia_gfid)) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->subvolumes[i] ==  mds_subvol) {
-                                if (!conf->subvolume_status[i]) {
-                                        gf_msg (this->name, GF_LOG_ERROR,
-                                                0,  DHT_MSG_HASHED_SUBVOL_DOWN,
-                                                "mds subvol is down for path "
-                                                " %s gfid is %s Unable to set xattr " ,
-                                                local->loc.path, gfid);
-                                        goto out;
-                                }
-                        }
-                }
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    int call_cnt = 0;
+    int ret = -1;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", data, out);
+
+    frame = data;
+    local = frame->local;
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", local, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO("dht", conf, out);
+
+    mds_subvol = local->mds_subvol;
+    call_cnt = conf->subvolume_cnt;
+
+    if (!__is_root_gfid(local->stbuf.ia_gfid) && (!mds_subvol)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NO_MDS_SUBVOL, "path=%s",
+                local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    if (!__is_root_gfid(local->stbuf.ia_gfid)) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvolumes[i] == mds_subvol) {
+                if (!conf->subvolume_status[i]) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            DHT_MSG_MDS_DOWN_UNABLE_TO_SET, "path=%s",
+                            local->loc.path, "gfid=%s", gfid, NULL);
+                    goto out;
+                }
+            }
+        }
+    }
+
+    for (i = 0; i < call_cnt; i++) {
+        subvol = conf->subvolumes[i];
+        if (!subvol || subvol == mds_subvol)
+            continue;
+        if (__is_root_gfid(local->stbuf.ia_gfid)) {
+            ret = syncop_setattr(
+                subvol, &local->loc, &local->stbuf,
+                (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL,
+                NULL, NULL, NULL);
+        } else {
+            ret = syncop_setattr(
+                subvol, &local->loc, &local->mds_stbuf,
+                (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL,
+                NULL, NULL, NULL);
         }
 
-        for (i = 0; i < call_cnt; i++) {
-                subvol = conf->subvolumes[i];
-                if (!subvol || subvol == mds_subvol)
-                        continue;
-                if (__is_root_gfid (local->stbuf.ia_gfid)) {
-                        ret = syncop_setattr (subvol, &local->loc, &local->stbuf,
-                                              (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE),
-                                              NULL, NULL, NULL, NULL);
-                } else {
-                        ret = syncop_setattr (subvol, &local->loc, &local->mds_stbuf,
-                                              (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
-                                              NULL, NULL, NULL, NULL);
-                }
-
-                if (ret) {
-                        gf_uuid_unparse(local->loc.gfid, gfid);
+        if (ret) {
+            gf_uuid_unparse(local->loc.gfid, gfid);
 
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                DHT_MSG_DIR_ATTR_HEAL_FAILED,
-                                "Directory attr heal failed. Failed to set"
-                                " uid/gid on path %s on subvol %s, gfid = %s ",
-                                local->loc.path, subvol->name, gfid);
-                }
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_DIR_ATTR_HEAL_FAILED, "path=%s", local->loc.path,
+                    "subvol=%s", subvol->name, "gfid=%s", gfid, NULL);
         }
+    }
 out:
-        return 0;
+    return 0;
 }
 
 int
-dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data)
+dht_dir_attr_heal_done(int ret, call_frame_t *sync_frame, void *data)
 {
-        DHT_STACK_DESTROY (sync_frame);
-        return 0;
+    DHT_STACK_DESTROY(sync_frame);
+    return 0;
 }
 
 /* EXIT: dht_update_commit_hash_for_layout */
-int
-dht_update_commit_hash_for_layout_done (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       dict_t *xdata)
+static int
+dht_update_commit_hash_for_layout_done(call_frame_t *frame, void *cookie,
+                                       xlator_t *this, int32_t op_ret,
+                                       int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        /* preserve oldest error */
-        if (op_ret && !local->op_ret) {
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-        }
+    /* preserve oldest error */
+    if (op_ret && !local->op_ret) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+    }
 
-        DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
-                          local->op_errno, NULL);
+    DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_update_commit_hash_for_layout_unlock (call_frame_t *frame, xlator_t *this)
+static int
+dht_update_commit_hash_for_layout_unlock(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t  *local = NULL;
-        int ret = 0;
+    dht_local_t *local = NULL;
+    int ret = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        ret = dht_unlock_inodelk (frame, local->lock[0].layout.my_layout.locks,
-                                   local->lock[0].layout.my_layout.lk_count,
-                                   dht_update_commit_hash_for_layout_done);
-        if (ret < 0) {
-                /* preserve oldest error, just ... */
-                if (!local->op_ret) {
-                        local->op_errno = errno;
-                        local->op_ret = -1;
-                }
+    ret = dht_unlock_inodelk(frame, local->lock[0].layout.my_layout.locks,
+                             local->lock[0].layout.my_layout.lk_count,
+                             dht_update_commit_hash_for_layout_done);
+    if (ret < 0) {
+        /* preserve oldest error, just ... */
+        if (!local->op_ret) {
+            local->op_errno = errno;
+            local->op_ret = -1;
+        }
 
-                gf_msg (this->name, GF_LOG_WARNING, errno,
-                        DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                        "Winding unlock failed: stale locks left on brick"
-                        " %s", local->loc.path);
+        gf_smsg(this->name, GF_LOG_WARNING, errno, DHT_MSG_WIND_UNLOCK_FAILED,
+                "path=%s", local->loc.path, NULL);
 
-                dht_update_commit_hash_for_layout_done (frame, NULL, this,
-                                                        0, 0, NULL);
-        }
+        dht_update_commit_hash_for_layout_done(frame, NULL, this, 0, 0, NULL);
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_update_commit_hash_for_layout_cbk (call_frame_t *frame, void *cookie,
-                                       xlator_t *this, int op_ret,
-                                       int op_errno, dict_t *xdata)
+static int
+dht_update_commit_hash_for_layout_cbk(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int op_ret, int op_errno,
+                                      dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        /* store first failure, just because */
-        if (op_ret && !local->op_ret) {
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    LOCK(&frame->lock);
+    /* store first failure, just because */
+    if (op_ret && !local->op_ret) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+    }
+    UNLOCK(&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
+    this_call_cnt = dht_frame_return(frame);
 
-        if (is_last_call (this_call_cnt)) {
-                dht_update_commit_hash_for_layout_unlock (frame, this);
-        }
+    if (is_last_call(this_call_cnt)) {
+        dht_update_commit_hash_for_layout_unlock(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_update_commit_hash_for_layout_resume (call_frame_t *frame, void *cookie,
-                                          xlator_t *this, int32_t op_ret,
-                                          int32_t op_errno, dict_t *xdata)
+static int
+dht_update_commit_hash_for_layout_resume(call_frame_t *frame, void *cookie,
+                                         xlator_t *this, int32_t op_ret,
+                                         int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t   *local = NULL;
-        int            count = 1, ret = -1, i = 0, j = 0;
-        dht_conf_t    *conf = NULL;
-        dht_layout_t  *layout = NULL;
-        int32_t       *disk_layout = NULL;
-        dict_t        **xattr = NULL;
-
-        local = frame->local;
-        conf = frame->this->private;
-        count = conf->local_subvols_cnt;
-        layout = local->layout;
-
-        if (op_ret < 0) {
-                goto err_done;
-        }
-
-        /* We precreate the xattr list as we cannot change call count post the
-         * first wind as we may never continue from there. So we finish prep
-         * work before winding the setxattrs */
-        xattr = GF_CALLOC (count, sizeof (*xattr), gf_common_mt_char);
-        if (!xattr) {
-                local->op_errno = errno;
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1, i = 0, j = 0;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    int32_t *disk_layout = NULL;
+    dict_t **xattr = NULL;
+
+    local = frame->local;
+    conf = frame->this->private;
+    count = conf->local_subvols_cnt;
+    layout = local->layout;
+
+    if (op_ret < 0) {
+        goto err_done;
+    }
+
+    /* We precreate the xattr list as we cannot change call count post the
+     * first wind as we may never continue from there. So we finish prep
+     * work before winding the setxattrs */
+    xattr = GF_CALLOC(count, sizeof(*xattr), gf_common_mt_char);
+    if (!xattr) {
+        local->op_errno = errno;
+
+        gf_smsg(this->name, GF_LOG_WARNING, errno, DHT_MSG_COMMIT_HASH_FAILED,
+                "allocation-failed path=%s", local->loc.path, NULL);
+
+        goto err;
+    }
+
+    for (i = 0; i < count; i++) {
+        /* find the layout index for the subvolume */
+        ret = dht_layout_index_for_subvol(layout, conf->local_subvols[i]);
+        if (ret < 0) {
+            local->op_errno = ENOENT;
 
-                gf_msg (this->name, GF_LOG_WARNING, errno,
-                        DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                        "Directory commit hash update failed:"
-                        " %s: Allocation failed", local->loc.path);
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_COMMIT_HASH_FAILED,
+                    "path=%s", local->loc.path, "subvol=%s",
+                    conf->local_subvols[i]->name, "find-disk-layout-failed",
+                    NULL);
 
-                goto err;
+            goto err;
         }
+        j = ret;
 
-        for (i = 0; i < count; i++) {
-                /* find the layout index for the subvolume */
-                ret = dht_layout_index_for_subvol (layout,
-                                                   conf->local_subvols[i]);
-                if (ret < 0) {
-                        local->op_errno = ENOENT;
-
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                                "Directory commit hash update failed:"
-                                " %s: (subvol %s) Failed to find disk layout",
-                                local->loc.path, conf->local_subvols[i]->name);
-
-                        goto err;
-                }
-                j = ret;
-
-                /* update the commit hash for the layout */
-                layout->list[j].commit_hash = layout->commit_hash;
+        /* update the commit hash for the layout */
+        layout->list[j].commit_hash = layout->commit_hash;
 
-                /* extract the current layout */
-                ret = dht_disk_layout_extract (this, layout, j, &disk_layout);
-                if (ret == -1) {
-                        local->op_errno = errno;
-
-                        gf_msg (this->name, GF_LOG_WARNING, errno,
-                                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                                "Directory commit hash update failed:"
-                                " %s: (subvol %s) Failed to extract disk"
-                                " layout", local->loc.path,
-                                conf->local_subvols[i]->name);
+        /* extract the current layout */
+        ret = dht_disk_layout_extract(this, layout, j, &disk_layout);
+        if (ret == -1) {
+            local->op_errno = errno;
 
-                        goto err;
-                }
+            gf_smsg(this->name, GF_LOG_WARNING, errno,
+                    DHT_MSG_COMMIT_HASH_FAILED, "path=%s", local->loc.path,
+                    "subvol=%s", conf->local_subvols[i]->name,
+                    "extract-disk-layout-failed", NULL);
 
-                xattr[i] = dict_new ();
-                if (!xattr[i]) {
-                        local->op_errno = errno;
+            goto err;
+        }
 
-                        gf_msg (this->name, GF_LOG_WARNING, errno,
-                                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                                "Directory commit hash update failed:"
-                                " %s: Allocation failed", local->loc.path);
+        xattr[i] = dict_new();
+        if (!xattr[i]) {
+            local->op_errno = errno;
 
-                        goto err;
-                }
+            gf_smsg(this->name, GF_LOG_WARNING, errno,
+                    DHT_MSG_COMMIT_HASH_FAILED, "path=%s Allocation-failed",
+                    local->loc.path, NULL);
 
-                ret = dict_set_bin (xattr[i], conf->xattr_name,
-                                    disk_layout, 4 * 4);
-                if (ret != 0) {
-                        local->op_errno = ENOMEM;
+            goto err;
+        }
 
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                                "Directory self heal xattr failed:"
-                                "%s: (subvol %s) Failed to set xattr"
-                                " dictionary,", local->loc.path,
-                                conf->local_subvols[i]->name);
+        ret = dict_set_bin(xattr[i], conf->xattr_name, disk_layout, 4 * 4);
+        if (ret != 0) {
+            local->op_errno = ENOMEM;
 
-                        goto err;
-                }
-                disk_layout = NULL;
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    DHT_MSG_DIR_SELFHEAL_XATTR_FAILED, "path=%s",
+                    local->loc.path, "subvol=%s", conf->local_subvols[i]->name,
+                    "set-xattr-failed", NULL);
 
-                gf_msg_trace (this->name, 0,
-                              "setting commit hash %u on subvolume %s"
-                              " for %s", layout->list[j].commit_hash,
-                              conf->local_subvols[i]->name, local->loc.path);
+            goto err;
         }
+        disk_layout = NULL;
 
-        /* wind the setting of the commit hash across the local subvols */
-        local->call_cnt = count;
-        local->op_ret = 0;
-        local->op_errno = 0;
+        gf_msg_trace(this->name, 0,
+                     "setting commit hash %u on subvolume %s"
+                     " for %s",
+                     layout->list[j].commit_hash, conf->local_subvols[i]->name,
+                     local->loc.path);
+    }
+
+    /* wind the setting of the commit hash across the local subvols */
+    local->call_cnt = count;
+    local->op_ret = 0;
+    local->op_errno = 0;
+    for (i = 0; i < count; i++) {
+        STACK_WIND(frame, dht_update_commit_hash_for_layout_cbk,
+                   conf->local_subvols[i],
+                   conf->local_subvols[i]->fops->setxattr, &local->loc,
+                   xattr[i], 0, NULL);
+    }
+    for (i = 0; i < count; i++)
+        dict_unref(xattr[i]);
+    GF_FREE(xattr);
+
+    return 0;
+err:
+    if (xattr) {
         for (i = 0; i < count; i++) {
-                STACK_WIND (frame, dht_update_commit_hash_for_layout_cbk,
-                            conf->local_subvols[i],
-                            conf->local_subvols[i]->fops->setxattr,
-                            &local->loc, xattr[i], 0, NULL);
-
+            if (xattr[i])
+                dict_unref(xattr[i]);
         }
-        for (i = 0; i < count; i++)
-                dict_unref (xattr[i]);
-        GF_FREE (xattr);
 
-        return 0;
-err:
-        if (xattr) {
-                for (i = 0; i < count; i++) {
-                        if (xattr[i])
-                                dict_unref (xattr[i]);
-                }
+        GF_FREE(xattr);
+    }
 
-                GF_FREE (xattr);
-        }
-
-        GF_FREE (disk_layout);
+    GF_FREE(disk_layout);
 
-        local->op_ret = -1;
+    local->op_ret = -1;
 
-        dht_update_commit_hash_for_layout_unlock (frame, this);
+    dht_update_commit_hash_for_layout_unlock(frame, this);
 
-        return 0;
+    return 0;
 err_done:
-        local->op_ret = -1;
+    local->op_ret = -1;
 
-        dht_update_commit_hash_for_layout_done (frame, NULL, this, 0, 0, NULL);
+    dht_update_commit_hash_for_layout_done(frame, NULL, this, 0, 0, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* ENTER: dht_update_commit_hash_for_layout (see EXIT above)
@@ -2764,55 +2549,52 @@ err_done:
  *  - Unlock and return.
  */
 int
-dht_update_commit_hash_for_layout (call_frame_t *frame)
+dht_update_commit_hash_for_layout(call_frame_t *frame)
 {
-        dht_local_t   *local = NULL;
-        int            count = 1, ret = -1, i = 0;
-        dht_lock_t   **lk_array = NULL;
-        dht_conf_t    *conf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO (frame->this->name, frame->local, err);
-
-        local = frame->local;
-        conf = frame->this->private;
-
-        if (!conf->defrag)
-                goto err;
-
-        count = conf->local_subvols_cnt;
-        lk_array = GF_CALLOC (count, sizeof (*lk_array),
-                              gf_common_mt_char);
-        if (lk_array == NULL)
-                goto err;
-
-        for (i = 0; i < count; i++) {
-                lk_array[i] = dht_lock_new (frame->this,
-                                            conf->local_subvols[i],
-                                            &local->loc, F_WRLCK,
-                                            DHT_LAYOUT_HEAL_DOMAIN, NULL,
-                                            FAIL_ON_ANY_ERROR);
-                if (lk_array[i] == NULL)
-                        goto err;
-        }
-
-        local->lock[0].layout.my_layout.locks = lk_array;
-        local->lock[0].layout.my_layout.lk_count = count;
-
-        ret = dht_blocking_inodelk (frame, lk_array, count,
-                                    dht_update_commit_hash_for_layout_resume);
-        if (ret < 0) {
-                local->lock[0].layout.my_layout.locks = NULL;
-                local->lock[0].layout.my_layout.lk_count = 0;
-                goto err;
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1, i = 0;
+    dht_lock_t **lk_array = NULL;
+    dht_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
+
+    local = frame->local;
+    conf = frame->this->private;
+
+    if (!conf->defrag)
+        goto err;
+
+    count = conf->local_subvols_cnt;
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_char);
+    if (lk_array == NULL)
+        goto err;
+
+    for (i = 0; i < count; i++) {
+        lk_array[i] = dht_lock_new(frame->this, conf->local_subvols[i],
+                                   &local->loc, F_WRLCK, DHT_LAYOUT_HEAL_DOMAIN,
+                                   NULL, FAIL_ON_ANY_ERROR);
+        if (lk_array[i] == NULL)
+            goto err;
+    }
+
+    local->lock[0].layout.my_layout.locks = lk_array;
+    local->lock[0].layout.my_layout.lk_count = count;
+
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_update_commit_hash_for_layout_resume);
+    if (ret < 0) {
+        local->lock[0].layout.my_layout.locks = NULL;
+        local->lock[0].layout.my_layout.lk_count = 0;
+        goto err;
+    }
+
+    return 0;
 err:
-        if (lk_array != NULL) {
-                dht_lock_array_free (lk_array, count);
-                GF_FREE (lk_array);
-        }
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
 
-        return -1;
+    return -1;
 }
diff --git a/xlators/cluster/dht/src/dht-shared.c b/xlators/cluster/dht/src/dht-shared.c
index b5f790b111b..bb72b0ffbb5 100644
--- a/xlators/cluster/dht/src/dht-shared.c
+++ b/xlators/cluster/dht/src/dht-shared.c
@@ -8,1231 +8,1097 @@
   cases as published by the Free Software Foundation.
 */
 
-
 /* TODO: add NS locking */
-#include "statedump.h"
+#include <glusterfs/statedump.h>
 #include "dht-common.h"
 #include "dht-messages.h"
 
 #ifndef MAX
-#define MAX(a, b) (((a) > (b))?(a):(b))
+#define MAX(a, b) (((a) > (b)) ? (a) : (b))
 #endif
 
-#define GF_DECIDE_DEFRAG_THROTTLE_COUNT(throttle_count, conf) {         \
-                                                                        \
-                pthread_mutex_lock (&conf->defrag->dfq_mutex);          \
-                                                                        \
-                if (!strcasecmp (conf->dthrottle, "lazy"))              \
-                        conf->defrag->recon_thread_count = 1;           \
-                                                                        \
-                throttle_count =                                        \
-                    MAX ((sysconf(_SC_NPROCESSORS_ONLN) - 4), 4);       \
-                                                                        \
-                if (!strcasecmp (conf->dthrottle, "normal"))            \
-                        conf->defrag->recon_thread_count =              \
-                                                 (throttle_count / 2);  \
-                                                                        \
-                if (!strcasecmp (conf->dthrottle, "aggressive"))        \
-                        conf->defrag->recon_thread_count =              \
-                                                 throttle_count;        \
-                                                                        \
-                pthread_mutex_unlock (&conf->defrag->dfq_mutex);        \
-        }                                                               \
-
 /* TODO:
    - use volumename in xattr instead of "dht"
    - use NS locks
    - handle all cases in self heal layout reconstruction
    - complete linkfile selfheal
 */
-struct volume_options options[];
-
-extern dht_methods_t dht_methods;
 
-void
-dht_layout_dump (dht_layout_t  *layout, const char *prefix)
+static void
+dht_layout_dump(dht_layout_t *layout, const char *prefix)
 {
-
-        char    key[GF_DUMP_MAX_BUF_LEN];
-        int     i = 0;
-
-        if (!layout)
-                goto out;
-        if (!prefix)
-                goto out;
-
-        gf_proc_dump_build_key(key, prefix, "cnt");
-        gf_proc_dump_write(key, "%d", layout->cnt);
-        gf_proc_dump_build_key(key, prefix, "preset");
-        gf_proc_dump_write(key, "%d", layout->preset);
-        gf_proc_dump_build_key(key, prefix, "gen");
-        gf_proc_dump_write(key, "%d", layout->gen);
-        if (layout->type != IA_INVAL) {
-                gf_proc_dump_build_key(key, prefix, "inode type");
-                gf_proc_dump_write(key, "%d", layout->type);
-        }
-
-        if  (!IA_ISDIR (layout->type))
-                goto out;
-
-        for (i = 0; i < layout->cnt; i++) {
-                gf_proc_dump_build_key(key, prefix,"list[%d].err", i);
-                gf_proc_dump_write(key, "%d", layout->list[i].err);
-                gf_proc_dump_build_key(key, prefix,"list[%d].start", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].start);
-                gf_proc_dump_build_key(key, prefix,"list[%d].stop", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].stop);
-                if (layout->list[i].xlator) {
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.type", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->type);
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.name", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->name);
-                }
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+
+    if (!layout)
+        goto out;
+
+    gf_proc_dump_build_key(key, prefix, "cnt");
+    gf_proc_dump_write(key, "%d", layout->cnt);
+    gf_proc_dump_build_key(key, prefix, "preset");
+    gf_proc_dump_write(key, "%d", layout->preset);
+    gf_proc_dump_build_key(key, prefix, "gen");
+    gf_proc_dump_write(key, "%d", layout->gen);
+    if (layout->type != IA_INVAL) {
+        gf_proc_dump_build_key(key, prefix, "inode type");
+        gf_proc_dump_write(key, "%d", layout->type);
+    }
+
+    if (!IA_ISDIR(layout->type))
+        goto out;
+
+    for (i = 0; i < layout->cnt; i++) {
+        gf_proc_dump_build_key(key, prefix, "list[%d].err", i);
+        gf_proc_dump_write(key, "%d", layout->list[i].err);
+        gf_proc_dump_build_key(key, prefix, "list[%d].start", i);
+        gf_proc_dump_write(key, "0x%x", layout->list[i].start);
+        gf_proc_dump_build_key(key, prefix, "list[%d].stop", i);
+        gf_proc_dump_write(key, "0x%x", layout->list[i].stop);
+        if (layout->list[i].xlator) {
+            gf_proc_dump_build_key(key, prefix, "list[%d].xlator.type", i);
+            gf_proc_dump_write(key, "%s", layout->list[i].xlator->type);
+            gf_proc_dump_build_key(key, prefix, "list[%d].xlator.name", i);
+            gf_proc_dump_write(key, "%s", layout->list[i].xlator->name);
         }
+    }
 
 out:
-        return;
+    return;
 }
 
-
 int32_t
-dht_priv_dump (xlator_t *this)
+dht_priv_dump(xlator_t *this)
 {
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
-        char            key[GF_DUMP_MAX_BUF_LEN];
-        int             i = 0;
-        dht_conf_t      *conf = NULL;
-        int             ret = -1;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
 
-        if (!this)
-                goto out;
+    if (!this)
+        goto out;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        ret = TRY_LOCK(&conf->subvolume_lock);
-        if (ret != 0) {
-                return ret;
+    ret = TRY_LOCK(&conf->subvolume_lock);
+    if (ret != 0) {
+        return ret;
+    }
+
+    gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
+    gf_proc_dump_build_key(key_prefix, "xlator.cluster.dht", "%s.priv",
+                           this->name);
+    gf_proc_dump_write("subvol_cnt", "%d", conf->subvolume_cnt);
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        snprintf(key, sizeof(key), "subvolumes[%d]", i);
+        gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
+                           conf->subvolumes[i]->name);
+        if (conf->file_layouts && conf->file_layouts[i]) {
+            snprintf(key, sizeof(key), "file_layouts[%d]", i);
+            dht_layout_dump(conf->file_layouts[i], key);
         }
-
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
-        gf_proc_dump_build_key(key_prefix,"xlator.cluster.dht","%s.priv",
-                               this->name);
-        gf_proc_dump_write("subvol_cnt","%d", conf->subvolume_cnt);
+        if (conf->dir_layouts && conf->dir_layouts[i]) {
+            snprintf(key, sizeof(key), "dir_layouts[%d]", i);
+            dht_layout_dump(conf->dir_layouts[i], key);
+        }
+        if (conf->subvolume_status) {
+            snprintf(key, sizeof(key), "subvolume_status[%d]", i);
+            gf_proc_dump_write(key, "%d", (int)conf->subvolume_status[i]);
+        }
+    }
+
+    gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
+    gf_proc_dump_write("gen", "%d", conf->gen);
+    gf_proc_dump_write("min_free_disk", "%lf", conf->min_free_disk);
+    gf_proc_dump_write("min_free_inodes", "%lf", conf->min_free_inodes);
+    gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
+    gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
+    gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
+    gf_proc_dump_write("use-readdirp", "%d", conf->use_readdirp);
+
+    if (conf->du_stats && conf->subvolume_status) {
         for (i = 0; i < conf->subvolume_cnt; i++) {
-                snprintf (key, sizeof (key), "subvolumes[%d]", i);
-                gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
-                                   conf->subvolumes[i]->name);
-                if (conf->file_layouts && conf->file_layouts[i]){
-                        snprintf (key, sizeof (key), "file_layouts[%d]", i);
-                        dht_layout_dump(conf->file_layouts[i], key);
-                }
-                if (conf->dir_layouts && conf->dir_layouts[i]) {
-                        snprintf (key, sizeof (key), "dir_layouts[%d]", i);
-                        dht_layout_dump(conf->dir_layouts[i], key);
-                }
-                if (conf->subvolume_status) {
-
-                        snprintf (key, sizeof (key), "subvolume_status[%d]", i);
-                        gf_proc_dump_write(key, "%d",
-                                           (int)conf->subvolume_status[i]);
-                }
+            if (!conf->subvolume_status[i])
+                continue;
 
-        }
+            snprintf(key, sizeof(key), "subvolumes[%d]", i);
+            gf_proc_dump_write(key, "%s", conf->subvolumes[i]->name);
+
+            snprintf(key, sizeof(key), "du_stats[%d].avail_percent", i);
+            gf_proc_dump_write(key, "%lf", conf->du_stats[i].avail_percent);
+
+            snprintf(key, sizeof(key), "du_stats[%d].avail_space", i);
+            gf_proc_dump_write(key, "%" PRIu64, conf->du_stats[i].avail_space);
 
-        gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
-        gf_proc_dump_write("gen", "%d", conf->gen);
-        gf_proc_dump_write("min_free_disk", "%lf", conf->min_free_disk);
-        gf_proc_dump_write("min_free_inodes", "%lf", conf->min_free_inodes);
-        gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
-        gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
-        gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
-        gf_proc_dump_write("use-readdirp", "%d", conf->use_readdirp);
-
-        if (conf->du_stats && conf->subvolume_status) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (!conf->subvolume_status[i])
-                                continue;
-
-                        snprintf (key, sizeof (key), "subvolumes[%d]", i);
-                        gf_proc_dump_write (key, "%s",
-                                            conf->subvolumes[i]->name);
-
-                        snprintf (key, sizeof (key),
-                                  "du_stats[%d].avail_percent", i);
-                        gf_proc_dump_write (key, "%lf",
-                                            conf->du_stats[i].avail_percent);
-
-                        snprintf (key, sizeof (key), "du_stats[%d].avail_space",
-                                  i);
-                        gf_proc_dump_write (key, "%lu",
-                                            conf->du_stats[i].avail_space);
-
-                        snprintf (key, sizeof (key),
-                                  "du_stats[%d].avail_inodes", i);
-                        gf_proc_dump_write (key, "%lf",
-                                            conf->du_stats[i].avail_inodes);
-
-                        snprintf (key, sizeof (key), "du_stats[%d].log", i);
-                        gf_proc_dump_write (key, "%lu",
-                                            conf->du_stats[i].log);
-                }
+            snprintf(key, sizeof(key), "du_stats[%d].avail_inodes", i);
+            gf_proc_dump_write(key, "%lf", conf->du_stats[i].avail_inodes);
+
+            snprintf(key, sizeof(key), "du_stats[%d].log", i);
+            gf_proc_dump_write(key, "%" PRIu32, conf->du_stats[i].log);
         }
+    }
 
-        if (conf->last_stat_fetch.tv_sec)
-                gf_proc_dump_write("last_stat_fetch", "%s",
-                                    ctime(&conf->last_stat_fetch.tv_sec));
+    if (conf->last_stat_fetch)
+        gf_proc_dump_write("last_stat_fetch", "%s",
+                           ctime(&conf->last_stat_fetch));
 
-        UNLOCK(&conf->subvolume_lock);
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-dht_inodectx_dump (xlator_t *this, inode_t *inode)
+dht_inodectx_dump(xlator_t *this, inode_t *inode)
 {
-        int             ret = -1;
-        dht_layout_t    *layout = NULL;
+    int ret = -1;
+    dht_layout_t *layout = NULL;
 
-        if (!this)
-                goto out;
-        if (!inode)
-                goto out;
+    if (!this)
+        goto out;
+    if (!inode)
+        goto out;
 
-        ret = dht_inode_ctx_layout_get (inode, this, &layout);
+    ret = dht_inode_ctx_layout_get(inode, this, &layout);
 
-        if ((ret != 0) || !layout)
-                return ret;
+    if ((ret != 0) || !layout)
+        return ret;
 
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
-        dht_layout_dump(layout, "layout");
+    gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
+    dht_layout_dump(layout, "layout");
 
 out:
-        return ret;
+    return ret;
 }
 
 void
-dht_fini (xlator_t *this)
+dht_fini(xlator_t *this)
 {
-        int         i = 0;
-        dht_conf_t *conf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-        conf = this->private;
-        this->private = NULL;
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                dict_unref(conf->leaf_to_subvol);
-
-                /* allocated in dht_init_subvolumes() */
-                GF_FREE (conf->subvolumes);
-                GF_FREE (conf->subvolume_status);
-                GF_FREE (conf->last_event);
-                GF_FREE (conf->subvol_up_time);
-                GF_FREE (conf->du_stats);
-                GF_FREE (conf->decommissioned_bricks);
-
-                /* allocated in dht_init() */
-                GF_FREE (conf->mds_xattr_key);
-                GF_FREE (conf->link_xattr_name);
-                GF_FREE (conf->commithash_xattr_name);
-                GF_FREE (conf->wild_xattr_name);
-
-                /* allocated in dht_init_regex() */
-                if (conf->rsync_regex_valid)
-                        regfree (&conf->rsync_regex);
-                if (conf->extra_regex_valid)
-                        regfree (&conf->extra_regex);
-
-                synclock_destroy (&conf->link_lock);
-
-                if (conf->lock_pool)
-                        mem_pool_destroy (conf->lock_pool);
-
-                GF_FREE (conf);
+    int i = 0;
+    dht_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+
+    conf = this->private;
+    this->private = NULL;
+    if (conf) {
+        if (conf->file_layouts) {
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                GF_FREE(conf->file_layouts[i]);
+            }
+            GF_FREE(conf->file_layouts);
         }
+
+        dict_unref(conf->leaf_to_subvol);
+
+        /* allocated in dht_init_subvolumes() */
+        GF_FREE(conf->subvolumes);
+        GF_FREE(conf->subvolume_status);
+        GF_FREE(conf->last_event);
+        GF_FREE(conf->subvol_up_time);
+        GF_FREE(conf->du_stats);
+        GF_FREE(conf->decommissioned_bricks);
+
+        /* allocated in dht_init() */
+        GF_FREE(conf->mds_xattr_key);
+        GF_FREE(conf->link_xattr_name);
+        GF_FREE(conf->commithash_xattr_name);
+        GF_FREE(conf->wild_xattr_name);
+
+        /* allocated in dht_init_regex() */
+        if (conf->rsync_regex_valid)
+            regfree(&conf->rsync_regex);
+        if (conf->extra_regex_valid)
+            regfree(&conf->extra_regex);
+
+        synclock_destroy(&conf->link_lock);
+
+        if (conf->lock_pool)
+            mem_pool_destroy(conf->lock_pool);
+
+        GF_FREE(conf);
+    }
 out:
-        return;
+    return;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
 
-        ret = xlator_mem_acct_init (this, gf_dht_mt_end + 1);
+    ret = xlator_mem_acct_init(this, gf_dht_mt_end + 1);
 
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_NO_MEMORY,
-                        "Memory accounting init failed");
-                return ret;
-        }
-out:
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+               "Memory accounting init failed");
         return ret;
+    }
+out:
+    return ret;
 }
 
-
-int
-dht_parse_decommissioned_bricks (xlator_t *this, dht_conf_t *conf,
-                                 const char *bricks)
+static int
+dht_parse_decommissioned_bricks(xlator_t *this, dht_conf_t *conf,
+                                const char *bricks)
 {
-        int         i  = 0;
-        int         ret  = -1;
-        char       *tmpstr = NULL;
-        char       *dup_brick = NULL;
-        char       *node = NULL;
-
-        if (!conf || !bricks)
-                goto out;
-
-        dup_brick = gf_strdup (bricks);
-        node = strtok_r (dup_brick, ",", &tmpstr);
-        while (node) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (!strcmp (conf->subvolumes[i]->name, node)) {
-                                conf->decommissioned_bricks[i] =
-                                        conf->subvolumes[i];
-                                        conf->decommission_subvols_cnt++;
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_SUBVOL_DECOMMISSION_INFO,
-                                        "decommissioning subvolume %s",
-                                        conf->subvolumes[i]->name);
-                                break;
-                        }
-                }
-                if (i == conf->subvolume_cnt) {
-                        /* Wrong node given. */
-                        goto out;
-                }
-                node = strtok_r (NULL, ",", &tmpstr);
+    int i = 0;
+    int ret = -1;
+    char *tmpstr = NULL;
+    char *dup_brick = NULL;
+    char *node = NULL;
+
+    if (!conf || !bricks)
+        goto out;
+
+    dup_brick = gf_strdup(bricks);
+    if (dup_brick == NULL) {
+        goto out;
+    }
+
+    node = strtok_r(dup_brick, ",", &tmpstr);
+    while (node) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (!strcmp(conf->subvolumes[i]->name, node)) {
+                conf->decommissioned_bricks[i] = conf->subvolumes[i];
+                conf->decommission_subvols_cnt++;
+                gf_msg(this->name, GF_LOG_INFO, 0,
+                       DHT_MSG_SUBVOL_DECOMMISSION_INFO,
+                       "decommissioning subvolume %s",
+                       conf->subvolumes[i]->name);
+                break;
+            }
+        }
+        if (i == conf->subvolume_cnt) {
+            /* Wrong node given. */
+            goto out;
         }
+        node = strtok_r(NULL, ",", &tmpstr);
+    }
 
-        ret = 0;
-        conf->decommission_in_progress = 1;
+    ret = 0;
+    conf->decommission_in_progress = 1;
 out:
-        GF_FREE (dup_brick);
+    GF_FREE(dup_brick);
 
-        return ret;
+    return ret;
 }
 
-int
-dht_decommissioned_remove (xlator_t *this, dht_conf_t *conf)
+static void
+dht_decommissioned_remove(xlator_t *this, dht_conf_t *conf)
 {
-        int         i  = 0;
-        int         ret  = -1;
+    int i = 0;
 
-        if (!conf)
-                goto out;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->decommissioned_bricks[i]) {
-                        conf->decommissioned_bricks[i] = NULL;
-                        conf->decommission_subvols_cnt--;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->decommissioned_bricks[i]) {
+            conf->decommissioned_bricks[i] = NULL;
+            conf->decommission_subvols_cnt--;
         }
-
-        ret = 0;
-out:
-
-        return ret;
+    }
 }
-void
-dht_init_regex (xlator_t *this, dict_t *odict, char *name,
-                regex_t *re, gf_boolean_t *re_valid, dht_conf_t *conf)
+
+static void
+dht_init_regex(xlator_t *this, dict_t *odict, char *name, regex_t *re,
+               gf_boolean_t *re_valid, dht_conf_t *conf)
 {
-        char       *temp_str = NULL;
+    char *temp_str = NULL;
 
-        if (dict_get_str (odict, name, &temp_str) != 0) {
-                if (strcmp(name,"rsync-hash-regex")) {
-                        return;
-                }
-                temp_str = "^\\.(.+)\\.[^.]+$";
+    if (dict_get_str(odict, name, &temp_str) != 0) {
+        if (strcmp(name, "rsync-hash-regex")) {
+            return;
+        }
+        temp_str = "^\\.(.+)\\.[^.]+$";
+    }
+
+    LOCK(&conf->lock);
+    {
+        if (*re_valid) {
+            regfree(re);
+            *re_valid = _gf_false;
+        }
+
+        if (!strcmp(temp_str, "none")) {
+            goto unlock;
         }
 
-        LOCK (&conf->lock);
-        {
-                if (*re_valid) {
-                        regfree(re);
-                        *re_valid = _gf_false;
-                }
-
-                if (!strcmp(temp_str, "none")) {
-                        goto unlock;
-                }
-
-                if (regcomp(re, temp_str, REG_EXTENDED) == 0) {
-                        gf_msg_debug (this->name, 0,
-                                      "using regex %s = %s", name, temp_str);
-                        *re_valid = _gf_true;
-                } else {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_REGEX_INFO,
-                                "compiling regex %s failed", temp_str);
-                }
+        if (regcomp(re, temp_str, REG_EXTENDED) == 0) {
+            gf_msg_debug(this->name, 0, "using regex %s = %s", name, temp_str);
+            *re_valid = _gf_true;
+        } else {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_REGEX_INFO,
+                   "compiling regex %s failed", temp_str);
         }
+    }
 unlock:
-        UNLOCK (&conf->lock);
+    UNLOCK(&conf->lock);
 }
 
 int
 dht_set_subvol_range(xlator_t *this)
 {
-        int ret = -1;
-        dht_conf_t *conf = NULL;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf)
-                goto out;
+    if (!conf)
+        goto out;
 
-        conf->leaf_to_subvol = dict_new();
-        if (!conf->leaf_to_subvol)
-                goto out;
+    conf->leaf_to_subvol = dict_new();
+    if (!conf->leaf_to_subvol)
+        goto out;
 
-        ret = glusterfs_reachable_leaves(this, conf->leaf_to_subvol);
+    ret = glusterfs_reachable_leaves(this, conf->leaf_to_subvol);
 
 out:
-        return ret;
+    return ret;
 }
 
-int
-dht_configure_throttle (xlator_t *this, dht_conf_t *conf, char *temp_str)
+static int
+dht_configure_throttle(xlator_t *this, dht_conf_t *conf, char *temp_str)
 {
-        int     rebal_thread_count      = 0;
-        int     ret                     = 0;
-
-        pthread_mutex_lock (&conf->defrag->dfq_mutex);
-        {
-        if (!strcasecmp (temp_str, "lazy")) {
-                conf->defrag->recon_thread_count = 1;
-        } else if (!strcasecmp (temp_str, "normal")) {
-                conf->defrag->recon_thread_count = 2;
-        } else if (!strcasecmp (temp_str, "aggressive")) {
-                conf->defrag->recon_thread_count = MAX (MAX_REBAL_THREADS - 4, 4);
-        } else if ((gf_string2int (temp_str, &rebal_thread_count) == 0)) {
-                if ((rebal_thread_count > 0) && (rebal_thread_count <= MAX_REBAL_THREADS)) {
-                        gf_msg (this->name, GF_LOG_INFO, 0, 0,
-                                "rebal thread count configured to %d",
-                                rebal_thread_count);
-                                conf->defrag->recon_thread_count = rebal_thread_count;
-                } else {
-                        gf_msg(this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_INVALID_OPTION,
-                                "Invalid option: Reconfigure: "
-                                "rebal-throttle should be "
-                                "within range of 0 and maximum number of"
-                                " cores available");
-                                ret = -1;
-                                pthread_mutex_unlock (&conf->defrag->dfq_mutex);
-                                goto out;
-                }
+    int rebal_thread_count = 0;
+    int ret = 0;
+
+    pthread_mutex_lock(&conf->defrag->dfq_mutex);
+    {
+        if (!strcasecmp(temp_str, "lazy")) {
+            conf->defrag->recon_thread_count = 1;
+        } else if (!strcasecmp(temp_str, "normal")) {
+            conf->defrag->recon_thread_count = 2;
+        } else if (!strcasecmp(temp_str, "aggressive")) {
+            conf->defrag->recon_thread_count = MAX(MAX_REBAL_THREADS - 4, 4);
+        } else if ((gf_string2int(temp_str, &rebal_thread_count) == 0)) {
+            if ((rebal_thread_count > 0) &&
+                (rebal_thread_count <= MAX_REBAL_THREADS)) {
+                conf->defrag->recon_thread_count = rebal_thread_count;
+                pthread_mutex_unlock(&conf->defrag->dfq_mutex);
+                gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                       "rebal thread count configured to %d",
+                       rebal_thread_count);
+                goto out;
+            } else {
+                pthread_mutex_unlock(&conf->defrag->dfq_mutex);
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                       "Invalid option: Reconfigure: "
+                       "rebal-throttle should be "
+                       "within range of 0 and maximum number of"
+                       " cores available");
+                ret = -1;
+                goto out;
+            }
         } else {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_INVALID_OPTION,
-                        "Invalid option: Reconfigure: "
-                        "rebal-throttle should be {lazy|normal|aggressive}"
-                        " or a number up to the number of cores available,"
-                        " not (%s), defaulting to (%d)",
-                        temp_str, conf->dthrottle);
-                        ret = -1;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                   "Invalid option: Reconfigure: "
+                   "rebal-throttle should be {lazy|normal|aggressive}"
+                   " or a number up to the number of cores available,"
+                   " not (%s), defaulting to (%d)",
+                   temp_str, conf->dthrottle);
+            ret = -1;
         }
-        pthread_mutex_unlock (&conf->defrag->dfq_mutex);
+    }
+    pthread_mutex_unlock(&conf->defrag->dfq_mutex);
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-dht_reconfigure (xlator_t *this, dict_t *options)
+dht_reconfigure(xlator_t *this, dict_t *options)
 {
-        dht_conf_t      *conf = NULL;
-        char            *temp_str = NULL;
-        gf_boolean_t     search_unhashed;
-        int              ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", options, out);
-
-        conf = this->private;
-        if (!conf)
-                return 0;
-
-        if (dict_get_str (options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean*/
-                if (strcasecmp (temp_str, "auto")) {
-                        if (!gf_string2boolean (temp_str, &search_unhashed)) {
-                                gf_msg_debug(this->name, 0, "Reconfigure: "
-                                             "lookup-unhashed reconfigured(%s)",
-                                             temp_str);
-                                conf->search_unhashed = search_unhashed;
-                        } else {
-                                gf_msg(this->name, GF_LOG_ERROR, 0,
-                                       DHT_MSG_INVALID_OPTION,
-                                       "Invalid option: Reconfigure: "
-                                       "lookup-unhashed should be boolean,"
-                                       " not (%s), defaulting to (%d)",
-                                       temp_str, conf->search_unhashed);
-                                ret = -1;
-                                goto out;
-                        }
-                } else {
-                        gf_msg_debug(this->name, 0, "Reconfigure:"
-                                     " lookup-unhashed reconfigured auto ");
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-                }
-        }
+    dht_conf_t *conf = NULL;
+    char *temp_str = NULL;
+    gf_boolean_t search_unhashed;
+    int ret = -1;
 
-        GF_OPTION_RECONF ("lookup-optimize", conf->lookup_optimize, options,
-                          bool, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", options, out);
 
-        GF_OPTION_RECONF ("min-free-disk", conf->min_free_disk, options,
-                          percent_or_size, out);
-        /* option can be any one of percent or bytes */
-        conf->disk_unit = 0;
-        if (conf->min_free_disk < 100.0)
-                conf->disk_unit = 'p';
+    conf = this->private;
+    if (!conf)
+        return 0;
 
-        GF_OPTION_RECONF ("min-free-inodes", conf->min_free_inodes, options,
-                          percent, out);
+    if (dict_get_str(options, "lookup-unhashed", &temp_str) == 0) {
+        /* If option is not "auto", other options _should_ be boolean*/
+        if (strcasecmp(temp_str, "auto")) {
+            if (!gf_string2boolean(temp_str, &search_unhashed)) {
+                gf_msg_debug(this->name, 0,
+                             "Reconfigure: "
+                             "lookup-unhashed reconfigured(%s)",
+                             temp_str);
+                conf->search_unhashed = search_unhashed;
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                       "Invalid option: Reconfigure: "
+                       "lookup-unhashed should be boolean,"
+                       " not (%s), defaulting to (%d)",
+                       temp_str, conf->search_unhashed);
+                ret = -1;
+                goto out;
+            }
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "Reconfigure:"
+                         " lookup-unhashed reconfigured auto ");
+            conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+        }
+    }
 
-        GF_OPTION_RECONF ("directory-layout-spread", conf->dir_spread_cnt,
-                          options, uint32, out);
+    GF_OPTION_RECONF("lookup-optimize", conf->lookup_optimize, options, bool,
+                     out);
 
-        GF_OPTION_RECONF ("readdir-optimize", conf->readdir_optimize, options,
-                          bool, out);
-        GF_OPTION_RECONF ("randomize-hash-range-by-gfid",
-                          conf->randomize_by_gfid,
-                          options, bool, out);
+    GF_OPTION_RECONF("min-free-disk", conf->min_free_disk, options,
+                     percent_or_size, out);
+    /* option can be any one of percent or bytes */
+    conf->disk_unit = 0;
+    if (conf->min_free_disk < 100.0)
+        conf->disk_unit = 'p';
 
-        GF_OPTION_RECONF ("lock-migration", conf->lock_migration_enabled,
-                          options, bool, out);
+    GF_OPTION_RECONF("min-free-inodes", conf->min_free_inodes, options, percent,
+                     out);
 
-        GF_OPTION_RECONF ("force-migration", conf->force_migration,
-                          options, bool, out);
+    GF_OPTION_RECONF("directory-layout-spread", conf->dir_spread_cnt, options,
+                     uint32, out);
 
+    GF_OPTION_RECONF("readdir-optimize", conf->readdir_optimize, options, bool,
+                     out);
+    GF_OPTION_RECONF("randomize-hash-range-by-gfid", conf->randomize_by_gfid,
+                     options, bool, out);
 
-        if (conf->defrag) {
-                if (dict_get_str (options, "rebal-throttle", &temp_str) == 0) {
-                        ret = dht_configure_throttle (this, conf, temp_str);
-                        if (ret == -1)
-                                goto out;
-                }
-        }
+    GF_OPTION_RECONF("lock-migration", conf->lock_migration_enabled, options,
+                     bool, out);
 
-        if (conf->defrag) {
-                conf->defrag->lock_migration_enabled =
-                                        conf->lock_migration_enabled;
-        }
+    GF_OPTION_RECONF("force-migration", conf->force_migration, options, bool,
+                     out);
 
-        if (conf->defrag) {
-                GF_OPTION_RECONF ("rebalance-stats", conf->defrag->stats,
-                                  options, bool, out);
+    if (conf->defrag) {
+        if (dict_get_str(options, "rebal-throttle", &temp_str) == 0) {
+            ret = dht_configure_throttle(this, conf, temp_str);
+            if (ret == -1)
+                goto out;
         }
+    }
+
+    if (conf->defrag) {
+        conf->defrag->lock_migration_enabled = conf->lock_migration_enabled;
+    }
+
+    if (conf->defrag) {
+        GF_OPTION_RECONF("rebalance-stats", conf->defrag->stats, options, bool,
+                         out);
+    }
+
+    if (dict_get_str(options, "decommissioned-bricks", &temp_str) == 0) {
+        ret = dht_parse_decommissioned_bricks(this, conf, temp_str);
+        if (ret == -1)
+            goto out;
+    } else {
+        dht_decommissioned_remove(this, conf);
+    }
+
+    dht_init_regex(this, options, "rsync-hash-regex", &conf->rsync_regex,
+                   &conf->rsync_regex_valid, conf);
+    dht_init_regex(this, options, "extra-hash-regex", &conf->extra_regex,
+                   &conf->extra_regex_valid, conf);
+
+    GF_OPTION_RECONF("weighted-rebalance", conf->do_weighting, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("use-readdirp", conf->use_readdirp, options, bool, out);
+    ret = 0;
+out:
+    return ret;
+}
 
-        if (dict_get_str (options, "decommissioned-bricks", &temp_str) == 0) {
-                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
-                if (ret == -1)
-                        goto out;
-        } else {
-                ret = dht_decommissioned_remove (this, conf);
-                if (ret == -1)
-                        goto out;
+static int
+gf_defrag_pattern_list_fill(xlator_t *this, gf_defrag_info_t *defrag,
+                            char *data)
+{
+    int ret = -1;
+    char *tmp_str = NULL;
+    char *tmp_str1 = NULL;
+    char *dup_str = NULL;
+    char *num = NULL;
+    char *pattern_str = NULL;
+    char *pattern = NULL;
+    gf_defrag_pattern_list_t *temp_list = NULL;
+    gf_defrag_pattern_list_t *pattern_list = NULL;
+
+    if (!this || !defrag || !data)
+        goto out;
+
+    /* Get the pattern for pattern list. "pattern:<optional-size>"
+     * eg: *avi, *pdf:10MB, *:1TB
+     */
+    pattern_str = strtok_r(data, ",", &tmp_str);
+    while (pattern_str) {
+        dup_str = gf_strdup(pattern_str);
+        if (!dup_str)
+            goto out;
+        pattern_list = GF_CALLOC(1, sizeof(gf_defrag_pattern_list_t), 1);
+        if (!pattern_list) {
+            goto out;
         }
+        pattern = strtok_r(dup_str, ":", &tmp_str1);
+        num = strtok_r(NULL, ":", &tmp_str1);
+        if (!pattern)
+            goto out;
+        if (!num) {
+            if (gf_string2bytesize_uint64(pattern, &pattern_list->size) == 0) {
+                pattern = "*";
+            }
+        } else if (gf_string2bytesize_uint64(num, &pattern_list->size) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                   "Invalid option. Defrag pattern:"
+                   " Invalid number format \"%s\"",
+                   num);
+            goto out;
+        }
+        memcpy(pattern_list->path_pattern, pattern, strlen(dup_str));
 
-        dht_init_regex (this, options, "rsync-hash-regex",
-                        &conf->rsync_regex, &conf->rsync_regex_valid, conf);
-        dht_init_regex (this, options, "extra-hash-regex",
-                        &conf->extra_regex, &conf->extra_regex_valid, conf);
+        if (!defrag->defrag_pattern)
+            temp_list = NULL;
+        else
+            temp_list = defrag->defrag_pattern;
 
-        GF_OPTION_RECONF ("weighted-rebalance", conf->do_weighting, options,
-                          bool, out);
+        pattern_list->next = temp_list;
 
-        GF_OPTION_RECONF ("use-readdirp", conf->use_readdirp, options,
-                          bool, out);
-        ret = 0;
-out:
-        return ret;
-}
+        defrag->defrag_pattern = pattern_list;
+        pattern_list = NULL;
 
-static int
-gf_defrag_pattern_list_fill (xlator_t *this, gf_defrag_info_t *defrag, char *data)
-{
-        int                    ret = -1;
-        char                  *tmp_str = NULL;
-        char                  *tmp_str1 = NULL;
-        char                  *dup_str = NULL;
-        char                  *num = NULL;
-        char                  *pattern_str = NULL;
-        char                  *pattern = NULL;
-        gf_defrag_pattern_list_t *temp_list = NULL;
-        gf_defrag_pattern_list_t *pattern_list = NULL;
-
-        if (!this || !defrag || !data)
-                goto out;
+        GF_FREE(dup_str);
+        dup_str = NULL;
 
-        /* Get the pattern for pattern list. "pattern:<optional-size>"
-         * eg: *avi, *pdf:10MB, *:1TB
-         */
-        pattern_str = strtok_r (data, ",", &tmp_str);
-        while (pattern_str) {
-                dup_str = gf_strdup (pattern_str);
-                pattern_list = GF_CALLOC (1, sizeof (gf_defrag_pattern_list_t),
-                                        1);
-                if (!pattern_list) {
-                        goto out;
-                }
-                pattern = strtok_r (dup_str, ":", &tmp_str1);
-                num = strtok_r (NULL, ":", &tmp_str1);
-                if (!pattern)
-                        goto out;
-                if (!num) {
-                        if (gf_string2bytesize_uint64(pattern, &pattern_list->size)
-                             == 0) {
-                                pattern = "*";
-                        }
-                } else if (gf_string2bytesize_uint64 (num, &pattern_list->size) != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_INVALID_OPTION,
-                                "Invalid option. Defrag pattern:"
-                                " Invalid number format \"%s\"", num);
-                        goto out;
-                }
-                memcpy (pattern_list->path_pattern, pattern, strlen (dup_str));
-
-                if (!defrag->defrag_pattern)
-                        temp_list = NULL;
-                else
-                        temp_list = defrag->defrag_pattern;
-
-                pattern_list->next = temp_list;
-
-                defrag->defrag_pattern = pattern_list;
-                pattern_list = NULL;
-
-                GF_FREE (dup_str);
-                dup_str = NULL;
-
-                pattern_str = strtok_r (NULL, ",", &tmp_str);
-        }
+        pattern_str = strtok_r(NULL, ",", &tmp_str);
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret)
-                GF_FREE (pattern_list);
-        GF_FREE (dup_str);
+    if (ret)
+        GF_FREE(pattern_list);
+    GF_FREE(dup_str);
 
-        return ret;
+    return ret;
 }
 
-
-
-int
-dht_init_methods (xlator_t *this)
+static int
+dht_init_methods(xlator_t *this)
 {
-        int ret                  = -1;
-        dht_conf_t      *conf    = NULL;
-        dht_methods_t   *methods = NULL;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
+    GF_VALIDATE_OR_GOTO("dht", this, err);
 
-        conf = this->private;
-        methods = &(conf->methods);
+    conf = this->private;
+    methods = &(conf->methods);
 
-        methods->migration_get_dst_subvol = dht_migration_get_dst_subvol;
-        methods->migration_needed = dht_migration_needed;
-        methods->migration_other  = NULL;
-        methods->layout_search    = dht_layout_search;
+    methods->migration_get_dst_subvol = dht_migration_get_dst_subvol;
+    methods->migration_other = NULL;
+    methods->layout_search = dht_layout_search;
 
-        ret = 0;
+    ret = 0;
 err:
-        return ret;
+    return ret;
 }
 
 int
-dht_init (xlator_t *this)
+dht_init(xlator_t *this)
 {
-        dht_conf_t                      *conf           = NULL;
-        char                            *temp_str       = NULL;
-        int                              ret            = -1;
-        int                              i              = 0;
-        gf_defrag_info_t                *defrag         = NULL;
-        int                              cmd            = 0;
-        char                            *node_uuid      = NULL;
-        uint32_t                         commit_hash    = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-
-        if (!this->children) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        DHT_MSG_INVALID_CONFIGURATION,
-                        "Distribute needs more than one subvolume");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_INVALID_CONFIGURATION,
-                        "dangling volume. check volfile");
-        }
-
-        conf = GF_CALLOC (1, sizeof (*conf), gf_dht_mt_dht_conf_t);
-        if (!conf) {
-                goto err;
-        }
-
-        LOCK_INIT (&conf->subvolume_lock);
-        LOCK_INIT (&conf->layout_lock);
-        LOCK_INIT (&conf->lock);
-        synclock_init (&conf->link_lock, SYNC_LOCK_DEFAULT);
-
-        /* We get the commit-hash to set only for rebalance process */
-        if (dict_get_uint32 (this->options,
-                             "commit-hash", &commit_hash) == 0) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_COMMIT_HASH_INFO, "%s using commit hash %u",
-                        __func__, commit_hash);
-                conf->vol_commit_hash = commit_hash;
-                conf->vch_forced = _gf_true;
-        }
-
-        ret = dict_get_int32 (this->options, "rebalance-cmd", &cmd);
-
-        if (cmd) {
-                defrag = GF_CALLOC (1, sizeof (gf_defrag_info_t),
-                                    gf_defrag_info_mt);
-
-                GF_VALIDATE_OR_GOTO (this->name, defrag, err);
-
-                LOCK_INIT (&defrag->lock);
-
-                defrag->is_exiting = 0;
-
-                conf->defrag = defrag;
-                defrag->this = this;
-
-                ret = dict_get_str (this->options, "node-uuid", &node_uuid);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_INVALID_CONFIGURATION,
-                                "Invalid volume configuration: "
-                                "node-uuid not specified");
-                        goto err;
-                }
+    dht_conf_t *conf = NULL;
+    char *temp_str = NULL;
+    int ret = -1;
+    int i = 0;
+    gf_defrag_info_t *defrag = NULL;
+    int cmd = 0;
+    char *node_uuid = NULL;
+    uint32_t commit_hash = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", this, err);
+
+    if (!this->children) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_INVALID_CONFIGURATION,
+               "Distribute needs more than one subvolume");
+        return -1;
+    }
 
-                if (gf_uuid_parse (node_uuid, defrag->node_uuid)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_INVALID_OPTION, "Invalid option:"
-                                " Cannot parse glusterd node uuid");
-                        goto err;
-                }
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_CONFIGURATION,
+               "dangling volume. check volfile");
+    }
 
-                defrag->cmd = cmd;
+    conf = GF_CALLOC(1, sizeof(*conf), gf_dht_mt_dht_conf_t);
+    if (!conf) {
+        goto err;
+    }
 
-                defrag->stats = _gf_false;
+    LOCK_INIT(&conf->subvolume_lock);
+    LOCK_INIT(&conf->layout_lock);
+    LOCK_INIT(&conf->lock);
+    synclock_init(&conf->link_lock, SYNC_LOCK_DEFAULT);
 
-                defrag->queue = NULL;
+    /* We get the commit-hash to set only for rebalance process */
+    if (dict_get_uint32(this->options, "commit-hash", &commit_hash) == 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_COMMIT_HASH_INFO,
+               "%s using commit hash %u", __func__, commit_hash);
+        conf->vol_commit_hash = commit_hash;
+        conf->vch_forced = _gf_true;
+    }
 
-                defrag->crawl_done = 0;
+    ret = dict_get_int32(this->options, "rebalance-cmd", &cmd);
 
-                defrag->global_error = 0;
+    if (cmd) {
+        defrag = GF_CALLOC(1, sizeof(gf_defrag_info_t), gf_defrag_info_mt);
 
-                defrag->q_entry_count = 0;
+        GF_VALIDATE_OR_GOTO(this->name, defrag, err);
 
-                defrag->wakeup_crawler = 0;
+        LOCK_INIT(&defrag->lock);
 
-                pthread_mutex_init (&defrag->dfq_mutex, 0);
-                pthread_cond_init  (&defrag->parallel_migration_cond, 0);
-                pthread_cond_init  (&defrag->rebalance_crawler_alarm, 0);
-                pthread_cond_init  (&defrag->df_wakeup_thread, 0);
+        defrag->is_exiting = 0;
 
-                pthread_mutex_init (&defrag->fc_mutex, 0);
-                pthread_cond_init  (&defrag->fc_wakeup_cond, 0);
+        conf->defrag = defrag;
+        defrag->this = this;
 
-                defrag->global_error = 0;
+        ret = dict_get_str(this->options, "node-uuid", &node_uuid);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_CONFIGURATION,
+                   "Invalid volume configuration: "
+                   "node-uuid not specified");
+            goto err;
         }
 
-        conf->use_fallocate = 1;
-
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto")) {
-                        gf_boolean_t    search_unhashed_bool;
-                        ret = gf_string2boolean (temp_str, &search_unhashed_bool);
-                        if (ret == -1) {
-                                goto err;
-                        }
-                        conf->search_unhashed = search_unhashed_bool
-                                              ? GF_DHT_LOOKUP_UNHASHED_ON
-                                              : GF_DHT_LOOKUP_UNHASHED_OFF;
-                }
-                else {
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-                }
+        if (gf_uuid_parse(node_uuid, defrag->node_uuid)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                   "Invalid option:"
+                   " Cannot parse glusterd node uuid");
+            goto err;
         }
 
-        GF_OPTION_INIT ("lookup-optimize", conf->lookup_optimize, bool, err);
+        defrag->cmd = cmd;
 
-        GF_OPTION_INIT ("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool,
-                        err);
+        defrag->stats = _gf_false;
 
-        GF_OPTION_INIT ("use-readdirp", conf->use_readdirp, bool, err);
+        defrag->queue = NULL;
 
-        GF_OPTION_INIT ("min-free-disk", conf->min_free_disk, percent_or_size,
-                        err);
+        defrag->crawl_done = 0;
 
-        GF_OPTION_INIT ("min-free-inodes", conf->min_free_inodes, percent,
-                        err);
+        defrag->global_error = 0;
 
-        conf->dir_spread_cnt = conf->subvolume_cnt;
-        GF_OPTION_INIT ("directory-layout-spread", conf->dir_spread_cnt,
-                        uint32, err);
+        defrag->q_entry_count = 0;
 
-        GF_OPTION_INIT ("assert-no-child-down", conf->assert_no_child_down,
-                        bool, err);
+        defrag->wakeup_crawler = 0;
 
-        GF_OPTION_INIT ("readdir-optimize", conf->readdir_optimize, bool, err);
+        pthread_mutex_init(&defrag->dfq_mutex, 0);
+        pthread_cond_init(&defrag->parallel_migration_cond, 0);
+        pthread_cond_init(&defrag->rebalance_crawler_alarm, 0);
+        pthread_cond_init(&defrag->df_wakeup_thread, 0);
 
+        pthread_mutex_init(&defrag->fc_mutex, 0);
+        pthread_cond_init(&defrag->fc_wakeup_cond, 0);
 
-        GF_OPTION_INIT ("lock-migration", conf->lock_migration_enabled,
-                         bool, err);
+        defrag->global_error = 0;
+    }
 
-        GF_OPTION_INIT ("force-migration", conf->force_migration,
-                        bool, err);
+    conf->use_fallocate = 1;
 
+    conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
+    if (dict_get_str(this->options, "lookup-unhashed", &temp_str) == 0) {
+        /* If option is not "auto", other options _should_ be boolean */
+        if (strcasecmp(temp_str, "auto")) {
+            gf_boolean_t search_unhashed_bool;
+            ret = gf_string2boolean(temp_str, &search_unhashed_bool);
+            if (ret == -1) {
+                goto err;
+            }
+            conf->search_unhashed = search_unhashed_bool
+                                        ? GF_DHT_LOOKUP_UNHASHED_ON
+                                        : GF_DHT_LOOKUP_UNHASHED_OFF;
+        } else {
+            conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+        }
+    }
 
-        if (defrag) {
-              defrag->lock_migration_enabled = conf->lock_migration_enabled;
+    GF_OPTION_INIT("lookup-optimize", conf->lookup_optimize, bool, err);
 
-              GF_OPTION_INIT ("rebalance-stats", defrag->stats, bool, err);
-                if (dict_get_str (this->options, "rebalance-filter", &temp_str)
-                    == 0) {
-                        if (gf_defrag_pattern_list_fill (this, defrag, temp_str)
-                            == -1) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_INVALID_OPTION,
-                                        "Invalid option:"
-                                        " Cannot parse rebalance-filter (%s)",
-                                        temp_str);
+    GF_OPTION_INIT("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool, err);
 
-                                goto err;
-                        }
-                }
-        }
+    GF_OPTION_INIT("use-readdirp", conf->use_readdirp, bool, err);
 
-        /* option can be any one of percent or bytes */
-        conf->disk_unit = 0;
-        if (conf->min_free_disk < 100)
-                conf->disk_unit = 'p';
+    GF_OPTION_INIT("min-free-disk", conf->min_free_disk, percent_or_size, err);
 
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
+    GF_OPTION_INIT("min-free-inodes", conf->min_free_inodes, percent, err);
 
-        if (cmd) {
-                ret = dht_init_local_subvolumes (this, conf);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_INIT_LOCAL_SUBVOL_FAILED,
-                                "dht_init_local_subvolumes failed");
-                        goto err;
-                }
-        }
+    conf->dir_spread_cnt = conf->subvolume_cnt;
+    GF_OPTION_INIT("directory-layout-spread", conf->dir_spread_cnt, uint32,
+                   err);
 
-        if (dict_get_str (this->options, "decommissioned-bricks", &temp_str) == 0) {
-                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
-                if (ret == -1)
-                        goto err;
-        }
+    GF_OPTION_INIT("assert-no-child-down", conf->assert_no_child_down, bool,
+                   err);
 
-        dht_init_regex (this, this->options, "rsync-hash-regex",
-                        &conf->rsync_regex, &conf->rsync_regex_valid, conf);
-        dht_init_regex (this, this->options, "extra-hash-regex",
-                        &conf->extra_regex, &conf->extra_regex_valid, conf);
+    GF_OPTION_INIT("readdir-optimize", conf->readdir_optimize, bool, err);
 
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
+    GF_OPTION_INIT("lock-migration", conf->lock_migration_enabled, bool, err);
+
+    GF_OPTION_INIT("force-migration", conf->force_migration, bool, err);
 
+    if (defrag) {
+        defrag->lock_migration_enabled = conf->lock_migration_enabled;
 
-        conf->gen = 1;
+        GF_OPTION_INIT("rebalance-stats", defrag->stats, bool, err);
+        if (dict_get_str(this->options, "rebalance-filter", &temp_str) == 0) {
+            if (gf_defrag_pattern_list_fill(this, defrag, temp_str) == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                       "Invalid option:"
+                       " Cannot parse rebalance-filter (%s)",
+                       temp_str);
 
-        this->local_pool = mem_pool_new (dht_local_t, 512);
-        if (!this->local_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        DHT_MSG_NO_MEMORY,
-                        " DHT initialisation failed. "
-                        "failed to create local_t's memory pool");
                 goto err;
+            }
         }
-
-        GF_OPTION_INIT ("randomize-hash-range-by-gfid",
-                        conf->randomize_by_gfid, bool, err);
-
-        if (defrag) {
-                GF_OPTION_INIT ("rebal-throttle", temp_str, str, err);
-                if (temp_str) {
-                        ret = dht_configure_throttle (this, conf, temp_str);
-                        if (ret == -1)
-                                goto err;
-                }
+    }
+
+    /* option can be any one of percent or bytes */
+    conf->disk_unit = 0;
+    if (conf->min_free_disk < 100)
+        conf->disk_unit = 'p';
+
+    ret = dht_init_subvolumes(this, conf);
+    if (ret == -1) {
+        goto err;
+    }
+
+    if (cmd) {
+        ret = dht_init_local_subvolumes(this, conf);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_INIT_LOCAL_SUBVOL_FAILED,
+                   "dht_init_local_subvolumes failed");
+            goto err;
         }
-
-        GF_OPTION_INIT ("xattr-name", conf->xattr_name, str, err);
-        gf_asprintf (&conf->mds_xattr_key, "%s."DHT_MDS_STR, conf->xattr_name);
-        gf_asprintf (&conf->link_xattr_name, "%s."DHT_LINKFILE_STR,
-                     conf->xattr_name);
-        gf_asprintf (&conf->commithash_xattr_name, "%s."DHT_COMMITHASH_STR,
-                     conf->xattr_name);
-        gf_asprintf (&conf->wild_xattr_name, "%s*", conf->xattr_name);
-        if (!conf->link_xattr_name || !conf->wild_xattr_name) {
+    }
+
+    if (dict_get_str(this->options, "decommissioned-bricks", &temp_str) == 0) {
+        ret = dht_parse_decommissioned_bricks(this, conf, temp_str);
+        if (ret == -1)
+            goto err;
+    }
+
+    dht_init_regex(this, this->options, "rsync-hash-regex", &conf->rsync_regex,
+                   &conf->rsync_regex_valid, conf);
+    dht_init_regex(this, this->options, "extra-hash-regex", &conf->extra_regex,
+                   &conf->extra_regex_valid, conf);
+
+    ret = dht_layouts_init(this, conf);
+    if (ret == -1) {
+        goto err;
+    }
+
+    conf->gen = 1;
+
+    this->local_pool = mem_pool_new(dht_local_t, 512);
+    if (!this->local_pool) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               " DHT initialisation failed. "
+               "failed to create local_t's memory pool");
+        goto err;
+    }
+
+    GF_OPTION_INIT("randomize-hash-range-by-gfid", conf->randomize_by_gfid,
+                   bool, err);
+
+    if (defrag) {
+        GF_OPTION_INIT("rebal-throttle", temp_str, str, err);
+        if (temp_str) {
+            ret = dht_configure_throttle(this, conf, temp_str);
+            if (ret == -1)
                 goto err;
         }
+    }
 
-        GF_OPTION_INIT ("weighted-rebalance", conf->do_weighting, bool, err);
+    GF_OPTION_INIT("xattr-name", conf->xattr_name, str, err);
+    gf_asprintf(&conf->mds_xattr_key, "%s." DHT_MDS_STR, conf->xattr_name);
+    gf_asprintf(&conf->link_xattr_name, "%s." DHT_LINKFILE_STR,
+                conf->xattr_name);
+    gf_asprintf(&conf->commithash_xattr_name, "%s." DHT_COMMITHASH_STR,
+                conf->xattr_name);
+    gf_asprintf(&conf->wild_xattr_name, "%s*", conf->xattr_name);
+    if (!conf->link_xattr_name || !conf->wild_xattr_name) {
+        goto err;
+    }
 
-        conf->lock_pool = mem_pool_new (dht_lock_t, 512);
-        if (!conf->lock_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_INIT_FAILED,
-                        "failed to create lock mem_pool, failing "
-                        "initialization");
-                goto err;
-        }
+    GF_OPTION_INIT("weighted-rebalance", conf->do_weighting, bool, err);
 
-        this->private = conf;
+    conf->lock_pool = mem_pool_new(dht_lock_t, 512);
+    if (!conf->lock_pool) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INIT_FAILED,
+               "failed to create lock mem_pool, failing "
+               "initialization");
+        goto err;
+    }
 
-        if (dht_set_subvol_range(this))
-                goto err;
+    this->private = conf;
 
-        if (dht_init_methods (this))
-                goto err;
+    if (dht_set_subvol_range(this))
+        goto err;
 
-        return 0;
+    if (dht_init_methods(this))
+        goto err;
+
+    return 0;
 
 err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
+    if (conf) {
+        if (conf->file_layouts) {
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                GF_FREE(conf->file_layouts[i]);
+            }
+            GF_FREE(conf->file_layouts);
+        }
 
-                GF_FREE (conf->subvolumes);
+        GF_FREE(conf->subvolumes);
 
-                GF_FREE (conf->subvolume_status);
+        GF_FREE(conf->subvolume_status);
 
-                GF_FREE (conf->du_stats);
+        GF_FREE(conf->du_stats);
 
-                GF_FREE (conf->defrag);
+        GF_FREE(conf->defrag);
 
-                GF_FREE (conf->xattr_name);
-                GF_FREE (conf->link_xattr_name);
-                GF_FREE (conf->wild_xattr_name);
-                GF_FREE (conf->mds_xattr_key);
+        GF_FREE(conf->xattr_name);
+        GF_FREE(conf->link_xattr_name);
+        GF_FREE(conf->wild_xattr_name);
+        GF_FREE(conf->mds_xattr_key);
 
-                if (conf->lock_pool)
-                        mem_pool_destroy (conf->lock_pool);
+        if (conf->lock_pool)
+            mem_pool_destroy(conf->lock_pool);
 
-                GF_FREE (conf);
-        }
+        GF_FREE(conf);
+    }
 
-        return -1;
+    return -1;
 }
 
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "on",
-          .description = "This option if set to ON, does a lookup through "
-          "all the sub-volumes, in case a lookup didn't return any result "
-          "from the hash subvolume. If set to OFF, it does not do a lookup "
-          "on the remaining subvolumes.",
-          .op_version = {1},
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
-          .level = OPT_STATUS_BASIC,
-        },
-        { .key = {"lookup-optimize"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "This option if set to ON enables the optimization "
-          "of -ve lookups, by not doing a lookup on non-hashed subvolumes for "
-          "files, in case the hashed subvolume does not return any result. "
-          "This option disregards the lookup-unhashed setting, when enabled.",
-          .op_version = {GD_OP_VERSION_3_7_2},
-          .level = OPT_STATUS_ADVANCED,
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
-        },
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-          .default_value = "10%",
-          .description = "Percentage/Size of disk space, after which the "
-          "process starts balancing out the cluster, and logs will appear "
-          "in log files",
-          .op_version = {1},
-          .level = OPT_STATUS_BASIC,
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
-        },
-        { .key  = {"min-free-inodes"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .default_value = "5%",
-          .description = "after system has only N% of inodes, warnings "
-          "starts to appear in log files",
-          .op_version = {1},
-          .level = OPT_STATUS_BASIC,
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
-        },
-        { .key = {"unhashed-sticky-bit"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key = {"use-readdirp"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "This option if set to ON, forces the use of "
-          "readdirp, and hence also displays the stats of the files.",
-          .level = OPT_STATUS_ADVANCED,
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
-        },
-        { .key = {"assert-no-child-down"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option if set to ON, in the event of "
-          "CHILD_DOWN, will call exit."
-        },
-        { .key  = {"directory-layout-spread"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 1,
-          .validate = GF_OPT_VALIDATE_MIN,
-          .description = "Specifies the directory layout spread. Takes number "
-                         "of subvolumes as default value.",
-
-          .op_version = {2},
-        },
-        { .key  = {"decommissioned-bricks"},
-          .type = GF_OPTION_TYPE_ANY,
-          .description = "This option if set to ON, decommissions "
-          "the brick, so that no new data is allowed to be created "
-          "on that brick.",
-          .level = OPT_STATUS_ADVANCED,
-        },
-        { .key  = {"rebalance-cmd"},
-          .type = GF_OPTION_TYPE_INT,
-        },
-        { .key = {"commit-hash"},
-          .type = GF_OPTION_TYPE_INT,
-        },
-        { .key = {"node-uuid"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-        { .key = {"rebalance-stats"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option if set to ON displays and logs the "
-          " time taken for migration of each file, during the rebalance "
-          "process. If set to OFF, the rebalance logs will only display the "
-          "time spent in each directory.",
-          .op_version = {2},
-          .level = OPT_STATUS_BASIC,
-        },
-        { .key = {"readdir-optimize"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option if set to ON enables the optimization "
-          "that allows DHT to requests non-first subvolumes to filter out "
-          "directory entries.",
-          .op_version = {1},
-          .level = OPT_STATUS_ADVANCED,
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
-        },
-        { .key = {"rsync-hash-regex"},
-          .type = GF_OPTION_TYPE_STR,
-          /* Setting a default here doesn't work.  See dht_init_regex. */
-          .description = "Regular expression for stripping temporary-file "
-          "suffix and prefix used by rsync, to prevent relocation when the "
-          "file is renamed.",
-          .op_version = {3},
-          .level = OPT_STATUS_BASIC,
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
-        },
-        { .key = {"extra-hash-regex"},
-          .type = GF_OPTION_TYPE_STR,
-          /* Setting a default here doesn't work.  See dht_init_regex. */
-          .description = "Regular expression for stripping temporary-file "
-          "suffix and prefix used by an application, to prevent relocation when "
-          "the file is renamed.",
-          .op_version = {3},
-          .level = OPT_STATUS_BASIC,
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
-        },
-        { .key = {"rebalance-filter"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-
-        { .key = {"xattr-name"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "trusted.glusterfs.dht",
-          .description = "Base for extended attributes used by this "
-          "translator instance, to avoid conflicts with others above or "
-          "below it.",
-          .op_version = {3},
-        },
-
-        { .key = {"weighted-rebalance"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "When enabled, files will be allocated to bricks "
-          "with a probability proportional to their size.  Otherwise, all "
-          "bricks will have the same probability (legacy behavior).",
-          .op_version  = {GD_OP_VERSION_3_6_0},
-          .level = OPT_STATUS_BASIC,
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
-        },
-
-        /* NUFA option */
-        { .key  = {"local-volume-name"},
-          .type = GF_OPTION_TYPE_XLATOR
-        },
-
-        /* tier options */
-        { .key  = {"tier-pause"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-
-        { .key  = {"tier-promote-frequency"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "120",
-        },
-
-        { .key  = {"tier-demote-frequency"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "3600",
-        },
-
-        { .key  = {"write-freq-threshold"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "0",
-        },
-
-        { .key  = {"read-freq-threshold"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "0",
-        },
-        { .key         = {"watermark-hi"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .default_value = "90",
-        },
-        { .key         = {"watermark-low"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .default_value = "75",
-        },
-        { .key         = {"tier-mode"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "test",
-        },
-        { .key         = {"tier-compact"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key         = {"tier-hot-compact-frequency"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "604800",
-          .description = "Frequency to compact DBs on hot tier in system"
-        },
-        { .key         = {"tier-cold-compact-frequency"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "604800",
-          .description = "Frequency to compact DBs on cold tier in system"
-        },
-        { .key         = {"tier-max-mb"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "4000",
-        },
-        { .key         = {"tier-max-promote-file-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "0",
-        },
-        { .key         = {"tier-max-files"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "10000",
-        },
-        { .key         = {"tier-query-limit"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "100",
-        },
-        /* switch option */
-        { .key  = {"pattern.switch.case"},
-          .type = GF_OPTION_TYPE_ANY
-        },
-
-        { .key =  {"randomize-hash-range-by-gfid"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Use gfid of directory to determine the subvolume "
-          "from which hash ranges are allocated starting with 0. "
-          "Note that we still use a directory/file's name to determine the "
-          "subvolume to which it hashes",
-          .op_version  = {GD_OP_VERSION_3_6_0},
-        },
-
-        { .key =  {"rebal-throttle"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "normal",
-          .description = " Sets the maximum number of parallel file migrations "
-                         "allowed on a node during the rebalance operation. The"
-                         " default value is normal and allows a max of "
-                         "[($(processing units) - 4) / 2), 2]  files to be "
-                         "migrated at a time. Lazy will allow only one file to "
-                         "be migrated at a time and aggressive will allow "
-                         "max of [($(processing units) - 4) / 2), 4]",
-          .op_version  = {GD_OP_VERSION_3_7_0},
-          .level = OPT_STATUS_BASIC,
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
-
-        },
-
-        { .key =  {"lock-migration"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = " If enabled this feature will migrate the posix locks"
-                         " associated with a file during rebalance",
-          .op_version  = {GD_OP_VERSION_3_8_0},
-          .level = OPT_STATUS_ADVANCED,
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
-        },
-
-        { .key =  {"force-migration"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "If disabled, rebalance will not migrate files that "
-                         "are being written to by an application",
-          .op_version  = {GD_OP_VERSION_4_0_0},
-          .level = OPT_STATUS_ADVANCED,
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
-        },
-
-        { .key  = {NULL} },
+struct volume_options dht_options[] = {
+    {
+        .key = {"lookup-unhashed"},
+        .value = {"auto", "yes", "no", "enable", "disable", "1", "0", "on",
+                  "off"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "on",
+        .description =
+            "This option if set to ON, does a lookup through "
+            "all the sub-volumes, in case a lookup didn't return any result "
+            "from the hash subvolume. If set to OFF, it does not do a lookup "
+            "on the remaining subvolumes.",
+        .op_version = {1},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+        .level = OPT_STATUS_BASIC,
+    },
+    {.key = {"lookup-optimize"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description =
+         "This option if set to ON enables the optimization "
+         "of -ve lookups, by not doing a lookup on non-hashed subvolumes for "
+         "files, in case the hashed subvolume does not return any result. "
+         "This option disregards the lookup-unhashed setting, when enabled.",
+     .op_version = {GD_OP_VERSION_3_7_2},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"min-free-disk"},
+     .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
+     .default_value = "10%",
+     .description =
+         "Percentage/Size of disk space, after which the "
+         "process starts balancing out the cluster, and logs will appear "
+         "in log files",
+     .op_version = {1},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"min-free-inodes"},
+     .type = GF_OPTION_TYPE_PERCENT,
+     .default_value = "5%",
+     .description = "after system has only N% of inodes, warnings "
+                    "starts to appear in log files",
+     .op_version = {1},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {
+        .key = {"unhashed-sticky-bit"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+    },
+    {.key = {"use-readdirp"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "This option if set to ON, forces the use of "
+                    "readdirp, and hence also displays the stats of the files.",
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"assert-no-child-down"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option if set to ON, in the event of "
+                    "CHILD_DOWN, will call exit."},
+    {
+        .key = {"directory-layout-spread"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 1,
+        .validate = GF_OPT_VALIDATE_MIN,
+        .description = "Specifies the directory layout spread. Takes number "
+                       "of subvolumes as default value.",
+
+        .op_version = {2},
+    },
+    {
+        .key = {"decommissioned-bricks"},
+        .type = GF_OPTION_TYPE_ANY,
+        .description =
+            "This option if set to ON, decommissions "
+            "the brick, so that no new data is allowed to be created "
+            "on that brick.",
+        .level = OPT_STATUS_ADVANCED,
+    },
+    {
+        .key = {"rebalance-cmd"},
+        .type = GF_OPTION_TYPE_INT,
+    },
+    {
+        .key = {"commit-hash"},
+        .type = GF_OPTION_TYPE_INT,
+    },
+    {
+        .key = {"node-uuid"},
+        .type = GF_OPTION_TYPE_STR,
+    },
+    {
+        .key = {"rebalance-stats"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description =
+            "This option if set to ON displays and logs the "
+            " time taken for migration of each file, during the rebalance "
+            "process. If set to OFF, the rebalance logs will only display the "
+            "time spent in each directory.",
+        .op_version = {2},
+        .level = OPT_STATUS_BASIC,
+    },
+    {.key = {"readdir-optimize"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description =
+         "This option if set to ON enables the optimization "
+         "that allows DHT to requests non-first subvolumes to filter out "
+         "directory entries.",
+     .op_version = {1},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"rsync-hash-regex"},
+     .type = GF_OPTION_TYPE_STR,
+     /* Setting a default here doesn't work.  See dht_init_regex. */
+     .description =
+         "Regular expression for stripping temporary-file "
+         "suffix and prefix used by rsync, to prevent relocation when the "
+         "file is renamed.",
+     .op_version = {3},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"extra-hash-regex"},
+     .type = GF_OPTION_TYPE_STR,
+     /* Setting a default here doesn't work.  See dht_init_regex. */
+     .description =
+         "Regular expression for stripping temporary-file "
+         "suffix and prefix used by an application, to prevent relocation when "
+         "the file is renamed.",
+     .op_version = {3},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {
+        .key = {"rebalance-filter"},
+        .type = GF_OPTION_TYPE_STR,
+    },
+
+    {
+        .key = {"xattr-name"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "trusted.glusterfs.dht",
+        .description =
+            "Base for extended attributes used by this "
+            "translator instance, to avoid conflicts with others above or "
+            "below it.",
+        .op_version = {3},
+    },
+
+    {.key = {"weighted-rebalance"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description =
+         "When enabled, files will be allocated to bricks "
+         "with a probability proportional to their size.  Otherwise, all "
+         "bricks will have the same probability (legacy behavior).",
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+
+    /* NUFA option */
+    {.key = {"local-volume-name"}, .type = GF_OPTION_TYPE_XLATOR},
+
+    /* switch option */
+    {.key = {"pattern.switch.case"}, .type = GF_OPTION_TYPE_ANY},
+
+    {
+        .key = {"randomize-hash-range-by-gfid"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description =
+            "Use gfid of directory to determine the subvolume "
+            "from which hash ranges are allocated starting with 0. "
+            "Note that we still use a directory/file's name to determine the "
+            "subvolume to which it hashes",
+        .op_version = {GD_OP_VERSION_3_6_0},
+    },
+
+    {.key = {"rebal-throttle"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "normal",
+     .description = " Sets the maximum number of parallel file migrations "
+                    "allowed on a node during the rebalance operation. The"
+                    " default value is normal and allows a max of "
+                    "[($(processing units) - 4) / 2), 2]  files to be "
+                    "migrated at a time. Lazy will allow only one file to "
+                    "be migrated at a time and aggressive will allow "
+                    "max of [($(processing units) - 4) / 2), 4]",
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
+
+    },
+
+    {.key = {"lock-migration"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = " If enabled this feature will migrate the posix locks"
+                    " associated with a file during rebalance",
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+
+    {.key = {"force-migration"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "If disabled, rebalance will not migrate files that "
+                    "are being written to by an application",
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+
+    {.key = {NULL}},
 };
+
+#define NUM_DHT_OPTIONS (sizeof(dht_options) / sizeof(dht_options[0]))
+
+extern struct volume_options options[NUM_DHT_OPTIONS]
+    __attribute__((alias("dht_options")));
diff --git a/xlators/cluster/dht/src/dht.c b/xlators/cluster/dht/src/dht.c
index 90962636d18..53de8292704 100644
--- a/xlators/cluster/dht/src/dht.c
+++ b/xlators/cluster/dht/src/dht.c
@@ -8,78 +8,116 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include "statedump.h"
 #include "dht-common.h"
 
-class_methods_t class_methods = {
-        .init           = dht_init,
-        .fini           = dht_fini,
-        .reconfigure    = dht_reconfigure,
-        .notify         = dht_notify
+struct xlator_fops dht_pt_fops = {
+    /* we need to keep mkdir to make sure we
+       have layout on new directory */
+    .mkdir = dht_pt_mkdir,
+    .getxattr = dht_pt_getxattr,
+    .fgetxattr = dht_pt_fgetxattr,
+
+    /* required to trace fop properly in changelog */
+    .rename = dht_pt_rename,
+
+    /* FIXME: commenting the '.lookup()' below made some of
+       the failing tests to pass. I would remove the below
+       line, but keeping it here as a reminder for people
+       to check for issues if they find concerns with DHT
+       pass-through logic  */
+    /*
+      .lookup = dht_lookup,
+      .readdir = dht_readdir,
+      .readdirp = dht_readdirp,
+    */
+    /* Keeping above as commented, mainly to support the
+       usecase of a gluster volume getting to 1x(anytype),
+       due to remove-brick (shrinking) exercise. In that case,
+       we would need above fops to be available, so we can
+       handle the case of dangling linkto files (if any) */
 };
 
 struct xlator_fops fops = {
-        .ipc         = dht_ipc,
-        .lookup      = dht_lookup,
-        .mknod       = dht_mknod,
-        .create      = dht_create,
+    .ipc = dht_ipc,
+    .lookup = dht_lookup,
+    .mknod = dht_mknod,
+    .create = dht_create,
 
-        .open        = dht_open,
-        .statfs      = dht_statfs,
-        .opendir     = dht_opendir,
-        .readdir     = dht_readdir,
-        .readdirp    = dht_readdirp,
-        .fsyncdir    = dht_fsyncdir,
-        .symlink     = dht_symlink,
-        .unlink      = dht_unlink,
-        .link        = dht_link,
-        .mkdir       = dht_mkdir,
-        .rmdir       = dht_rmdir,
-        .rename      = dht_rename,
-        .entrylk     = dht_entrylk,
-        .fentrylk    = dht_fentrylk,
+    .open = dht_open,
+    .statfs = dht_statfs,
+    .opendir = dht_opendir,
+    .readdir = dht_readdir,
+    .readdirp = dht_readdirp,
+    .fsyncdir = dht_fsyncdir,
+    .symlink = dht_symlink,
+    .unlink = dht_unlink,
+    .link = dht_link,
+    .mkdir = dht_mkdir,
+    .rmdir = dht_rmdir,
+    .rename = dht_rename,
+    .entrylk = dht_entrylk,
+    .fentrylk = dht_fentrylk,
 
-        /* Inode read operations */
-        .stat        = dht_stat,
-        .fstat       = dht_fstat,
-        .access      = dht_access,
-        .readlink    = dht_readlink,
-        .getxattr    = dht_getxattr,
-        .fgetxattr    = dht_fgetxattr,
-        .readv       = dht_readv,
-        .flush       = dht_flush,
-        .fsync       = dht_fsync,
-        .inodelk     = dht_inodelk,
-        .finodelk    = dht_finodelk,
-        .lk          = dht_lk,
-        .lease       = dht_lease,
+    /* Inode read operations */
+    .stat = dht_stat,
+    .fstat = dht_fstat,
+    .access = dht_access,
+    .readlink = dht_readlink,
+    .getxattr = dht_getxattr,
+    .fgetxattr = dht_fgetxattr,
+    .readv = dht_readv,
+    .flush = dht_flush,
+    .fsync = dht_fsync,
+    .inodelk = dht_inodelk,
+    .finodelk = dht_finodelk,
+    .lk = dht_lk,
+    .lease = dht_lease,
 
-        /* Inode write operations */
-        .fremovexattr = dht_fremovexattr,
-        .removexattr = dht_removexattr,
-        .setxattr    = dht_setxattr,
-        .fsetxattr   = dht_fsetxattr,
-        .truncate    = dht_truncate,
-        .ftruncate   = dht_ftruncate,
-        .writev      = dht_writev,
-        .xattrop     = dht_xattrop,
-        .fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
-        .fsetattr    = dht_fsetattr,
-	.fallocate   = dht_fallocate,
-	.discard     = dht_discard,
-        .zerofill    = dht_zerofill,
+    /* Inode write operations */
+    .fremovexattr = dht_fremovexattr,
+    .removexattr = dht_removexattr,
+    .setxattr = dht_setxattr,
+    .fsetxattr = dht_fsetxattr,
+    .truncate = dht_truncate,
+    .ftruncate = dht_ftruncate,
+    .writev = dht_writev,
+    .xattrop = dht_xattrop,
+    .fxattrop = dht_fxattrop,
+    .setattr = dht_setattr,
+    .fsetattr = dht_fsetattr,
+    .fallocate = dht_fallocate,
+    .discard = dht_discard,
+    .zerofill = dht_zerofill,
 };
 
 struct xlator_dumpops dumpops = {
-        .priv = dht_priv_dump,
-        .inodectx = dht_inodectx_dump,
+    .priv = dht_priv_dump,
+    .inodectx = dht_inodectx_dump,
 };
 
-
 struct xlator_cbks cbks = {
-        .release    = dht_release,
-//      .releasedir = dht_releasedir,
-        .forget     = dht_forget
+    .release = dht_release,
+    //      .releasedir = dht_releasedir,
+    .forget = dht_forget,
+};
+
+extern int32_t
+mem_acct_init(xlator_t *this);
+
+extern struct volume_options dht_options[];
+
+xlator_api_t xlator_api = {
+    .init = dht_init,
+    .fini = dht_fini,
+    .notify = dht_notify,
+    .reconfigure = dht_reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = dht_options,
+    .identifier = "distribute",
+    .pass_through_fops = &dht_pt_fops,
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/cluster/dht/src/dht.sym b/xlators/cluster/dht/src/dht.sym
deleted file mode 100644
index 780b5fc0387..00000000000
--- a/xlators/cluster/dht/src/dht.sym
+++ /dev/null
@@ -1,8 +0,0 @@
-fops
-cbks
-class_methods
-dht_methods
-options
-mem_acct_init
-reconfigure
-dumpops
diff --git a/xlators/cluster/dht/src/nufa.c b/xlators/cluster/dht/src/nufa.c
index afa446584ba..3648a564840 100644
--- a/xlators/cluster/dht/src/nufa.c
+++ b/xlators/cluster/dht/src/nufa.c
@@ -8,681 +8,650 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "dht-common.h"
 
 /* TODO: all 'TODO's in dht.c holds good */
 
-extern struct volume_options options[];
+extern struct volume_options dht_options[];
 
 int
-nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                       struct iatt *postparent)
+nufa_local_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *stbuf, dict_t *xattr,
+                      struct iatt *postparent)
 {
-        xlator_t     *subvol      = NULL;
-        char          is_linkfile = 0;
-        char          is_dir      = 0;
-        dht_conf_t   *conf        = NULL;
-        dht_local_t  *local       = NULL;
-        loc_t        *loc         = NULL;
-        int           i           = 0;
-        xlator_t     *prev        = NULL;
-        int           call_cnt    = 0;
-        int           ret         = 0;
-
-        conf  = this->private;
-
-        prev  = cookie;
-        local = frame->local;
-        loc   = &local->loc;
-
-        if (ENTRY_MISSING (op_ret, op_errno)) {
-                if (conf->search_unhashed) {
-                        local->op_errno = ENOENT;
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+    xlator_t *subvol = NULL;
+    char is_linkfile = 0;
+    char is_dir = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    int i = 0;
+    xlator_t *prev = NULL;
+    int call_cnt = 0;
+    int ret = 0;
+
+    conf = this->private;
+
+    prev = cookie;
+    local = frame->local;
+    loc = &local->loc;
+
+    if (ENTRY_MISSING(op_ret, op_errno)) {
+        if (conf->search_unhashed) {
+            local->op_errno = ENOENT;
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
         }
-
-        if (op_ret == -1)
-                goto out;
-
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                         conf->link_xattr_name);
-        is_dir      = check_is_dir (inode, stbuf, xattr);
-
-        if (!is_dir && !is_linkfile) {
-                /* non-directory and not a linkfile */
-                ret = dht_layout_preset (this, prev, inode);
-                if (ret < 0) {
-                        gf_msg_debug (this->name, 0,
-                                      "could not set pre-set layout for subvol"
-                                      " %s", prev->name);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                goto out;
+    }
+
+    if (op_ret == -1)
+        goto out;
+
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
+    is_dir = check_is_dir(inode, stbuf, xattr);
+
+    if (!is_dir && !is_linkfile) {
+        /* non-directory and not a linkfile */
+        ret = dht_layout_preset(this, prev, inode);
+        if (ret < 0) {
+            gf_msg_debug(this->name, 0,
+                         "could not set pre-set layout for subvol"
+                         " %s",
+                         prev->name);
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (is_dir) {
-                call_cnt        = conf->subvolume_cnt;
-                local->call_cnt = call_cnt;
-
-                local->inode = inode_ref (inode);
-                local->xattr = dict_ref (xattr);
-
-                local->op_ret = 0;
-                local->op_errno = 0;
-
-                local->layout = dht_layout_new (this, conf->subvolume_cnt);
-                if (!local->layout) {
-                        op_ret   = -1;
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND_COOKIE (frame, dht_lookup_dir_cbk,
-                                           conf->subvolumes[i],
-                                           conf->subvolumes[i],
-                                           conf->subvolumes[i]->fops->lookup,
-                                           &local->loc, local->xattr_req);
-                }
-        }
+        goto out;
+    }
 
-        if (is_linkfile) {
-                subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
+    if (is_dir) {
+        call_cnt = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
 
-                if (!subvol) {
-                        gf_msg_debug (this->name, 0,
-                                      "linkfile has no link subvolume. path=%s",
-                                      loc->path);
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+        local->inode = inode_ref(inode);
+        local->xattr = dict_ref(xattr);
 
-                STACK_WIND_COOKIE (frame, dht_lookup_linkfile_cbk, subvol,
-                                   subvol, subvol->fops->lookup,
-                                   &local->loc, local->xattr_req);
+        local->op_ret = 0;
+        local->op_errno = 0;
+
+        local->layout = dht_layout_new(this, conf->subvolume_cnt);
+        if (!local->layout) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto err;
         }
 
-        return 0;
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
+        }
+    }
 
-out:
-        if (!local->hashed_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              local->loc.path);
-                local->op_errno = ENOENT;
-                dht_lookup_everywhere (frame, this, loc);
-                return 0;
+    if (is_linkfile) {
+        subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
+
+        if (!subvol) {
+            gf_msg_debug(this->name, 0,
+                         "linkfile has no link subvolume. path=%s", loc->path);
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
         }
 
-        STACK_WIND_COOKIE (frame, dht_lookup_cbk, local->hashed_subvol,
-                           local->hashed_subvol,
-                           local->hashed_subvol->fops->lookup,
-                           &local->loc, local->xattr_req);
+        STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, local->xattr_req);
+    }
 
+    return 0;
+
+out:
+    if (!local->hashed_subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     local->loc.path);
+        local->op_errno = ENOENT;
+        dht_lookup_everywhere(frame, this, loc);
         return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, dht_lookup_cbk, local->hashed_subvol,
+                      local->hashed_subvol, local->hashed_subvol->fops->lookup,
+                      &local->loc, local->xattr_req);
+
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno,
-                          inode, stbuf, xattr, postparent);
-        return 0;
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     postparent);
+    return 0;
 }
 
 int
-nufa_lookup (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, dict_t *xattr_req)
+nufa_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret    = -1;
-        int           op_errno = -1;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_LOOKUP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (xattr_req) {
+        local->xattr_req = dict_ref(xattr_req);
+    } else {
+        local->xattr_req = dict_new();
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
+
+    local->hashed_subvol = hashed_subvol;
+
+    if (is_revalidate(loc)) {
+        layout = local->layout;
+        if (!layout) {
+            gf_msg_debug(this->name, 0,
+                         "revalidate lookup without cache. "
+                         "path=%s",
+                         loc->path);
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (xattr_req) {
-                local->xattr_req = dict_ref (xattr_req);
-        } else {
-                local->xattr_req = dict_new ();
+        if (layout->gen && (layout->gen < conf->gen)) {
+            gf_msg_debug(this->name, 0, "incomplete layout failure for path=%s",
+                         loc->path);
+            dht_layout_unref(this, local->layout);
+            goto do_fresh_lookup;
         }
 
-        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
-
-        local->hashed_subvol = hashed_subvol;
-
-        if (is_revalidate (loc)) {
-                layout = local->layout;
-                if (!layout) {
-                        gf_msg_debug (this->name, 0,
-                                      "revalidate lookup without cache. "
-                                      "path=%s", loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                if (layout->gen && (layout->gen < conf->gen)) {
-                        gf_msg_debug (this->name, 0,
-                                      "incomplete layout failure for path=%s",
-                                      loc->path);
-                        dht_layout_unref (this, local->layout);
-                        goto do_fresh_lookup;
-                }
-
-                local->inode = inode_ref (loc->inode);
-
-                local->call_cnt = layout->cnt;
-                call_cnt = local->call_cnt;
-
-                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
-                 *       revalidates directly go to the cached-subvolume.
-                 */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dict value.");
-                        op_errno = -1;
-                        goto err;
-                }
-
-                for (i = 0; i < layout->cnt; i++) {
-                        subvol = layout->list[i].xlator;
-
-                        STACK_WIND_COOKIE (frame, dht_revalidate_cbk, subvol,
-                                           subvol, subvol->fops->lookup,
-                                           loc, local->xattr_req);
-
-                        if (!--call_cnt)
-                                break;
-                }
-        } else {
-        do_fresh_lookup:
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dict value.");
-                        op_errno = -1;
-                        goto err;
-                }
-
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->link_xattr_name, 256);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dict value.");
-                        op_errno = -1;
-                        goto err;
-                }
-
-                /* Send it to only local volume */
-                STACK_WIND_COOKIE (frame, nufa_local_lookup_cbk,
-                                   ((xlator_t *)conf->private),
-                                   ((xlator_t *)conf->private),
-                                   ((xlator_t *)conf->private)->fops->lookup,
-                                   loc, local->xattr_req);
-        }
+        local->inode = inode_ref(loc->inode);
 
-        return 0;
+        local->call_cnt = layout->cnt;
+        call_cnt = local->call_cnt;
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
-        return 0;
-}
+        /* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
+         *       revalidates directly go to the cached-subvolume.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dict value.");
+            op_errno = -1;
+            goto err;
+        }
 
-int
-nufa_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int op_ret, int op_errno,
-                                 inode_t *inode, struct iatt *stbuf,
-                                 struct iatt *preparent,
-                                 struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
+        for (i = 0; i < layout->cnt; i++) {
+            subvol = layout->list[i].xlator;
 
-        local = frame->local;
+            STACK_WIND_COOKIE(frame, dht_revalidate_cbk, subvol, subvol,
+                              subvol->fops->lookup, loc, local->xattr_req);
 
-        if (op_ret == -1)
-                goto err;
+            if (!--call_cnt)
+                break;
+        }
+    } else {
+    do_fresh_lookup:
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dict value.");
+            op_errno = -1;
+            goto err;
+        }
+
+        ret = dict_set_uint32(local->xattr_req, conf->link_xattr_name, 256);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dict value.");
+            op_errno = -1;
+            goto err;
+        }
 
-        STACK_WIND_COOKIE (frame, dht_create_cbk, local->cached_subvol,
-                           local->cached_subvol, local->cached_subvol->fops->create,
-                           &local->loc, local->flags, local->mode, local->umask,
-                           local->fd, local->params);
+        /* Send it to only local volume */
+        STACK_WIND_COOKIE(
+            frame, nufa_local_lookup_cbk, ((xlator_t *)conf->private),
+            ((xlator_t *)conf->private),
+            ((xlator_t *)conf->private)->fops->lookup, loc, local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-nufa_create (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, int32_t flags, mode_t mode,
-             mode_t umask, fd_t *fd, dict_t *params)
+nufa_create_linkfile_create_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int op_ret, int op_errno,
+                                inode_t *inode, struct iatt *stbuf,
+                                struct iatt *preparent, struct iatt *postparent,
+                                dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
+    dht_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    local = frame->local;
 
-        conf  = this->private;
+    if (op_ret == -1)
+        goto err;
 
-        dht_get_du_info (frame, this, loc);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, local->cached_subvol,
+                      local->cached_subvol, local->cached_subvol->fops->create,
+                      &local->loc, local->flags, local->mode, local->umask,
+                      local->fd, local->params);
 
-        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    return 0;
 
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        avail_subvol = conf->private;
-        if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private,
-                                                        local);
-        }
+err:
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
+}
 
-        if (subvol != avail_subvol) {
-                /* create a link file instead of actual file */
-                local->params = dict_ref (params);
-                local->mode = mode;
-                local->flags = flags;
-                local->umask = umask;
-                local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame, nufa_create_linkfile_create_cbk,
-                                     this, avail_subvol, subvol, loc);
-                return 0;
-        }
+int
+nufa_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            mode_t mode, mode_t umask, fd_t *fd, dict_t *params)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_CREATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    avail_subvol = conf->private;
+    if (dht_is_subvol_filled(this, (xlator_t *)conf->private)) {
+        avail_subvol = dht_free_disk_available_subvol(
+            this, (xlator_t *)conf->private, local);
+    }
+
+    if (subvol != avail_subvol) {
+        /* create a link file instead of actual file */
+        local->params = dict_ref(params);
+        local->mode = mode;
+        local->flags = flags;
+        local->umask = umask;
+        local->cached_subvol = avail_subvol;
+        dht_linkfile_create(frame, nufa_create_linkfile_create_cbk, this,
+                            avail_subvol, subvol, loc);
+        return 0;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND_COOKIE (frame, dht_create_cbk, subvol,
-                           subvol, subvol->fops->create,
-                           loc, flags, mode, umask, fd, params);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                      subvol->fops->create, loc, flags, mode, umask, fd,
+                      params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-nufa_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, inode_t *inode,
-                         struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
+nufa_mknod_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
-        if (!local || !local->cached_subvol) {
-                op_errno = EINVAL;
-                op_ret = -1;
-                goto err;
-        }
+    local = frame->local;
+    if (!local || !local->cached_subvol) {
+        op_errno = EINVAL;
+        op_ret = -1;
+        goto err;
+    }
 
-        if (op_ret >= 0) {
-                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
-                            (void *)local->cached_subvol, local->cached_subvol,
-                            local->cached_subvol->fops->mknod,
-                            &local->loc, local->mode, local->rdev,
-                            local->umask, local->params);
+    if (op_ret >= 0) {
+        STACK_WIND_COOKIE(
+            frame, dht_newfile_cbk, (void *)local->cached_subvol,
+            local->cached_subvol, local->cached_subvol->fops->mknod,
+            &local->loc, local->mode, local->rdev, local->umask, local->params);
 
-                return 0;
-        }
+        return 0;
+    }
 err:
-        WIPE (postparent);
-        WIPE (preparent);
+    WIPE(postparent);
+    WIPE(preparent);
 
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
-                          inode, stbuf, preparent, postparent, xdata);
-        return 0;
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
-
 int
-nufa_mknod (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
+nufa_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+           dev_t rdev, mode_t umask, dict_t *params)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf  = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        /* Consider the disksize in consideration */
-        avail_subvol = conf->private;
-        if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private,
-                                                        local);
-        }
-
-        if (avail_subvol != subvol) {
-                /* Create linkfile first */
-
-                local->params = dict_ref (params);
-                local->mode = mode;
-                local->umask = umask;
-                local->rdev = rdev;
-                local->cached_subvol = avail_subvol;
-
-                dht_linkfile_create (frame, nufa_mknod_linkfile_cbk, this,
-                                     avail_subvol, subvol, loc);
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKNOD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    /* Consider the disksize in consideration */
+    avail_subvol = conf->private;
+    if (dht_is_subvol_filled(this, (xlator_t *)conf->private)) {
+        avail_subvol = dht_free_disk_available_subvol(
+            this, (xlator_t *)conf->private, local);
+    }
+
+    if (avail_subvol != subvol) {
+        /* Create linkfile first */
+
+        local->params = dict_ref(params);
+        local->mode = mode;
+        local->umask = umask;
+        local->rdev = rdev;
+        local->cached_subvol = avail_subvol;
+
+        dht_linkfile_create(frame, nufa_mknod_linkfile_cbk, this, avail_subvol,
+                            subvol, loc);
+        return 0;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
-                           subvol->fops->mknod, loc, mode, rdev, umask,
-                           params);
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                      subvol->fops->mknod, loc, mode, rdev, umask, params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 gf_boolean_t
-same_first_part (char *str1, char term1, char *str2, char term2)
+same_first_part(char *str1, char term1, char *str2, char term2)
 {
-        gf_boolean_t    ended1;
-        gf_boolean_t    ended2;
-
-        for (;;) {
-                ended1 = ((*str1 == '\0') || (*str1 == term1));
-                ended2 = ((*str2 == '\0') || (*str2 == term2));
-                if (ended1 && ended2) {
-                        return _gf_true;
-                }
-                if (ended1 || ended2 || (*str1 != *str2)) {
-                        return _gf_false;
-                }
-                ++str1;
-                ++str2;
+    gf_boolean_t ended1;
+    gf_boolean_t ended2;
+
+    for (;;) {
+        ended1 = ((*str1 == '\0') || (*str1 == term1));
+        ended2 = ((*str2 == '\0') || (*str2 == term2));
+        if (ended1 && ended2) {
+            return _gf_true;
         }
+        if (ended1 || ended2 || (*str1 != *str2)) {
+            return _gf_false;
+        }
+        ++str1;
+        ++str2;
+    }
 }
 
 typedef struct nufa_args {
-        xlator_t    *this;
-        char        *volname;
-        gf_boolean_t addr_match;
+    xlator_t *this;
+    char *volname;
+    gf_boolean_t addr_match;
 } nufa_args_t;
 
 static void
-nufa_find_local_brick (xlator_t *xl, void *data)
+nufa_find_local_brick(xlator_t *xl, void *data)
 {
-        nufa_args_t     *args = data;
-        xlator_t        *this = args->this;
-        char            *local_volname = args->volname;
-        gf_boolean_t    addr_match = args->addr_match;
-        char            *brick_host = NULL;
-        dht_conf_t      *conf = this->private;
-        int             ret = -1;
-
-        /*This means a local subvol was already found. We pick the first brick
-         * that is local*/
-        if (conf->private)
-                return;
-
-        if (strcmp (xl->name, local_volname) == 0) {
-                conf->private = xl;
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_SUBVOL_INFO,
-                        "Using specified subvol %s",
-                        local_volname);
-                return;
-        }
-
-        if (!addr_match)
-                return;
-
-        ret = dict_get_str (xl->options, "remote-host", &brick_host);
-        if ((ret == 0) &&
-            (gf_is_same_address (local_volname, brick_host) ||
-             gf_is_local_addr (brick_host))) {
-                conf->private = xl;
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_SUBVOL_INFO, "Using the first local "
-                        "subvol %s", xl->name);
-                return;
-        }
-
+    nufa_args_t *args = data;
+    xlator_t *this = args->this;
+    char *local_volname = args->volname;
+    gf_boolean_t addr_match = args->addr_match;
+    char *brick_host = NULL;
+    dht_conf_t *conf = this->private;
+    int ret = -1;
+
+    /*This means a local subvol was already found. We pick the first brick
+     * that is local*/
+    if (conf->private)
+        return;
+
+    if (strcmp(xl->name, local_volname) == 0) {
+        conf->private = xl;
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "Using specified subvol %s", local_volname);
+        return;
+    }
+
+    if (!addr_match)
+        return;
+
+    ret = dict_get_str(xl->options, "remote-host", &brick_host);
+    if ((ret == 0) && (gf_is_same_address(local_volname, brick_host) ||
+                       gf_is_local_addr(brick_host))) {
+        conf->private = xl;
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "Using the first local "
+               "subvol %s",
+               xl->name);
+        return;
+    }
 }
 
 static void
-nufa_to_dht (xlator_t *this)
+nufa_to_dht(xlator_t *this)
 {
-        GF_ASSERT (this);
-        GF_ASSERT (this->fops);
+    GF_ASSERT(this);
+    GF_ASSERT(this->fops);
 
-        this->fops->lookup = dht_lookup;
-        this->fops->create = dht_create;
-        this->fops->mknod  = dht_mknod;
+    this->fops->lookup = dht_lookup;
+    this->fops->create = dht_create;
+    this->fops->mknod = dht_mknod;
 }
 
 int
-nufa_find_local_subvol (xlator_t *this,
-                        void (*fn) (xlator_t *each, void* data), void *data)
+nufa_find_local_subvol(xlator_t *this, void (*fn)(xlator_t *each, void *data),
+                       void *data)
 {
-        int             ret = -1;
-        dht_conf_t      *conf = this->private;
-        xlator_list_t   *trav = NULL;
-        xlator_t        *parent = NULL;
-        xlator_t        *candidate = NULL;
-
-        xlator_foreach_depth_first (this, fn, data);
-        if (!conf->private) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_BRICK_ERROR, "Couldn't find a local "
-                        "brick");
-                return -1;
+    int ret = -1;
+    dht_conf_t *conf = this->private;
+    xlator_list_t *trav = NULL;
+    xlator_t *parent = NULL;
+    xlator_t *candidate = NULL;
+
+    xlator_foreach_depth_first(this, fn, data);
+    if (!conf->private) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_BRICK_ERROR,
+               "Couldn't find a local "
+               "brick");
+        return -1;
+    }
+
+    candidate = conf->private;
+    trav = candidate->parents;
+    while (trav) {
+        parent = trav->xlator;
+        if (strcmp(parent->type, "cluster/nufa") == 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                   "Found local subvol, "
+                   "%s",
+                   candidate->name);
+            ret = 0;
+            conf->private = candidate;
+            break;
         }
 
-        candidate = conf->private;
-        trav = candidate->parents;
-        while (trav) {
-
-                parent = trav->xlator;
-                if (strcmp (parent->type, "cluster/nufa") == 0) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_SUBVOL_INFO, "Found local subvol, "
-                                "%s", candidate->name);
-                        ret = 0;
-                        conf->private = candidate;
-                        break;
-                }
-
-                candidate = parent;
-                trav = parent->parents;
-        }
+        candidate = parent;
+        trav = parent->parents;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-nufa_init (xlator_t *this)
+nufa_init(xlator_t *this)
 {
-        data_t        *data = NULL;
-        char          *local_volname = NULL;
-        int            ret = -1;
-        char           my_hostname[256];
-        gf_boolean_t   addr_match = _gf_false;
-        nufa_args_t    args = {0, };
-
-        ret = dht_init(this);
-        if (ret) {
-                return ret;
-        }
-
-        if ((data = dict_get (this->options, "local-volume-name"))) {
-                local_volname = data->data;
-
-        } else {
-                addr_match = _gf_true;
-                local_volname = "localhost";
-                ret = gethostname (my_hostname, 256);
-                if (ret == 0)
-                        local_volname = my_hostname;
-
-                else
-                        gf_msg (this->name, GF_LOG_WARNING, errno,
-                                DHT_MSG_GET_HOSTNAME_FAILED,
-                                "could not find hostname");
-
-        }
-
-        args.this = this;
-        args.volname = local_volname;
-        args.addr_match = addr_match;
-        ret = nufa_find_local_subvol (this, nufa_find_local_brick, &args);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_SUBVOL_INFO,
-                        "Unable to find local subvolume, switching "
-                        "to dht mode");
-                nufa_to_dht (this);
-        }
-        return 0;
+    data_t *data = NULL;
+    char *local_volname = NULL;
+    int ret = -1;
+    char my_hostname[256];
+    gf_boolean_t addr_match = _gf_false;
+    nufa_args_t args = {
+        0,
+    };
+
+    ret = dht_init(this);
+    if (ret) {
+        return ret;
+    }
+
+    if ((data = dict_get(this->options, "local-volume-name"))) {
+        local_volname = data->data;
+
+    } else {
+        addr_match = _gf_true;
+        local_volname = "localhost";
+        ret = gethostname(my_hostname, 256);
+        if (ret == 0)
+            local_volname = my_hostname;
+
+        else
+            gf_msg(this->name, GF_LOG_WARNING, errno,
+                   DHT_MSG_GET_HOSTNAME_FAILED, "could not find hostname");
+    }
+
+    args.this = this;
+    args.volname = local_volname;
+    args.addr_match = addr_match;
+    ret = nufa_find_local_subvol(this, nufa_find_local_brick, &args);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "Unable to find local subvolume, switching "
+               "to dht mode");
+        nufa_to_dht(this);
+    }
+    return 0;
 }
 
 dht_methods_t dht_methods = {
-        .migration_get_dst_subvol = dht_migration_get_dst_subvol,
-        .migration_needed = dht_migration_needed,
-        .layout_search   = dht_layout_search,
-};
-
-class_methods_t class_methods = {
-        .init           = nufa_init,
-        .fini           = dht_fini,
-        .reconfigure    = dht_reconfigure,
-        .notify         = dht_notify
+    .migration_get_dst_subvol = dht_migration_get_dst_subvol,
+    .layout_search = dht_layout_search,
 };
 
-
 struct xlator_fops fops = {
-        .lookup      = nufa_lookup,
-        .create      = nufa_create,
-        .mknod       = nufa_mknod,
-
-        .stat        = dht_stat,
-        .fstat       = dht_fstat,
-        .truncate    = dht_truncate,
-        .ftruncate   = dht_ftruncate,
-        .access      = dht_access,
-        .readlink    = dht_readlink,
-        .setxattr    = dht_setxattr,
-        .getxattr    = dht_getxattr,
-        .removexattr = dht_removexattr,
-        .open        = dht_open,
-        .readv       = dht_readv,
-        .writev      = dht_writev,
-        .flush       = dht_flush,
-        .fsync       = dht_fsync,
-        .statfs      = dht_statfs,
-        .lk          = dht_lk,
-        .opendir     = dht_opendir,
-        .readdir     = dht_readdir,
-        .readdirp    = dht_readdirp,
-        .fsyncdir    = dht_fsyncdir,
-        .symlink     = dht_symlink,
-        .unlink      = dht_unlink,
-        .link        = dht_link,
-        .mkdir       = dht_mkdir,
-        .rmdir       = dht_rmdir,
-        .rename      = dht_rename,
-        .inodelk     = dht_inodelk,
-        .finodelk    = dht_finodelk,
-        .entrylk     = dht_entrylk,
-        .fentrylk    = dht_fentrylk,
-        .xattrop     = dht_xattrop,
-        .fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
+    .lookup = nufa_lookup,
+    .create = nufa_create,
+    .mknod = nufa_mknod,
+
+    .stat = dht_stat,
+    .fstat = dht_fstat,
+    .truncate = dht_truncate,
+    .ftruncate = dht_ftruncate,
+    .access = dht_access,
+    .readlink = dht_readlink,
+    .setxattr = dht_setxattr,
+    .getxattr = dht_getxattr,
+    .removexattr = dht_removexattr,
+    .open = dht_open,
+    .readv = dht_readv,
+    .writev = dht_writev,
+    .flush = dht_flush,
+    .fsync = dht_fsync,
+    .statfs = dht_statfs,
+    .lk = dht_lk,
+    .opendir = dht_opendir,
+    .readdir = dht_readdir,
+    .readdirp = dht_readdirp,
+    .fsyncdir = dht_fsyncdir,
+    .symlink = dht_symlink,
+    .unlink = dht_unlink,
+    .link = dht_link,
+    .mkdir = dht_mkdir,
+    .rmdir = dht_rmdir,
+    .rename = dht_rename,
+    .inodelk = dht_inodelk,
+    .finodelk = dht_finodelk,
+    .entrylk = dht_entrylk,
+    .fentrylk = dht_fentrylk,
+    .xattrop = dht_xattrop,
+    .fxattrop = dht_fxattrop,
+    .setattr = dht_setattr,
 };
 
-
-struct xlator_cbks cbks = {
-        .forget     = dht_forget
+struct xlator_cbks cbks = {.forget = dht_forget};
+extern int32_t
+mem_acct_init(xlator_t *this);
+
+xlator_api_t xlator_api = {
+    .init = nufa_init,
+    .fini = dht_fini,
+    .notify = dht_notify,
+    .reconfigure = dht_reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = dht_options,
+    .identifier = "nufa",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/cluster/dht/src/nufa.sym b/xlators/cluster/dht/src/nufa.sym
deleted file mode 100644
index 780b5fc0387..00000000000
--- a/xlators/cluster/dht/src/nufa.sym
+++ /dev/null
@@ -1,8 +0,0 @@
-fops
-cbks
-class_methods
-dht_methods
-options
-mem_acct_init
-reconfigure
-dumpops
diff --git a/xlators/cluster/dht/src/switch.c b/xlators/cluster/dht/src/switch.c
index 9795ff4af4d..207d109a025 100644
--- a/xlators/cluster/dht/src/switch.c
+++ b/xlators/cluster/dht/src/switch.c
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "dht-common.h"
 #include "dht-mem-types.h"
 
@@ -17,895 +16,876 @@
 #include <fnmatch.h>
 #include <string.h>
 
-extern struct volume_options options[];
+extern struct volume_options dht_options[];
 
 struct switch_sched_array {
-        xlator_t *xl;
-        int32_t   eligible;
-        int32_t   considered;
+    xlator_t *xl;
+    int32_t eligible;
+    int32_t considered;
 };
 
 /* Select one of this struct based on the path's pattern match */
 struct switch_struct {
-        struct switch_struct      *next;
-        struct switch_sched_array *array;
-        int32_t                    node_index; /* Index of the node in
-                                                  this pattern. */
-        int32_t                    num_child;  /* Total num of child nodes
-                                                  with this pattern. */
-        char                       path_pattern[256];
+    struct switch_struct *next;
+    struct switch_sched_array *array;
+    int32_t node_index; /* Index of the node in
+                           this pattern. */
+    int32_t num_child;  /* Total num of child nodes
+                           with this pattern. */
+    char path_pattern[256];
 };
 
 /* TODO: all 'TODO's in dht.c holds good */
 /* This function should return child node as '*:subvolumes' is inserterd */
 
 static int32_t
-gf_switch_valid_child (xlator_t *this, const char *child)
+gf_switch_valid_child(xlator_t *this, const char *child)
 {
-        xlator_list_t *children = NULL;
-        int32_t        ret = 0;
-
-        children = this->children;
-        while (children) {
-                if (!strcmp (child, children->xlator->name)) {
-                        ret = 1;
-                        break;
-                }
-                children = children->next;
+    xlator_list_t *children = NULL;
+    int32_t ret = 0;
+
+    children = this->children;
+    while (children) {
+        if (!strcmp(child, children->xlator->name)) {
+            ret = 1;
+            break;
         }
+        children = children->next;
+    }
 
-        return ret;
+    return ret;
 }
 
 static xlator_t *
-get_switch_matching_subvol (const char *path, dht_conf_t *conf,
-                            xlator_t *hashed_subvol)
+get_switch_matching_subvol(const char *path, dht_conf_t *conf,
+                           xlator_t *hashed_subvol)
 {
-        struct switch_struct *cond      = NULL;
-        struct switch_struct *trav      = NULL;
-        char                 *pathname  = NULL;
-        int                   idx       = 0;
-        xlator_t             *subvol    = NULL;
-
-        cond = conf->private;
-        subvol = hashed_subvol;
-        if (!cond)
-                goto out;
-
-        pathname = gf_strdup (path);
-        if (!pathname)
-                goto out;
-
-        trav = cond;
-        while (trav) {
-                if (fnmatch (trav->path_pattern,
-                             pathname, FNM_NOESCAPE) == 0) {
-                        for (idx = 0; idx < trav->num_child; idx++) {
-                                if (trav->array[idx].xl == hashed_subvol)
-                                        goto out;
-                        }
-                        idx = trav->node_index++;
-                        trav->node_index %= trav->num_child;
-                        subvol = trav->array[idx].xl;
-                        goto out;
-                }
-                trav = trav->next;
+    struct switch_struct *cond = NULL;
+    struct switch_struct *trav = NULL;
+    char *pathname = NULL;
+    int idx = 0;
+    xlator_t *subvol = NULL;
+
+    cond = conf->private;
+    subvol = hashed_subvol;
+    if (!cond)
+        goto out;
+
+    pathname = gf_strdup(path);
+    if (!pathname)
+        goto out;
+
+    trav = cond;
+    while (trav) {
+        if (fnmatch(trav->path_pattern, pathname, FNM_NOESCAPE) == 0) {
+            for (idx = 0; idx < trav->num_child; idx++) {
+                if (trav->array[idx].xl == hashed_subvol)
+                    goto out;
+            }
+            idx = trav->node_index++;
+            trav->node_index %= trav->num_child;
+            subvol = trav->array[idx].xl;
+            goto out;
         }
+        trav = trav->next;
+    }
 out:
-        GF_FREE (pathname);
+    GF_FREE(pathname);
 
-        return subvol;
+    return subvol;
 }
 
-
 int
-switch_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno,
-                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                         struct iatt *postparent)
+switch_local_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent)
 {
-        xlator_t     *subvol      = NULL;
-        char          is_linkfile = 0;
-        char          is_dir      = 0;
-        dht_conf_t   *conf        = NULL;
-        dht_local_t  *local       = NULL;
-        loc_t        *loc         = NULL;
-        int           i           = 0;
-        xlator_t     *prev        = NULL;
-        int           call_cnt    = 0;
-        int           ret         = 0;
-
-        conf  = this->private;
-
-        prev  = cookie;
-        local = frame->local;
-        loc   = &local->loc;
-
-        if (ENTRY_MISSING (op_ret, op_errno)) {
-                if (conf->search_unhashed) {
-                        local->op_errno = ENOENT;
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+    xlator_t *subvol = NULL;
+    char is_linkfile = 0;
+    char is_dir = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    int i = 0;
+    xlator_t *prev = NULL;
+    int call_cnt = 0;
+    int ret = 0;
+
+    conf = this->private;
+
+    prev = cookie;
+    local = frame->local;
+    loc = &local->loc;
+
+    if (ENTRY_MISSING(op_ret, op_errno)) {
+        if (conf->search_unhashed) {
+            local->op_errno = ENOENT;
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
         }
-
-        if (op_ret == -1)
-                goto out;
-
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                         conf->link_xattr_name);
-        is_dir      = check_is_dir (inode, stbuf, xattr);
-
-        if (!is_dir && !is_linkfile) {
-                /* non-directory and not a linkfile */
-
-                ret = dht_layout_preset (this, prev, inode);
-                if (ret < 0) {
-                        gf_msg_debug (this->name, 0,
-                                      "could not set pre-set layout "
-                                      "for subvol %s",
-                                      prev->name);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                goto out;
+    }
+
+    if (op_ret == -1)
+        goto out;
+
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
+    is_dir = check_is_dir(inode, stbuf, xattr);
+
+    if (!is_dir && !is_linkfile) {
+        /* non-directory and not a linkfile */
+
+        ret = dht_layout_preset(this, prev, inode);
+        if (ret < 0) {
+            gf_msg_debug(this->name, 0,
+                         "could not set pre-set layout "
+                         "for subvol %s",
+                         prev->name);
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (is_dir) {
-                call_cnt        = conf->subvolume_cnt;
-                local->call_cnt = call_cnt;
+        goto out;
+    }
 
-                local->inode = inode_ref (inode);
-                local->xattr = dict_ref (xattr);
+    if (is_dir) {
+        call_cnt = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
 
-                local->op_ret = 0;
-                local->op_errno = 0;
+        local->inode = inode_ref(inode);
+        local->xattr = dict_ref(xattr);
 
-                local->layout = dht_layout_new (this, conf->subvolume_cnt);
-                if (!local->layout) {
-                        op_ret   = -1;
-                        op_errno = ENOMEM;
-                        gf_msg_debug (this->name, 0,
-                                      "memory allocation failed :(");
-                        goto err;
-                }
+        local->op_ret = 0;
+        local->op_errno = 0;
 
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND_COOKIE (frame, dht_lookup_dir_cbk,
-                                           conf->subvolumes[i],
-                                           conf->subvolumes[i],
-                                           conf->subvolumes[i]->fops->lookup,
-                                           &local->loc, local->xattr_req);
-                }
+        local->layout = dht_layout_new(this, conf->subvolume_cnt);
+        if (!local->layout) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg_debug(this->name, 0, "memory allocation failed :(");
+            goto err;
         }
 
-        if (is_linkfile) {
-                subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
+        }
+    }
 
-                if (!subvol) {
-                        gf_msg_debug (this->name, 0,
-                                      "linkfile has no link subvolume.path=%s",
-                                      loc->path);
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+    if (is_linkfile) {
+        subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
 
-                STACK_WIND_COOKIE (frame, dht_lookup_linkfile_cbk, subvol,
-                                   subvol, subvol->fops->lookup,
-                                   &local->loc, local->xattr_req);
+        if (!subvol) {
+            gf_msg_debug(this->name, 0,
+                         "linkfile has no link subvolume.path=%s", loc->path);
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
         }
 
-        return 0;
+        STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, local->xattr_req);
+    }
+
+    return 0;
 
 out:
-        if (!local->hashed_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              local->loc.path);
-                local->op_errno = ENOENT;
-                dht_lookup_everywhere (frame, this, loc);
-                return 0;
-        }
+    if (!local->hashed_subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     local->loc.path);
+        local->op_errno = ENOENT;
+        dht_lookup_everywhere(frame, this, loc);
+        return 0;
+    }
 
-        STACK_WIND_COOKIE (frame, dht_lookup_cbk, local->hashed_subvol,
-                           local->hashed_subvol,
-                           local->hashed_subvol->fops->lookup,
-                           &local->loc, local->xattr_req);
+    STACK_WIND_COOKIE(frame, dht_lookup_cbk, local->hashed_subvol,
+                      local->hashed_subvol, local->hashed_subvol->fops->lookup,
+                      &local->loc, local->xattr_req);
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno,
-                          inode, stbuf, xattr, NULL);
-        return 0;
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     NULL);
+    return 0;
 }
 
 int
-switch_lookup (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, dict_t *xattr_req)
+switch_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xattr_req)
 {
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *cached_subvol = NULL;
-        xlator_t     *subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret    = -1;
-        int           op_errno = -1;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_LOOKUP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (xattr_req) {
+        local->xattr_req = dict_ref(xattr_req);
+    } else {
+        local->xattr_req = dict_new();
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
+    cached_subvol = local->cached_subvol;
+
+    local->hashed_subvol = hashed_subvol;
+
+    if (is_revalidate(loc)) {
+        layout = local->layout;
+        if (!layout) {
+            gf_msg_debug(this->name, 0,
+                         "revalidate lookup without cache. path=%s", loc->path);
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (xattr_req) {
-                local->xattr_req = dict_ref (xattr_req);
-        } else {
-                local->xattr_req = dict_new ();
+        if (layout->gen && (layout->gen < conf->gen)) {
+            gf_msg_debug(this->name, 0, "incomplete layout failure for path=%s",
+                         loc->path);
+            dht_layout_unref(this, local->layout);
+            goto do_fresh_lookup;
         }
 
-        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
-        cached_subvol = local->cached_subvol;
+        local->inode = inode_ref(loc->inode);
 
-        local->hashed_subvol = hashed_subvol;
+        local->call_cnt = layout->cnt;
+        call_cnt = local->call_cnt;
 
-        if (is_revalidate (loc)) {
-                layout = local->layout;
-                if (!layout) {
-                        gf_msg_debug(this->name, 0,
-                                     "revalidate lookup without cache. path=%s",
-                                     loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                if (layout->gen && (layout->gen < conf->gen)) {
-                        gf_msg_debug (this->name, 0,
-                                      "incomplete layout failure for path=%s",
-                                      loc->path);
-                        dht_layout_unref (this, local->layout);
-                        goto do_fresh_lookup;
-                }
-
-                local->inode    = inode_ref (loc->inode);
-
-                local->call_cnt = layout->cnt;
-                call_cnt = local->call_cnt;
+        /* NOTE: we don't require 'trusted.glusterfs.dht.linkto'
+         * attribute, revalidates directly go to the cached-subvolume.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                   "failed to set dict value for %s", conf->xattr_name);
 
-                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto'
-                 * attribute, revalidates directly go to the cached-subvolume.
-                 */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret < 0)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "failed to set dict value for %s",
-                                conf->xattr_name);
+        for (i = 0; i < layout->cnt; i++) {
+            subvol = layout->list[i].xlator;
 
-                for (i = 0; i < layout->cnt; i++) {
-                        subvol = layout->list[i].xlator;
+            STACK_WIND_COOKIE(frame, dht_revalidate_cbk, subvol, subvol,
+                              subvol->fops->lookup, loc, local->xattr_req);
 
-                        STACK_WIND_COOKIE (frame, dht_revalidate_cbk, subvol,
-                                           subvol, subvol->fops->lookup,
-                                           loc, local->xattr_req);
+            if (!--call_cnt)
+                break;
+        }
+    } else {
+    do_fresh_lookup:
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                   "failed to set dict value for %s", conf->xattr_name);
+
+        ret = dict_set_uint32(local->xattr_req, conf->link_xattr_name, 256);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_DICT_SET_FAILED,
+                   "failed to set dict value for %s", conf->link_xattr_name);
+
+        if (!hashed_subvol) {
+            gf_msg_debug(this->name, 0,
+                         "no subvolume in layout for path=%s, "
+                         "checking on all the subvols to see if "
+                         "it is a directory",
+                         loc->path);
+            call_cnt = conf->subvolume_cnt;
+            local->call_cnt = call_cnt;
+
+            local->layout = dht_layout_new(this, conf->subvolume_cnt);
+            if (!local->layout) {
+                op_errno = ENOMEM;
+                goto err;
+            }
+
+            for (i = 0; i < call_cnt; i++) {
+                STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk,
+                                  conf->subvolumes[i], conf->subvolumes[i],
+                                  conf->subvolumes[i]->fops->lookup,
+                                  &local->loc, local->xattr_req);
+            }
+            return 0;
+        }
 
-                        if (!--call_cnt)
-                                break;
-                }
+        /*  */
+        cached_subvol = get_switch_matching_subvol(loc->path, conf,
+                                                   hashed_subvol);
+        if (cached_subvol == hashed_subvol) {
+            STACK_WIND_COOKIE(frame, dht_lookup_cbk, hashed_subvol,
+                              hashed_subvol, hashed_subvol->fops->lookup, loc,
+                              local->xattr_req);
         } else {
-        do_fresh_lookup:
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret < 0)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "failed to set dict value for %s",
-                                conf->xattr_name);
-
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->link_xattr_name, 256);
-                if (ret < 0)
-                        gf_msg (this->name, GF_LOG_WARNING, EINVAL,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "failed to set dict value for %s",
-                                conf->link_xattr_name);
-
-                if (!hashed_subvol) {
-                        gf_msg_debug (this->name, 0,
-                                      "no subvolume in layout for path=%s, "
-                                      "checking on all the subvols to see if "
-                                      "it is a directory", loc->path);
-                        call_cnt        = conf->subvolume_cnt;
-                        local->call_cnt = call_cnt;
-
-                        local->layout = dht_layout_new (this,
-                                                        conf->subvolume_cnt);
-                        if (!local->layout) {
-                                op_errno = ENOMEM;
-                                goto err;
-                        }
-
-                        for (i = 0; i < call_cnt; i++) {
-                                STACK_WIND_COOKIE (frame, dht_lookup_dir_cbk,
-                                                   conf->subvolumes[i],
-                                                   conf->subvolumes[i],
-                                                   conf->subvolumes[i]->fops->lookup,
-                                                   &local->loc, local->xattr_req);
-                        }
-                        return 0;
-                }
-
-                /*  */
-                cached_subvol = get_switch_matching_subvol (loc->path, conf,
-                                                            hashed_subvol);
-                if (cached_subvol == hashed_subvol) {
-                        STACK_WIND_COOKIE (frame, dht_lookup_cbk, hashed_subvol,
-                                           hashed_subvol,
-                                           hashed_subvol->fops->lookup,
-                                           loc, local->xattr_req);
-                } else {
-                        STACK_WIND_COOKIE (frame, switch_local_lookup_cbk,
-                                           cached_subvol, cached_subvol,
-                                           cached_subvol->fops->lookup,
-                                           loc, local->xattr_req);
-                }
+            STACK_WIND_COOKIE(frame, switch_local_lookup_cbk, cached_subvol,
+                              cached_subvol, cached_subvol->fops->lookup, loc,
+                              local->xattr_req);
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
-        return 0;
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-switch_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                   xlator_t *this, int op_ret, int op_errno,
-                                   inode_t *inode, struct iatt *stbuf,
-                                   struct iatt *preparent,
-                                   struct iatt *postparent, dict_t *xdata)
+switch_create_linkfile_create_cbk(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int op_ret, int op_errno,
+                                  inode_t *inode, struct iatt *stbuf,
+                                  struct iatt *preparent,
+                                  struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret == -1)
-                goto err;
+    if (op_ret == -1)
+        goto err;
 
-        STACK_WIND_COOKIE (frame, dht_create_cbk, local->cached_subvol,
-                           local->cached_subvol, local->cached_subvol->fops->create,
-                           &local->loc, local->flags, local->mode, local->umask,
-                           local->fd, local->params);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, local->cached_subvol,
+                      local->cached_subvol, local->cached_subvol->fops->create,
+                      &local->loc, local->flags, local->mode, local->umask,
+                      local->fd, local->params);
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 int
-switch_create (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, int32_t flags, mode_t mode,
-               mode_t umask, fd_t *fd, dict_t *params)
+switch_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *params)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf  = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
-        if (dht_is_subvol_filled (this, avail_subvol)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol,
-                                                        local);
-        }
-
-        if (subvol != avail_subvol) {
-                /* create a link file instead of actual file */
-                local->mode = mode;
-                local->flags = flags;
-                local->umask = umask;
-                local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame, switch_create_linkfile_create_cbk,
-                                     this, avail_subvol, subvol, loc);
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_CREATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    avail_subvol = get_switch_matching_subvol(loc->path, conf, subvol);
+    if (dht_is_subvol_filled(this, avail_subvol)) {
+        avail_subvol = dht_free_disk_available_subvol(this, avail_subvol,
+                                                      local);
+    }
+
+    if (subvol != avail_subvol) {
+        /* create a link file instead of actual file */
+        local->mode = mode;
+        local->flags = flags;
+        local->umask = umask;
+        local->cached_subvol = avail_subvol;
+        dht_linkfile_create(frame, switch_create_linkfile_create_cbk, this,
+                            avail_subvol, subvol, loc);
+        return 0;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND_COOKIE (frame, dht_create_cbk, subvol, subvol,
-                           subvol->fops->create, loc, flags, mode, umask, fd,
-                           params);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                      subvol->fops->create, loc, flags, mode, umask, fd,
+                      params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-switch_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, inode_t *inode,
-                           struct iatt *stbuf, struct iatt *preparent,
-                           struct iatt *postparent, dict_t *xdata)
+switch_mknod_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, inode_t *inode,
+                          struct iatt *stbuf, struct iatt *preparent,
+                          struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
-        if (!local || !local->cached_subvol) {
-                op_errno = EINVAL;
-                op_ret = -1;
-                goto err;
-        }
+    local = frame->local;
+    if (!local || !local->cached_subvol) {
+        op_errno = EINVAL;
+        op_ret = -1;
+        goto err;
+    }
 
-        if (op_ret >= 0) {
-                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
-                            (void *)local->cached_subvol, local->cached_subvol,
-                            local->cached_subvol->fops->mknod,
-                            &local->loc, local->mode, local->rdev,
-                            local->umask, local->params);
+    if (op_ret >= 0) {
+        STACK_WIND_COOKIE(
+            frame, dht_newfile_cbk, (void *)local->cached_subvol,
+            local->cached_subvol, local->cached_subvol->fops->mknod,
+            &local->loc, local->mode, local->rdev, local->umask, local->params);
 
-                return 0;
-        }
-err:
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
-                          inode, stbuf, preparent, postparent, xdata);
         return 0;
+    }
+err:
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
-
 int
-switch_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, mode_t umask, dict_t *params)
+switch_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             dev_t rdev, mode_t umask, dict_t *params)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf  = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        /* Consider the disksize in consideration */
-        avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
-        if (dht_is_subvol_filled (this, avail_subvol)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol,
-                                                        local);
-        }
-
-        if (avail_subvol != subvol) {
-                /* Create linkfile first */
-
-                local->params = dict_ref (params);
-                local->mode = mode;
-                local->umask = umask;
-                local->rdev = rdev;
-                local->cached_subvol = avail_subvol;
-
-                dht_linkfile_create (frame, switch_mknod_linkfile_cbk,
-                                     this, avail_subvol, subvol, loc);
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKNOD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    /* Consider the disksize in consideration */
+    avail_subvol = get_switch_matching_subvol(loc->path, conf, subvol);
+    if (dht_is_subvol_filled(this, avail_subvol)) {
+        avail_subvol = dht_free_disk_available_subvol(this, avail_subvol,
+                                                      local);
+    }
+
+    if (avail_subvol != subvol) {
+        /* Create linkfile first */
+
+        local->params = dict_ref(params);
+        local->mode = mode;
+        local->umask = umask;
+        local->rdev = rdev;
+        local->cached_subvol = avail_subvol;
+
+        dht_linkfile_create(frame, switch_mknod_linkfile_cbk, this,
+                            avail_subvol, subvol, loc);
+        return 0;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
-                           subvol->fops->mknod, loc, mode, rdev, umask,
-                           params);
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                      subvol->fops->mknod, loc, mode, rdev, umask, params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 void
-switch_fini (xlator_t *this)
+switch_fini(xlator_t *this)
 {
-        dht_conf_t           *conf = NULL;
-        struct switch_struct *trav = NULL;
-        struct switch_struct *prev = NULL;
-
-        conf = this->private;
-
-        if (conf) {
-                trav = (struct switch_struct *)conf->private;
-                conf->private = NULL;
-                while (trav) {
-                        GF_FREE (trav->array);
-                        prev = trav;
-                        trav = trav->next;
-                        GF_FREE (prev);
-                }
+    dht_conf_t *conf = NULL;
+    struct switch_struct *trav = NULL;
+    struct switch_struct *prev = NULL;
+
+    conf = this->private;
+
+    if (conf) {
+        trav = (struct switch_struct *)conf->private;
+        conf->private = NULL;
+        while (trav) {
+            GF_FREE(trav->array);
+            prev = trav;
+            trav = trav->next;
+            GF_FREE(prev);
         }
+    }
 
-        dht_fini(this);
+    dht_fini(this);
 }
 
 int
-set_switch_pattern (xlator_t *this, dht_conf_t *conf,
-                    const char *pattern_str)
+set_switch_pattern(xlator_t *this, dht_conf_t *conf, const char *pattern_str)
 {
-        int                         flag = 0;
-        int                         idx = 0;
-        int                         index = 0;
-        int                         child_count = 0;
-        char                       *tmp = NULL;
-        char                       *tmp1 = NULL;
-        char                       *child = NULL;
-        char                       *tmp_str = NULL;
-        char                       *tmp_str1 = NULL;
-        char                       *dup_str = NULL;
-        char                       *dup_childs = NULL;
-        char                       *switch_str = NULL;
-        char                       *pattern = NULL;
-        char                       *childs = NULL;
-        char                       *option_string = NULL;
-        struct switch_struct        *switch_buf = NULL;
-        struct switch_struct        *switch_opt = NULL;
-        struct switch_struct        *trav = NULL;
-        struct switch_sched_array  *switch_buf_array = NULL;
-        xlator_list_t              *trav_xl = NULL;
-
-        trav_xl = this->children;
-        while (trav_xl) {
-                index++;
-                trav_xl = trav_xl->next;
+    int flag = 0;
+    int idx = 0;
+    int index = 0;
+    int child_count = 0;
+    char *tmp = NULL;
+    char *tmp1 = NULL;
+    char *child = NULL;
+    char *tmp_str = NULL;
+    char *tmp_str1 = NULL;
+    char *dup_str = NULL;
+    char *dup_childs = NULL;
+    char *switch_str = NULL;
+    char *pattern = NULL;
+    char *childs = NULL;
+    char *option_string = NULL;
+    size_t pattern_length;
+    struct switch_struct *switch_buf = NULL;
+    struct switch_struct *switch_opt = NULL;
+    struct switch_struct *trav = NULL;
+    struct switch_sched_array *switch_buf_array = NULL;
+    xlator_list_t *trav_xl = NULL;
+
+    trav_xl = this->children;
+    while (trav_xl) {
+        index++;
+        trav_xl = trav_xl->next;
+    }
+    child_count = index;
+    switch_buf_array = GF_CALLOC((index + 1), sizeof(struct switch_sched_array),
+                                 gf_switch_mt_switch_sched_array);
+    if (!switch_buf_array)
+        goto err;
+
+    trav_xl = this->children;
+    index = 0;
+
+    while (trav_xl) {
+        switch_buf_array[index].xl = trav_xl->xlator;
+        switch_buf_array[index].eligible = 1;
+        trav_xl = trav_xl->next;
+        index++;
+    }
+
+    /*  *jpg:child1,child2;*mpg:child3;*:child4,child5,child6 */
+
+    /* Get the pattern for considering switch case.
+       "option block-size *avi:10MB" etc */
+    option_string = gf_strdup(pattern_str);
+    if (option_string == NULL) {
+        goto err;
+    }
+    switch_str = strtok_r(option_string, ";", &tmp_str);
+    while (switch_str) {
+        dup_str = gf_strdup(switch_str);
+        if (dup_str == NULL) {
+            goto err;
         }
-        child_count = index;
-        switch_buf_array = GF_CALLOC ((index + 1),
-                                      sizeof (struct switch_sched_array),
-                                      gf_switch_mt_switch_sched_array);
-        if (!switch_buf_array)
-                goto err;
-
-        trav_xl = this->children;
-        index = 0;
-
-        while (trav_xl) {
-                switch_buf_array[index].xl = trav_xl->xlator;
-                switch_buf_array[index].eligible = 1;
-                trav_xl = trav_xl->next;
-                index++;
+        switch_opt = GF_CALLOC(1, sizeof(struct switch_struct),
+                               gf_switch_mt_switch_struct);
+        if (!switch_opt) {
+            GF_FREE(dup_str);
+            goto err;
         }
 
-        /*  *jpg:child1,child2;*mpg:child3;*:child4,child5,child6 */
-
-        /* Get the pattern for considering switch case.
-           "option block-size *avi:10MB" etc */
-        option_string = gf_strdup (pattern_str);
-        switch_str = strtok_r (option_string, ";", &tmp_str);
-        while (switch_str) {
-                dup_str = gf_strdup (switch_str);
-                switch_opt = GF_CALLOC (1, sizeof (struct switch_struct),
-                                        gf_switch_mt_switch_struct);
-                if (!switch_opt) {
-                        GF_FREE (dup_str);
-                        goto err;
-                }
+        pattern = strtok_r(dup_str, ":", &tmp_str1);
+        childs = strtok_r(NULL, ":", &tmp_str1);
+        if (strncmp(pattern, "*", 2) == 0) {
+            gf_msg("switch", GF_LOG_INFO, 0, DHT_MSG_SWITCH_PATTERN_INFO,
+                   "'*' pattern will be taken by default "
+                   "for all the unconfigured child nodes,"
+                   " hence neglecting current option");
+            switch_str = strtok_r(NULL, ";", &tmp_str);
+            GF_FREE(switch_opt);
+            switch_opt = NULL;
+            GF_FREE(dup_str);
+            continue;
+        }
+        GF_FREE(dup_str);
+
+        pattern_length = strlen(pattern);
+        if (pattern_length >= (sizeof(switch_opt->path_pattern))) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_SET_SWITCH_PATTERN_ERROR, "Pattern (%s) too long",
+                   pattern);
+            goto err;
+        }
+        memcpy(switch_opt->path_pattern, pattern, pattern_length);
+        switch_opt->path_pattern[pattern_length] = '\0';
 
-                pattern = strtok_r (dup_str, ":", &tmp_str1);
-                childs = strtok_r (NULL, ":", &tmp_str1);
-                if (strncmp (pattern, "*", 2) == 0) {
-                        gf_msg ("switch", GF_LOG_INFO, 0,
-                                DHT_MSG_SWITCH_PATTERN_INFO,
-                                "'*' pattern will be taken by default "
-                                "for all the unconfigured child nodes,"
-                                " hence neglecting current option");
-                        switch_str = strtok_r (NULL, ";", &tmp_str);
-                        GF_FREE (switch_opt);
-                        GF_FREE (dup_str);
-                        continue;
-                }
-                GF_FREE (dup_str);
-                memcpy (switch_opt->path_pattern, pattern, strlen (pattern));
-                if (childs) {
-                        dup_childs = gf_strdup (childs);
-                        child = strtok_r (dup_childs, ",", &tmp);
-                        while (child) {
-                                if (gf_switch_valid_child (this, child)) {
-                                        idx++;
-                                        child = strtok_r (NULL, ",", &tmp);
-                                } else {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                DHT_MSG_SUBVOL_ERROR,
-                                                "%s is not a subvolume of %s. "
-                                                "pattern can only be scheduled "
-                                                "only to a subvolume of %s",
-                                                child, this->name, this->name);
-                                        goto err;
-                                }
-                        }
-                        GF_FREE (dup_childs);
-                        child = strtok_r (childs, ",", &tmp1);
-                        switch_opt->num_child = idx;
-                        switch_opt->array = GF_CALLOC (1, (idx *
-                                                           sizeof (struct switch_sched_array)),
-                                                       gf_switch_mt_switch_sched_array);
-                        if (!switch_opt->array)
-                                goto err;
-                        idx = 0;
-                        while (child) {
-                                for (index = 0; index < child_count; index++) {
-                                        if (strcmp (switch_buf_array[index].xl->name,
-                                                    child) == 0) {
-                                                gf_msg_debug ("switch", 0,
-                                                              "'%s' pattern will be "
-                                                              "scheduled to \"%s\"",
-                                                              switch_opt->path_pattern, child);
-                                                /*
-                                                  if (switch_buf_array[index-1].considered) {
-                                                  gf_msg_debug ("switch", 0,
-                                                  "ambiguity found, exiting");
-                                                  return -1;
-                                                  }
-                                                */
-                                                switch_opt->array[idx].xl = switch_buf_array[index].xl;
-                                                switch_buf_array[index].considered = 1;
-                                                idx++;
-                                                break;
-                                        }
-                                }
-                                child = strtok_r (NULL, ",", &tmp1);
-                        }
+        if (childs) {
+            dup_childs = gf_strdup(childs);
+            if (dup_childs == NULL) {
+                goto err;
+            }
+            child = strtok_r(dup_childs, ",", &tmp);
+            while (child) {
+                if (gf_switch_valid_child(this, child)) {
+                    idx++;
+                    child = strtok_r(NULL, ",", &tmp);
                 } else {
-                        /* error */
-                        gf_msg ("switch", GF_LOG_ERROR, 0,
-                                DHT_MSG_SET_SWITCH_PATTERN_ERROR,
-                                "Check \"scheduler.switch.case\" "
-                                "option in unify volume. Exiting");
-                        goto err;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SUBVOL_ERROR,
+                           "%s is not a subvolume of %s. "
+                           "pattern can only be scheduled "
+                           "only to a subvolume of %s",
+                           child, this->name, this->name);
+                    GF_FREE(dup_childs);
+                    goto err;
                 }
-
-                /* Link it to the main structure */
-                if (switch_buf) {
-                        /* there are already few entries */
-                        trav = switch_buf;
-                        while (trav->next)
-                                trav = trav->next;
-                        trav->next = switch_opt;
-                } else {
-                        /* First entry */
-                        switch_buf = switch_opt;
+            }
+            GF_FREE(dup_childs);
+            child = strtok_r(childs, ",", &tmp1);
+            switch_opt->num_child = idx;
+            switch_opt->array = GF_CALLOC(
+                1, (idx * sizeof(struct switch_sched_array)),
+                gf_switch_mt_switch_sched_array);
+            if (!switch_opt->array)
+                goto err;
+            idx = 0;
+            while (child) {
+                for (index = 0; index < child_count; index++) {
+                    if (strcmp(switch_buf_array[index].xl->name, child) == 0) {
+                        gf_msg_debug("switch", 0,
+                                     "'%s' pattern will be "
+                                     "scheduled to \"%s\"",
+                                     switch_opt->path_pattern, child);
+                        /*
+                          if (switch_buf_array[index-1].considered) {
+                          gf_msg_debug ("switch", 0,
+                          "ambiguity found, exiting");
+                          return -1;
+                          }
+                        */
+                        switch_opt->array[idx].xl = switch_buf_array[index].xl;
+                        switch_buf_array[index].considered = 1;
+                        idx++;
+                        break;
+                    }
                 }
-                switch_opt = NULL;
-                switch_str = strtok_r (NULL, ";", &tmp_str);
+                child = strtok_r(NULL, ",", &tmp1);
+            }
+        } else {
+            /* error */
+            gf_msg("switch", GF_LOG_ERROR, 0, DHT_MSG_SET_SWITCH_PATTERN_ERROR,
+                   "Check \"scheduler.switch.case\" "
+                   "option in unify volume. Exiting");
+            goto err;
         }
 
-        /* Now, all the pattern based considerations done, so for all the
-         * remaining pattern, '*' to all the remaining child nodes
-         */
-        {
-                for (index=0; index < child_count; index++) {
-                        /* check for considered flag */
-                        if (switch_buf_array[index].considered)
-                                continue;
-                        flag++;
-                }
-                if (!flag) {
-                        gf_msg ("switch", GF_LOG_ERROR, 0,
-                                DHT_MSG_SET_SWITCH_PATTERN_ERROR,
-                                "No nodes left for pattern '*'. Exiting");
-                        goto err;
-                }
-                switch_opt = GF_CALLOC (1, sizeof (struct switch_struct),
-                                        gf_switch_mt_switch_struct);
-                if (!switch_opt)
-                        goto err;
-
-                /* Add the '*' pattern to the array */
-                memcpy (switch_opt->path_pattern, "*", 2);
-                switch_opt->num_child = flag;
-                switch_opt->array =
-                        GF_CALLOC (1,
-                                   flag * sizeof (struct switch_sched_array),
-                                   gf_switch_mt_switch_sched_array);
-                if (!switch_opt->array)
-                        goto err;
-                flag = 0;
-                for (index=0; index < child_count; index++) {
-                        /* check for considered flag */
-                        if (switch_buf_array[index].considered)
-                                continue;
-                        gf_msg_debug ("switch", 0, "'%s'"
-                                      " pattern will be scheduled to \"%s\"",
-                                      switch_opt->path_pattern,
-                                      switch_buf_array[index].xl->name);
-
-                        switch_opt->array[flag].xl =
-                                switch_buf_array[index].xl;
-                        switch_buf_array[index].considered = 1;
-                        flag++;
-                }
-                if (switch_buf) {
-                        /* there are already few entries */
-                        trav = switch_buf;
-                        while (trav->next)
-                                trav = trav->next;
-                        trav->next = switch_opt;
-                } else {
-                        /* First entry */
-                        switch_buf = switch_opt;
-                }
-                switch_opt = NULL;
+        /* Link it to the main structure */
+        if (switch_buf) {
+            /* there are already few entries */
+            trav = switch_buf;
+            while (trav->next)
+                trav = trav->next;
+            trav->next = switch_opt;
+        } else {
+            /* First entry */
+            switch_buf = switch_opt;
+        }
+        switch_opt = NULL;
+        switch_str = strtok_r(NULL, ";", &tmp_str);
+    }
+
+    /* Now, all the pattern based considerations done, so for all the
+     * remaining pattern, '*' to all the remaining child nodes
+     */
+    {
+        for (index = 0; index < child_count; index++) {
+            /* check for considered flag */
+            if (switch_buf_array[index].considered)
+                continue;
+            flag++;
+        }
+        if (!flag) {
+            gf_msg("switch", GF_LOG_ERROR, 0, DHT_MSG_SET_SWITCH_PATTERN_ERROR,
+                   "No nodes left for pattern '*'. Exiting");
+            goto err;
+        }
+        switch_opt = GF_CALLOC(1, sizeof(struct switch_struct),
+                               gf_switch_mt_switch_struct);
+        if (!switch_opt)
+            goto err;
+
+        /* Add the '*' pattern to the array */
+        memcpy(switch_opt->path_pattern, "*", 2);
+        switch_opt->num_child = flag;
+        switch_opt->array = GF_CALLOC(1,
+                                      flag * sizeof(struct switch_sched_array),
+                                      gf_switch_mt_switch_sched_array);
+        if (!switch_opt->array)
+            goto err;
+        flag = 0;
+        for (index = 0; index < child_count; index++) {
+            /* check for considered flag */
+            if (switch_buf_array[index].considered)
+                continue;
+            gf_msg_debug("switch", 0,
+                         "'%s'"
+                         " pattern will be scheduled to \"%s\"",
+                         switch_opt->path_pattern,
+                         switch_buf_array[index].xl->name);
+
+            switch_opt->array[flag].xl = switch_buf_array[index].xl;
+            switch_buf_array[index].considered = 1;
+            flag++;
+        }
+        if (switch_buf) {
+            /* there are already few entries */
+            trav = switch_buf;
+            while (trav->next)
+                trav = trav->next;
+            trav->next = switch_opt;
+        } else {
+            /* First entry */
+            switch_buf = switch_opt;
         }
-        /* */
-        conf->private = switch_buf;
+        switch_opt = NULL;
+    }
+    /* */
+    conf->private = switch_buf;
 
-        return 0;
+    GF_FREE(option_string);
+    return 0;
 err:
-        GF_FREE (switch_buf_array);
-        GF_FREE (switch_opt);
+    GF_FREE(switch_buf_array);
+    GF_FREE(switch_opt);
+    GF_FREE(option_string);
 
-        if (switch_buf) {
-                trav = switch_buf;
-                while (trav) {
-                        GF_FREE (trav->array);
-                        switch_opt = trav;
-                        trav = trav->next;
-                        GF_FREE (switch_opt);
-                }
+    if (switch_buf) {
+        trav = switch_buf;
+        while (trav) {
+            GF_FREE(trav->array);
+            switch_opt = trav;
+            trav = trav->next;
+            GF_FREE(switch_opt);
         }
-        return -1;
+    }
+    return -1;
 }
 
-
 int32_t
-switch_init (xlator_t *this)
+switch_init(xlator_t *this)
 {
-        dht_conf_t            *conf = NULL;
-        data_t                *data = NULL;
-        int                    ret = -1;
+    dht_conf_t *conf = NULL;
+    data_t *data = NULL;
+    int ret = -1;
+
+    ret = dht_init(this);
+    if (ret) {
+        return ret;
+    }
+    conf = this->private;
 
-        ret = dht_init(this);
+    data = dict_get(this->options, "pattern.switch.case");
+    if (data) {
+        /* TODO: */
+        ret = set_switch_pattern(this, conf, data->data);
         if (ret) {
-                return ret;
-        }
-        conf = this->private;
-
-        data = dict_get (this->options, "pattern.switch.case");
-        if (data) {
-                /* TODO: */
-                ret = set_switch_pattern (this, conf, data->data);
-                if (ret) {
-                        goto err;
-                }
+            goto err;
         }
+    }
 
-        this->private = conf;
-        return 0;
+    this->private = conf;
+    return 0;
 
 err:
-        dht_fini(this);
-        return -1;
+    dht_fini(this);
+    return -1;
 }
 
-
-class_methods_t class_methods = {
-        .init           = switch_init,
-        .fini           = switch_fini,
-        .reconfigure    = dht_reconfigure,
-        .notify         = dht_notify
-};
-
-
 struct xlator_fops fops = {
-        .lookup      = switch_lookup,
-        .create      = switch_create,
-        .mknod       = switch_mknod,
-
-        .stat        = dht_stat,
-        .fstat       = dht_fstat,
-        .truncate    = dht_truncate,
-        .ftruncate   = dht_ftruncate,
-        .access      = dht_access,
-        .readlink    = dht_readlink,
-        .setxattr    = dht_setxattr,
-        .getxattr    = dht_getxattr,
-        .removexattr = dht_removexattr,
-        .open        = dht_open,
-        .readv       = dht_readv,
-        .writev      = dht_writev,
-        .flush       = dht_flush,
-        .fsync       = dht_fsync,
-        .statfs      = dht_statfs,
-        .lk          = dht_lk,
-        .opendir     = dht_opendir,
-        .readdir     = dht_readdir,
-        .readdirp    = dht_readdirp,
-        .fsyncdir    = dht_fsyncdir,
-        .symlink     = dht_symlink,
-        .unlink      = dht_unlink,
-        .link        = dht_link,
-        .mkdir       = dht_mkdir,
-        .rmdir       = dht_rmdir,
-        .rename      = dht_rename,
-        .inodelk     = dht_inodelk,
-        .finodelk    = dht_finodelk,
-        .entrylk     = dht_entrylk,
-        .fentrylk    = dht_fentrylk,
-        .xattrop     = dht_xattrop,
-        .fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
+    .lookup = switch_lookup,
+    .create = switch_create,
+    .mknod = switch_mknod,
+
+    .stat = dht_stat,
+    .fstat = dht_fstat,
+    .truncate = dht_truncate,
+    .ftruncate = dht_ftruncate,
+    .access = dht_access,
+    .readlink = dht_readlink,
+    .setxattr = dht_setxattr,
+    .getxattr = dht_getxattr,
+    .removexattr = dht_removexattr,
+    .open = dht_open,
+    .readv = dht_readv,
+    .writev = dht_writev,
+    .flush = dht_flush,
+    .fsync = dht_fsync,
+    .statfs = dht_statfs,
+    .lk = dht_lk,
+    .opendir = dht_opendir,
+    .readdir = dht_readdir,
+    .readdirp = dht_readdirp,
+    .fsyncdir = dht_fsyncdir,
+    .symlink = dht_symlink,
+    .unlink = dht_unlink,
+    .link = dht_link,
+    .mkdir = dht_mkdir,
+    .rmdir = dht_rmdir,
+    .rename = dht_rename,
+    .inodelk = dht_inodelk,
+    .finodelk = dht_finodelk,
+    .entrylk = dht_entrylk,
+    .fentrylk = dht_fentrylk,
+    .xattrop = dht_xattrop,
+    .fxattrop = dht_fxattrop,
+    .setattr = dht_setattr,
 };
 
-
-struct xlator_cbks cbks = {
-        .forget     = dht_forget
+struct xlator_cbks cbks = {.forget = dht_forget};
+extern int32_t
+mem_acct_init(xlator_t *this);
+
+xlator_api_t xlator_api = {
+    .init = switch_init,
+    .fini = switch_fini,
+    .notify = dht_notify,
+    .reconfigure = dht_reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = dht_options,
+    .identifier = "switch",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/cluster/dht/src/switch.sym b/xlators/cluster/dht/src/switch.sym
deleted file mode 100644
index 780b5fc0387..00000000000
--- a/xlators/cluster/dht/src/switch.sym
+++ /dev/null
@@ -1,8 +0,0 @@
-fops
-cbks
-class_methods
-dht_methods
-options
-mem_acct_init
-reconfigure
-dumpops
diff --git a/xlators/cluster/dht/src/tier-common.c b/xlators/cluster/dht/src/tier-common.c
deleted file mode 100644
index c6b54d21d8e..00000000000
--- a/xlators/cluster/dht/src/tier-common.c
+++ /dev/null
@@ -1,1249 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "libxlator.h"
-#include "dht-common.h"
-#include "defaults.h"
-#include "tier-common.h"
-#include "tier.h"
-
-int
-dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno,
-              inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata);
-
-
-int
-tier_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno,
-              inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        loc_t        *oldloc = NULL;
-        loc_t        *newloc = NULL;
-
-        local = frame->local;
-
-        oldloc = &local->loc;
-        newloc = &local->loc2;
-
-        if (op_ret == -1) {
-                /* No continuation on DHT inode missing errors, as we should
-                 * then have a good stbuf that states P2 happened. We would
-                 * get inode missing if, the file completed migrated between
-                 * the lookup and the link call */
-                goto out;
-        }
-
-        if (local->call_cnt != 1) {
-                goto out;
-        }
-
-        local->call_cnt = 2;
-
-        /* Do this on the hot tier now */
-
-        STACK_WIND (frame, tier_link_cbk, local->cached_subvol,
-                    local->cached_subvol->fops->link,
-                    oldloc, newloc, xdata);
-
-        return 0;
-
-out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf,
-                          preparent, postparent, NULL);
-
-        return 0;
-}
-
-
-int
-tier_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        xlator_t    *cached_subvol = NULL;
-        xlator_t    *hashed_subvol = NULL;
-        int          op_errno = -1;
-        int          ret = -1;
-        dht_local_t *local = NULL;
-        dht_conf_t   *conf           = NULL;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (newloc, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, oldloc, NULL, GF_FOP_LINK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->call_cnt = 1;
-
-        cached_subvol = local->cached_subvol;
-
-        if (!cached_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", oldloc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        hashed_subvol = TIER_HASHED_SUBVOL;
-
-        ret = loc_copy (&local->loc2, newloc);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        if (hashed_subvol == cached_subvol) {
-                STACK_WIND (frame, dht_link_cbk,
-                            cached_subvol, cached_subvol->fops->link,
-                            oldloc, newloc, xdata);
-                return 0;
-        }
-
-
-        /* Create hardlinks to both the data file on the hot tier
-           and the linkto file on the cold tier */
-
-        gf_uuid_copy (local->gfid, oldloc->inode->gfid);
-
-        STACK_WIND (frame, tier_link_cbk,
-                    hashed_subvol, hashed_subvol->fops->link,
-                    oldloc, newloc, xdata);
-
-        return 0;
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-                          NULL);
-        return 0;
-}
-
-
-
-int
-tier_create_unlink_stale_linkto_cbk (call_frame_t *frame, void *cookie,
-                                     xlator_t *this, int op_ret, int op_errno,
-                                     struct iatt *preparent,
-                                     struct iatt *postparent, dict_t *xdata)
-{
-
-        dht_local_t     *local = NULL;
-
-        local = frame->local;
-
-        if (local->params) {
-                dict_del (local->params, GLUSTERFS_INTERNAL_FOP_KEY);
-        }
-
-        DHT_STACK_UNWIND (create, frame, -1, local->op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
-
-        return 0;
-}
-
-int
-tier_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno,
-                 fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        xlator_t     *prev           = NULL;
-        int           ret            = -1;
-        dht_local_t  *local          = NULL;
-        xlator_t     *hashed_subvol  = NULL;
-        dht_conf_t   *conf           = NULL;
-
-        local = frame->local;
-        conf  = this->private;
-
-        hashed_subvol = TIER_HASHED_SUBVOL;
-
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (op_ret == -1) {
-                if (local->linked == _gf_true && local->xattr_req) {
-                        local->op_errno = op_errno;
-                        local->op_ret = op_ret;
-                        ret = dht_fill_dict_to_avoid_unlink_of_migrating_file
-                              (local->xattr_req);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "Failed to set dictionary value to "
-                                        "unlink of migrating file");
-                                goto out;
-                        }
-
-                        STACK_WIND (frame,
-                                    tier_create_unlink_stale_linkto_cbk,
-                                    hashed_subvol,
-                                    hashed_subvol->fops->unlink,
-                                    &local->loc, 0, local->xattr_req);
-                        return 0;
-                }
-                goto out;
-        }
-
-        prev = cookie;
-
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           preparent, 0);
-
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
-
-        ret = dht_layout_preset (this, prev, inode);
-        if (ret != 0) {
-                gf_msg_debug (this->name, 0,
-                              "could not set preset layout for subvol %s",
-                              prev->name);
-                op_ret   = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        local->op_errno = op_errno;
-
-        if (local->linked == _gf_true) {
-                local->stbuf = *stbuf;
-                dht_linkfile_attr_heal (frame, this);
-        }
-out:
-        if (local) {
-                if (local->xattr_req) {
-                        dict_del (local->xattr_req, TIER_LINKFILE_GFID);
-                }
-        }
-
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-
-        DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode,
-                          stbuf, preparent, postparent, xdata);
-
-        return 0;
-}
-
-int
-tier_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent,
-                                dict_t *xdata)
-{
-        dht_local_t     *local             = NULL;
-        xlator_t        *cached_subvol     = NULL;
-        dht_conf_t      *conf              = NULL;
-        int              ret               = -1;
-        unsigned char   *gfid              = NULL;
-
-        local = frame->local;
-        if (!local) {
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (op_ret == -1) {
-                local->op_errno = op_errno;
-                goto err;
-        }
-
-        conf = this->private;
-        if (!conf) {
-                local->op_errno = EINVAL;
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        cached_subvol = TIER_UNHASHED_SUBVOL;
-
-        if (local->params) {
-                dict_del (local->params, conf->link_xattr_name);
-                dict_del (local->params, GLUSTERFS_INTERNAL_FOP_KEY);
-        }
-
-        /*
-         * We will delete the linkfile if data file creation fails.
-         * When deleting this stale linkfile, there is a possibility
-         * for a race between this linkfile deletion and a stale
-         * linkfile deletion triggered by another lookup from different
-         * client.
-         *
-         * For eg:
-         *
-         *     Client 1                        Client 2
-         *
-         * 1   linkfile created for foo
-         *
-         * 2   data file creation failed
-         *
-         * 3                                   creating a file with same name
-         *
-         * 4                                   lookup before creation deleted
-         *                                     the linkfile created by client1
-         *                                     considering as a stale linkfile.
-         *
-         * 5                                   New linkfile created for foo
-         *                                     with different gfid.
-         *
-         * 6 Trigger linkfile deletion as
-         *   data file creation failed.
-         *
-         * 7 Linkfile deleted which is
-         *   created by client2.
-         *
-         * 8                                   Data file created.
-         *
-         * With this race, we will end up having a file in a non-hashed subvol
-         * without a linkfile in hashed subvol.
-         *
-         * To avoid this, we store the gfid of linkfile created by client, So
-         * If we delete the linkfile , we validate gfid of existing file with
-         * stored value from posix layer.
-         *
-         * Storing this value in local->xattr_req as local->params was also used
-         * to create the data file. During the linkfile deletion we will use
-         * local->xattr_req dictionary.
-         */
-        if (!local->xattr_req) {
-                local->xattr_req = dict_new ();
-                if (!local->xattr_req) {
-                        local->op_errno = ENOMEM;
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-        }
-
-        gfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
-        if (!gfid) {
-                local->op_errno = ENOMEM;
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        gf_uuid_copy (gfid, stbuf->ia_gfid);
-        ret = dict_set_dynptr (local->xattr_req, TIER_LINKFILE_GFID,
-                                   gfid, sizeof (uuid_t));
-        if (ret) {
-                GF_FREE (gfid);
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value"
-                        " : key = %s", TIER_LINKFILE_GFID);
-        }
-
-        STACK_WIND_COOKIE (frame, tier_create_cbk, cached_subvol,
-                           cached_subvol, cached_subvol->fops->create,
-                           &local->loc, local->flags, local->mode,
-                           local->umask, local->fd, local->params);
-
-        return 0;
-err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL, NULL);
-        return 0;
-}
-
-gf_boolean_t
-tier_is_hot_tier_decommissioned (xlator_t *this)
-{
-        dht_conf_t              *conf        = NULL;
-        xlator_t                *hot_tier    = NULL;
-        int                      i           = 0;
-
-        conf = this->private;
-        hot_tier = conf->subvolumes[1];
-
-        if (conf->decommission_subvols_cnt) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->decommissioned_bricks[i] &&
-                                conf->decommissioned_bricks[i] == hot_tier)
-                                return _gf_true;
-                }
-        }
-
-        return _gf_false;
-}
-
-int
-tier_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *params)
-{
-        int                     op_errno            = -1;
-        dht_local_t             *local              = NULL;
-        dht_conf_t              *conf               = NULL;
-        xlator_t                *hot_subvol         = NULL;
-        xlator_t                *cold_subvol        = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-
-        cold_subvol = TIER_HASHED_SUBVOL;
-        hot_subvol = TIER_UNHASHED_SUBVOL;
-
-        if (conf->subvolumes[0] != cold_subvol) {
-                hot_subvol = conf->subvolumes[0];
-        }
-        /*
-         * if hot tier full, write to cold.
-         * Also if hot tier is full, create in cold
-         */
-        if (dht_is_subvol_filled (this, hot_subvol) ||
-            tier_is_hot_tier_decommissioned (this)) {
-                gf_msg_debug (this->name, 0,
-                              "creating %s on %s", loc->path,
-                              cold_subvol->name);
-
-                STACK_WIND_COOKIE (frame, tier_create_cbk, cold_subvol,
-                                   cold_subvol, cold_subvol->fops->create,
-                                   loc, flags, mode, umask, fd, params);
-        } else {
-                local->params = dict_ref (params);
-                local->flags = flags;
-                local->mode = mode;
-                local->umask = umask;
-                local->cached_subvol = hot_subvol;
-                local->hashed_subvol = cold_subvol;
-
-                gf_msg_debug (this->name, 0,
-                              "creating %s on %s (link at %s)", loc->path,
-                              hot_subvol->name, cold_subvol->name);
-
-                dht_linkfile_create (frame, tier_create_linkfile_create_cbk,
-                                     this, hot_subvol, cold_subvol, loc);
-
-                goto out;
-        }
-out:
-        return 0;
-
-err:
-
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL, NULL);
-
-        return 0;
-}
-
-int
-tier_unlink_nonhashed_linkfile_cbk (call_frame_t *frame, void *cookie,
-                                    xlator_t *this, int op_ret, int op_errno,
-                                    struct iatt *preparent,
-                                    struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        xlator_t     *prev = NULL;
-
-        local = frame->local;
-        prev  = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if ((op_ret == -1) && (op_errno != ENOENT)) {
-                        local->op_errno = op_errno;
-                        local->op_ret = op_ret;
-                        gf_msg_debug (this->name, op_errno,
-                                      "Unlink link: subvolume %s"
-                                      " returned -1",
-                                      prev->name);
-                        goto unlock;
-                }
-
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (local->op_ret == -1)
-                goto err;
-        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
-
-
-        return 0;
-
-err:
-        DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno,
-                          NULL, NULL, NULL);
-        return 0;
-}
-
-int
-tier_unlink_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, inode_t *inode,
-                        struct iatt *preparent, dict_t *xdata,
-                        struct iatt *postparent)
-{
-        dht_local_t             *local             = NULL;
-        xlator_t                *prev              = NULL;
-        dht_conf_t              *conf              = NULL;
-        xlator_t                *hot_subvol        = NULL;
-
-        local = frame->local;
-        prev  = cookie;
-        conf = this->private;
-        hot_subvol = TIER_UNHASHED_SUBVOL;
-
-        if (!op_ret) {
-                /*
-                 * linkfile present on hot tier. unlinking the linkfile
-                 */
-                STACK_WIND_COOKIE (frame, tier_unlink_nonhashed_linkfile_cbk,
-                                   hot_subvol, hot_subvol, hot_subvol->fops->unlink,
-                                   &local->loc, local->flags, NULL);
-                return 0;
-        }
-
-        LOCK (&frame->lock);
-        {
-                if (op_errno == ENOENT) {
-                        local->op_ret   = 0;
-                        local->op_errno = op_errno;
-                } else {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                }
-                gf_msg_debug (this->name, op_errno,
-                              "Lookup : subvolume %s returned -1",
-                               prev->name);
-        }
-
-        UNLOCK (&frame->lock);
-
-        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, xdata);
-
-        return 0;
-}
-
-int
-tier_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        xlator_t     *prev = NULL;
-
-        local = frame->local;
-        prev  = cookie;
-
-        LOCK (&frame->lock);
-        {
-                /* Ignore EINVAL for tier to ignore error when the file
-                        does not exist on the other tier  */
-                if ((op_ret == -1) && !((op_errno == ENOENT) ||
-                                        (op_errno == EINVAL))) {
-                        local->op_errno = op_errno;
-                        local->op_ret   = op_ret;
-                        gf_msg_debug (this->name, op_errno,
-                                      "Unlink link: subvolume %s"
-                                      " returned -1",
-                                      prev->name);
-                        goto unlock;
-                }
-
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (local->op_ret == -1)
-                goto err;
-
-        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, xdata);
-
-        return 0;
-
-err:
-        DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno,
-                          NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-tier_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t     *local          = NULL;
-        xlator_t        *prev           = NULL;
-        struct iatt     *stbuf          = NULL;
-        dht_conf_t      *conf           = NULL;
-        int              ret            = -1;
-        xlator_t        *hot_tier       = NULL;
-        xlator_t        *cold_tier      = NULL;
-
-        local = frame->local;
-        prev  = cookie;
-        conf  = this->private;
-
-        cold_tier = TIER_HASHED_SUBVOL;
-        hot_tier = TIER_UNHASHED_SUBVOL;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno == ENOENT) {
-                                local->op_ret = 0;
-                        } else {
-                                local->op_ret   = -1;
-                                local->op_errno = op_errno;
-                        }
-                        gf_msg_debug (this->name, op_errno,
-                                      "Unlink: subvolume %s returned -1"
-                                      " with errno = %d",
-                                       prev->name, op_errno);
-                        goto unlock;
-                }
-
-                local->op_ret = 0;
-
-                local->postparent = *postparent;
-                local->preparent = *preparent;
-
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->preparent, 0);
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (local->op_ret)
-                goto out;
-
-        if (cold_tier != local->cached_subvol) {
-                /*
-                 * File is present in hot tier, so there will be
-                 * a link file on cold tier, deleting the linkfile
-                 * from cold tier
-                 */
-                STACK_WIND_COOKIE (frame, tier_unlink_linkfile_cbk, cold_tier,
-                                   cold_tier, cold_tier->fops->unlink,
-                                   &local->loc,
-                                   local->flags, xdata);
-                return 0;
-        }
-
-        ret = dict_get_bin (xdata, DHT_IATT_IN_XDATA_KEY, (void **) &stbuf);
-        if (!ret && stbuf && ((IS_DHT_MIGRATION_PHASE2 (stbuf)) ||
-                      IS_DHT_MIGRATION_PHASE1 (stbuf))) {
-                /*
-                 * File is migrating from cold to hot tier.
-                 * Delete the destination linkfile.
-                 */
-                STACK_WIND_COOKIE (frame, tier_unlink_lookup_cbk, hot_tier,
-                                   hot_tier, hot_tier->fops->lookup,
-                                   &local->loc, NULL);
-                return 0;
-
-        }
-
-out:
-        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, xdata);
-
-        return 0;
-}
-
-int
-tier_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-            dict_t *xdata)
-{
-        xlator_t                *cached_subvol = NULL;
-        xlator_t                *hashed_subvol = NULL;
-        dht_conf_t              *conf          = NULL;
-        int                      op_errno      = -1;
-        dht_local_t             *local         = NULL;
-        int                      ret           = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_UNLINK);
-        if (!local) {
-                op_errno = ENOMEM;
-
-                goto err;
-        }
-
-        hashed_subvol = TIER_HASHED_SUBVOL;
-
-        cached_subvol = local->cached_subvol;
-        if (!cached_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        local->flags = xflag;
-        if (IA_ISREG (loc->inode->ia_type) &&
-            (hashed_subvol == cached_subvol)) {
-                /*
-                 * File resides in cold tier. We need to stat
-                 * the file to see if it is being promoted.
-                 * If yes we need to delete the destination
-                 * file as well.
-                 *
-                 * Currently we are doing this check only for
-                 * regular files.
-                 */
-                xdata = xdata ? dict_ref (xdata) : dict_new ();
-                if (xdata) {
-                        ret = dict_set_int8 (xdata, DHT_IATT_IN_XDATA_KEY, 1);
-                        if (ret) {
-                                gf_msg_debug (this->name, 0,
-                                        "Failed to set dictionary key %s",
-                                         DHT_IATT_IN_XDATA_KEY);
-                        }
-                }
-        }
-
-        /*
-         * File is on hot tier, delete the data file first, then
-         * linkfile from cold.
-         */
-        STACK_WIND_COOKIE (frame, tier_unlink_cbk, cached_subvol,
-                           cached_subvol, cached_subvol->fops->unlink, loc,
-                           xflag, xdata);
-        if (xdata)
-                dict_unref (xdata);
-        return 0;
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-
-        return 0;
-}
-
-int
-tier_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries,
-                 dict_t *xdata)
-{
-        gf_dirent_t   entries;
-        gf_dirent_t  *orig_entry = NULL;
-        gf_dirent_t  *entry = NULL;
-        int           count = 0;
-
-        INIT_LIST_HEAD (&entries.list);
-
-        if (op_ret < 0)
-                goto unwind;
-
-        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
-                entry = gf_dirent_for_name (orig_entry->d_name);
-                if (!entry) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                DHT_MSG_NO_MEMORY,
-                                "Memory allocation failed ");
-                        goto unwind;
-                }
-
-                entry->d_off  = orig_entry->d_off;
-                entry->d_ino  = orig_entry->d_ino;
-                entry->d_type = orig_entry->d_type;
-                entry->d_len  = orig_entry->d_len;
-
-                list_add_tail (&entry->list, &entries.list);
-                count++;
-        }
-        op_ret = count;
-
-unwind:
-        if (op_ret < 0)
-                op_ret = 0;
-
-        DHT_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, NULL);
-
-        gf_dirent_free (&entries);
-
-        return 0;
-}
-
-int
-tier_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-                  int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        gf_dirent_t   entries;
-        gf_dirent_t  *orig_entry = NULL;
-        gf_dirent_t  *entry = NULL;
-        xlator_t     *prev = NULL;
-        xlator_t     *next_subvol = NULL;
-        off_t         next_offset = 0;
-        int           count = 0;
-        dht_conf_t   *conf   = NULL;
-        int           ret    = 0;
-        inode_table_t           *itable = NULL;
-        inode_t                 *inode = NULL;
-
-        INIT_LIST_HEAD (&entries.list);
-        prev = cookie;
-        local = frame->local;
-        itable = local->fd ? local->fd->inode->table : NULL;
-
-        conf  = this->private;
-        GF_VALIDATE_OR_GOTO(this->name, conf, unwind);
-
-        if (op_ret < 0)
-                goto done;
-
-        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
-                next_offset = orig_entry->d_off;
-
-                if (IA_ISINVAL(orig_entry->d_stat.ia_type)) {
-                        /*stat failed somewhere- ignore this entry*/
-                        continue;
-                }
-
-                entry = gf_dirent_for_name (orig_entry->d_name);
-                if (!entry) {
-
-                        goto unwind;
-                }
-
-                entry->d_off  = orig_entry->d_off;
-                entry->d_stat = orig_entry->d_stat;
-                entry->d_ino  = orig_entry->d_ino;
-                entry->d_type = orig_entry->d_type;
-                entry->d_len  = orig_entry->d_len;
-
-                if (orig_entry->dict)
-                        entry->dict = dict_ref (orig_entry->dict);
-
-                if (check_is_linkfile (NULL, (&orig_entry->d_stat),
-                                       orig_entry->dict,
-                                       conf->link_xattr_name)) {
-                        goto entries;
-
-                } else if (IA_ISDIR(entry->d_stat.ia_type)) {
-                        if (orig_entry->inode) {
-                                dht_inode_ctx_time_update (orig_entry->inode,
-                                                           this, &entry->d_stat,
-                                                           1);
-                        }
-                } else {
-                        if (orig_entry->inode) {
-                                ret = dht_layout_preset (this, prev,
-                                                         orig_entry->inode);
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                                DHT_MSG_LAYOUT_SET_FAILED,
-                                                "failed to link the layout "
-                                                "in inode");
-
-                                entry->inode = inode_ref (orig_entry->inode);
-                        } else if (itable) {
-                                /*
-                                 * orig_entry->inode might be null if any upper
-                                 * layer xlators below client set to null, to
-                                 * force a lookup on the inode even if the inode
-                                 * is present in the inode table. In that case
-                                 * we just update the ctx to make sure we didn't
-                                 * missed anything.
-                                 */
-                                inode = inode_find (itable,
-                                                    orig_entry->d_stat.ia_gfid);
-                                if (inode) {
-                                        ret = dht_layout_preset
-                                                (this, TIER_HASHED_SUBVOL,
-                                                 inode);
-                                        if (ret)
-                                                gf_msg (this->name,
-                                                     GF_LOG_WARNING, 0,
-                                                     DHT_MSG_LAYOUT_SET_FAILED,
-                                                     "failed to link the layout"
-                                                     " in inode");
-                                        inode_unref (inode);
-                                        inode = NULL;
-                                }
-                        }
-                }
-
-entries:
-                list_add_tail (&entry->list, &entries.list);
-                count++;
-        }
-        op_ret = count;
-
-done:
-        if (count == 0) {
-                /* non-zero next_offset means that
-                   EOF is not yet hit on the current subvol
-                */
-                if (next_offset != 0) {
-                        next_subvol = prev;
-                } else {
-                        goto unwind;
-                }
-
-                STACK_WIND_COOKIE (frame, tier_readdirp_cbk, next_subvol,
-                                   next_subvol, next_subvol->fops->readdirp,
-                                   local->fd, local->size, next_offset,
-                                   local->xattr);
-                return 0;
-        }
-
-unwind:
-        if (op_ret < 0)
-                op_ret = 0;
-
-        DHT_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries, NULL);
-
-        gf_dirent_free (&entries);
-
-        return 0;
-}
-
-int
-tier_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t yoff, int whichop, dict_t *dict)
-{
-        dht_local_t  *local         = NULL;
-        int           op_errno      = -1;
-        xlator_t     *hashed_subvol = NULL;
-        int           ret           = 0;
-        dht_conf_t   *conf          = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, NULL, NULL, whichop);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->fd = fd_ref (fd);
-        local->size = size;
-        local->xattr_req = (dict) ? dict_ref (dict) : NULL;
-
-        hashed_subvol = TIER_HASHED_SUBVOL;
-
-
-        /* TODO: do proper readdir */
-        if (whichop == GF_FOP_READDIRP) {
-                if (dict)
-                        local->xattr = dict_ref (dict);
-                else
-                        local->xattr = dict_new ();
-
-                if (local->xattr) {
-                        ret = dict_set_uint32 (local->xattr,
-                                               conf->link_xattr_name, 256);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "Failed to set dictionary value"
-                                        " : key = %s",
-                                        conf->link_xattr_name);
-
-                }
-
-                STACK_WIND_COOKIE (frame, tier_readdirp_cbk, hashed_subvol,
-                                   hashed_subvol, hashed_subvol->fops->readdirp,
-                                   fd, size, yoff, local->xattr);
-
-        } else {
-                STACK_WIND_COOKIE (frame, tier_readdir_cbk, hashed_subvol,
-                                   hashed_subvol, hashed_subvol->fops->readdir,
-                                   fd, size, yoff, local->xattr);
-        }
-
-        return 0;
-
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
-
-        return 0;
-}
-
-int
-tier_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t yoff, dict_t *xdata)
-{
-        int          op = GF_FOP_READDIR;
-        dht_conf_t  *conf = NULL;
-        int          i = 0;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->subvolume_status[i]) {
-                        op = GF_FOP_READDIRP;
-                        break;
-                }
-        }
-
-        if (conf->use_readdirp)
-                op = GF_FOP_READDIRP;
-
-out:
-        tier_do_readdir (frame, this, fd, size, yoff, op, 0);
-        return 0;
-}
-
-int
-tier_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-               off_t yoff, dict_t *dict)
-{
-        tier_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIRP, dict);
-        return 0;
-}
-
-int
-tier_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct statvfs *statvfs,
-                dict_t *xdata)
-{
-        gf_boolean_t            event              = _gf_false;
-        qdstatfs_action_t       action             = qdstatfs_action_OFF;
-        dht_local_t             *local             = NULL;
-        int                     this_call_cnt      = 0;
-        int                     bsize              = 0;
-        int                     frsize             = 0;
-        GF_UNUSED int           ret                = 0;
-        unsigned long           new_usage          = 0;
-        unsigned long           cur_usage          = 0;
-        xlator_t                *prev              = NULL;
-        dht_conf_t              *conf              = NULL;
-        tier_statvfs_t          *tier_stat         = NULL;
-
-        prev = cookie;
-        local = frame->local;
-        GF_ASSERT (local);
-
-        conf = this->private;
-
-        if (xdata)
-                ret = dict_get_int8 (xdata, "quota-deem-statfs",
-                                     (int8_t *)&event);
-
-        tier_stat = &local->tier_statvfs;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        goto unlock;
-                }
-                if (!statvfs) {
-                        op_errno = EINVAL;
-                        local->op_ret = -1;
-                        goto unlock;
-                }
-                local->op_ret = 0;
-
-                if (local->quota_deem_statfs) {
-                        if (event == _gf_true) {
-                                action = qdstatfs_action_COMPARE;
-                        } else {
-                                action = qdstatfs_action_NEGLECT;
-                        }
-                } else {
-
-                        if (event == _gf_true) {
-                                action = qdstatfs_action_REPLACE;
-                                local->quota_deem_statfs = _gf_true;
-                        }
-                }
-
-                if (local->quota_deem_statfs) {
-                        switch (action) {
-                        case qdstatfs_action_NEGLECT:
-                                goto unlock;
-
-                        case qdstatfs_action_REPLACE:
-                                local->statvfs = *statvfs;
-                                goto unlock;
-
-                        case qdstatfs_action_COMPARE:
-                                new_usage = statvfs->f_blocks -
-                                             statvfs->f_bfree;
-                                cur_usage = local->statvfs.f_blocks -
-                                             local->statvfs.f_bfree;
-
-                                /* Take the max of the usage from subvols */
-                                if (new_usage >= cur_usage)
-                                        local->statvfs = *statvfs;
-                                goto unlock;
-
-                        default:
-                                break;
-                        }
-                }
-
-                if (local->statvfs.f_bsize != 0) {
-                        bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
-                        frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
-                        dht_normalize_stats(&local->statvfs, bsize, frsize);
-                        dht_normalize_stats(statvfs, bsize, frsize);
-                } else {
-                        local->statvfs.f_bsize    = statvfs->f_bsize;
-                        local->statvfs.f_frsize   = statvfs->f_frsize;
-                }
-
-                if (prev == TIER_HASHED_SUBVOL) {
-                        local->statvfs.f_blocks   = statvfs->f_blocks;
-                        local->statvfs.f_files    = statvfs->f_files;
-                        local->statvfs.f_fsid     = statvfs->f_fsid;
-                        local->statvfs.f_flag     = statvfs->f_flag;
-                        local->statvfs.f_namemax  = statvfs->f_namemax;
-                        tier_stat->blocks_used    = (statvfs->f_blocks - statvfs->f_bfree);
-                        tier_stat->pblocks_used   = (statvfs->f_blocks - statvfs->f_bavail);
-                        tier_stat->files_used     = (statvfs->f_files - statvfs->f_ffree);
-                        tier_stat->pfiles_used    = (statvfs->f_files - statvfs->f_favail);
-                        tier_stat->hashed_fsid    = statvfs->f_fsid;
-                } else {
-                        tier_stat->unhashed_fsid      = statvfs->f_fsid;
-                        tier_stat->unhashed_blocks_used    = (statvfs->f_blocks - statvfs->f_bfree);
-                        tier_stat->unhashed_pblocks_used   = (statvfs->f_blocks - statvfs->f_bavail);
-                        tier_stat->unhashed_files_used     = (statvfs->f_files - statvfs->f_ffree);
-                        tier_stat->unhashed_pfiles_used    = (statvfs->f_files - statvfs->f_favail);
-                }
-
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                if (tier_stat->unhashed_fsid != tier_stat->hashed_fsid) {
-                        tier_stat->blocks_used    += tier_stat->unhashed_blocks_used;
-                        tier_stat->pblocks_used   += tier_stat->unhashed_pblocks_used;
-                        tier_stat->files_used     += tier_stat->unhashed_files_used;
-                        tier_stat->pfiles_used    += tier_stat->unhashed_pfiles_used;
-                }
-                local->statvfs.f_bfree = local->statvfs.f_blocks -
-                        tier_stat->blocks_used;
-                local->statvfs.f_bavail = local->statvfs.f_blocks -
-                        tier_stat->pblocks_used;
-                local->statvfs.f_ffree = local->statvfs.f_files -
-                        tier_stat->files_used;
-                local->statvfs.f_favail = local->statvfs.f_files -
-                        tier_stat->pfiles_used;
-                DHT_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->statvfs, xdata);
-        }
-
-        return 0;
-}
-
-
-int
-tier_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-        dht_local_t      *local  = NULL;
-        dht_conf_t       *conf = NULL;
-        int               op_errno = -1;
-        int               i = -1;
-        inode_t          *inode         = NULL;
-        inode_table_t    *itable        = NULL;
-        uuid_t            root_gfid     = {0, };
-        loc_t             newloc        = {0, };
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, NULL, NULL, GF_FOP_STATFS);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        if (loc->inode && !IA_ISDIR (loc->inode->ia_type)) {
-                itable = loc->inode->table;
-                if (!itable) {
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                loc = &local->loc2;
-                root_gfid[15] = 1;
-
-                inode = inode_find (itable, root_gfid);
-                if (!inode) {
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                dht_build_root_loc (inode, &newloc);
-                loc = &newloc;
-        }
-
-        local->call_cnt = conf->subvolume_cnt;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND_COOKIE (frame, tier_statfs_cbk, conf->subvolumes[i],
-                                   conf->subvolumes[i],
-                                   conf->subvolumes[i]->fops->statfs, loc,
-                                   xdata);
-        }
-
-        return 0;
-
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
-
-        return 0;
-}
diff --git a/xlators/cluster/dht/src/tier-common.h b/xlators/cluster/dht/src/tier-common.h
deleted file mode 100644
index 26e01d7f141..00000000000
--- a/xlators/cluster/dht/src/tier-common.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _TIER_COMMON_H_
-#define _TIER_COMMON_H_
-/* Function definitions */
-int
-tier_create_unlink_stale_linkto_cbk (call_frame_t *frame, void *cookie,
-                                     xlator_t *this, int op_ret, int op_errno,
-                                     struct iatt *preparent,
-                                     struct iatt *postparent, dict_t *xdata);
-
-int
-tier_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno,
-                 fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                 struct iatt *preparent, struct iatt *postparent,
-                 dict_t *xdata);
-
-int
-tier_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                 xlator_t *this,
-                                 int32_t op_ret, int32_t op_errno,
-                                 inode_t *inode, struct iatt *stbuf,
-                                 struct iatt *preparent,
-                                 struct iatt *postparent,
-                                 dict_t *xdata);
-
-int
-tier_create (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, int32_t flags, mode_t mode,
-             mode_t umask, fd_t *fd, dict_t *params);
-
-int32_t
-tier_unlink (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, int xflag, dict_t *xdata);
-
-int32_t
-tier_readdirp (call_frame_t *frame,
-               xlator_t *this,
-               fd_t     *fd,
-               size_t    size, off_t off, dict_t *dict);
-
-int
-tier_readdir (call_frame_t *frame,
-              xlator_t *this, fd_t *fd, size_t size,
-              off_t yoff, dict_t *xdata);
-
-
-
-int
-tier_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata);
-
-
-int
-tier_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
-
-
-#endif
-
diff --git a/xlators/cluster/dht/src/tier.c b/xlators/cluster/dht/src/tier.c
deleted file mode 100644
index 5fa50a26d80..00000000000
--- a/xlators/cluster/dht/src/tier.c
+++ /dev/null
@@ -1,3326 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include <dlfcn.h>
-
-#include "dht-common.h"
-#include "tier.h"
-#include "tier-common.h"
-#include "syscall.h"
-#include "events.h"
-#include "tier-ctr-interface.h"
-
-/*Hard coded DB info*/
-static gfdb_db_type_t dht_tier_db_type = GFDB_SQLITE3;
-/*Hard coded DB info*/
-
-/*Mutex for updating the data movement stats*/
-static pthread_mutex_t dm_stat_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-/* Stores the path location of promotion query files */
-static char *promotion_qfile;
-/* Stores the path location of demotion query files */
-static char *demotion_qfile;
-
-static void *libhandle;
-static gfdb_methods_t gfdb_methods;
-
-#define DB_QUERY_RECORD_SIZE 4096
-
-/*
- * Closes all the fds and frees the qfile_array
- * */
-static void
-qfile_array_free (tier_qfile_array_t *qfile_array)
-{
-        ssize_t i = 0;
-
-        if (qfile_array) {
-                if (qfile_array->fd_array) {
-                        for (i = 0; i < qfile_array->array_size; i++) {
-                                if (qfile_array->fd_array[i] != -1) {
-                                        sys_close (qfile_array->fd_array[i]);
-                                }
-                        }
-                }
-                GF_FREE (qfile_array->fd_array);
-        }
-        GF_FREE (qfile_array);
-}
-
-
-/* Create a new query file list with given size */
-static tier_qfile_array_t *
-qfile_array_new (ssize_t array_size)
-{
-        int ret                         = -1;
-        tier_qfile_array_t *qfile_array   = NULL;
-        ssize_t i                       = 0;
-
-        GF_VALIDATE_OR_GOTO ("tier", (array_size > 0), out);
-
-        qfile_array = GF_CALLOC (1, sizeof (tier_qfile_array_t),
-                                gf_tier_mt_qfile_array_t);
-        if (!qfile_array) {
-                gf_msg ("tier", GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to allocate memory for tier_qfile_array_t");
-                goto out;
-        }
-
-        qfile_array->fd_array = GF_CALLOC (array_size, sizeof (int),
-                                        gf_dht_mt_int32_t);
-        if (!qfile_array->fd_array) {
-                gf_msg ("tier", GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to allocate memory for "
-                        "tier_qfile_array_t->fd_array");
-                goto out;
-        }
-
-        /* Init all the fds to -1 */
-        for (i = 0; i < array_size; i++) {
-                qfile_array->fd_array[i] = -1;
-        }
-
-        qfile_array->array_size = array_size;
-        qfile_array->next_index = 0;
-
-        /* Set exhausted count to list size as the list is empty */
-        qfile_array->exhausted_count = qfile_array->array_size;
-
-        ret = 0;
-out:
-        if (ret) {
-                qfile_array_free (qfile_array);
-                qfile_array = NULL;
-        }
-        return qfile_array;
-}
-
-
-/* Checks if the query file list is empty or totally exhausted. */
-static gf_boolean_t
-is_qfile_array_empty (tier_qfile_array_t *qfile_array)
-{
-        return (qfile_array->exhausted_count == qfile_array->array_size) ?
-                _gf_true : _gf_false;
-}
-
-
-/* Shifts the next_fd pointer to the next available fd in the list */
-static void
-shift_next_index (tier_qfile_array_t *qfile_array)
-{
-        int qfile_fd = 0;
-        int spin_count = 0;
-
-        if (is_qfile_array_empty (qfile_array)) {
-                return;
-        }
-
-        do {
-                /* change next_index in a rotional manner */
-                (qfile_array->next_index == (qfile_array->array_size - 1)) ?
-                qfile_array->next_index = 0 : qfile_array->next_index++;
-
-                qfile_fd = (qfile_array->fd_array[qfile_array->next_index]);
-
-                spin_count++;
-
-        } while ((qfile_fd == -1) && (spin_count < qfile_array->array_size));
-
-}
-
-/*
- * This is a non-thread safe function to read query records
- * from a list of query files in a Round-Robin manner.
- * As in when the query files get exhuasted they are closed.
- * Returns:
- * 0   if all the query records in all the query files of the list are
- *     exhausted.
- * > 0 if a query record is successfully read. Indicates the size of the query
- *     record read.
- * < 0  if there was failure
- * */
-static int
-read_query_record_list (tier_qfile_array_t *qfile_array,
-                          gfdb_query_record_t **query_record)
-{
-        int ret = -1;
-        int qfile_fd = 0;
-
-        GF_VALIDATE_OR_GOTO ("tier", qfile_array, out);
-        GF_VALIDATE_OR_GOTO ("tier", qfile_array->fd_array, out);
-
-        do {
-                if (is_qfile_array_empty (qfile_array)) {
-                        ret = 0;
-                        break;
-                }
-
-                qfile_fd = qfile_array->fd_array[qfile_array->next_index];
-                ret = gfdb_methods.gfdb_read_query_record
-                        (qfile_fd, query_record);
-                if (ret <= 0) {
-                        /*The qfile_fd has reached EOF or
-                         * there was an error.
-                         * 1. Close the exhausted fd
-                         * 2. increment the exhausted count
-                         * 3. shift next_qfile to next qfile
-                         **/
-                         sys_close (qfile_fd);
-                         qfile_array->fd_array[qfile_array->next_index] = -1;
-                         qfile_array->exhausted_count++;
-                         /* shift next_qfile to next qfile */
-                         shift_next_index (qfile_array);
-                         continue;
-                } else {
-                        /* shift next_qfile to next qfile */
-                        shift_next_index (qfile_array);
-                        break;
-                }
-        } while (1);
-out:
-        return ret;
-}
-
-
-/* Check and update the watermark every WM_INTERVAL seconds */
-#define WM_INTERVAL            5
-#define WM_INTERVAL_EMERG      1
-
-static int
-tier_check_same_node (xlator_t *this, loc_t *loc, gf_defrag_info_t *defrag)
-{
-        int         ret                     = -1;
-        dict_t     *dict                    = NULL;
-        char       *uuid_str                = NULL;
-        uuid_t      node_uuid               = {0,};
-
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, defrag, out);
-
-        if (syncop_getxattr (this, loc, &dict, GF_XATTR_NODE_UUID_KEY,
-                             NULL, NULL)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Unable to get NODE_UUID_KEY %s %s\n",
-                        loc->name, loc->path);
-                goto out;
-        }
-
-        if (dict_get_str (dict, GF_XATTR_NODE_UUID_KEY, &uuid_str) < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to get node-uuids for %s", loc->path);
-                goto out;
-        }
-
-
-        if (gf_uuid_parse (uuid_str, node_uuid)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "uuid_parse failed for %s", loc->path);
-                goto out;
-        }
-
-        if (gf_uuid_compare (node_uuid, defrag->node_uuid)) {
-                gf_msg_debug (this->name, 0,
-                              "%s does not belong to this node", loc->path);
-                ret = 1;
-                goto out;
-        }
-
-        ret = 0;
-out:
-        if (dict)
-                dict_unref(dict);
-
-        return ret;
-}
-
-int
-tier_get_fs_stat (xlator_t *this, loc_t *root_loc)
-{
-        int                     ret = 0;
-        gf_defrag_info_t       *defrag = NULL;
-        dht_conf_t             *conf   = NULL;
-        dict_t                 *xdata  = NULL;
-        struct statvfs          statfs = {0, };
-        gf_tier_conf_t         *tier_conf = NULL;
-
-
-        conf = this->private;
-        if (!conf) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "conf is NULL");
-                ret = -1;
-                goto exit;
-        }
-
-        defrag = conf->defrag;
-        if (!defrag) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "defrag is NULL");
-                ret = -1;
-                goto exit;
-        }
-
-        tier_conf = &defrag->tier_conf;
-
-        xdata = dict_new ();
-        if (!xdata) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        DHT_MSG_NO_MEMORY,
-                        "failed to allocate dictionary");
-                ret = -1;
-                goto exit;
-        }
-
-        ret = dict_set_int8 (xdata, GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set "
-                        GF_INTERNAL_IGNORE_DEEM_STATFS" in dict");
-                ret = -1;
-                goto exit;
-        }
-
-        /* Find how much free space is on the hot subvolume.
-         * Then see if that value */
-        /* is less than or greater than user defined watermarks.
-         * Stash results in */
-        /* the tier_conf data structure. */
-
-        ret = syncop_statfs (conf->subvolumes[1], root_loc, &statfs,
-                             xdata, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Unable to obtain statfs.");
-                goto exit;
-        }
-
-        pthread_mutex_lock (&dm_stat_mutex);
-
-        tier_conf->block_size = statfs.f_bsize;
-        tier_conf->blocks_total = statfs.f_blocks;
-        tier_conf->blocks_used = statfs.f_blocks - statfs.f_bfree;
-
-        tier_conf->percent_full = GF_PERCENTAGE(tier_conf->blocks_used,
-                                                statfs.f_blocks);
-        pthread_mutex_unlock (&dm_stat_mutex);
-
-exit:
-        if (xdata)
-                dict_unref (xdata);
-        return ret;
-}
-
-static void
-tier_send_watermark_event (const char *volname,
-                           tier_watermark_op_t old_wm,
-                           tier_watermark_op_t new_wm)
-{
-        if (old_wm == TIER_WM_LOW || old_wm == TIER_WM_NONE) {
-                if (new_wm == TIER_WM_MID) {
-                        gf_event (EVENT_TIER_WATERMARK_RAISED_TO_MID,
-                                  "vol=%s", volname);
-                } else if (new_wm == TIER_WM_HI) {
-                        gf_event (EVENT_TIER_WATERMARK_HI, "vol=%s", volname);
-                }
-        } else if (old_wm == TIER_WM_MID) {
-                if (new_wm == TIER_WM_LOW) {
-                        gf_event (EVENT_TIER_WATERMARK_DROPPED_TO_LOW,
-                                  "vol=%s", volname);
-                } else if (new_wm == TIER_WM_HI) {
-                        gf_event (EVENT_TIER_WATERMARK_HI, "vol=%s", volname);
-                }
-        } else if (old_wm == TIER_WM_HI) {
-                if (new_wm == TIER_WM_MID) {
-                        gf_event (EVENT_TIER_WATERMARK_DROPPED_TO_MID,
-                                  "vol=%s", volname);
-                } else if (new_wm == TIER_WM_LOW) {
-                        gf_event (EVENT_TIER_WATERMARK_DROPPED_TO_LOW,
-                                  "vol=%s", volname);
-                }
-        }
-}
-
-int
-tier_check_watermark (xlator_t *this)
-{
-        int                     ret       = -1;
-        gf_defrag_info_t       *defrag    = NULL;
-        dht_conf_t             *conf      = NULL;
-        gf_tier_conf_t         *tier_conf = NULL;
-        tier_watermark_op_t     wm        = TIER_WM_NONE;
-
-        conf = this->private;
-        if (!conf)
-                goto exit;
-
-        defrag = conf->defrag;
-        if (!defrag)
-                goto exit;
-
-        tier_conf = &defrag->tier_conf;
-
-        if (tier_conf->percent_full < tier_conf->watermark_low) {
-                wm = TIER_WM_LOW;
-
-        } else if (tier_conf->percent_full < tier_conf->watermark_hi) {
-                wm = TIER_WM_MID;
-
-        } else {
-                wm = TIER_WM_HI;
-        }
-
-        if (wm != tier_conf->watermark_last) {
-
-                tier_send_watermark_event (tier_conf->volname,
-                                           tier_conf->watermark_last,
-                                           wm);
-
-                tier_conf->watermark_last = wm;
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Tier watermark now %d", wm);
-        }
-
-        ret = 0;
-
-exit:
-        return ret;
-}
-
-
-static gf_boolean_t
-is_hot_tier_full (gf_tier_conf_t *tier_conf)
-{
-        if (tier_conf && (tier_conf->mode == TIER_MODE_WM) &&
-           (tier_conf->watermark_last == TIER_WM_HI))
-                return _gf_true;
-
-        return _gf_false;
-}
-
-int
-tier_do_migration (xlator_t *this, int promote)
-{
-        gf_defrag_info_t       *defrag = NULL;
-        dht_conf_t             *conf   = NULL;
-        long                    rand = 0;
-        int                     migrate = 0;
-        gf_tier_conf_t         *tier_conf = NULL;
-
-        conf = this->private;
-        if (!conf)
-                goto exit;
-
-        defrag = conf->defrag;
-        if (!defrag)
-                goto exit;
-
-        if (tier_check_watermark (this) != 0) {
-                gf_msg (this->name, GF_LOG_CRITICAL, errno,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to get watermark");
-                goto exit;
-        }
-
-        tier_conf = &defrag->tier_conf;
-
-        switch (tier_conf->watermark_last) {
-        case TIER_WM_LOW:
-                migrate = promote ? 1 : 0;
-                break;
-        case TIER_WM_HI:
-                migrate = promote ? 0 : 1;
-                break;
-        case TIER_WM_MID:
-                /* coverity[DC.WEAK_CRYPTO] */
-                rand = random() % 100;
-                if (promote) {
-                        migrate = (rand > tier_conf->percent_full);
-                } else {
-                        migrate = (rand <= tier_conf->percent_full);
-                }
-                break;
-        }
-
-exit:
-        return migrate;
-}
-
-int
-tier_migrate (xlator_t *this, int is_promotion, dict_t *migrate_data,
-              loc_t *loc, gf_tier_conf_t *tier_conf)
-{
-        int ret = -1;
-
-        pthread_mutex_lock (&tier_conf->pause_mutex);
-        if (is_promotion)
-                tier_conf->promote_in_progress = 1;
-        else
-                tier_conf->demote_in_progress = 1;
-        pthread_mutex_unlock (&tier_conf->pause_mutex);
-
-        /* Data migration */
-        ret = syncop_setxattr (this, loc, migrate_data, 0,
-                               NULL, NULL);
-
-        pthread_mutex_lock (&tier_conf->pause_mutex);
-        if (is_promotion)
-                tier_conf->promote_in_progress = 0;
-        else
-                tier_conf->demote_in_progress = 0;
-        pthread_mutex_unlock (&tier_conf->pause_mutex);
-
-        return ret;
-}
-
-/* returns  _gf_true: if file can be promoted
- * returns _gf_false: if file cannot be promoted
- */
-static gf_boolean_t
-tier_can_promote_file (xlator_t *this, char const *file_name,
-                       struct iatt *current, gf_defrag_info_t *defrag)
-{
-        gf_boolean_t ret = _gf_false;
-        fsblkcnt_t estimated_usage = 0;
-
-        if (defrag->tier_conf.tier_max_promote_size &&
-            (current->ia_size > defrag->tier_conf.tier_max_promote_size)) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "File %s (gfid:%s) with size (%lu) exceeds maxsize "
-                        "(%d) for promotion. File will not be promoted.",
-                        file_name,
-                        uuid_utoa(current->ia_gfid),
-                        current->ia_size,
-                        defrag->tier_conf.tier_max_promote_size);
-                goto err;
-        }
-
-        /* bypass further validations for TEST mode */
-        if (defrag->tier_conf.mode != TIER_MODE_WM) {
-                ret = _gf_true;
-                goto err;
-        }
-
-        /* convert the file size to blocks as per the block size of the
-         * destination tier
-         * NOTE: add (block_size - 1) to get the correct block size when
-         *       there is a remainder after a modulo
-         */
-        estimated_usage = ((current->ia_size + defrag->tier_conf.block_size - 1) /
-                                defrag->tier_conf.block_size) +
-                                defrag->tier_conf.blocks_used;
-
-        /* test if the estimated block usage goes above HI watermark */
-        if (GF_PERCENTAGE (estimated_usage, defrag->tier_conf.blocks_total) >=
-                        defrag->tier_conf.watermark_hi) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Estimated block count consumption on "
-                        "hot tier (%lu) exceeds hi watermark (%d%%). "
-                        "File will not be promoted.",
-                        estimated_usage,
-                        defrag->tier_conf.watermark_hi);
-                goto err;
-        }
-        ret = _gf_true;
-err:
-        return ret;
-}
-
-static int
-tier_set_migrate_data (dict_t *migrate_data)
-{
-        int    failed = 1;
-
-
-        failed = dict_set_str (migrate_data, GF_XATTR_FILE_MIGRATE_KEY, "force");
-        if (failed) {
-                goto bail_out;
-        }
-
-        /* Flag to suggest the xattr call is from migrator */
-        failed = dict_set_str (migrate_data, "from.migrator", "yes");
-        if (failed) {
-                goto bail_out;
-        }
-
-        /* Flag to suggest its a tiering migration
-         * The reason for this dic key-value is that
-         * promotions and demotions are multithreaded
-         * so the original frame from gf_defrag_start()
-         * is not carried. A new frame will be created when
-         * we do syncop_setxattr(). This doesnot have the
-         * frame->root->pid of the original frame. So we pass
-         * this dic key-value when we do syncop_setxattr() to do
-         * data migration and set the frame->root->pid to
-         * GF_CLIENT_PID_TIER_DEFRAG in dht_setxattr() just before
-         * calling dht_start_rebalance_task() */
-        failed = dict_set_str (migrate_data, TIERING_MIGRATION_KEY, "yes");
-        if (failed) {
-                goto bail_out;
-        }
-
-        failed = 0;
-
-bail_out:
-        return failed;
-}
-
-static char *
-tier_get_parent_path (xlator_t          *this,
-                      loc_t             *p_loc,
-                      struct iatt       *par_stbuf,
-                      int               *per_link_status)
-{
-        int     ret             = -1;
-        char    *parent_path    = NULL;
-        dict_t  *xdata_request  = NULL;
-        dict_t  *xdata_response = NULL;
-
-
-        xdata_request = dict_new ();
-        if (!xdata_request) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to create xdata_request dict");
-                goto err;
-        }
-        ret = dict_set_int32 (xdata_request,
-                              GET_ANCESTRY_PATH_KEY, 42);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to set value to dict : key %s \n",
-                        GET_ANCESTRY_PATH_KEY);
-                goto err;
-        }
-
-        ret = syncop_lookup (this, p_loc, par_stbuf, NULL,
-                             xdata_request, &xdata_response);
-        /* When the parent gfid is a stale entry, the lookup
-         * will fail and stop the demotion process.
-         * The parent gfid can be stale when a huge folder is
-         * deleted while the files within it are being migrated
-         */
-        if (ret == -ESTALE) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_STALE_LOOKUP,
-                        "Stale entry in parent lookup for %s",
-                        uuid_utoa (p_loc->gfid));
-                *per_link_status = 1;
-                goto err;
-        } else if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Error in parent lookup for %s",
-                        uuid_utoa (p_loc->gfid));
-                *per_link_status = -1;
-                goto err;
-        }
-        ret = dict_get_str (xdata_response, GET_ANCESTRY_PATH_KEY, &parent_path);
-        if (ret || !parent_path) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to get parent path for %s",
-                        uuid_utoa (p_loc->gfid));
-                *per_link_status = -1;
-                goto err;
-        }
-
-err:
-        if (xdata_request) {
-                dict_unref (xdata_request);
-        }
-
-        if (xdata_response) {
-                dict_unref (xdata_response);
-                xdata_response = NULL;
-        }
-
-        return parent_path;
-}
-
-static int
-tier_get_file_name_and_path (xlator_t                   *this,
-                             uuid_t                     gfid,
-                             gfdb_link_info_t           *link_info,
-                             char const                 *parent_path,
-                             loc_t                      *loc,
-                             int                        *per_link_status)
-{
-        int             ret     = -1;
-
-        loc->name = gf_strdup (link_info->file_name);
-        if (!loc->name) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR, "Memory "
-                        "allocation failed for %s",
-                        uuid_utoa (gfid));
-                *per_link_status = -1;
-                goto err;
-        }
-        ret = gf_asprintf((char **)&(loc->path), "%s/%s", parent_path, loc->name);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR, "Failed to "
-                        "construct file path for %s %s\n",
-                        parent_path, loc->name);
-                *per_link_status = -1;
-                goto err;
-        }
-
-        ret = 0;
-
-err:
-        return ret;
-}
-
-static int
-tier_lookup_file (xlator_t    *this,
-                  loc_t       *p_loc,
-                  loc_t       *loc,
-                  struct iatt *current,
-                  int         *per_link_status)
-{
-        int     ret = -1;
-
-        ret = syncop_lookup (this, loc, current, NULL, NULL, NULL);
-
-        /* The file may be deleted even when the parent
-         * is available and the lookup will
-         * return a stale entry which would stop the
-         * migration. so if its a stale entry, then skip
-         * the file and keep migrating.
-         */
-        if (ret == -ESTALE) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_STALE_LOOKUP,
-                        "Stale lookup for %s",
-                        uuid_utoa (p_loc->gfid));
-                *per_link_status = 1;
-                goto err;
-        } else if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_LOG_TIER_ERROR, "Failed to "
-                        "lookup file %s\n", loc->name);
-                *per_link_status = -1;
-                goto err;
-        }
-        ret = 0;
-
-err:
-        return ret;
-}
-
-static gf_boolean_t
-tier_is_file_already_at_destination (xlator_t           *src_subvol,
-                                     query_cbk_args_t   *query_cbk_args,
-                                     dht_conf_t         *conf,
-                                     int                *per_link_status)
-{
-        gf_boolean_t    at_destination = _gf_true;
-
-        if (src_subvol == NULL) {
-                *per_link_status = 1;
-                goto err;
-        }
-        if (query_cbk_args->is_promotion &&
-            src_subvol == conf->subvolumes[1]) {
-                *per_link_status = 1;
-                goto err;
-        }
-
-        if (!query_cbk_args->is_promotion &&
-            src_subvol == conf->subvolumes[0]) {
-                *per_link_status = 1;
-                goto err;
-        }
-        at_destination = _gf_false;
-
-err:
-        return at_destination;
-}
-
-static void
-tier_update_migration_counters (query_cbk_args_t        *query_cbk_args,
-                                gf_defrag_info_t        *defrag,
-                                uint64_t                *total_migrated_bytes,
-                                int                     *total_files)
-{
-        if (query_cbk_args->is_promotion) {
-                defrag->total_files_promoted++;
-                *total_migrated_bytes +=
-                        defrag->tier_conf.st_last_promoted_size;
-                pthread_mutex_lock (&dm_stat_mutex);
-                defrag->tier_conf.blocks_used +=
-                        defrag->tier_conf.st_last_promoted_size;
-                pthread_mutex_unlock (&dm_stat_mutex);
-        } else {
-                defrag->total_files_demoted++;
-                *total_migrated_bytes +=
-                        defrag->tier_conf.st_last_demoted_size;
-                pthread_mutex_lock (&dm_stat_mutex);
-                defrag->tier_conf.blocks_used -=
-                        defrag->tier_conf.st_last_demoted_size;
-                pthread_mutex_unlock (&dm_stat_mutex);
-        }
-        if (defrag->tier_conf.blocks_total) {
-                pthread_mutex_lock (&dm_stat_mutex);
-                defrag->tier_conf.percent_full =
-                        GF_PERCENTAGE (defrag->tier_conf.blocks_used,
-                                        defrag->tier_conf.blocks_total);
-                pthread_mutex_unlock (&dm_stat_mutex);
-        }
-
-        (*total_files)++;
-}
-
-static int
-tier_migrate_link (xlator_t            *this,
-                   dht_conf_t          *conf,
-                   uuid_t              gfid,
-                   gfdb_link_info_t    *link_info,
-                   gf_defrag_info_t    *defrag,
-                   query_cbk_args_t    *query_cbk_args,
-                   dict_t              *migrate_data,
-                   int                 *per_link_status,
-                   int                 *total_files,
-                   uint64_t            *total_migrated_bytes)
-{
-        int             ret             = -1;
-        struct iatt     current         = {0,};
-        struct iatt     par_stbuf       = {0,};
-        loc_t           p_loc           = {0,};
-        loc_t           loc             = {0,};
-        xlator_t        *src_subvol     = NULL;
-        inode_t         *linked_inode   = NULL;
-        char            *parent_path    = NULL;
-
-
-        /* Lookup for parent and get the path of parent */
-        gf_uuid_copy (p_loc.gfid, link_info->pargfid);
-        p_loc.inode = inode_new (defrag->root_inode->table);
-        if (!p_loc.inode) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to create reference to inode"
-                        " for %s", uuid_utoa (p_loc.gfid));
-
-                *per_link_status = -1;
-                goto err;
-        }
-
-        parent_path = tier_get_parent_path (this, &p_loc, &par_stbuf,
-                                            per_link_status);
-        if (!parent_path) {
-                goto err;
-        }
-
-        linked_inode = inode_link (p_loc.inode, NULL, NULL, &par_stbuf);
-        inode_unref (p_loc.inode);
-        p_loc.inode = linked_inode;
-
-
-        /* Preparing File Inode */
-        gf_uuid_copy (loc.gfid, gfid);
-        loc.inode = inode_new (defrag->root_inode->table);
-        gf_uuid_copy (loc.pargfid, link_info->pargfid);
-        loc.parent = inode_ref (p_loc.inode);
-
-        /* Get filename and Construct file path */
-        if (tier_get_file_name_and_path (this, gfid, link_info,
-                                         parent_path, &loc, per_link_status) != 0) {
-                goto err;
-        }
-        gf_uuid_copy (loc.parent->gfid, link_info->pargfid);
-
-        /* lookup file inode */
-        if (tier_lookup_file (this, &p_loc, &loc,
-                              &current, per_link_status) != 0) {
-                goto err;
-        }
-
-        if (query_cbk_args->is_promotion) {
-                if (!tier_can_promote_file (this,
-                                            link_info->file_name,
-                                            &current,
-                                            defrag)) {
-                        *per_link_status = 1;
-                        goto err;
-                }
-        }
-
-        linked_inode = inode_link (loc.inode, NULL, NULL, &current);
-        inode_unref (loc.inode);
-        loc.inode = linked_inode;
-
-
-        /*
-         * Do not promote/demote if file already is where it
-         * should be. It means another brick moved the file
-         * so is not an error. So we set per_link_status = 1
-         * so that we ignore counting this.
-         */
-        src_subvol = dht_subvol_get_cached (this, loc.inode);
-
-        if (tier_is_file_already_at_destination (src_subvol, query_cbk_args,
-                                                 conf, per_link_status)) {
-                goto err;
-        }
-
-        gf_msg_debug (this->name, 0, "Tier %s: src_subvol %s file %s",
-                      (query_cbk_args->is_promotion ?  "promote" : "demote"),
-                      src_subvol->name,
-                      loc.path);
-
-
-        ret = tier_check_same_node (this, &loc, defrag);
-        if (ret != 0) {
-                if (ret < 0) {
-                        *per_link_status = -1;
-                        goto err;
-                }
-                ret = 0;
-                /* By setting per_link_status to 1 we are
-                 * ignoring this status and will not be counting
-                 * this file for migration */
-                *per_link_status = 1;
-                goto err;
-        }
-
-        gf_uuid_copy (loc.gfid, loc.inode->gfid);
-
-        if (gf_defrag_get_pause_state (&defrag->tier_conf) != TIER_RUNNING) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Tiering paused. "
-                        "Exiting tier_migrate_link");
-                goto err;
-        }
-
-        ret = tier_migrate (this, query_cbk_args->is_promotion,
-                            migrate_data, &loc, &defrag->tier_conf);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_LOG_TIER_ERROR, "Failed to "
-                        "migrate %s ", loc.path);
-                *per_link_status = -1;
-                goto err;
-        }
-
-        tier_update_migration_counters (query_cbk_args, defrag,
-                                        total_migrated_bytes, total_files);
-
-        ret = 0;
-
-err:
-        GF_FREE ((char *) loc.name);
-        loc.name = NULL;
-        loc_wipe (&loc);
-        loc_wipe (&p_loc);
-
-        if ((*total_files >= defrag->tier_conf.max_migrate_files)
-            || (*total_migrated_bytes > defrag->tier_conf.max_migrate_bytes)) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Reached cycle migration limit."
-                        "migrated bytes %"PRId64" files %d",
-                        *total_migrated_bytes,
-                        *total_files);
-                ret = -1;
-        }
-
-        return ret;
-}
-
-
-static int
-tier_migrate_using_query_file (void *_args)
-{
-        int ret                                 = -1;
-        query_cbk_args_t *query_cbk_args        = (query_cbk_args_t *) _args;
-        xlator_t *this                          = NULL;
-        gf_defrag_info_t *defrag                = NULL;
-        gfdb_query_record_t *query_record       = NULL;
-        gfdb_link_info_t *link_info             = NULL;
-        dict_t *migrate_data                    = NULL;
-        /*
-         * per_file_status and per_link_status
-         *  0  : success
-         * -1 : failure
-         *  1  : ignore the status and dont count for migration
-         * */
-        int per_file_status                     = 0;
-        int per_link_status                     = 0;
-        int total_status                        = 0;
-        dht_conf_t   *conf                      = NULL;
-        uint64_t total_migrated_bytes           = 0;
-        int total_files                         = 0;
-        loc_t root_loc                          = { 0 };
-        gfdb_time_t  start_time                 = { 0 };
-        gfdb_time_t  current_time               = { 0 };
-        int total_time                          = 0;
-        int max_time                            = 0;
-        gf_boolean_t emergency_demote_mode      = _gf_false;
-
-
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args, out);
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->this, out);
-        this = query_cbk_args->this;
-        GF_VALIDATE_OR_GOTO (this->name, query_cbk_args->defrag, out);
-        GF_VALIDATE_OR_GOTO (this->name, query_cbk_args->qfile_array, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        conf = this->private;
-
-        defrag = query_cbk_args->defrag;
-        migrate_data = dict_new ();
-        if (!migrate_data)
-                goto out;
-
-        emergency_demote_mode = (!query_cbk_args->is_promotion &&
-                                 is_hot_tier_full(&defrag->tier_conf));
-
-        if (tier_set_migrate_data (migrate_data) != 0) {
-                goto out;
-        }
-
-        dht_build_root_loc (defrag->root_inode, &root_loc);
-
-        ret = gettimeofday (&start_time, NULL);
-        if (query_cbk_args->is_promotion) {
-                max_time = defrag->tier_conf.tier_promote_frequency;
-        } else {
-                max_time = defrag->tier_conf.tier_demote_frequency;
-        }
-
-        /* Per file */
-        while ((ret = read_query_record_list (query_cbk_args->qfile_array,
-              &query_record)) != 0) {
-
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "Failed to fetch query record "
-                                "from query file");
-                        goto out;
-                }
-
-                if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                        ret = -1;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "Exiting tier migration as"
-                                "defrag status is not started");
-                        goto out;
-                }
-
-                ret = gettimeofday (&current_time, NULL);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "Could not get current time.");
-                        goto out;
-                }
-
-                total_time = current_time.tv_sec - start_time.tv_sec;
-                if (total_time > max_time) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_LOG_TIER_STATUS,
-                                "Max cycle time reached. Exiting migration.");
-                        goto out;
-                }
-
-                per_file_status      = 0;
-                per_link_status      = 0;
-
-                if (gf_defrag_get_pause_state (&defrag->tier_conf)
-                    != TIER_RUNNING) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_LOG_TIER_STATUS,
-                                "Tiering paused. "
-                                "Exiting tier_migrate_using_query_file");
-                        break;
-                }
-
-                if (defrag->tier_conf.mode == TIER_MODE_WM) {
-                        ret = tier_get_fs_stat (this, &root_loc);
-                        if (ret != 0) {
-                                gfdb_methods.gfdb_query_record_free (query_record);
-                                query_record = NULL;
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_STATUS,
-                                        "tier_get_fs_stat() FAILED ... "
-                                        "skipping file migrations until next cycle");
-                                break;
-                        }
-
-                        if (!tier_do_migration (this, query_cbk_args->is_promotion)) {
-                                gfdb_methods.gfdb_query_record_free (query_record);
-                                query_record = NULL;
-
-                                /* We have crossed the high watermark. Stop processing
-                                 * files if this is a promotion cycle so demotion gets
-                                 * a chance to start if not already running*/
-
-                                if (query_cbk_args->is_promotion &&
-                                    is_hot_tier_full (&defrag->tier_conf)) {
-
-                                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                                DHT_MSG_LOG_TIER_STATUS,
-                                                "High watermark crossed during "
-                                                "promotion. Exiting "
-                                                "tier_migrate_using_query_file");
-                                        break;
-                                }
-                                continue;
-                        }
-                }
-
-                per_link_status = 0;
-
-                /* For now we only support single link migration. And we will
-                 * ignore other hard links in the link info list of query record
-                 * TODO: Multiple hard links migration */
-                if (!list_empty (&query_record->link_list)) {
-                        link_info = list_first_entry (&query_record->link_list,
-                                                      gfdb_link_info_t, list);
-                }
-                if (link_info != NULL) {
-                        if (tier_migrate_link (this,
-                                               conf,
-                                               query_record->gfid,
-                                               link_info,
-                                               defrag,
-                                               query_cbk_args,
-                                               migrate_data,
-                                               &per_link_status,
-                                               &total_files,
-                                               &total_migrated_bytes) != 0) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_LOG_TIER_STATUS,
-                                        "%s failed for %s(gfid:%s)",
-                                        (query_cbk_args->is_promotion ?
-                                         "Promotion" : "Demotion"),
-                                        link_info->file_name,
-                                        uuid_utoa (query_record->gfid));
-                        }
-                }
-                per_file_status = per_link_status;
-
-                if (per_file_status < 0) {/* Failure */
-                        pthread_mutex_lock (&dm_stat_mutex);
-                        defrag->total_failures++;
-                        pthread_mutex_unlock (&dm_stat_mutex);
-                } else if (per_file_status == 0) {/* Success */
-                        pthread_mutex_lock (&dm_stat_mutex);
-                        defrag->total_files++;
-                        pthread_mutex_unlock (&dm_stat_mutex);
-                } else if (per_file_status == 1) {/* Ignore */
-                        per_file_status = 0;
-                        /* Since this attempt was ignored we
-                         * decrement the lookup count*/
-                        pthread_mutex_lock (&dm_stat_mutex);
-                        defrag->num_files_lookedup--;
-                        pthread_mutex_unlock (&dm_stat_mutex);
-                }
-                total_status = total_status + per_file_status;
-                per_link_status = 0;
-                per_file_status = 0;
-
-                gfdb_methods.gfdb_query_record_free (query_record);
-                query_record = NULL;
-
-                /* If we are demoting and the entry watermark was HI, then
-                 * we are done with emergency demotions if the current
-                 * watermark has fallen below hi-watermark level
-                 */
-                if (emergency_demote_mode) {
-                        if (tier_check_watermark (this) == 0) {
-                                if (!is_hot_tier_full (&defrag->tier_conf)) {
-                                        break;
-                                }
-                        }
-                }
-        }
-
-out:
-        if (migrate_data)
-                dict_unref (migrate_data);
-
-        gfdb_methods.gfdb_query_record_free (query_record);
-        query_record = NULL;
-
-        return total_status;
-}
-
-
-/* This is the call back function per record/file from data base */
-static int
-tier_gf_query_callback (gfdb_query_record_t *gfdb_query_record,
-                        void *_args) {
-        int ret = -1;
-        query_cbk_args_t *query_cbk_args = _args;
-
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args, out);
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->defrag, out);
-        GF_VALIDATE_OR_GOTO ("tier", (query_cbk_args->query_fd > 0), out);
-
-        ret = gfdb_methods.gfdb_write_query_record (query_cbk_args->query_fd,
-                                                        gfdb_query_record);
-        if (ret) {
-                gf_msg ("tier", GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Failed writing query record to query file");
-                goto out;
-        }
-
-        pthread_mutex_lock (&dm_stat_mutex);
-        query_cbk_args->defrag->num_files_lookedup++;
-        pthread_mutex_unlock (&dm_stat_mutex);
-
-        ret = 0;
-out:
-        return ret;
-}
-
-
-
-
-/* Create query file in tier process */
-static int
-tier_process_self_query (tier_brick_list_t *local_brick, void *args)
-{
-        int ret                                         = -1;
-        char *db_path                                   = NULL;
-        query_cbk_args_t *query_cbk_args                = NULL;
-        xlator_t *this                                  = NULL;
-        gfdb_conn_node_t *conn_node                     = NULL;
-        dict_t *params_dict                             = NULL;
-        dict_t *ctr_ipc_dict                            = NULL;
-        gfdb_brick_info_t *gfdb_brick_info              = args;
-
-        /*Init of all the essentials*/
-        GF_VALIDATE_OR_GOTO ("tier", gfdb_brick_info , out);
-        query_cbk_args = gfdb_brick_info->_query_cbk_args;
-
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->this, out);
-        this = query_cbk_args->this;
-
-        GF_VALIDATE_OR_GOTO (this->name,
-                             gfdb_brick_info->_query_cbk_args, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick->xlator, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick->brick_db_path, out);
-
-        db_path = local_brick->brick_db_path;
-
-        /*Preparing DB parameters before init_db i.e getting db connection*/
-        params_dict = dict_new ();
-        if (!params_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "DB Params cannot initialized");
-                goto out;
-        }
-        SET_DB_PARAM_TO_DICT(this->name, params_dict,
-                             (char *) gfdb_methods.get_db_path_key(),
-                             db_path, ret, out);
-
-        /*Get the db connection*/
-        conn_node = gfdb_methods.init_db ((void *)params_dict, dht_tier_db_type);
-        if (!conn_node) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "FATAL: Failed initializing db operations");
-                         goto out;
-        }
-
-        /* Query for eligible files from db */
-        query_cbk_args->query_fd = open (local_brick->qfile_path,
-                        O_WRONLY | O_CREAT | O_APPEND,
-                        S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (query_cbk_args->query_fd < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to open query file %s",
-                        local_brick->qfile_path);
-                goto out;
-        }
-        if (!gfdb_brick_info->_gfdb_promote) {
-                if (query_cbk_args->defrag->tier_conf.watermark_last ==
-                        TIER_WM_HI) {
-                        /* emergency demotion mode */
-                        ret = gfdb_methods.find_all (conn_node,
-                                tier_gf_query_callback,
-                                (void *)query_cbk_args,
-                                query_cbk_args->defrag->tier_conf.
-                                        query_limit);
-                } else {
-                        if (query_cbk_args->defrag->write_freq_threshold == 0 &&
-                            query_cbk_args->defrag->read_freq_threshold == 0) {
-                                ret = gfdb_methods.find_unchanged_for_time (
-                                                conn_node,
-                                                tier_gf_query_callback,
-                                                (void *)query_cbk_args,
-                                                gfdb_brick_info->time_stamp);
-                        } else {
-                                ret = gfdb_methods.find_unchanged_for_time_freq (
-                                        conn_node,
-                                        tier_gf_query_callback,
-                                        (void *)query_cbk_args,
-                                        gfdb_brick_info->time_stamp,
-                                        query_cbk_args->defrag->
-                                                        write_freq_threshold,
-                                        query_cbk_args->defrag->
-                                                        read_freq_threshold,
-                                        _gf_false);
-                        }
-                }
-        } else {
-                if (query_cbk_args->defrag->write_freq_threshold == 0 &&
-                        query_cbk_args->defrag->read_freq_threshold == 0) {
-                        ret = gfdb_methods.find_recently_changed_files (
-                                conn_node,
-                                tier_gf_query_callback,
-                                (void *)query_cbk_args,
-                                gfdb_brick_info->time_stamp);
-                } else {
-                        ret = gfdb_methods.find_recently_changed_files_freq (
-                                conn_node,
-                                tier_gf_query_callback,
-                                (void *)query_cbk_args,
-                                gfdb_brick_info->time_stamp,
-                                query_cbk_args->defrag->write_freq_threshold,
-                                query_cbk_args->defrag->read_freq_threshold,
-                                _gf_false);
-                }
-        }
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "FATAL: query from db failed");
-                        goto out;
-        }
-
-        /*Clear the heat on the DB entries*/
-        /*Preparing ctr_ipc_dict*/
-        ctr_ipc_dict = dict_new ();
-        if (!ctr_ipc_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "ctr_ipc_dict cannot initialized");
-                goto out;
-        }
-
-        SET_DB_PARAM_TO_DICT(this->name, ctr_ipc_dict,
-                             GFDB_IPC_CTR_KEY, GFDB_IPC_CTR_CLEAR_OPS,
-                             ret, out);
-
-        ret = syncop_ipc (local_brick->xlator, GF_IPC_TARGET_CTR, ctr_ipc_dict,
-                                                                        NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR, "Failed clearing the heat "
-                        "on db %s error %d", local_brick->brick_db_path, ret);
-                goto out;
-        }
-
-        ret = 0;
-out:
-        if (params_dict) {
-                dict_unref (params_dict);
-                params_dict = NULL;
-        }
-
-        if (ctr_ipc_dict) {
-                dict_unref (ctr_ipc_dict);
-                ctr_ipc_dict = NULL;
-        }
-
-        if (query_cbk_args && query_cbk_args->query_fd >= 0) {
-                sys_close (query_cbk_args->query_fd);
-                query_cbk_args->query_fd = -1;
-        }
-        gfdb_methods.fini_db (conn_node);
-
-        return ret;
-}
-
-
-
-
-
-/*Ask CTR to create the query file*/
-static int
-tier_process_ctr_query (tier_brick_list_t *local_brick, void *args)
-{
-        int ret                                         = -1;
-        query_cbk_args_t *query_cbk_args                = NULL;
-        xlator_t *this                                  = NULL;
-        dict_t *ctr_ipc_in_dict                         = NULL;
-        dict_t *ctr_ipc_out_dict                        = NULL;
-        gfdb_brick_info_t *gfdb_brick_info              = args;
-        gfdb_ipc_ctr_params_t *ipc_ctr_params           = NULL;
-        int count                                       = 0;
-
-        /*Init of all the essentials*/
-        GF_VALIDATE_OR_GOTO ("tier", gfdb_brick_info , out);
-        query_cbk_args = gfdb_brick_info->_query_cbk_args;
-
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->this, out);
-        this = query_cbk_args->this;
-
-        GF_VALIDATE_OR_GOTO (this->name,
-                             gfdb_brick_info->_query_cbk_args, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick->xlator, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick->brick_db_path, out);
-
-
-        /*Preparing ctr_ipc_in_dict*/
-        ctr_ipc_in_dict = dict_new ();
-        if (!ctr_ipc_in_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "ctr_ipc_in_dict cannot initialized");
-                goto out;
-        }
-
-        ipc_ctr_params = GF_CALLOC (1, sizeof (gfdb_ipc_ctr_params_t),
-                                        gf_tier_mt_ipc_ctr_params_t);
-        if (!ipc_ctr_params) {
-                goto out;
-        }
-
-        /* set all the query params*/
-        ipc_ctr_params->is_promote = gfdb_brick_info->_gfdb_promote;
-
-        ipc_ctr_params->write_freq_threshold =
-                query_cbk_args->defrag->write_freq_threshold;
-
-        ipc_ctr_params->read_freq_threshold =
-                query_cbk_args->defrag->read_freq_threshold;
-
-        ipc_ctr_params->query_limit =
-                query_cbk_args->defrag->tier_conf.query_limit;
-
-        ipc_ctr_params->emergency_demote =
-                (!gfdb_brick_info->_gfdb_promote &&
-                 query_cbk_args->defrag->tier_conf.watermark_last ==
-                        TIER_WM_HI);
-
-        memcpy (&ipc_ctr_params->time_stamp,
-                gfdb_brick_info->time_stamp,
-                sizeof (gfdb_time_t));
-
-        SET_DB_PARAM_TO_DICT (this->name, ctr_ipc_in_dict,
-                              GFDB_IPC_CTR_KEY, GFDB_IPC_CTR_QUERY_OPS,
-                              ret, out);
-
-
-        SET_DB_PARAM_TO_DICT (this->name, ctr_ipc_in_dict,
-                              GFDB_IPC_CTR_GET_QFILE_PATH,
-                              local_brick->qfile_path,
-                              ret, out);
-
-        ret = dict_set_bin (ctr_ipc_in_dict, GFDB_IPC_CTR_GET_QUERY_PARAMS,
-                                ipc_ctr_params, sizeof (*ipc_ctr_params));
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, LG_MSG_SET_PARAM_FAILED,
-                        "Failed setting %s to params dictionary",
-                        GFDB_IPC_CTR_GET_QUERY_PARAMS);
-                GF_FREE (ipc_ctr_params);
-                goto out;
-        }
-        ipc_ctr_params = NULL;
-
-        ret = syncop_ipc (local_brick->xlator, GF_IPC_TARGET_CTR,
-                                ctr_ipc_in_dict, &ctr_ipc_out_dict);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_IPC_TIER_ERROR, "Failed query on %s ret %d",
-                        local_brick->brick_db_path, ret);
-                goto out;
-        }
-
-        ret = dict_get_int32(ctr_ipc_out_dict, GFDB_IPC_CTR_RET_QUERY_COUNT,
-                                &count);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR, "Failed getting count "
-                        "of records on %s",
-                        local_brick->brick_db_path);
-                goto out;
-        }
-
-        if (count < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR, "Failed query on %s",
-                        local_brick->brick_db_path);
-                ret = -1;
-                goto out;
-        }
-
-        pthread_mutex_lock (&dm_stat_mutex);
-        query_cbk_args->defrag->num_files_lookedup = count;
-        pthread_mutex_unlock (&dm_stat_mutex);
-
-        ret = 0;
-out:
-
-        if (ctr_ipc_in_dict) {
-                dict_unref(ctr_ipc_in_dict);
-                ctr_ipc_in_dict = NULL;
-        }
-
-        if (ctr_ipc_out_dict) {
-                dict_unref(ctr_ipc_out_dict);
-                ctr_ipc_out_dict = NULL;
-        }
-
-        GF_FREE (ipc_ctr_params);
-
-        return ret;
-}
-
-
-
-
-/* This is the call back function for each brick from hot/cold bricklist
- * It picks up each bricks db and queries for eligible files for migration.
- * The list of eligible files are populated in appropriate query files*/
-static int
-tier_process_brick (tier_brick_list_t *local_brick, void *args) {
-        int ret = -1;
-        dict_t *ctr_ipc_in_dict = NULL;
-        dict_t *ctr_ipc_out_dict = NULL;
-        char *strval = NULL;
-
-        GF_VALIDATE_OR_GOTO ("tier", local_brick, out);
-
-        GF_VALIDATE_OR_GOTO ("tier", local_brick->xlator, out);
-
-        if (dht_tier_db_type == GFDB_SQLITE3) {
-
-                /*Preparing ctr_ipc_in_dict*/
-                ctr_ipc_in_dict = dict_new ();
-                if (!ctr_ipc_in_dict) {
-                        gf_msg ("tier", GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "ctr_ipc_in_dict cannot initialized");
-                        goto out;
-                }
-
-                ret = dict_set_str (ctr_ipc_in_dict, GFDB_IPC_CTR_KEY,
-                                    GFDB_IPC_CTR_GET_DB_PARAM_OPS);
-                if (ret) {
-                        gf_msg ("tier", GF_LOG_ERROR, 0,\
-                                LG_MSG_SET_PARAM_FAILED, "Failed to set %s "
-                                "to params dictionary", GFDB_IPC_CTR_KEY);
-                        goto out;
-                }
-
-                ret = dict_set_str (ctr_ipc_in_dict,
-                                    GFDB_IPC_CTR_GET_DB_PARAM_OPS, "");
-                if (ret) {
-                        gf_msg ("tier", GF_LOG_ERROR, 0,\
-                                LG_MSG_SET_PARAM_FAILED, "Failed to set %s "
-                                "to params dictionary",
-                                GFDB_IPC_CTR_GET_DB_PARAM_OPS);
-                        goto out;
-                }
-
-                ret = dict_set_str (ctr_ipc_in_dict,
-                                    GFDB_IPC_CTR_GET_DB_KEY, "journal_mode");
-                if (ret) {
-                        gf_msg ("tier", GF_LOG_ERROR, 0,
-                                LG_MSG_SET_PARAM_FAILED, "Failed to set %s "
-                                "to params dictionary",
-                                GFDB_IPC_CTR_GET_DB_KEY);
-                        goto out;
-                }
-
-
-
-                ret = syncop_ipc (local_brick->xlator, GF_IPC_TARGET_CTR,
-                                ctr_ipc_in_dict, &ctr_ipc_out_dict);
-                if (ret || ctr_ipc_out_dict == NULL) {
-                        gf_msg ("tier", GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR, "Failed to get "
-                                "journal_mode of sql db %s",
-                                local_brick->brick_db_path);
-                        goto out;
-                }
-
-                ret = dict_get_str (ctr_ipc_out_dict, "journal_mode", &strval);
-                if (ret) {
-                        gf_msg ("tier", GF_LOG_ERROR, 0,
-                                LG_MSG_GET_PARAM_FAILED, "Failed to get %s "
-                                "from params dictionary"
-                                "journal_mode", strval);
-                        goto out;
-                }
-
-                if (strval && (strncmp(strval, "wal", strlen ("wal")) == 0)) {
-                        ret = tier_process_self_query (local_brick, args);
-                        if (ret) {
-                                goto out;
-                        }
-                } else {
-                        ret = tier_process_ctr_query (local_brick, args);
-                        if (ret) {
-                                goto out;
-                        }
-                }
-                ret = 0;
-
-        } else {
-                ret = tier_process_self_query (local_brick, args);
-                if (ret) {
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
-        if  (ctr_ipc_in_dict)
-                dict_unref (ctr_ipc_in_dict);
-
-        if  (ctr_ipc_out_dict)
-                dict_unref (ctr_ipc_out_dict);
-
-        return ret;
-}
-
-
-
-
-static int
-tier_build_migration_qfile (migration_args_t *args,
-                            query_cbk_args_t *query_cbk_args,
-                            gf_boolean_t is_promotion)
-{
-        gfdb_time_t               current_time;
-        gfdb_brick_info_t         gfdb_brick_info;
-        gfdb_time_t               time_in_past;
-        int                       ret = -1;
-        tier_brick_list_t         *local_brick = NULL;
-        int                       i = 0;
-        time_in_past.tv_sec = args->freq_time;
-        time_in_past.tv_usec = 0;
-
-        ret = gettimeofday (&current_time, NULL);
-        if (ret == -1) {
-                gf_msg (args->this->name, GF_LOG_ERROR, errno,
-                        DHT_MSG_SYS_CALL_GET_TIME_FAILED,
-                        "Failed to get current time");
-                goto out;
-        }
-        time_in_past.tv_sec = current_time.tv_sec - time_in_past.tv_sec;
-
-        /* The migration daemon may run a varying numberof usec after the */
-        /* sleep call triggers. A file may be registered in CTR some number */
-        /* of usec X after the daemon started and missed in the subsequent */
-        /* cycle if the daemon starts Y usec after the period in seconds */
-        /* where Y>X. Normalize away this problem by always setting usec */
-        /* to 0. */
-        time_in_past.tv_usec = 0;
-
-        gfdb_brick_info.time_stamp = &time_in_past;
-        gfdb_brick_info._gfdb_promote = is_promotion;
-        gfdb_brick_info._query_cbk_args = query_cbk_args;
-
-        list_for_each_entry (local_brick, args->brick_list, list) {
-
-                /* Construct query file path for this brick
-                 * i.e
-                 * /var/run/gluster/xlator_name/
-                 * {promote/demote}-brickname-indexinbricklist
-                 * So that no two query files will have same path even
-                 * bricks have the same name
-                 * */
-                snprintf (local_brick->qfile_path, PATH_MAX , "%s-%s-%d",
-                        GET_QFILE_PATH (gfdb_brick_info._gfdb_promote),
-                        local_brick->brick_name, i);
-
-                /* Delete any old query files for this brick */
-                sys_unlink (local_brick->qfile_path);
-
-                ret = tier_process_brick (local_brick,
-                                          &gfdb_brick_info);
-                if (ret) {
-                        gf_msg (args->this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_BRICK_QUERY_FAILED,
-                                "Brick %s query failed\n",
-                                local_brick->brick_db_path);
-                }
-                i++;
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-static int
-tier_migrate_files_using_qfile (migration_args_t *comp,
-                                query_cbk_args_t *query_cbk_args)
-{
-        int ret                                 = -1;
-        tier_brick_list_t *local_brick          = NULL;
-        tier_brick_list_t *temp                 = NULL;
-        char query_file_path_err[PATH_MAX+128]  = {0,};
-        struct tm tm                            = {0,};
-        gfdb_time_t current_time                = {0,};
-        char time_str[128]                      = {0,};
-        ssize_t qfile_array_size                = 0;
-        int count                               = 0;
-        int temp_fd                             = 0;
-        gf_tier_conf_t  *tier_conf              = NULL;
-
-        tier_conf = &(query_cbk_args->defrag->tier_conf);
-
-        /* Time format for error query files */
-        gettimeofday (&current_time, NULL);
-        gmtime_r (&current_time.tv_sec, &tm);
-        strftime (time_str, sizeof (time_str), "%F-%T", &tm);
-
-        /* Build the qfile list */
-        list_for_each_entry_safe (local_brick, temp, comp->brick_list, list) {
-                qfile_array_size++;
-        }
-        query_cbk_args->qfile_array = qfile_array_new (qfile_array_size);
-        if (!query_cbk_args->qfile_array) {
-                gf_msg ("tier", GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR, "Failed to create new "
-                        "qfile_array");
-                goto out;
-        }
-
-        /*Open all qfiles*/
-        count = 0;
-        query_cbk_args->qfile_array->exhausted_count = 0;
-        list_for_each_entry_safe (local_brick, temp, comp->brick_list, list) {
-                temp_fd = query_cbk_args->qfile_array->fd_array[count];
-                temp_fd = open (local_brick->qfile_path, O_RDONLY,
-                                S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-                if (temp_fd < 0) {
-                        gf_msg ("tier", GF_LOG_ERROR, errno,
-                                DHT_MSG_LOG_TIER_ERROR, "Failed to open "
-                                "%s to the query file",
-                                local_brick->qfile_path);
-                        query_cbk_args->qfile_array->exhausted_count++;
-                }
-                query_cbk_args->qfile_array->fd_array[count] = temp_fd;
-                count++;
-        }
-
-        /* Moving the query file index to the next, so that we won't the same
-         * query file every cycle as the first one */
-        query_cbk_args->qfile_array->next_index =
-                        (query_cbk_args->is_promotion) ?
-                        tier_conf->last_promote_qfile_index :
-                        tier_conf->last_demote_qfile_index;
-        shift_next_index (query_cbk_args->qfile_array);
-        if (query_cbk_args->is_promotion) {
-                tier_conf->last_promote_qfile_index =
-                        query_cbk_args->qfile_array->next_index;
-        } else {
-                tier_conf->last_demote_qfile_index =
-                        query_cbk_args->qfile_array->next_index;
-        }
-
-        /* Migrate files using query file list */
-        ret = tier_migrate_using_query_file ((void *)query_cbk_args);
-out:
-        qfile_array_free (query_cbk_args->qfile_array);
-
-        /* If there is an error rename all the query files to .err files
-         * with a timestamp for better debugging */
-        if (ret) {
-                list_for_each_entry_safe (local_brick, temp, comp->brick_list,
-                                        list) {
-                        /* rename error qfile*/
-                        snprintf (query_file_path_err,
-                                  sizeof (query_file_path_err),
-                                  "%s-%s.err", local_brick->qfile_path,
-                                  time_str);
-                        if (sys_rename (local_brick->qfile_path,
-                                        query_file_path_err) == -1)
-                                gf_msg_debug ("tier", 0, "rename "
-                                              "failed");
-                }
-        }
-
-        query_cbk_args->qfile_array = NULL;
-
-        return ret;
-}
-
-
-
-int
-tier_demote (migration_args_t *demotion_args)
-{
-        query_cbk_args_t query_cbk_args;
-        int ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("tier", demotion_args, out);
-        GF_VALIDATE_OR_GOTO ("tier", demotion_args->this, out);
-        GF_VALIDATE_OR_GOTO (demotion_args->this->name,
-                             demotion_args->brick_list, out);
-        GF_VALIDATE_OR_GOTO (demotion_args->this->name,
-                             demotion_args->defrag, out);
-
-        THIS = demotion_args->this;
-
-        query_cbk_args.this = demotion_args->this;
-        query_cbk_args.defrag = demotion_args->defrag;
-        query_cbk_args.is_promotion = 0;
-
-        /*Build the query file using bricklist*/
-        ret = tier_build_migration_qfile (demotion_args, &query_cbk_args,
-                                          _gf_false);
-        if (ret)
-                goto out;
-
-        /* Migrate files using the query file */
-        ret = tier_migrate_files_using_qfile (demotion_args,
-                                              &query_cbk_args);
-        if (ret)
-                goto out;
-
-out:
-        demotion_args->return_value = ret;
-        return ret;
-}
-
-
-int
-tier_promote (migration_args_t *promotion_args)
-{
-        int ret = -1;
-        query_cbk_args_t query_cbk_args;
-
-        GF_VALIDATE_OR_GOTO ("tier", promotion_args->this, out);
-        GF_VALIDATE_OR_GOTO (promotion_args->this->name,
-                             promotion_args->brick_list, out);
-        GF_VALIDATE_OR_GOTO (promotion_args->this->name,
-                             promotion_args->defrag, out);
-
-        THIS = promotion_args->this;
-
-        query_cbk_args.this = promotion_args->this;
-        query_cbk_args.defrag = promotion_args->defrag;
-        query_cbk_args.is_promotion = 1;
-
-        /*Build the query file using bricklist*/
-        ret = tier_build_migration_qfile (promotion_args, &query_cbk_args,
-                                          _gf_true);
-        if (ret)
-                goto out;
-
-        /* Migrate files using the query file */
-        ret = tier_migrate_files_using_qfile (promotion_args, &query_cbk_args);
-        if (ret)
-                goto out;
-
-out:
-        promotion_args->return_value = ret;
-        return ret;
-}
-
-
-/*
- * Command the CTR on a brick to compact the local database using an IPC
- */
-static int
-tier_process_self_compact (tier_brick_list_t *local_brick, void *args)
-{
-        int ret                                         = -1;
-        char *db_path                                   = NULL;
-        query_cbk_args_t *query_cbk_args                = NULL;
-        xlator_t *this                                  = NULL;
-        gfdb_conn_node_t *conn_node                     = NULL;
-        dict_t *params_dict                             = NULL;
-        dict_t *ctr_ipc_dict                            = NULL;
-        gfdb_brick_info_t *gfdb_brick_info              = args;
-
-        /*Init of all the essentials*/
-        GF_VALIDATE_OR_GOTO ("tier", gfdb_brick_info , out);
-        query_cbk_args = gfdb_brick_info->_query_cbk_args;
-
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->this, out);
-        this = query_cbk_args->this;
-
-        GF_VALIDATE_OR_GOTO (this->name,
-                             gfdb_brick_info->_query_cbk_args, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick->xlator, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, local_brick->brick_db_path, out);
-
-        db_path = local_brick->brick_db_path;
-
-        /*Preparing DB parameters before init_db i.e getting db connection*/
-        params_dict = dict_new ();
-        if (!params_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "DB Params cannot initialized");
-                goto out;
-        }
-        SET_DB_PARAM_TO_DICT (this->name, params_dict,
-                             (char *) gfdb_methods.get_db_path_key(), db_path,
-                             ret, out);
-
-        /*Get the db connection*/
-        conn_node = gfdb_methods.init_db ((void *)params_dict,
-                                          dht_tier_db_type);
-        if (!conn_node) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "FATAL: Failed initializing db operations");
-                         goto out;
-        }
-
-        ret = 0;
-
-        /*Preparing ctr_ipc_dict*/
-        ctr_ipc_dict = dict_new ();
-        if (!ctr_ipc_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "ctr_ipc_dict cannot initialized");
-                goto out;
-        }
-
-        ret = dict_set_int32 (ctr_ipc_dict, "compact_active",
-                              query_cbk_args->defrag->
-                              tier_conf.compact_active);
-
-        if (ret) {
-                gf_msg ("tier", GF_LOG_ERROR, 0,
-                        LG_MSG_SET_PARAM_FAILED, "Failed to set %s "
-                        "to params dictionary",
-                        "compact_active");
-                goto out;
-        }
-
-        ret = dict_set_int32 (ctr_ipc_dict, "compact_mode_switched",
-                              query_cbk_args->defrag->
-                              tier_conf.compact_mode_switched);
-
-        if (ret) {
-                gf_msg ("tier", GF_LOG_ERROR, 0,
-                        LG_MSG_SET_PARAM_FAILED, "Failed to set %s "
-                        "to params dictionary",
-                        "compact_mode_switched");
-                goto out;
-        }
-
-        SET_DB_PARAM_TO_DICT(this->name, ctr_ipc_dict,
-                             GFDB_IPC_CTR_KEY, GFDB_IPC_CTR_SET_COMPACT_PRAGMA,
-                             ret, out);
-
-        gf_msg (this->name, GF_LOG_TRACE, 0, DHT_MSG_LOG_TIER_STATUS,
-                "Starting Compaction IPC");
-
-        ret = syncop_ipc (local_brick->xlator, GF_IPC_TARGET_CTR, ctr_ipc_dict,
-                          NULL);
-
-        gf_msg (this->name, GF_LOG_TRACE, 0, DHT_MSG_LOG_TIER_STATUS,
-                "Ending Compaction IPC");
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR, "Failed compaction "
-                        "on db %s error %d", local_brick->brick_db_path, ret);
-                goto out;
-        }
-
-        gf_msg (this->name, GF_LOG_TRACE, 0, DHT_MSG_LOG_TIER_STATUS,
-                "SUCCESS: %s Compaction", local_brick->brick_name);
-
-        ret = 0;
-out:
-        if (params_dict) {
-                dict_unref (params_dict);
-                params_dict = NULL;
-        }
-
-        if (ctr_ipc_dict) {
-                dict_unref (ctr_ipc_dict);
-                ctr_ipc_dict = NULL;
-        }
-
-        gfdb_methods.fini_db (conn_node);
-
-        return ret;
-}
-
-/*
- * This is the call back function for each brick from hot/cold bricklist.
- * It determines the database type on each brick and calls the corresponding
- * function to prepare the compaction IPC.
- */
-static int
-tier_compact_db_brick (tier_brick_list_t *local_brick, void *args)
-{
-        int ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("tier", local_brick, out);
-
-        GF_VALIDATE_OR_GOTO ("tier", local_brick->xlator, out);
-
-        ret = tier_process_self_compact (local_brick, args);
-        if (ret) {
-                gf_msg ("tier", GF_LOG_INFO, 0,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Brick %s did not compact",
-                        local_brick->brick_name);
-                goto out;
-        }
-
-        ret = 0;
-
-out:
-
-        return ret;
-}
-
-static int
-tier_send_compact (migration_args_t *args,
-                   query_cbk_args_t *query_cbk_args)
-{
-        gfdb_time_t               current_time;
-        gfdb_brick_info_t         gfdb_brick_info;
-        gfdb_time_t               time_in_past;
-        int                       ret = -1;
-        tier_brick_list_t         *local_brick = NULL;
-
-        time_in_past.tv_sec = args->freq_time;
-        time_in_past.tv_usec = 0;
-
-        ret = gettimeofday (&current_time, NULL);
-        if (ret == -1) {
-                gf_msg (args->this->name, GF_LOG_ERROR, errno,
-                        DHT_MSG_SYS_CALL_GET_TIME_FAILED,
-                        "Failed to get current time");
-                goto out;
-        }
-        time_in_past.tv_sec = current_time.tv_sec - time_in_past.tv_sec;
-
-        /* The migration daemon may run a varying numberof usec after the sleep
-           call triggers. A file may be registered in CTR some number of usec X
-           after the daemon started and missed in the subsequent cycle if the
-           daemon starts Y usec after the period in seconds where Y>X. Normalize
-           away this problem by always setting usec to 0. */
-        time_in_past.tv_usec = 0;
-
-        gfdb_brick_info.time_stamp = &time_in_past;
-
-        /* This is meant to say we are always compacting at this point */
-        /* We simply borrow the promotion flag to do this */
-        gfdb_brick_info._gfdb_promote = 1;
-
-        gfdb_brick_info._query_cbk_args = query_cbk_args;
-
-        list_for_each_entry (local_brick, args->brick_list, list) {
-
-                gf_msg (args->this->name, GF_LOG_TRACE, 0,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Start compaction for %s",
-                        local_brick->brick_name);
-
-                ret = tier_compact_db_brick (local_brick,
-                                             &gfdb_brick_info);
-                if (ret) {
-                        gf_msg (args->this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_BRICK_QUERY_FAILED,
-                                "Brick %s compaction failed\n",
-                                local_brick->brick_db_path);
-                }
-
-                gf_msg (args->this->name, GF_LOG_TRACE, 0,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "End compaction for %s",
-                        local_brick->brick_name);
-
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-static int
-tier_compact (void *args)
-{
-        int ret = -1;
-        query_cbk_args_t query_cbk_args;
-        migration_args_t *compaction_args = args;
-
-        GF_VALIDATE_OR_GOTO ("tier", compaction_args->this, out);
-        GF_VALIDATE_OR_GOTO (compaction_args->this->name,
-                             compaction_args->brick_list, out);
-        GF_VALIDATE_OR_GOTO (compaction_args->this->name,
-                             compaction_args->defrag, out);
-
-        THIS = compaction_args->this;
-
-        query_cbk_args.this = compaction_args->this;
-        query_cbk_args.defrag = compaction_args->defrag;
-        query_cbk_args.is_compaction = 1;
-
-        /* Send the compaction pragma out to all the bricks on the bricklist. */
-        /* tier_get_bricklist ensures all bricks on the list are local to */
-        /* this node. */
-        ret = tier_send_compact (compaction_args, &query_cbk_args);
-        if (ret)
-                goto out;
-
-        ret = 0;
-out:
-        compaction_args->return_value = ret;
-        return ret;
- }
-
-static int
-tier_get_bricklist (xlator_t *xl, struct list_head *local_bricklist_head)
-{
-        xlator_list_t           *child = NULL;
-        char                    *rv        = NULL;
-        char                    *rh        = NULL;
-        char                    *brickname = NULL;
-        char                    db_name[PATH_MAX] = "";
-        int                     ret = 0;
-        tier_brick_list_t       *local_brick = NULL;
-
-        GF_VALIDATE_OR_GOTO ("tier", xl, out);
-        GF_VALIDATE_OR_GOTO ("tier", local_bricklist_head, out);
-
-        /*
-         * This function obtains remote subvolumes and filters out only
-         * those running on the same node as the tier daemon.
-         */
-        if (strcmp(xl->type, "protocol/client") == 0) {
-                ret = dict_get_str (xl->options, "remote-host", &rh);
-                if (ret < 0)
-                        goto out;
-
-                if (gf_is_local_addr (rh)) {
-
-                       local_brick = GF_CALLOC (1, sizeof(tier_brick_list_t),
-                                                gf_tier_mt_bricklist_t);
-                        if (!local_brick) {
-                                goto out;
-                        }
-
-                        ret = dict_get_str (xl->options, "remote-subvolume",
-                                           &rv);
-                        if (ret < 0)
-                                goto out;
-
-                        brickname = strrchr(rv, '/') + 1;
-                        snprintf(db_name, sizeof(db_name), "%s.db",
-                                 brickname);
-
-                        local_brick->brick_db_path =
-                                GF_CALLOC (PATH_MAX, 1, gf_common_mt_char);
-                        if (!local_brick->brick_db_path) {
-                                gf_msg ("tier", GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_STATUS,
-                                        "Failed to allocate memory for"
-                                        " bricklist.");
-                                goto out;
-                        }
-
-                        snprintf(local_brick->brick_db_path,
-                                PATH_MAX, "%s/%s/%s", rv,
-                                GF_HIDDEN_PATH, db_name);
-
-                        local_brick->xlator = xl;
-
-                        snprintf (local_brick->brick_name,
-                                NAME_MAX, "%s", brickname);
-
-                        list_add_tail (&(local_brick->list),
-                                        local_bricklist_head);
-
-                        ret = 0;
-                        goto out;
-                }
-        }
-
-        for (child = xl->children; child; child = child->next) {
-                ret = tier_get_bricklist (child->xlator, local_bricklist_head);
-                if (ret) {
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
-
-        if (ret) {
-                if (local_brick) {
-                        GF_FREE (local_brick->brick_db_path);
-                }
-                GF_FREE (local_brick);
-        }
-
-        return ret;
-}
-
-int
-tier_get_freq_demote (gf_tier_conf_t *tier_conf)
-{
-        if ((tier_conf->mode == TIER_MODE_WM) &&
-            (tier_conf->watermark_last == TIER_WM_HI))
-                return DEFAULT_DEMOTE_DEGRADED;
-        else
-                return tier_conf->tier_demote_frequency;
-}
-
-int
-tier_get_freq_promote (gf_tier_conf_t *tier_conf)
-{
-        return tier_conf->tier_promote_frequency;
-}
-
-int
-tier_get_freq_compact_hot (gf_tier_conf_t *tier_conf)
-{
-        return tier_conf->tier_compact_hot_frequency;
-}
-
-int
-tier_get_freq_compact_cold (gf_tier_conf_t *tier_conf)
-{
-        return tier_conf->tier_compact_cold_frequency;
-}
-
-static int
-tier_check_demote (gfdb_time_t  current_time, int freq)
-{
-        return ((current_time.tv_sec % freq) == 0) ?
-                _gf_true : _gf_false;
-}
-
-static gf_boolean_t
-tier_check_promote (gf_tier_conf_t   *tier_conf,
-                    gfdb_time_t  current_time,
-                    int freq)
-{
-        if ((tier_conf->mode == TIER_MODE_WM) &&
-            (tier_conf->watermark_last == TIER_WM_HI))
-                return _gf_false;
-
-        else
-                return ((current_time.tv_sec % freq) == 0) ?
-                        _gf_true : _gf_false;
-}
-
-static gf_boolean_t
-tier_check_compact (gf_tier_conf_t   *tier_conf,
-                    gfdb_time_t  current_time,
-                    int freq_compact)
-{
-
-        if (!(tier_conf->compact_active ||
-              tier_conf->compact_mode_switched))
-                return _gf_false;
-
-
-        return ((current_time.tv_sec % freq_compact) == 0) ?
-                _gf_true : _gf_false;
-}
-
-
-void
-clear_bricklist (struct list_head *brick_list)
-{
-        tier_brick_list_t  *local_brick      = NULL;
-        tier_brick_list_t  *temp             = NULL;
-
-        if (list_empty(brick_list)) {
-                return;
-        }
-
-        list_for_each_entry_safe (local_brick, temp, brick_list, list) {
-                list_del (&local_brick->list);
-                GF_FREE (local_brick->brick_db_path);
-                GF_FREE (local_brick);
-        }
-}
-
-
-static void
-set_brick_list_qpath (struct list_head *brick_list, gf_boolean_t is_cold)
-{
-
-        tier_brick_list_t *local_brick = NULL;
-        int i                          = 0;
-
-        GF_VALIDATE_OR_GOTO ("tier", brick_list, out);
-
-        list_for_each_entry (local_brick, brick_list, list) {
-
-                /* Construct query file path for this brick
-                 * i.e
-                 * /var/run/gluster/xlator_name/
-                 * {promote/demote}-brickname-indexinbricklist
-                 * So that no two query files will have same path even
-                 * bricks have the same name
-                 * */
-                snprintf (local_brick->qfile_path, PATH_MAX , "%s-%s-%d",
-                        GET_QFILE_PATH (is_cold),
-                        local_brick->brick_name, i);
-                i++;
-        }
-out:
-        return;
-}
-
-static int
-tier_prepare_compact (migration_args_t *args, gfdb_time_t current_time)
-{
-        xlator_t *this = NULL;
-        dht_conf_t *conf                        = NULL;
-        gf_defrag_info_t *defrag                = NULL;
-        gf_tier_conf_t *tier_conf               = NULL;
-        gf_boolean_t is_hot_tier = args->is_hot_tier;
-        int freq = 0;
-        int ret = -1;
-        const char *tier_type = is_hot_tier ? "hot" : "cold";
-
-        this = args->this;
-
-        conf = this->private;
-
-        defrag = conf->defrag;
-
-        tier_conf = &defrag->tier_conf;
-
-        freq = is_hot_tier ? tier_get_freq_compact_hot (tier_conf) :
-                tier_get_freq_compact_cold (tier_conf);
-
-        defrag->tier_conf.compact_mode_switched = is_hot_tier ?
-                defrag->tier_conf.compact_mode_switched_hot :
-                defrag->tier_conf.compact_mode_switched_cold;
-
-        gf_msg(this->name, GF_LOG_TRACE, 0,
-               DHT_MSG_LOG_TIER_STATUS,
-               "Compact mode %i",
-               defrag->tier_conf.compact_mode_switched);
-
-        if (tier_check_compact (tier_conf, current_time,
-                                freq)) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Start compaction on %s tier",
-                        tier_type);
-
-                args->freq_time = freq;
-                ret = tier_compact (args);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR, "Compaction failed on "
-                                "%s tier", tier_type);
-                        goto out;
-                }
-
-                gf_msg (this->name, GF_LOG_INFO, 0, DHT_MSG_LOG_TIER_STATUS,
-                        "End compaction on %s tier", tier_type);
-
-                if (is_hot_tier) {
-                        defrag->tier_conf.compact_mode_switched_hot =
-                                _gf_false;
-                } else {
-                        defrag->tier_conf.compact_mode_switched_cold =
-                                _gf_false;
-                }
-        }
-
-out:
-        return ret;
-}
-
-static int
-tier_get_wm_interval(tier_mode_t mode, tier_watermark_op_t wm)
-{
-        if (mode == TIER_MODE_WM && wm == TIER_WM_HI)
-                return WM_INTERVAL_EMERG;
-
-        return WM_INTERVAL;
-}
-
-/*
- * Main tiering loop. This is called from the promotion and the
- * demotion threads spawned in tier_start().
- *
- * Every second, wake from sleep to perform tasks.
- * 1. Check trigger to migrate data.
- * 2. Check for state changes (pause, unpause, stop).
- */
-static void
-*tier_run (void *in_args)
-{
-        dht_conf_t *conf                        = NULL;
-        gfdb_time_t  current_time               = { 0 };
-        int freq                                = 0;
-        int ret                                 = 0;
-        xlator_t *any                           = NULL;
-        xlator_t *xlator                        = NULL;
-        gf_tier_conf_t *tier_conf               = NULL;
-        loc_t root_loc                          = { 0 };
-        int check_watermark                     = 0;
-        gf_defrag_info_t *defrag                = NULL;
-        xlator_t  *this                         = NULL;
-        migration_args_t *args = in_args;
-        GF_VALIDATE_OR_GOTO ("tier", args, out);
-        GF_VALIDATE_OR_GOTO ("tier", args->brick_list, out);
-
-        this = args->this;
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
-
-        conf   = this->private;
-        GF_VALIDATE_OR_GOTO ("tier", conf, out);
-
-        defrag = conf->defrag;
-        GF_VALIDATE_OR_GOTO ("tier", defrag, out);
-
-        if (list_empty (args->brick_list)) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Brick list for tier is empty. Exiting.");
-                goto out;
-        }
-
-        defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
-        tier_conf = &defrag->tier_conf;
-
-        dht_build_root_loc (defrag->root_inode, &root_loc);
-
-        while (1) {
-
-                /*
-                 * Check if a graph switch occured. If so, stop migration
-                 * thread. It will need to be restarted manually.
-                 */
-                any = THIS->ctx->active->first;
-                xlator = xlator_search_by_name (any, this->name);
-
-                if (xlator != this) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_LOG_TIER_STATUS,
-                                "Detected graph switch. Exiting migration "
-                                "daemon.");
-                        goto out;
-                }
-
-                gf_defrag_check_pause_tier (tier_conf);
-
-                sleep(1);
-
-                if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                        ret = 1;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "defrag->defrag_status != "
-                                "GF_DEFRAG_STATUS_STARTED");
-                        goto out;
-                }
-
-                if (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER ||
-                    defrag->cmd == GF_DEFRAG_CMD_DETACH_START) {
-                        ret = 0;
-                        defrag->defrag_status =
-                                GF_DEFRAG_STATUS_COMPLETE;
-                        gf_msg (this->name, GF_LOG_DEBUG, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "defrag->defrag_cmd == "
-                                "GF_DEFRAG_CMD_START_DETACH_TIER");
-                        goto out;
-                }
-
-                if (gf_defrag_get_pause_state (&defrag->tier_conf) !=
-                    TIER_RUNNING)
-                        continue;
-
-                /* To have proper synchronization amongst all
-                 * brick holding nodes, so that promotion and demotions
-                 * start atomicly w.r.t promotion/demotion frequency
-                 * period, all nodes should have thier system time
-                 * in-sync with each other either manually set or
-                 * using a NTP server*/
-                ret = gettimeofday (&current_time, NULL);
-                if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                DHT_MSG_SYS_CALL_GET_TIME_FAILED,
-                                "Failed to get current time");
-                        goto out;
-                }
-
-                check_watermark++;
-
-                /* emergency demotion requires frequent watermark monitoring */
-                if (check_watermark >=
-                        tier_get_wm_interval(tier_conf->mode,
-                                             tier_conf->watermark_last)) {
-                        check_watermark = 0;
-                        if (tier_conf->mode == TIER_MODE_WM) {
-                                ret = tier_get_fs_stat (this, &root_loc);
-                                if (ret != 0) {
-                                        continue;
-                                }
-                                ret = tier_check_watermark (this);
-                                if (ret != 0) {
-                                        gf_msg (this->name, GF_LOG_CRITICAL, errno,
-                                                DHT_MSG_LOG_TIER_ERROR,
-                                                "Failed to get watermark");
-                                        continue;
-                                }
-                        }
-                }
-
-                if (args->is_promotion) {
-                        freq = tier_get_freq_promote (tier_conf);
-
-                        if (tier_check_promote (tier_conf, current_time, freq)) {
-                                args->freq_time = freq;
-                                ret = tier_promote (args);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                DHT_MSG_LOG_TIER_ERROR,
-                                                "Promotion failed");
-                                }
-                        }
-                } else if (args->is_compaction) {
-                        tier_prepare_compact (args, current_time);
-                } else {
-                        freq = tier_get_freq_demote (tier_conf);
-
-                        if (tier_check_demote (current_time, freq)) {
-                                args->freq_time = freq;
-                                ret = tier_demote (args);
-                                if (ret) {
-                                        gf_msg (this->name,
-                                                GF_LOG_ERROR,
-                                                0,
-                                                DHT_MSG_LOG_TIER_ERROR,
-                                                "Demotion failed");
-                                }
-                        }
-                }
-
-                /* Check the statfs immediately after the processing threads
-                   return */
-                check_watermark = WM_INTERVAL;
-        }
-
-        ret = 0;
-out:
-
-        args->return_value = ret;
-
-        return NULL;
-}
-
-int
-tier_start (xlator_t *this, gf_defrag_info_t *defrag)
-{
-        pthread_t promote_thread;
-        pthread_t demote_thread;
-        pthread_t hot_compact_thread;
-        pthread_t cold_compact_thread;
-        int ret = -1;
-        struct list_head bricklist_hot          = { 0 };
-        struct list_head bricklist_cold         = { 0 };
-        migration_args_t promotion_args         = { 0 };
-        migration_args_t demotion_args          = { 0 };
-        migration_args_t hot_compaction_args    = { 0 };
-        migration_args_t cold_compaction_args   = { 0 };
-        dht_conf_t *conf                        = NULL;
-
-        INIT_LIST_HEAD ((&bricklist_hot));
-        INIT_LIST_HEAD ((&bricklist_cold));
-
-        conf   = this->private;
-
-        tier_get_bricklist (conf->subvolumes[1], &bricklist_hot);
-        set_brick_list_qpath (&bricklist_hot, _gf_false);
-
-        demotion_args.this = this;
-        demotion_args.brick_list = &bricklist_hot;
-        demotion_args.defrag = defrag;
-        demotion_args.is_promotion = _gf_false;
-        demotion_args.is_compaction = _gf_false;
-
-        ret = gf_thread_create (&demote_thread,
-                                NULL, &tier_run, &demotion_args, "tierdem");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to start demotion thread.");
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                goto cleanup;
-        }
-
-        tier_get_bricklist (conf->subvolumes[0], &bricklist_cold);
-        set_brick_list_qpath (&bricklist_cold, _gf_true);
-
-        promotion_args.this = this;
-        promotion_args.brick_list = &bricklist_cold;
-        promotion_args.defrag = defrag;
-        promotion_args.is_promotion = _gf_true;
-
-        ret = gf_thread_create (&promote_thread, NULL, &tier_run,
-                                &promotion_args, "tierpro");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to start promotion thread.");
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                goto waitforspawned;
-        }
-
-        hot_compaction_args.this = this;
-        hot_compaction_args.brick_list = &bricklist_hot;
-        hot_compaction_args.defrag = defrag;
-        hot_compaction_args.is_promotion = _gf_false;
-        hot_compaction_args.is_compaction = _gf_true;
-        hot_compaction_args.is_hot_tier = _gf_true;
-
-        ret = gf_thread_create (&hot_compact_thread, NULL, &tier_run,
-                                &hot_compaction_args, "tierhcom");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to start compaction thread.");
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                goto waitforspawnedpromote;
-        }
-
-        cold_compaction_args.this = this;
-        cold_compaction_args.brick_list = &bricklist_cold;
-        cold_compaction_args.defrag = defrag;
-        cold_compaction_args.is_promotion = _gf_false;
-        cold_compaction_args.is_compaction = _gf_true;
-        cold_compaction_args.is_hot_tier = _gf_false;
-
-        ret = gf_thread_create (&cold_compact_thread, NULL, &tier_run,
-                                &cold_compaction_args, "tierccom");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to start compaction thread.");
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                goto waitforspawnedhotcompact;
-        }
-        pthread_join (cold_compact_thread, NULL);
-
-waitforspawnedhotcompact:
-        pthread_join (hot_compact_thread, NULL);
-
-waitforspawnedpromote:
-        pthread_join (promote_thread, NULL);
-
-waitforspawned:
-        pthread_join (demote_thread, NULL);
-
-cleanup:
-        clear_bricklist (&bricklist_cold);
-        clear_bricklist (&bricklist_hot);
-        return ret;
-}
-
-int32_t
-tier_migration_needed (xlator_t *this)
-{
-        gf_defrag_info_t        *defrag = NULL;
-        dht_conf_t              *conf   = NULL;
-        int                      ret = 0;
-
-        conf = this->private;
-
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-        GF_VALIDATE_OR_GOTO (this->name, conf->defrag, out);
-
-        defrag = conf->defrag;
-
-        if ((defrag->cmd == GF_DEFRAG_CMD_START_TIER) ||
-            (defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER))
-                ret = 1;
-out:
-        return ret;
-}
-
-int32_t
-tier_migration_get_dst (xlator_t *this, dht_local_t *local)
-{
-        dht_conf_t              *conf   = NULL;
-        int32_t                  ret = -1;
-        gf_defrag_info_t        *defrag = NULL;
-
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        conf = this->private;
-
-        defrag = conf->defrag;
-
-        if (defrag && defrag->cmd == GF_DEFRAG_CMD_START_DETACH_TIER) {
-                local->rebalance.target_node = conf->subvolumes[0];
-
-        } else if (conf->subvolumes[0] == local->cached_subvol)
-                local->rebalance.target_node =
-                        conf->subvolumes[1];
-        else
-                local->rebalance.target_node =
-                        conf->subvolumes[0];
-
-        if (local->rebalance.target_node)
-                ret = 0;
-
-out:
-        return ret;
-}
-
-xlator_t *
-tier_search (xlator_t *this, dht_layout_t *layout, const char *name)
-{
-        xlator_t                *subvol = NULL;
-        dht_conf_t              *conf   = NULL;
-
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        conf = this->private;
-
-        subvol = TIER_HASHED_SUBVOL;
-
- out:
-        return subvol;
-}
-
-
-static int
-tier_load_externals (xlator_t *this)
-{
-        int               ret            = -1;
-        char *libpathfull = (LIBDIR "/libgfdb.so.0");
-        get_gfdb_methods_t get_gfdb_methods;
-
-        GF_VALIDATE_OR_GOTO ("this", this, out);
-
-        libhandle = dlopen (libpathfull, RTLD_NOW);
-        if (!libhandle) {
-                gf_msg(this->name, GF_LOG_ERROR, 0,
-                       DHT_MSG_LOG_TIER_ERROR,
-                       "Error loading libgfdb.so %s\n", dlerror());
-                ret = -1;
-                goto out;
-        }
-
-        get_gfdb_methods = dlsym (libhandle, "get_gfdb_methods");
-        if (!get_gfdb_methods) {
-                gf_msg(this->name, GF_LOG_ERROR, 0,
-                       DHT_MSG_LOG_TIER_ERROR,
-                       "Error loading get_gfdb_methods()");
-                ret = -1;
-                goto out;
-        }
-
-        get_gfdb_methods (&gfdb_methods);
-
-        ret = 0;
-
-out:
-        if (ret && libhandle)
-                dlclose (libhandle);
-
-        return ret;
-}
-
-static tier_mode_t
-tier_validate_mode (char *mode)
-{
-        int ret = -1;
-
-        if (strcmp (mode, "test") == 0) {
-                ret = TIER_MODE_TEST;
-        } else {
-                ret = TIER_MODE_WM;
-        }
-
-        return ret;
-}
-
-static gf_boolean_t
-tier_validate_compact_mode (char *mode)
-{
-        gf_boolean_t ret = _gf_false;
-
-        gf_msg ("tier", GF_LOG_INFO, 0, DHT_MSG_LOG_TIER_STATUS,
-                "tier_validate_compact_mode: mode = %s", mode);
-
-        if (!strcmp (mode, "on")) {
-                ret = _gf_true;
-        } else {
-                ret = _gf_false;
-        }
-
-        gf_msg ("tier", GF_LOG_ERROR, 0,
-                DHT_MSG_LOG_TIER_STATUS,
-                "tier_validate_compact_mode: ret = %i", ret);
-
-        return ret;
-}
-
-int
-tier_init_methods (xlator_t *this)
-{
-        int ret                  = -1;
-        dht_conf_t      *conf    = NULL;
-        dht_methods_t   *methods = NULL;
-
-        GF_VALIDATE_OR_GOTO ("tier", this, err);
-
-        conf = this->private;
-
-        methods = &(conf->methods);
-
-        methods->migration_get_dst_subvol = tier_migration_get_dst;
-        methods->migration_other   = tier_start;
-        methods->migration_needed  = tier_migration_needed;
-        methods->layout_search     = tier_search;
-
-        ret = 0;
-err:
-        return ret;
-}
-
-
-static void
-tier_save_vol_name (xlator_t *this)
-{
-        dht_conf_t       *conf           = NULL;
-        gf_defrag_info_t *defrag         = NULL;
-        char             *suffix         = NULL;
-        int               name_len       = 0;
-
-
-        conf = this->private;
-        defrag = conf->defrag;
-
-        suffix = strstr (this->name, "-tier-dht");
-
-        if (suffix)
-                name_len = suffix - this->name;
-        else
-                name_len = strlen (this->name);
-
-        if (name_len > GD_VOLUME_NAME_MAX)
-                name_len = GD_VOLUME_NAME_MAX;
-
-        strncpy (defrag->tier_conf.volname, this->name, name_len);
-        defrag->tier_conf.volname[name_len] = 0;
-}
-
-int
-tier_init (xlator_t *this)
-{
-        int               ret            = -1;
-        int               freq           = 0;
-        int               maxsize        = 0;
-        dht_conf_t       *conf           = NULL;
-        gf_defrag_info_t *defrag         = NULL;
-        char             *voldir         = NULL;
-        char             *mode           = NULL;
-        char             *paused         = NULL;
-        tier_mode_t	  tier_mode	 = DEFAULT_TIER_MODE;
-        gf_boolean_t	  compact_mode	 = _gf_false;
-
-        ret = dht_init (this);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "tier_init failed");
-                goto out;
-        }
-
-        conf = this->private;
-
-        ret = tier_init_methods (this);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "tier_init_methods failed");
-                goto out;
-        }
-
-        if (conf->subvolume_cnt != 2) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "Invalid number of subvolumes %d", conf->subvolume_cnt);
-                goto out;
-        }
-
-        /* if instatiated from client side initialization is complete. */
-        if (!conf->defrag) {
-                ret = 0;
-                goto out;
-        }
-
-        /* if instatiated from server side, load db libraries */
-        ret = tier_load_externals (this);
-        if (ret) {
-                gf_msg(this->name, GF_LOG_ERROR, 0,
-                       DHT_MSG_LOG_TIER_ERROR,
-                       "Could not load externals. Aborting");
-                goto out;
-        }
-
-        defrag = conf->defrag;
-
-        defrag->tier_conf.last_demote_qfile_index = 0;
-        defrag->tier_conf.last_promote_qfile_index = 0;
-
-        defrag->tier_conf.is_tier = 1;
-        defrag->this = this;
-
-        ret = dict_get_int32 (this->options,
-                              "tier-max-promote-file-size", &maxsize);
-        if (ret) {
-                maxsize = 0;
-        }
-
-        defrag->tier_conf.tier_max_promote_size = maxsize;
-
-        ret = dict_get_int32 (this->options,
-                              "tier-promote-frequency", &freq);
-        if (ret) {
-                freq = DEFAULT_PROMOTE_FREQ_SEC;
-        }
-
-        defrag->tier_conf.tier_promote_frequency = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "tier-demote-frequency", &freq);
-        if (ret) {
-                freq = DEFAULT_DEMOTE_FREQ_SEC;
-        }
-
-        defrag->tier_conf.tier_demote_frequency = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "tier-hot-compact-frequency", &freq);
-        if (ret) {
-                freq = DEFAULT_HOT_COMPACT_FREQ_SEC;
-        }
-
-        defrag->tier_conf.tier_compact_hot_frequency = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "tier-cold-compact-frequency", &freq);
-        if (ret) {
-                freq = DEFAULT_COLD_COMPACT_FREQ_SEC;
-        }
-
-        defrag->tier_conf.tier_compact_cold_frequency = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "watermark-hi", &freq);
-        if (ret) {
-                freq = DEFAULT_WM_HI;
-        }
-
-        defrag->tier_conf.watermark_hi = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "watermark-low", &freq);
-        if (ret) {
-                freq = DEFAULT_WM_LOW;
-        }
-
-        defrag->tier_conf.watermark_low = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "write-freq-threshold", &freq);
-        if (ret) {
-                freq = DEFAULT_WRITE_FREQ_SEC;
-        }
-
-        defrag->write_freq_threshold = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "read-freq-threshold", &freq);
-        if (ret) {
-                freq = DEFAULT_READ_FREQ_SEC;
-        }
-
-        defrag->read_freq_threshold = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "tier-max-mb", &freq);
-        if (ret) {
-                freq = DEFAULT_TIER_MAX_MIGRATE_MB;
-        }
-
-        defrag->tier_conf.max_migrate_bytes = (uint64_t) freq * 1024 * 1024;
-
-        ret = dict_get_int32 (this->options,
-                              "tier-max-files", &freq);
-        if (ret) {
-                freq = DEFAULT_TIER_MAX_MIGRATE_FILES;
-        }
-
-        defrag->tier_conf.max_migrate_files = freq;
-
-
-        ret = dict_get_int32 (this->options,
-                              "tier-query-limit",
-                              &(defrag->tier_conf.query_limit));
-        if (ret) {
-                defrag->tier_conf.query_limit =
-                        DEFAULT_TIER_QUERY_LIMIT;
-        }
-
-        ret = dict_get_str (this->options,
-                            "tier-compact", &mode);
-
-        if (ret) {
-                defrag->tier_conf.compact_active = DEFAULT_COMP_MODE;
-        } else {
-                compact_mode = tier_validate_compact_mode (mode);
-                /* If compaction is now active, we need to inform the bricks on
-                   the hot and cold tier of this. See dht-common.h for more. */
-                defrag->tier_conf.compact_active = compact_mode;
-                if (compact_mode) {
-                        defrag->tier_conf.compact_mode_switched_hot = _gf_true;
-                        defrag->tier_conf.compact_mode_switched_cold = _gf_true;
-                }
-        }
-
-        ret = dict_get_str (this->options,
-                            "tier-mode", &mode);
-        if (ret) {
-                defrag->tier_conf.mode = DEFAULT_TIER_MODE;
-        } else {
-                tier_mode = tier_validate_mode (mode);
-                defrag->tier_conf.mode = tier_mode;
-        }
-
-        pthread_mutex_init (&defrag->tier_conf.pause_mutex, 0);
-
-        gf_defrag_set_pause_state (&defrag->tier_conf, TIER_RUNNING);
-
-        ret = dict_get_str (this->options,
-                              "tier-pause", &paused);
-
-        if (paused && strcmp (paused, "on") == 0)
-                gf_defrag_set_pause_state (&defrag->tier_conf,
-                                           TIER_REQUEST_PAUSE);
-
-        ret = gf_asprintf(&voldir, "%s/%s",
-                          DEFAULT_VAR_RUN_DIRECTORY,
-                          this->name);
-        if (ret < 0)
-                goto out;
-
-        ret = mkdir_p(voldir, 0777, _gf_true);
-        if (ret == -1 && errno != EEXIST) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                       DHT_MSG_LOG_TIER_ERROR,
-                       "tier_init failed");
-
-                GF_FREE(voldir);
-                goto out;
-        }
-
-        GF_FREE(voldir);
-
-        ret = gf_asprintf (&promotion_qfile, "%s/%s/promote",
-                           DEFAULT_VAR_RUN_DIRECTORY,
-                           this->name);
-        if (ret < 0)
-                goto out;
-
-        ret = gf_asprintf (&demotion_qfile, "%s/%s/demote",
-                           DEFAULT_VAR_RUN_DIRECTORY,
-                           this->name);
-        if (ret < 0) {
-                GF_FREE (promotion_qfile);
-                goto out;
-        }
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                DHT_MSG_LOG_TIER_STATUS,
-               "Promote/demote frequency %d/%d "
-               "Write/Read freq thresholds %d/%d",
-               defrag->tier_conf.tier_promote_frequency,
-               defrag->tier_conf.tier_demote_frequency,
-               defrag->write_freq_threshold,
-               defrag->read_freq_threshold);
-
-        tier_save_vol_name (this);
-
-        ret = 0;
-
-out:
-
-        return ret;
-}
-
-int
-tier_cli_pause_done (int op_ret, call_frame_t *sync_frame, void *data)
-{
-        gf_msg ("tier", GF_LOG_INFO, 0,
-                DHT_MSG_TIER_PAUSED,
-                "Migrate file paused with op_ret %d", op_ret);
-
-        return op_ret;
-}
-
-int
-tier_cli_pause (void *data)
-{
-        gf_defrag_info_t        *defrag         = NULL;
-        xlator_t                *this           = NULL;
-        dht_conf_t              *conf           = NULL;
-        int                      ret            = -1;
-
-        this = data;
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, exit);
-
-        defrag = conf->defrag;
-        GF_VALIDATE_OR_GOTO (this->name, defrag, exit);
-
-        gf_defrag_pause_tier (this, defrag);
-
-        ret = 0;
-exit:
-        return ret;
-}
-
-int
-tier_reconfigure (xlator_t *this, dict_t *options)
-{
-        dht_conf_t       *conf            = NULL;
-        gf_defrag_info_t *defrag          = NULL;
-        char             *mode            = NULL;
-        int               migrate_mb      = 0;
-        gf_boolean_t      req_pause       = _gf_false;
-        int               ret             = 0;
-        call_frame_t            *frame    = NULL;
-        gf_boolean_t last_compact_setting = _gf_false;
-
-        conf = this->private;
-
-        if (conf->defrag) {
-                defrag = conf->defrag;
-                GF_OPTION_RECONF ("tier-max-promote-file-size",
-                                  defrag->tier_conf.tier_max_promote_size,
-                                  options, int32, out);
-
-                GF_OPTION_RECONF ("tier-promote-frequency",
-                                  defrag->tier_conf.tier_promote_frequency,
-                                  options, int32, out);
-
-                GF_OPTION_RECONF ("tier-demote-frequency",
-                                  defrag->tier_conf.tier_demote_frequency,
-                                  options, int32, out);
-
-                GF_OPTION_RECONF ("write-freq-threshold",
-                                  defrag->write_freq_threshold, options,
-                                  int32, out);
-
-                GF_OPTION_RECONF ("read-freq-threshold",
-                                  defrag->read_freq_threshold, options,
-                                  int32, out);
-
-                GF_OPTION_RECONF ("watermark-hi",
-                                  defrag->tier_conf.watermark_hi, options,
-                                  int32, out);
-
-                GF_OPTION_RECONF ("watermark-low",
-                                  defrag->tier_conf.watermark_low, options,
-                                  int32, out);
-
-                last_compact_setting = defrag->tier_conf.compact_active;
-
-                GF_OPTION_RECONF ("tier-compact",
-                                  defrag->tier_conf.compact_active, options,
-                                  bool, out);
-
-                if (last_compact_setting != defrag->tier_conf.compact_active) {
-                        defrag->tier_conf.compact_mode_switched_hot = _gf_true;
-                        defrag->tier_conf.compact_mode_switched_cold = _gf_true;
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_LOG_TIER_STATUS,
-                                "compact mode switched");
-                }
-
-                GF_OPTION_RECONF ("tier-hot-compact-frequency",
-                                  defrag->tier_conf.tier_compact_hot_frequency,
-                                  options, int32, out);
-
-                GF_OPTION_RECONF ("tier-cold-compact-frequency",
-                                  defrag->tier_conf.tier_compact_cold_frequency,
-                                  options, int32, out);
-
-                GF_OPTION_RECONF ("tier-mode",
-                                  mode, options,
-                                  str, out);
-                defrag->tier_conf.mode = tier_validate_mode (mode);
-
-                GF_OPTION_RECONF ("tier-max-mb",
-                                  migrate_mb, options,
-                                  int32, out);
-                defrag->tier_conf.max_migrate_bytes = (uint64_t) migrate_mb *
-                                                        1024 * 1024;
-
-                GF_OPTION_RECONF ("tier-max-files",
-                                  defrag->tier_conf.max_migrate_files, options,
-                                  int32, out);
-
-                GF_OPTION_RECONF ("tier-query-limit",
-                                  defrag->tier_conf.query_limit,
-                                  options, int32, out);
-
-                GF_OPTION_RECONF ("tier-pause",
-                                  req_pause, options,
-                                  bool, out);
-
-                if (req_pause == _gf_true) {
-
-                        frame = create_frame (this, this->ctx->pool);
-                        if (!frame)
-                                goto out;
-
-                        frame->root->pid = GF_CLIENT_PID_DEFRAG;
-
-                        ret = synctask_new (this->ctx->env, tier_cli_pause,
-                                            tier_cli_pause_done, frame, this);
-
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        "pause tier failed on reconfigure");
-                        }
-                } else {
-                        ret = gf_defrag_resume_tier (this, defrag);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        "resume tier failed on reconfigure");
-                        }
-                }
-
-        }
-
-out:
-        return dht_reconfigure (this, options);
-}
-
-void
-tier_fini (xlator_t *this)
-{
-        if (libhandle)
-                dlclose (libhandle);
-
-        GF_FREE (demotion_qfile);
-        GF_FREE (promotion_qfile);
-
-        dht_fini(this);
-}
-
-class_methods_t class_methods = {
-        .init           = tier_init,
-        .fini           = tier_fini,
-        .reconfigure    = tier_reconfigure,
-        .notify         = dht_notify
-};
-
-struct xlator_fops fops = {
-
-        .lookup      = dht_lookup,
-        .create      = tier_create,
-        .mknod       = dht_mknod,
-
-        .open        = dht_open,
-        .statfs      = tier_statfs,
-        .opendir     = dht_opendir,
-        .readdir     = tier_readdir,
-        .readdirp    = tier_readdirp,
-        .fsyncdir    = dht_fsyncdir,
-        .symlink     = dht_symlink,
-        .unlink      = tier_unlink,
-        .link        = tier_link,
-        .mkdir       = dht_mkdir,
-        .rmdir       = dht_rmdir,
-        .rename      = dht_rename,
-        .entrylk     = dht_entrylk,
-        .fentrylk    = dht_fentrylk,
-
-        /* Inode read operations */
-        .stat        = dht_stat,
-        .fstat       = dht_fstat,
-        .access      = dht_access,
-        .readlink    = dht_readlink,
-        .getxattr    = dht_getxattr,
-        .fgetxattr    = dht_fgetxattr,
-        .readv       = dht_readv,
-        .flush       = dht_flush,
-        .fsync       = dht_fsync,
-        .inodelk     = dht_inodelk,
-        .finodelk    = dht_finodelk,
-        .lk          = dht_lk,
-
-        /* Inode write operations */
-        .fremovexattr = dht_fremovexattr,
-        .removexattr = dht_removexattr,
-        .setxattr    = dht_setxattr,
-        .fsetxattr   = dht_fsetxattr,
-        .truncate    = dht_truncate,
-        .ftruncate   = dht_ftruncate,
-        .writev      = dht_writev,
-        .xattrop     = dht_xattrop,
-        .fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
-        .fsetattr    = dht_fsetattr,
-        .fallocate   = dht_fallocate,
-        .discard     = dht_discard,
-        .zerofill    = dht_zerofill,
-};
-
-struct xlator_cbks cbks = {
-        .release    = dht_release,
-        .forget     = dht_forget
-};
diff --git a/xlators/cluster/dht/src/tier.h b/xlators/cluster/dht/src/tier.h
deleted file mode 100644
index 764860e6884..00000000000
--- a/xlators/cluster/dht/src/tier.h
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _TIER_H_
-#define _TIER_H_
-
-
-/******************************************************************************/
-/* This is from dht-rebalancer.c as we dont have dht-rebalancer.h */
-#include "dht-common.h"
-#include "xlator.h"
-#include <signal.h>
-#include <fnmatch.h>
-#include <signal.h>
-
-/*
- * Size of timer wheel. We would not promote or demote less
- * frequently than this number.
- */
-#define TIMER_SECS 3600
-
-#include "gfdb_data_store.h"
-#include <ctype.h>
-#include <sys/stat.h>
-
-#define PROMOTION_QFILE "promotequeryfile"
-#define DEMOTION_QFILE "demotequeryfile"
-
-#define TIER_HASHED_SUBVOL   conf->subvolumes[0]
-#define TIER_UNHASHED_SUBVOL   conf->subvolumes[1]
-
-#define GET_QFILE_PATH(is_promotion)\
-        (is_promotion) ? promotion_qfile : demotion_qfile
-
-typedef struct tier_qfile_array {
-        int             *fd_array;
-        ssize_t         array_size;
-        ssize_t         next_index;
-        /* Indicate the number of exhuasted FDs*/
-        ssize_t         exhausted_count;
-} tier_qfile_array_t;
-
-
-typedef struct _query_cbk_args {
-        xlator_t                *this;
-        gf_defrag_info_t        *defrag;
-        /* This is write */
-        int                     query_fd;
-        int                     is_promotion;
-        int                     is_compaction;
-        /* This is for read */
-        tier_qfile_array_t       *qfile_array;
-} query_cbk_args_t;
-
-int
-gf_run_tier(xlator_t *this, gf_defrag_info_t *defrag);
-
-typedef struct gfdb_brick_info {
-        gfdb_time_t             *time_stamp;
-        gf_boolean_t            _gfdb_promote;
-        query_cbk_args_t        *_query_cbk_args;
-} gfdb_brick_info_t;
-
-typedef struct brick_list {
-        xlator_t          *xlator;
-        char              *brick_db_path;
-        char              brick_name[NAME_MAX];
-        char              qfile_path[PATH_MAX];
-        struct list_head  list;
-} tier_brick_list_t;
-
-typedef struct _dm_thread_args {
-        xlator_t                *this;
-        gf_defrag_info_t        *defrag;
-        struct list_head        *brick_list;
-        int                     freq_time;
-        int                     return_value;
-        int                     is_promotion;
-        int                     is_compaction;
-        gf_boolean_t            is_hot_tier;
-} migration_args_t;
-
-typedef enum tier_watermark_op_ {
-        TIER_WM_NONE = 0,
-        TIER_WM_LOW,
-        TIER_WM_HI,
-        TIER_WM_MID
-} tier_watermark_op_t;
-
-#define DEFAULT_PROMOTE_FREQ_SEC       120
-#define DEFAULT_DEMOTE_FREQ_SEC        120
-#define DEFAULT_HOT_COMPACT_FREQ_SEC   604800
-#define DEFAULT_COLD_COMPACT_FREQ_SEC  604800
-#define DEFAULT_DEMOTE_DEGRADED        1
-#define DEFAULT_WRITE_FREQ_SEC         0
-#define DEFAULT_READ_FREQ_SEC          0
-#define DEFAULT_WM_LOW                 75
-#define DEFAULT_WM_HI                  90
-#define DEFAULT_TIER_MODE              TIER_MODE_TEST
-#define DEFAULT_COMP_MODE              _gf_true
-#define DEFAULT_TIER_MAX_MIGRATE_MB    1000
-#define DEFAULT_TIER_MAX_MIGRATE_FILES 5000
-#define DEFAULT_TIER_QUERY_LIMIT       100
-
-#endif
diff --git a/xlators/cluster/dht/src/tier.sym b/xlators/cluster/dht/src/tier.sym
deleted file mode 100644
index 60205d145b6..00000000000
--- a/xlators/cluster/dht/src/tier.sym
+++ /dev/null
@@ -1,9 +0,0 @@
-fops
-cbks
-class_methods
-dht_methods
-tier_methods
-options
-mem_acct_init
-reconfigure
-dumpops
diff --git a/xlators/cluster/dht/src/unittest/dht_layout_mock.c b/xlators/cluster/dht/src/unittest/dht_layout_mock.c
index 6544f4208f5..771452963d1 100644
--- a/xlators/cluster/dht/src/unittest/dht_layout_mock.c
+++ b/xlators/cluster/dht/src/unittest/dht_layout_mock.c
@@ -7,66 +7,67 @@
   later), or the GNU General Public License, version 2 (GPLv2), in all
   cases as published by the Free Software Foundation.
 */
-#include "glusterfs.h"
-#include "xlator.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
 #include "dht-common.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 
 int
-dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
+dht_hash_compute(xlator_t *this, int type, const char *name, uint32_t *hash_p)
 {
     return 0;
 }
 
 int
-dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+dht_inode_ctx_layout_get(inode_t *inode, xlator_t *this, dht_layout_t **layout)
 {
     return 0;
 }
 
 int
-dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
-                          dht_layout_t *layout_int)
+dht_inode_ctx_layout_set(inode_t *inode, xlator_t *this,
+                         dht_layout_t *layout_int)
 {
     return 0;
 }
 
 int
-dict_get_ptr (dict_t *this, char *key, void **ptr)
+dict_get_ptr(dict_t *this, char *key, void **ptr)
 {
     return 0;
 }
 
 int
-dict_get_ptr_and_len (dict_t *this, char *key, void **ptr, int *len)
+dict_get_ptr_and_len(dict_t *this, char *key, void **ptr, int *len)
 {
     return 0;
 }
 
-int _gf_log (const char *domain, const char *file,
-             const char *function, int32_t line, gf_loglevel_t level,
-             const char *fmt, ...)
+int
+_gf_log(const char *domain, const char *file, const char *function,
+        int32_t line, gf_loglevel_t level, const char *fmt, ...)
 {
     return 0;
 }
 
-int _gf_log_callingfn (const char *domain, const char *file,
-                       const char *function, int32_t line, gf_loglevel_t level,
-                       const char *fmt, ...)
+int
+_gf_log_callingfn(const char *domain, const char *file, const char *function,
+                  int32_t line, gf_loglevel_t level, const char *fmt, ...)
 {
     return 0;
 }
 
-void gf_uuid_unparse(const uuid_t uu, char *out)
+void
+gf_uuid_unparse(const uuid_t uu, char *out)
 {
     // could call a will-return function here
     // to place the correct data in *out
 }
 
 int
-_gf_msg (const char *domain, const char *file, const char *function,
-         int32_t line, gf_loglevel_t level, int errnum, int trace,
-         uint64_t msgid, const char *fmt, ...)
+_gf_msg(const char *domain, const char *file, const char *function,
+        int32_t line, gf_loglevel_t level, int errnum, int trace,
+        uint64_t msgid, const char *fmt, ...)
 {
     return 0;
 }
diff --git a/xlators/cluster/dht/src/unittest/dht_layout_unittest.c b/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
index fb6445ce1f3..c94a1d0a2e1 100644
--- a/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
+++ b/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
@@ -9,8 +9,8 @@
 */
 
 #include "dht-common.h"
-#include "logging.h"
-#include "xlator.h"
+#include <glusterfs/logging.h>
+#include <glusterfs/xlator.h>
 
 #include <inttypes.h>
 #include <stdarg.h>
@@ -34,16 +34,16 @@ helper_xlator_init(uint32_t num_types)
     xl = test_calloc(1, sizeof(xlator_t));
     assert_non_null(xl);
     xl->mem_acct->num_types = num_types;
-    xl->mem_acct = test_calloc (sizeof(struct mem_acct)
-                                + sizeof(struct mem_acct_rec) + num_types);
+    xl->mem_acct = test_calloc(sizeof(struct mem_acct) +
+                               sizeof(struct mem_acct_rec) + num_types);
     assert_non_null(xl->mem_acct);
 
     xl->ctx = test_calloc(1, sizeof(glusterfs_ctx_t));
     assert_non_null(xl->ctx);
 
     for (i = 0; i < num_types; i++) {
-            ret = LOCK_INIT(&(xl->mem_acct.rec[i].lock));
-            assert_false(ret);
+        ret = LOCK_INIT(&(xl->mem_acct.rec[i].lock));
+        assert_false(ret);
     }
 
     ENSURE(num_types == xl->mem_acct.num_types);
@@ -58,8 +58,8 @@ helper_xlator_destroy(xlator_t *xl)
     int i, ret;
 
     for (i = 0; i < xl->mem_acct.num_types; i++) {
-            ret = LOCK_DESTROY(&(xl->mem_acct.rec[i].lock));
-            assert_int_equal(ret, 0);
+        ret = LOCK_DESTROY(&(xl->mem_acct.rec[i].lock));
+        assert_int_equal(ret, 0);
     }
 
     free(xl->mem_acct.rec);
@@ -76,7 +76,7 @@ test_dht_layout_new(void **state)
 {
     xlator_t *xl;
     dht_layout_t *layout;
-    dht_conf_t   *conf;
+    dht_conf_t *conf;
     int cnt;
 
     expect_assert_failure(dht_layout_new(NULL, 0));
@@ -90,7 +90,7 @@ test_dht_layout_new(void **state)
     assert_non_null(layout);
     assert_int_equal(layout->type, DHT_HASH_TYPE_DM);
     assert_int_equal(layout->cnt, cnt);
-    assert_int_equal(GF_ATOMIC_GET (layout->ref), 1);
+    assert_int_equal(GF_ATOMIC_GET(layout->ref), 1);
     assert_int_equal(layout->gen, 0);
     assert_int_equal(layout->spread_cnt, 0);
     free(layout);
@@ -107,7 +107,7 @@ test_dht_layout_new(void **state)
     assert_non_null(layout);
     assert_int_equal(layout->type, DHT_HASH_TYPE_DM);
     assert_int_equal(layout->cnt, cnt);
-    assert_int_equal(GF_ATOMIC_GET (layout->ref), 1);
+    assert_int_equal(GF_ATOMIC_GET(layout->ref), 1);
     assert_int_equal(layout->gen, conf->gen);
     assert_int_equal(layout->spread_cnt, conf->dir_spread_cnt);
     free(layout);
@@ -116,7 +116,9 @@ test_dht_layout_new(void **state)
     helper_xlator_destroy(xl);
 }
 
-int main(void) {
+int
+main(void)
+{
     const struct CMUnitTest xlator_dht_layout_tests[] = {
         unit_test(test_dht_layout_new),
     };
diff --git a/xlators/cluster/ec/src/ec-code-avx.c b/xlators/cluster/ec/src/ec-code-avx.c
index 92bd3e83c5e..70afaa00f54 100644
--- a/xlators/cluster/ec/src/ec-code-avx.c
+++ b/xlators/cluster/ec/src/ec-code-avx.c
@@ -34,10 +34,9 @@ ec_code_avx_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
                  uint32_t bit)
 {
     if (builder->linear) {
-        ec_code_intel_op_mov_m2avx(builder, REG_SI, REG_DX, 1,
-                                   idx * builder->width * builder->bits +
-                                   bit * builder->width,
-                                   dst);
+        ec_code_intel_op_mov_m2avx(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
     } else {
         if (builder->base != idx) {
             ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
@@ -81,10 +80,9 @@ ec_code_avx_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
                  uint32_t bit)
 {
     if (builder->linear) {
-        ec_code_intel_op_xor_m2avx(builder, REG_SI, REG_DX, 1,
-                                   idx * builder->width * builder->bits +
-                                   bit * builder->width,
-                                   dst);
+        ec_code_intel_op_xor_m2avx(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
     } else {
         if (builder->base != idx) {
             ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
@@ -96,21 +94,16 @@ ec_code_avx_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
     }
 }
 
-static char *ec_code_avx_needed_flags[] = {
-    "avx2",
-    NULL
-};
+static char *ec_code_avx_needed_flags[] = {"avx2", NULL};
 
-ec_code_gen_t ec_code_gen_avx = {
-    .name   = "avx",
-    .flags  = ec_code_avx_needed_flags,
-    .width  = 32,
-    .prolog = ec_code_avx_prolog,
-    .epilog = ec_code_avx_epilog,
-    .load   = ec_code_avx_load,
-    .store  = ec_code_avx_store,
-    .copy   = ec_code_avx_copy,
-    .xor2   = ec_code_avx_xor2,
-    .xor3   = ec_code_avx_xor3,
-    .xorm   = ec_code_avx_xorm
-};
+ec_code_gen_t ec_code_gen_avx = {.name = "avx",
+                                 .flags = ec_code_avx_needed_flags,
+                                 .width = 32,
+                                 .prolog = ec_code_avx_prolog,
+                                 .epilog = ec_code_avx_epilog,
+                                 .load = ec_code_avx_load,
+                                 .store = ec_code_avx_store,
+                                 .copy = ec_code_avx_copy,
+                                 .xor2 = ec_code_avx_xor2,
+                                 .xor3 = ec_code_avx_xor3,
+                                 .xorm = ec_code_avx_xorm};
diff --git a/xlators/cluster/ec/src/ec-code-c.c b/xlators/cluster/ec/src/ec-code-c.c
index 7387f3ea435..acdc665c2cf 100644
--- a/xlators/cluster/ec/src/ec-code-c.c
+++ b/xlators/cluster/ec/src/ec-code-c.c
@@ -16,12 +16,14 @@
 
 #define WIDTH (EC_METHOD_WORD_SIZE / sizeof(uint64_t))
 
-static void gf8_muladd_00(void *out, void *in)
+static void
+gf8_muladd_00(void *out, void *in)
 {
     memcpy(out, in, EC_METHOD_WORD_SIZE * 8);
 }
 
-static void gf8_muladd_01(void *out, void *in)
+static void
+gf8_muladd_01(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -42,7 +44,8 @@ static void gf8_muladd_01(void *out, void *in)
     }
 }
 
-static void gf8_muladd_02(void *out, void *in)
+static void
+gf8_muladd_02(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -83,7 +86,8 @@ static void gf8_muladd_02(void *out, void *in)
     }
 }
 
-static void gf8_muladd_03(void *out, void *in)
+static void
+gf8_muladd_03(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -126,7 +130,8 @@ static void gf8_muladd_03(void *out, void *in)
     }
 }
 
-static void gf8_muladd_04(void *out, void *in)
+static void
+gf8_muladd_04(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -169,7 +174,8 @@ static void gf8_muladd_04(void *out, void *in)
     }
 }
 
-static void gf8_muladd_05(void *out, void *in)
+static void
+gf8_muladd_05(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -210,7 +216,8 @@ static void gf8_muladd_05(void *out, void *in)
     }
 }
 
-static void gf8_muladd_06(void *out, void *in)
+static void
+gf8_muladd_06(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -253,7 +260,8 @@ static void gf8_muladd_06(void *out, void *in)
     }
 }
 
-static void gf8_muladd_07(void *out, void *in)
+static void
+gf8_muladd_07(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -299,7 +307,8 @@ static void gf8_muladd_07(void *out, void *in)
     }
 }
 
-static void gf8_muladd_08(void *out, void *in)
+static void
+gf8_muladd_08(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -340,7 +349,8 @@ static void gf8_muladd_08(void *out, void *in)
     }
 }
 
-static void gf8_muladd_09(void *out, void *in)
+static void
+gf8_muladd_09(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -383,7 +393,8 @@ static void gf8_muladd_09(void *out, void *in)
     }
 }
 
-static void gf8_muladd_0A(void *out, void *in)
+static void
+gf8_muladd_0A(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -424,7 +435,8 @@ static void gf8_muladd_0A(void *out, void *in)
     }
 }
 
-static void gf8_muladd_0B(void *out, void *in)
+static void
+gf8_muladd_0B(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -469,7 +481,8 @@ static void gf8_muladd_0B(void *out, void *in)
     }
 }
 
-static void gf8_muladd_0C(void *out, void *in)
+static void
+gf8_muladd_0C(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -513,7 +526,8 @@ static void gf8_muladd_0C(void *out, void *in)
     }
 }
 
-static void gf8_muladd_0D(void *out, void *in)
+static void
+gf8_muladd_0D(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -558,7 +572,8 @@ static void gf8_muladd_0D(void *out, void *in)
     }
 }
 
-static void gf8_muladd_0E(void *out, void *in)
+static void
+gf8_muladd_0E(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -604,7 +619,8 @@ static void gf8_muladd_0E(void *out, void *in)
     }
 }
 
-static void gf8_muladd_0F(void *out, void *in)
+static void
+gf8_muladd_0F(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -650,7 +666,8 @@ static void gf8_muladd_0F(void *out, void *in)
     }
 }
 
-static void gf8_muladd_10(void *out, void *in)
+static void
+gf8_muladd_10(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -694,7 +711,8 @@ static void gf8_muladd_10(void *out, void *in)
     }
 }
 
-static void gf8_muladd_11(void *out, void *in)
+static void
+gf8_muladd_11(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -735,7 +753,8 @@ static void gf8_muladd_11(void *out, void *in)
     }
 }
 
-static void gf8_muladd_12(void *out, void *in)
+static void
+gf8_muladd_12(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -779,7 +798,8 @@ static void gf8_muladd_12(void *out, void *in)
     }
 }
 
-static void gf8_muladd_13(void *out, void *in)
+static void
+gf8_muladd_13(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -823,7 +843,8 @@ static void gf8_muladd_13(void *out, void *in)
     }
 }
 
-static void gf8_muladd_14(void *out, void *in)
+static void
+gf8_muladd_14(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -867,7 +888,8 @@ static void gf8_muladd_14(void *out, void *in)
     }
 }
 
-static void gf8_muladd_15(void *out, void *in)
+static void
+gf8_muladd_15(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -910,7 +932,8 @@ static void gf8_muladd_15(void *out, void *in)
     }
 }
 
-static void gf8_muladd_16(void *out, void *in)
+static void
+gf8_muladd_16(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -956,7 +979,8 @@ static void gf8_muladd_16(void *out, void *in)
     }
 }
 
-static void gf8_muladd_17(void *out, void *in)
+static void
+gf8_muladd_17(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1002,7 +1026,8 @@ static void gf8_muladd_17(void *out, void *in)
     }
 }
 
-static void gf8_muladd_18(void *out, void *in)
+static void
+gf8_muladd_18(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1046,7 +1071,8 @@ static void gf8_muladd_18(void *out, void *in)
     }
 }
 
-static void gf8_muladd_19(void *out, void *in)
+static void
+gf8_muladd_19(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1090,7 +1116,8 @@ static void gf8_muladd_19(void *out, void *in)
     }
 }
 
-static void gf8_muladd_1A(void *out, void *in)
+static void
+gf8_muladd_1A(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1136,7 +1163,8 @@ static void gf8_muladd_1A(void *out, void *in)
     }
 }
 
-static void gf8_muladd_1B(void *out, void *in)
+static void
+gf8_muladd_1B(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1183,7 +1211,8 @@ static void gf8_muladd_1B(void *out, void *in)
     }
 }
 
-static void gf8_muladd_1C(void *out, void *in)
+static void
+gf8_muladd_1C(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1230,7 +1259,8 @@ static void gf8_muladd_1C(void *out, void *in)
     }
 }
 
-static void gf8_muladd_1D(void *out, void *in)
+static void
+gf8_muladd_1D(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1277,7 +1307,8 @@ static void gf8_muladd_1D(void *out, void *in)
     }
 }
 
-static void gf8_muladd_1E(void *out, void *in)
+static void
+gf8_muladd_1E(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1323,7 +1354,8 @@ static void gf8_muladd_1E(void *out, void *in)
     }
 }
 
-static void gf8_muladd_1F(void *out, void *in)
+static void
+gf8_muladd_1F(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1367,7 +1399,8 @@ static void gf8_muladd_1F(void *out, void *in)
     }
 }
 
-static void gf8_muladd_20(void *out, void *in)
+static void
+gf8_muladd_20(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1411,7 +1444,8 @@ static void gf8_muladd_20(void *out, void *in)
     }
 }
 
-static void gf8_muladd_21(void *out, void *in)
+static void
+gf8_muladd_21(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1454,7 +1488,8 @@ static void gf8_muladd_21(void *out, void *in)
     }
 }
 
-static void gf8_muladd_22(void *out, void *in)
+static void
+gf8_muladd_22(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1495,7 +1530,8 @@ static void gf8_muladd_22(void *out, void *in)
     }
 }
 
-static void gf8_muladd_23(void *out, void *in)
+static void
+gf8_muladd_23(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1536,7 +1572,8 @@ static void gf8_muladd_23(void *out, void *in)
     }
 }
 
-static void gf8_muladd_24(void *out, void *in)
+static void
+gf8_muladd_24(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1580,7 +1617,8 @@ static void gf8_muladd_24(void *out, void *in)
     }
 }
 
-static void gf8_muladd_25(void *out, void *in)
+static void
+gf8_muladd_25(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1623,7 +1661,8 @@ static void gf8_muladd_25(void *out, void *in)
     }
 }
 
-static void gf8_muladd_26(void *out, void *in)
+static void
+gf8_muladd_26(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1668,7 +1707,8 @@ static void gf8_muladd_26(void *out, void *in)
     }
 }
 
-static void gf8_muladd_27(void *out, void *in)
+static void
+gf8_muladd_27(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1709,7 +1749,8 @@ static void gf8_muladd_27(void *out, void *in)
     }
 }
 
-static void gf8_muladd_28(void *out, void *in)
+static void
+gf8_muladd_28(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1754,7 +1795,8 @@ static void gf8_muladd_28(void *out, void *in)
     }
 }
 
-static void gf8_muladd_29(void *out, void *in)
+static void
+gf8_muladd_29(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1799,7 +1841,8 @@ static void gf8_muladd_29(void *out, void *in)
     }
 }
 
-static void gf8_muladd_2A(void *out, void *in)
+static void
+gf8_muladd_2A(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1843,7 +1886,8 @@ static void gf8_muladd_2A(void *out, void *in)
     }
 }
 
-static void gf8_muladd_2B(void *out, void *in)
+static void
+gf8_muladd_2B(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1887,7 +1931,8 @@ static void gf8_muladd_2B(void *out, void *in)
     }
 }
 
-static void gf8_muladd_2C(void *out, void *in)
+static void
+gf8_muladd_2C(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1933,7 +1978,8 @@ static void gf8_muladd_2C(void *out, void *in)
     }
 }
 
-static void gf8_muladd_2D(void *out, void *in)
+static void
+gf8_muladd_2D(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -1979,7 +2025,8 @@ static void gf8_muladd_2D(void *out, void *in)
     }
 }
 
-static void gf8_muladd_2E(void *out, void *in)
+static void
+gf8_muladd_2E(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2024,7 +2071,8 @@ static void gf8_muladd_2E(void *out, void *in)
     }
 }
 
-static void gf8_muladd_2F(void *out, void *in)
+static void
+gf8_muladd_2F(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2069,7 +2117,8 @@ static void gf8_muladd_2F(void *out, void *in)
     }
 }
 
-static void gf8_muladd_30(void *out, void *in)
+static void
+gf8_muladd_30(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2113,7 +2162,8 @@ static void gf8_muladd_30(void *out, void *in)
     }
 }
 
-static void gf8_muladd_31(void *out, void *in)
+static void
+gf8_muladd_31(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2158,7 +2208,8 @@ static void gf8_muladd_31(void *out, void *in)
     }
 }
 
-static void gf8_muladd_32(void *out, void *in)
+static void
+gf8_muladd_32(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2202,7 +2253,8 @@ static void gf8_muladd_32(void *out, void *in)
     }
 }
 
-static void gf8_muladd_33(void *out, void *in)
+static void
+gf8_muladd_33(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2249,7 +2301,8 @@ static void gf8_muladd_33(void *out, void *in)
     }
 }
 
-static void gf8_muladd_34(void *out, void *in)
+static void
+gf8_muladd_34(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2296,7 +2349,8 @@ static void gf8_muladd_34(void *out, void *in)
     }
 }
 
-static void gf8_muladd_35(void *out, void *in)
+static void
+gf8_muladd_35(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2341,7 +2395,8 @@ static void gf8_muladd_35(void *out, void *in)
     }
 }
 
-static void gf8_muladd_36(void *out, void *in)
+static void
+gf8_muladd_36(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2385,7 +2440,8 @@ static void gf8_muladd_36(void *out, void *in)
     }
 }
 
-static void gf8_muladd_37(void *out, void *in)
+static void
+gf8_muladd_37(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2431,7 +2487,8 @@ static void gf8_muladd_37(void *out, void *in)
     }
 }
 
-static void gf8_muladd_38(void *out, void *in)
+static void
+gf8_muladd_38(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2476,7 +2533,8 @@ static void gf8_muladd_38(void *out, void *in)
     }
 }
 
-static void gf8_muladd_39(void *out, void *in)
+static void
+gf8_muladd_39(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2521,7 +2579,8 @@ static void gf8_muladd_39(void *out, void *in)
     }
 }
 
-static void gf8_muladd_3A(void *out, void *in)
+static void
+gf8_muladd_3A(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2569,7 +2628,8 @@ static void gf8_muladd_3A(void *out, void *in)
     }
 }
 
-static void gf8_muladd_3B(void *out, void *in)
+static void
+gf8_muladd_3B(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2614,7 +2674,8 @@ static void gf8_muladd_3B(void *out, void *in)
     }
 }
 
-static void gf8_muladd_3C(void *out, void *in)
+static void
+gf8_muladd_3C(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2659,7 +2720,8 @@ static void gf8_muladd_3C(void *out, void *in)
     }
 }
 
-static void gf8_muladd_3D(void *out, void *in)
+static void
+gf8_muladd_3D(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2704,7 +2766,8 @@ static void gf8_muladd_3D(void *out, void *in)
     }
 }
 
-static void gf8_muladd_3E(void *out, void *in)
+static void
+gf8_muladd_3E(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2748,7 +2811,8 @@ static void gf8_muladd_3E(void *out, void *in)
     }
 }
 
-static void gf8_muladd_3F(void *out, void *in)
+static void
+gf8_muladd_3F(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2793,7 +2857,8 @@ static void gf8_muladd_3F(void *out, void *in)
     }
 }
 
-static void gf8_muladd_40(void *out, void *in)
+static void
+gf8_muladd_40(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2837,7 +2902,8 @@ static void gf8_muladd_40(void *out, void *in)
     }
 }
 
-static void gf8_muladd_41(void *out, void *in)
+static void
+gf8_muladd_41(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2881,7 +2947,8 @@ static void gf8_muladd_41(void *out, void *in)
     }
 }
 
-static void gf8_muladd_42(void *out, void *in)
+static void
+gf8_muladd_42(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2922,7 +2989,8 @@ static void gf8_muladd_42(void *out, void *in)
     }
 }
 
-static void gf8_muladd_43(void *out, void *in)
+static void
+gf8_muladd_43(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -2963,7 +3031,8 @@ static void gf8_muladd_43(void *out, void *in)
     }
 }
 
-static void gf8_muladd_44(void *out, void *in)
+static void
+gf8_muladd_44(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3006,7 +3075,8 @@ static void gf8_muladd_44(void *out, void *in)
     }
 }
 
-static void gf8_muladd_45(void *out, void *in)
+static void
+gf8_muladd_45(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3047,7 +3117,8 @@ static void gf8_muladd_45(void *out, void *in)
     }
 }
 
-static void gf8_muladd_46(void *out, void *in)
+static void
+gf8_muladd_46(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3088,7 +3159,8 @@ static void gf8_muladd_46(void *out, void *in)
     }
 }
 
-static void gf8_muladd_47(void *out, void *in)
+static void
+gf8_muladd_47(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3131,7 +3203,8 @@ static void gf8_muladd_47(void *out, void *in)
     }
 }
 
-static void gf8_muladd_48(void *out, void *in)
+static void
+gf8_muladd_48(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3175,7 +3248,8 @@ static void gf8_muladd_48(void *out, void *in)
     }
 }
 
-static void gf8_muladd_49(void *out, void *in)
+static void
+gf8_muladd_49(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3219,7 +3293,8 @@ static void gf8_muladd_49(void *out, void *in)
     }
 }
 
-static void gf8_muladd_4A(void *out, void *in)
+static void
+gf8_muladd_4A(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3263,7 +3338,8 @@ static void gf8_muladd_4A(void *out, void *in)
     }
 }
 
-static void gf8_muladd_4B(void *out, void *in)
+static void
+gf8_muladd_4B(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3309,7 +3385,8 @@ static void gf8_muladd_4B(void *out, void *in)
     }
 }
 
-static void gf8_muladd_4C(void *out, void *in)
+static void
+gf8_muladd_4C(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3354,7 +3431,8 @@ static void gf8_muladd_4C(void *out, void *in)
     }
 }
 
-static void gf8_muladd_4D(void *out, void *in)
+static void
+gf8_muladd_4D(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3399,7 +3477,8 @@ static void gf8_muladd_4D(void *out, void *in)
     }
 }
 
-static void gf8_muladd_4E(void *out, void *in)
+static void
+gf8_muladd_4E(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3440,7 +3519,8 @@ static void gf8_muladd_4E(void *out, void *in)
     }
 }
 
-static void gf8_muladd_4F(void *out, void *in)
+static void
+gf8_muladd_4F(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3481,7 +3561,8 @@ static void gf8_muladd_4F(void *out, void *in)
     }
 }
 
-static void gf8_muladd_50(void *out, void *in)
+static void
+gf8_muladd_50(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3526,7 +3607,8 @@ static void gf8_muladd_50(void *out, void *in)
     }
 }
 
-static void gf8_muladd_51(void *out, void *in)
+static void
+gf8_muladd_51(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3567,7 +3649,8 @@ static void gf8_muladd_51(void *out, void *in)
     }
 }
 
-static void gf8_muladd_52(void *out, void *in)
+static void
+gf8_muladd_52(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3613,7 +3696,8 @@ static void gf8_muladd_52(void *out, void *in)
     }
 }
 
-static void gf8_muladd_53(void *out, void *in)
+static void
+gf8_muladd_53(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3654,7 +3738,8 @@ static void gf8_muladd_53(void *out, void *in)
     }
 }
 
-static void gf8_muladd_54(void *out, void *in)
+static void
+gf8_muladd_54(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3700,7 +3785,8 @@ static void gf8_muladd_54(void *out, void *in)
     }
 }
 
-static void gf8_muladd_55(void *out, void *in)
+static void
+gf8_muladd_55(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3745,7 +3831,8 @@ static void gf8_muladd_55(void *out, void *in)
     }
 }
 
-static void gf8_muladd_56(void *out, void *in)
+static void
+gf8_muladd_56(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3788,7 +3875,8 @@ static void gf8_muladd_56(void *out, void *in)
     }
 }
 
-static void gf8_muladd_57(void *out, void *in)
+static void
+gf8_muladd_57(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3832,7 +3920,8 @@ static void gf8_muladd_57(void *out, void *in)
     }
 }
 
-static void gf8_muladd_58(void *out, void *in)
+static void
+gf8_muladd_58(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3876,7 +3965,8 @@ static void gf8_muladd_58(void *out, void *in)
     }
 }
 
-static void gf8_muladd_59(void *out, void *in)
+static void
+gf8_muladd_59(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3920,7 +4010,8 @@ static void gf8_muladd_59(void *out, void *in)
     }
 }
 
-static void gf8_muladd_5A(void *out, void *in)
+static void
+gf8_muladd_5A(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -3965,7 +4056,8 @@ static void gf8_muladd_5A(void *out, void *in)
     }
 }
 
-static void gf8_muladd_5B(void *out, void *in)
+static void
+gf8_muladd_5B(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4012,7 +4104,8 @@ static void gf8_muladd_5B(void *out, void *in)
     }
 }
 
-static void gf8_muladd_5C(void *out, void *in)
+static void
+gf8_muladd_5C(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4057,7 +4150,8 @@ static void gf8_muladd_5C(void *out, void *in)
     }
 }
 
-static void gf8_muladd_5D(void *out, void *in)
+static void
+gf8_muladd_5D(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4104,7 +4198,8 @@ static void gf8_muladd_5D(void *out, void *in)
     }
 }
 
-static void gf8_muladd_5E(void *out, void *in)
+static void
+gf8_muladd_5E(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4151,7 +4246,8 @@ static void gf8_muladd_5E(void *out, void *in)
     }
 }
 
-static void gf8_muladd_5F(void *out, void *in)
+static void
+gf8_muladd_5F(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4197,7 +4293,8 @@ static void gf8_muladd_5F(void *out, void *in)
     }
 }
 
-static void gf8_muladd_60(void *out, void *in)
+static void
+gf8_muladd_60(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4241,7 +4338,8 @@ static void gf8_muladd_60(void *out, void *in)
     }
 }
 
-static void gf8_muladd_61(void *out, void *in)
+static void
+gf8_muladd_61(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4285,7 +4383,8 @@ static void gf8_muladd_61(void *out, void *in)
     }
 }
 
-static void gf8_muladd_62(void *out, void *in)
+static void
+gf8_muladd_62(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4331,7 +4430,8 @@ static void gf8_muladd_62(void *out, void *in)
     }
 }
 
-static void gf8_muladd_63(void *out, void *in)
+static void
+gf8_muladd_63(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4378,7 +4478,8 @@ static void gf8_muladd_63(void *out, void *in)
     }
 }
 
-static void gf8_muladd_64(void *out, void *in)
+static void
+gf8_muladd_64(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4422,7 +4523,8 @@ static void gf8_muladd_64(void *out, void *in)
     }
 }
 
-static void gf8_muladd_65(void *out, void *in)
+static void
+gf8_muladd_65(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4468,7 +4570,8 @@ static void gf8_muladd_65(void *out, void *in)
     }
 }
 
-static void gf8_muladd_66(void *out, void *in)
+static void
+gf8_muladd_66(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4515,7 +4618,8 @@ static void gf8_muladd_66(void *out, void *in)
     }
 }
 
-static void gf8_muladd_67(void *out, void *in)
+static void
+gf8_muladd_67(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4561,7 +4665,8 @@ static void gf8_muladd_67(void *out, void *in)
     }
 }
 
-static void gf8_muladd_68(void *out, void *in)
+static void
+gf8_muladd_68(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4607,7 +4712,8 @@ static void gf8_muladd_68(void *out, void *in)
     }
 }
 
-static void gf8_muladd_69(void *out, void *in)
+static void
+gf8_muladd_69(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4650,7 +4756,8 @@ static void gf8_muladd_69(void *out, void *in)
     }
 }
 
-static void gf8_muladd_6A(void *out, void *in)
+static void
+gf8_muladd_6A(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4695,7 +4802,8 @@ static void gf8_muladd_6A(void *out, void *in)
     }
 }
 
-static void gf8_muladd_6B(void *out, void *in)
+static void
+gf8_muladd_6B(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4739,7 +4847,8 @@ static void gf8_muladd_6B(void *out, void *in)
     }
 }
 
-static void gf8_muladd_6C(void *out, void *in)
+static void
+gf8_muladd_6C(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4783,7 +4892,8 @@ static void gf8_muladd_6C(void *out, void *in)
     }
 }
 
-static void gf8_muladd_6D(void *out, void *in)
+static void
+gf8_muladd_6D(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4827,7 +4937,8 @@ static void gf8_muladd_6D(void *out, void *in)
     }
 }
 
-static void gf8_muladd_6E(void *out, void *in)
+static void
+gf8_muladd_6E(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4872,7 +4983,8 @@ static void gf8_muladd_6E(void *out, void *in)
     }
 }
 
-static void gf8_muladd_6F(void *out, void *in)
+static void
+gf8_muladd_6F(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4917,7 +5029,8 @@ static void gf8_muladd_6F(void *out, void *in)
     }
 }
 
-static void gf8_muladd_70(void *out, void *in)
+static void
+gf8_muladd_70(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -4962,7 +5075,8 @@ static void gf8_muladd_70(void *out, void *in)
     }
 }
 
-static void gf8_muladd_71(void *out, void *in)
+static void
+gf8_muladd_71(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5007,7 +5121,8 @@ static void gf8_muladd_71(void *out, void *in)
     }
 }
 
-static void gf8_muladd_72(void *out, void *in)
+static void
+gf8_muladd_72(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5052,7 +5167,8 @@ static void gf8_muladd_72(void *out, void *in)
     }
 }
 
-static void gf8_muladd_73(void *out, void *in)
+static void
+gf8_muladd_73(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5093,7 +5209,8 @@ static void gf8_muladd_73(void *out, void *in)
     }
 }
 
-static void gf8_muladd_74(void *out, void *in)
+static void
+gf8_muladd_74(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5137,7 +5254,8 @@ static void gf8_muladd_74(void *out, void *in)
     }
 }
 
-static void gf8_muladd_75(void *out, void *in)
+static void
+gf8_muladd_75(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5182,7 +5300,8 @@ static void gf8_muladd_75(void *out, void *in)
     }
 }
 
-static void gf8_muladd_76(void *out, void *in)
+static void
+gf8_muladd_76(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5228,7 +5347,8 @@ static void gf8_muladd_76(void *out, void *in)
     }
 }
 
-static void gf8_muladd_77(void *out, void *in)
+static void
+gf8_muladd_77(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5274,7 +5394,8 @@ static void gf8_muladd_77(void *out, void *in)
     }
 }
 
-static void gf8_muladd_78(void *out, void *in)
+static void
+gf8_muladd_78(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5319,7 +5440,8 @@ static void gf8_muladd_78(void *out, void *in)
     }
 }
 
-static void gf8_muladd_79(void *out, void *in)
+static void
+gf8_muladd_79(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5365,7 +5487,8 @@ static void gf8_muladd_79(void *out, void *in)
     }
 }
 
-static void gf8_muladd_7A(void *out, void *in)
+static void
+gf8_muladd_7A(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5409,7 +5532,8 @@ static void gf8_muladd_7A(void *out, void *in)
     }
 }
 
-static void gf8_muladd_7B(void *out, void *in)
+static void
+gf8_muladd_7B(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5454,7 +5578,8 @@ static void gf8_muladd_7B(void *out, void *in)
     }
 }
 
-static void gf8_muladd_7C(void *out, void *in)
+static void
+gf8_muladd_7C(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5498,7 +5623,8 @@ static void gf8_muladd_7C(void *out, void *in)
     }
 }
 
-static void gf8_muladd_7D(void *out, void *in)
+static void
+gf8_muladd_7D(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5544,7 +5670,8 @@ static void gf8_muladd_7D(void *out, void *in)
     }
 }
 
-static void gf8_muladd_7E(void *out, void *in)
+static void
+gf8_muladd_7E(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5589,7 +5716,8 @@ static void gf8_muladd_7E(void *out, void *in)
     }
 }
 
-static void gf8_muladd_7F(void *out, void *in)
+static void
+gf8_muladd_7F(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5635,7 +5763,8 @@ static void gf8_muladd_7F(void *out, void *in)
     }
 }
 
-static void gf8_muladd_80(void *out, void *in)
+static void
+gf8_muladd_80(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5680,7 +5809,8 @@ static void gf8_muladd_80(void *out, void *in)
     }
 }
 
-static void gf8_muladd_81(void *out, void *in)
+static void
+gf8_muladd_81(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5724,7 +5854,8 @@ static void gf8_muladd_81(void *out, void *in)
     }
 }
 
-static void gf8_muladd_82(void *out, void *in)
+static void
+gf8_muladd_82(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5767,7 +5898,8 @@ static void gf8_muladd_82(void *out, void *in)
     }
 }
 
-static void gf8_muladd_83(void *out, void *in)
+static void
+gf8_muladd_83(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5814,7 +5946,8 @@ static void gf8_muladd_83(void *out, void *in)
     }
 }
 
-static void gf8_muladd_84(void *out, void *in)
+static void
+gf8_muladd_84(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5855,7 +5988,8 @@ static void gf8_muladd_84(void *out, void *in)
     }
 }
 
-static void gf8_muladd_85(void *out, void *in)
+static void
+gf8_muladd_85(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5901,7 +6035,8 @@ static void gf8_muladd_85(void *out, void *in)
     }
 }
 
-static void gf8_muladd_86(void *out, void *in)
+static void
+gf8_muladd_86(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5942,7 +6077,8 @@ static void gf8_muladd_86(void *out, void *in)
     }
 }
 
-static void gf8_muladd_87(void *out, void *in)
+static void
+gf8_muladd_87(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -5986,7 +6122,8 @@ static void gf8_muladd_87(void *out, void *in)
     }
 }
 
-static void gf8_muladd_88(void *out, void *in)
+static void
+gf8_muladd_88(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6030,7 +6167,8 @@ static void gf8_muladd_88(void *out, void *in)
     }
 }
 
-static void gf8_muladd_89(void *out, void *in)
+static void
+gf8_muladd_89(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6075,7 +6213,8 @@ static void gf8_muladd_89(void *out, void *in)
     }
 }
 
-static void gf8_muladd_8A(void *out, void *in)
+static void
+gf8_muladd_8A(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6116,7 +6255,8 @@ static void gf8_muladd_8A(void *out, void *in)
     }
 }
 
-static void gf8_muladd_8B(void *out, void *in)
+static void
+gf8_muladd_8B(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6163,7 +6303,8 @@ static void gf8_muladd_8B(void *out, void *in)
     }
 }
 
-static void gf8_muladd_8C(void *out, void *in)
+static void
+gf8_muladd_8C(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6204,7 +6345,8 @@ static void gf8_muladd_8C(void *out, void *in)
     }
 }
 
-static void gf8_muladd_8D(void *out, void *in)
+static void
+gf8_muladd_8D(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6247,7 +6389,8 @@ static void gf8_muladd_8D(void *out, void *in)
     }
 }
 
-static void gf8_muladd_8E(void *out, void *in)
+static void
+gf8_muladd_8E(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6288,7 +6431,8 @@ static void gf8_muladd_8E(void *out, void *in)
     }
 }
 
-static void gf8_muladd_8F(void *out, void *in)
+static void
+gf8_muladd_8F(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6331,7 +6475,8 @@ static void gf8_muladd_8F(void *out, void *in)
     }
 }
 
-static void gf8_muladd_90(void *out, void *in)
+static void
+gf8_muladd_90(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6376,7 +6521,8 @@ static void gf8_muladd_90(void *out, void *in)
     }
 }
 
-static void gf8_muladd_91(void *out, void *in)
+static void
+gf8_muladd_91(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6422,7 +6568,8 @@ static void gf8_muladd_91(void *out, void *in)
     }
 }
 
-static void gf8_muladd_92(void *out, void *in)
+static void
+gf8_muladd_92(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6466,7 +6613,8 @@ static void gf8_muladd_92(void *out, void *in)
     }
 }
 
-static void gf8_muladd_93(void *out, void *in)
+static void
+gf8_muladd_93(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6511,7 +6659,8 @@ static void gf8_muladd_93(void *out, void *in)
     }
 }
 
-static void gf8_muladd_94(void *out, void *in)
+static void
+gf8_muladd_94(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6554,7 +6703,8 @@ static void gf8_muladd_94(void *out, void *in)
     }
 }
 
-static void gf8_muladd_95(void *out, void *in)
+static void
+gf8_muladd_95(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6602,7 +6752,8 @@ static void gf8_muladd_95(void *out, void *in)
     }
 }
 
-static void gf8_muladd_96(void *out, void *in)
+static void
+gf8_muladd_96(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6647,7 +6798,8 @@ static void gf8_muladd_96(void *out, void *in)
     }
 }
 
-static void gf8_muladd_97(void *out, void *in)
+static void
+gf8_muladd_97(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6693,7 +6845,8 @@ static void gf8_muladd_97(void *out, void *in)
     }
 }
 
-static void gf8_muladd_98(void *out, void *in)
+static void
+gf8_muladd_98(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6737,7 +6890,8 @@ static void gf8_muladd_98(void *out, void *in)
     }
 }
 
-static void gf8_muladd_99(void *out, void *in)
+static void
+gf8_muladd_99(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6782,7 +6936,8 @@ static void gf8_muladd_99(void *out, void *in)
     }
 }
 
-static void gf8_muladd_9A(void *out, void *in)
+static void
+gf8_muladd_9A(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6827,7 +6982,8 @@ static void gf8_muladd_9A(void *out, void *in)
     }
 }
 
-static void gf8_muladd_9B(void *out, void *in)
+static void
+gf8_muladd_9B(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6870,7 +7026,8 @@ static void gf8_muladd_9B(void *out, void *in)
     }
 }
 
-static void gf8_muladd_9C(void *out, void *in)
+static void
+gf8_muladd_9C(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6913,7 +7070,8 @@ static void gf8_muladd_9C(void *out, void *in)
     }
 }
 
-static void gf8_muladd_9D(void *out, void *in)
+static void
+gf8_muladd_9D(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6956,7 +7114,8 @@ static void gf8_muladd_9D(void *out, void *in)
     }
 }
 
-static void gf8_muladd_9E(void *out, void *in)
+static void
+gf8_muladd_9E(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -6999,7 +7158,8 @@ static void gf8_muladd_9E(void *out, void *in)
     }
 }
 
-static void gf8_muladd_9F(void *out, void *in)
+static void
+gf8_muladd_9F(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7042,7 +7202,8 @@ static void gf8_muladd_9F(void *out, void *in)
     }
 }
 
-static void gf8_muladd_A0(void *out, void *in)
+static void
+gf8_muladd_A0(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7088,7 +7249,8 @@ static void gf8_muladd_A0(void *out, void *in)
     }
 }
 
-static void gf8_muladd_A1(void *out, void *in)
+static void
+gf8_muladd_A1(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7133,7 +7295,8 @@ static void gf8_muladd_A1(void *out, void *in)
     }
 }
 
-static void gf8_muladd_A2(void *out, void *in)
+static void
+gf8_muladd_A2(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7176,7 +7339,8 @@ static void gf8_muladd_A2(void *out, void *in)
     }
 }
 
-static void gf8_muladd_A3(void *out, void *in)
+static void
+gf8_muladd_A3(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7220,7 +7384,8 @@ static void gf8_muladd_A3(void *out, void *in)
     }
 }
 
-static void gf8_muladd_A4(void *out, void *in)
+static void
+gf8_muladd_A4(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7267,7 +7432,8 @@ static void gf8_muladd_A4(void *out, void *in)
     }
 }
 
-static void gf8_muladd_A5(void *out, void *in)
+static void
+gf8_muladd_A5(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7312,7 +7478,8 @@ static void gf8_muladd_A5(void *out, void *in)
     }
 }
 
-static void gf8_muladd_A6(void *out, void *in)
+static void
+gf8_muladd_A6(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7353,7 +7520,8 @@ static void gf8_muladd_A6(void *out, void *in)
     }
 }
 
-static void gf8_muladd_A7(void *out, void *in)
+static void
+gf8_muladd_A7(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7394,7 +7562,8 @@ static void gf8_muladd_A7(void *out, void *in)
     }
 }
 
-static void gf8_muladd_A8(void *out, void *in)
+static void
+gf8_muladd_A8(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7439,7 +7608,8 @@ static void gf8_muladd_A8(void *out, void *in)
     }
 }
 
-static void gf8_muladd_A9(void *out, void *in)
+static void
+gf8_muladd_A9(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7480,7 +7650,8 @@ static void gf8_muladd_A9(void *out, void *in)
     }
 }
 
-static void gf8_muladd_AA(void *out, void *in)
+static void
+gf8_muladd_AA(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7525,7 +7696,8 @@ static void gf8_muladd_AA(void *out, void *in)
     }
 }
 
-static void gf8_muladd_AB(void *out, void *in)
+static void
+gf8_muladd_AB(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7569,7 +7741,8 @@ static void gf8_muladd_AB(void *out, void *in)
     }
 }
 
-static void gf8_muladd_AC(void *out, void *in)
+static void
+gf8_muladd_AC(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7612,7 +7785,8 @@ static void gf8_muladd_AC(void *out, void *in)
     }
 }
 
-static void gf8_muladd_AD(void *out, void *in)
+static void
+gf8_muladd_AD(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7653,7 +7827,8 @@ static void gf8_muladd_AD(void *out, void *in)
     }
 }
 
-static void gf8_muladd_AE(void *out, void *in)
+static void
+gf8_muladd_AE(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7698,7 +7873,8 @@ static void gf8_muladd_AE(void *out, void *in)
     }
 }
 
-static void gf8_muladd_AF(void *out, void *in)
+static void
+gf8_muladd_AF(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7741,7 +7917,8 @@ static void gf8_muladd_AF(void *out, void *in)
     }
 }
 
-static void gf8_muladd_B0(void *out, void *in)
+static void
+gf8_muladd_B0(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7787,7 +7964,8 @@ static void gf8_muladd_B0(void *out, void *in)
     }
 }
 
-static void gf8_muladd_B1(void *out, void *in)
+static void
+gf8_muladd_B1(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7831,7 +8009,8 @@ static void gf8_muladd_B1(void *out, void *in)
     }
 }
 
-static void gf8_muladd_B2(void *out, void *in)
+static void
+gf8_muladd_B2(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7878,7 +8057,8 @@ static void gf8_muladd_B2(void *out, void *in)
     }
 }
 
-static void gf8_muladd_B3(void *out, void *in)
+static void
+gf8_muladd_B3(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7923,7 +8103,8 @@ static void gf8_muladd_B3(void *out, void *in)
     }
 }
 
-static void gf8_muladd_B4(void *out, void *in)
+static void
+gf8_muladd_B4(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -7966,7 +8147,8 @@ static void gf8_muladd_B4(void *out, void *in)
     }
 }
 
-static void gf8_muladd_B5(void *out, void *in)
+static void
+gf8_muladd_B5(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8011,7 +8193,8 @@ static void gf8_muladd_B5(void *out, void *in)
     }
 }
 
-static void gf8_muladd_B6(void *out, void *in)
+static void
+gf8_muladd_B6(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8057,7 +8240,8 @@ static void gf8_muladd_B6(void *out, void *in)
     }
 }
 
-static void gf8_muladd_B7(void *out, void *in)
+static void
+gf8_muladd_B7(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8101,7 +8285,8 @@ static void gf8_muladd_B7(void *out, void *in)
     }
 }
 
-static void gf8_muladd_B8(void *out, void *in)
+static void
+gf8_muladd_B8(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8146,7 +8331,8 @@ static void gf8_muladd_B8(void *out, void *in)
     }
 }
 
-static void gf8_muladd_B9(void *out, void *in)
+static void
+gf8_muladd_B9(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8191,7 +8377,8 @@ static void gf8_muladd_B9(void *out, void *in)
     }
 }
 
-static void gf8_muladd_BA(void *out, void *in)
+static void
+gf8_muladd_BA(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8236,7 +8423,8 @@ static void gf8_muladd_BA(void *out, void *in)
     }
 }
 
-static void gf8_muladd_BB(void *out, void *in)
+static void
+gf8_muladd_BB(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8280,7 +8468,8 @@ static void gf8_muladd_BB(void *out, void *in)
     }
 }
 
-static void gf8_muladd_BC(void *out, void *in)
+static void
+gf8_muladd_BC(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8325,7 +8514,8 @@ static void gf8_muladd_BC(void *out, void *in)
     }
 }
 
-static void gf8_muladd_BD(void *out, void *in)
+static void
+gf8_muladd_BD(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8370,7 +8560,8 @@ static void gf8_muladd_BD(void *out, void *in)
     }
 }
 
-static void gf8_muladd_BE(void *out, void *in)
+static void
+gf8_muladd_BE(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8413,7 +8604,8 @@ static void gf8_muladd_BE(void *out, void *in)
     }
 }
 
-static void gf8_muladd_BF(void *out, void *in)
+static void
+gf8_muladd_BF(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8459,7 +8651,8 @@ static void gf8_muladd_BF(void *out, void *in)
     }
 }
 
-static void gf8_muladd_C0(void *out, void *in)
+static void
+gf8_muladd_C0(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8503,7 +8696,8 @@ static void gf8_muladd_C0(void *out, void *in)
     }
 }
 
-static void gf8_muladd_C1(void *out, void *in)
+static void
+gf8_muladd_C1(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8547,7 +8741,8 @@ static void gf8_muladd_C1(void *out, void *in)
     }
 }
 
-static void gf8_muladd_C2(void *out, void *in)
+static void
+gf8_muladd_C2(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8591,7 +8786,8 @@ static void gf8_muladd_C2(void *out, void *in)
     }
 }
 
-static void gf8_muladd_C3(void *out, void *in)
+static void
+gf8_muladd_C3(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8636,7 +8832,8 @@ static void gf8_muladd_C3(void *out, void *in)
     }
 }
 
-static void gf8_muladd_C4(void *out, void *in)
+static void
+gf8_muladd_C4(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8680,7 +8877,8 @@ static void gf8_muladd_C4(void *out, void *in)
     }
 }
 
-static void gf8_muladd_C5(void *out, void *in)
+static void
+gf8_muladd_C5(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8723,7 +8921,8 @@ static void gf8_muladd_C5(void *out, void *in)
     }
 }
 
-static void gf8_muladd_C6(void *out, void *in)
+static void
+gf8_muladd_C6(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8771,7 +8970,8 @@ static void gf8_muladd_C6(void *out, void *in)
     }
 }
 
-static void gf8_muladd_C7(void *out, void *in)
+static void
+gf8_muladd_C7(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8815,7 +9015,8 @@ static void gf8_muladd_C7(void *out, void *in)
     }
 }
 
-static void gf8_muladd_C8(void *out, void *in)
+static void
+gf8_muladd_C8(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8859,7 +9060,8 @@ static void gf8_muladd_C8(void *out, void *in)
     }
 }
 
-static void gf8_muladd_C9(void *out, void *in)
+static void
+gf8_muladd_C9(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8902,7 +9104,8 @@ static void gf8_muladd_C9(void *out, void *in)
     }
 }
 
-static void gf8_muladd_CA(void *out, void *in)
+static void
+gf8_muladd_CA(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8948,7 +9151,8 @@ static void gf8_muladd_CA(void *out, void *in)
     }
 }
 
-static void gf8_muladd_CB(void *out, void *in)
+static void
+gf8_muladd_CB(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -8993,7 +9197,8 @@ static void gf8_muladd_CB(void *out, void *in)
     }
 }
 
-static void gf8_muladd_CC(void *out, void *in)
+static void
+gf8_muladd_CC(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9039,7 +9244,8 @@ static void gf8_muladd_CC(void *out, void *in)
     }
 }
 
-static void gf8_muladd_CD(void *out, void *in)
+static void
+gf8_muladd_CD(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9083,7 +9289,8 @@ static void gf8_muladd_CD(void *out, void *in)
     }
 }
 
-static void gf8_muladd_CE(void *out, void *in)
+static void
+gf8_muladd_CE(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9128,7 +9335,8 @@ static void gf8_muladd_CE(void *out, void *in)
     }
 }
 
-static void gf8_muladd_CF(void *out, void *in)
+static void
+gf8_muladd_CF(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9173,7 +9381,8 @@ static void gf8_muladd_CF(void *out, void *in)
     }
 }
 
-static void gf8_muladd_D0(void *out, void *in)
+static void
+gf8_muladd_D0(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9220,7 +9429,8 @@ static void gf8_muladd_D0(void *out, void *in)
     }
 }
 
-static void gf8_muladd_D1(void *out, void *in)
+static void
+gf8_muladd_D1(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9265,7 +9475,8 @@ static void gf8_muladd_D1(void *out, void *in)
     }
 }
 
-static void gf8_muladd_D2(void *out, void *in)
+static void
+gf8_muladd_D2(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9308,7 +9519,8 @@ static void gf8_muladd_D2(void *out, void *in)
     }
 }
 
-static void gf8_muladd_D3(void *out, void *in)
+static void
+gf8_muladd_D3(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9354,7 +9566,8 @@ static void gf8_muladd_D3(void *out, void *in)
     }
 }
 
-static void gf8_muladd_D4(void *out, void *in)
+static void
+gf8_muladd_D4(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9399,7 +9612,8 @@ static void gf8_muladd_D4(void *out, void *in)
     }
 }
 
-static void gf8_muladd_D5(void *out, void *in)
+static void
+gf8_muladd_D5(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9444,7 +9658,8 @@ static void gf8_muladd_D5(void *out, void *in)
     }
 }
 
-static void gf8_muladd_D6(void *out, void *in)
+static void
+gf8_muladd_D6(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9488,7 +9703,8 @@ static void gf8_muladd_D6(void *out, void *in)
     }
 }
 
-static void gf8_muladd_D7(void *out, void *in)
+static void
+gf8_muladd_D7(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9533,7 +9749,8 @@ static void gf8_muladd_D7(void *out, void *in)
     }
 }
 
-static void gf8_muladd_D8(void *out, void *in)
+static void
+gf8_muladd_D8(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9577,7 +9794,8 @@ static void gf8_muladd_D8(void *out, void *in)
     }
 }
 
-static void gf8_muladd_D9(void *out, void *in)
+static void
+gf8_muladd_D9(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9618,7 +9836,8 @@ static void gf8_muladd_D9(void *out, void *in)
     }
 }
 
-static void gf8_muladd_DA(void *out, void *in)
+static void
+gf8_muladd_DA(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9662,7 +9881,8 @@ static void gf8_muladd_DA(void *out, void *in)
     }
 }
 
-static void gf8_muladd_DB(void *out, void *in)
+static void
+gf8_muladd_DB(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9709,7 +9929,8 @@ static void gf8_muladd_DB(void *out, void *in)
     }
 }
 
-static void gf8_muladd_DC(void *out, void *in)
+static void
+gf8_muladd_DC(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9755,7 +9976,8 @@ static void gf8_muladd_DC(void *out, void *in)
     }
 }
 
-static void gf8_muladd_DD(void *out, void *in)
+static void
+gf8_muladd_DD(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9796,7 +10018,8 @@ static void gf8_muladd_DD(void *out, void *in)
     }
 }
 
-static void gf8_muladd_DE(void *out, void *in)
+static void
+gf8_muladd_DE(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9840,7 +10063,8 @@ static void gf8_muladd_DE(void *out, void *in)
     }
 }
 
-static void gf8_muladd_DF(void *out, void *in)
+static void
+gf8_muladd_DF(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9885,7 +10109,8 @@ static void gf8_muladd_DF(void *out, void *in)
     }
 }
 
-static void gf8_muladd_E0(void *out, void *in)
+static void
+gf8_muladd_E0(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9929,7 +10154,8 @@ static void gf8_muladd_E0(void *out, void *in)
     }
 }
 
-static void gf8_muladd_E1(void *out, void *in)
+static void
+gf8_muladd_E1(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -9975,7 +10201,8 @@ static void gf8_muladd_E1(void *out, void *in)
     }
 }
 
-static void gf8_muladd_E2(void *out, void *in)
+static void
+gf8_muladd_E2(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10016,7 +10243,8 @@ static void gf8_muladd_E2(void *out, void *in)
     }
 }
 
-static void gf8_muladd_E3(void *out, void *in)
+static void
+gf8_muladd_E3(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10063,7 +10291,8 @@ static void gf8_muladd_E3(void *out, void *in)
     }
 }
 
-static void gf8_muladd_E4(void *out, void *in)
+static void
+gf8_muladd_E4(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10108,7 +10337,8 @@ static void gf8_muladd_E4(void *out, void *in)
     }
 }
 
-static void gf8_muladd_E5(void *out, void *in)
+static void
+gf8_muladd_E5(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10153,7 +10383,8 @@ static void gf8_muladd_E5(void *out, void *in)
     }
 }
 
-static void gf8_muladd_E6(void *out, void *in)
+static void
+gf8_muladd_E6(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10194,7 +10425,8 @@ static void gf8_muladd_E6(void *out, void *in)
     }
 }
 
-static void gf8_muladd_E7(void *out, void *in)
+static void
+gf8_muladd_E7(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10241,7 +10473,8 @@ static void gf8_muladd_E7(void *out, void *in)
     }
 }
 
-static void gf8_muladd_E8(void *out, void *in)
+static void
+gf8_muladd_E8(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10287,7 +10520,8 @@ static void gf8_muladd_E8(void *out, void *in)
     }
 }
 
-static void gf8_muladd_E9(void *out, void *in)
+static void
+gf8_muladd_E9(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10332,7 +10566,8 @@ static void gf8_muladd_E9(void *out, void *in)
     }
 }
 
-static void gf8_muladd_EA(void *out, void *in)
+static void
+gf8_muladd_EA(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10373,7 +10608,8 @@ static void gf8_muladd_EA(void *out, void *in)
     }
 }
 
-static void gf8_muladd_EB(void *out, void *in)
+static void
+gf8_muladd_EB(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10417,7 +10653,8 @@ static void gf8_muladd_EB(void *out, void *in)
     }
 }
 
-static void gf8_muladd_EC(void *out, void *in)
+static void
+gf8_muladd_EC(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10458,7 +10695,8 @@ static void gf8_muladd_EC(void *out, void *in)
     }
 }
 
-static void gf8_muladd_ED(void *out, void *in)
+static void
+gf8_muladd_ED(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10502,7 +10740,8 @@ static void gf8_muladd_ED(void *out, void *in)
     }
 }
 
-static void gf8_muladd_EE(void *out, void *in)
+static void
+gf8_muladd_EE(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10548,7 +10787,8 @@ static void gf8_muladd_EE(void *out, void *in)
     }
 }
 
-static void gf8_muladd_EF(void *out, void *in)
+static void
+gf8_muladd_EF(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10592,7 +10832,8 @@ static void gf8_muladd_EF(void *out, void *in)
     }
 }
 
-static void gf8_muladd_F0(void *out, void *in)
+static void
+gf8_muladd_F0(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10638,7 +10879,8 @@ static void gf8_muladd_F0(void *out, void *in)
     }
 }
 
-static void gf8_muladd_F1(void *out, void *in)
+static void
+gf8_muladd_F1(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10684,7 +10926,8 @@ static void gf8_muladd_F1(void *out, void *in)
     }
 }
 
-static void gf8_muladd_F2(void *out, void *in)
+static void
+gf8_muladd_F2(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10730,7 +10973,8 @@ static void gf8_muladd_F2(void *out, void *in)
     }
 }
 
-static void gf8_muladd_F3(void *out, void *in)
+static void
+gf8_muladd_F3(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10774,7 +11018,8 @@ static void gf8_muladd_F3(void *out, void *in)
     }
 }
 
-static void gf8_muladd_F4(void *out, void *in)
+static void
+gf8_muladd_F4(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10817,7 +11062,8 @@ static void gf8_muladd_F4(void *out, void *in)
     }
 }
 
-static void gf8_muladd_F5(void *out, void *in)
+static void
+gf8_muladd_F5(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10860,7 +11106,8 @@ static void gf8_muladd_F5(void *out, void *in)
     }
 }
 
-static void gf8_muladd_F6(void *out, void *in)
+static void
+gf8_muladd_F6(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10904,7 +11151,8 @@ static void gf8_muladd_F6(void *out, void *in)
     }
 }
 
-static void gf8_muladd_F7(void *out, void *in)
+static void
+gf8_muladd_F7(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10947,7 +11195,8 @@ static void gf8_muladd_F7(void *out, void *in)
     }
 }
 
-static void gf8_muladd_F8(void *out, void *in)
+static void
+gf8_muladd_F8(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -10992,7 +11241,8 @@ static void gf8_muladd_F8(void *out, void *in)
     }
 }
 
-static void gf8_muladd_F9(void *out, void *in)
+static void
+gf8_muladd_F9(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -11039,7 +11289,8 @@ static void gf8_muladd_F9(void *out, void *in)
     }
 }
 
-static void gf8_muladd_FA(void *out, void *in)
+static void
+gf8_muladd_FA(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -11085,7 +11336,8 @@ static void gf8_muladd_FA(void *out, void *in)
     }
 }
 
-static void gf8_muladd_FB(void *out, void *in)
+static void
+gf8_muladd_FB(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -11129,7 +11381,8 @@ static void gf8_muladd_FB(void *out, void *in)
     }
 }
 
-static void gf8_muladd_FC(void *out, void *in)
+static void
+gf8_muladd_FC(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -11175,7 +11428,8 @@ static void gf8_muladd_FC(void *out, void *in)
     }
 }
 
-static void gf8_muladd_FD(void *out, void *in)
+static void
+gf8_muladd_FD(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -11221,7 +11475,8 @@ static void gf8_muladd_FD(void *out, void *in)
     }
 }
 
-static void gf8_muladd_FE(void *out, void *in)
+static void
+gf8_muladd_FE(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -11268,7 +11523,8 @@ static void gf8_muladd_FE(void *out, void *in)
     }
 }
 
-static void gf8_muladd_FF(void *out, void *in)
+static void
+gf8_muladd_FF(void *out, void *in)
 {
     unsigned int i;
     uint64_t *in_ptr = (uint64_t *)in;
@@ -11315,75 +11571,65 @@ static void gf8_muladd_FF(void *out, void *in)
 }
 
 static void (*gf8_muladd[])(void *out, void *in) = {
-    gf8_muladd_00, gf8_muladd_01, gf8_muladd_02, gf8_muladd_03,
-    gf8_muladd_04, gf8_muladd_05, gf8_muladd_06, gf8_muladd_07,
-    gf8_muladd_08, gf8_muladd_09, gf8_muladd_0A, gf8_muladd_0B,
-    gf8_muladd_0C, gf8_muladd_0D, gf8_muladd_0E, gf8_muladd_0F,
-    gf8_muladd_10, gf8_muladd_11, gf8_muladd_12, gf8_muladd_13,
-    gf8_muladd_14, gf8_muladd_15, gf8_muladd_16, gf8_muladd_17,
-    gf8_muladd_18, gf8_muladd_19, gf8_muladd_1A, gf8_muladd_1B,
-    gf8_muladd_1C, gf8_muladd_1D, gf8_muladd_1E, gf8_muladd_1F,
-    gf8_muladd_20, gf8_muladd_21, gf8_muladd_22, gf8_muladd_23,
-    gf8_muladd_24, gf8_muladd_25, gf8_muladd_26, gf8_muladd_27,
-    gf8_muladd_28, gf8_muladd_29, gf8_muladd_2A, gf8_muladd_2B,
-    gf8_muladd_2C, gf8_muladd_2D, gf8_muladd_2E, gf8_muladd_2F,
-    gf8_muladd_30, gf8_muladd_31, gf8_muladd_32, gf8_muladd_33,
-    gf8_muladd_34, gf8_muladd_35, gf8_muladd_36, gf8_muladd_37,
-    gf8_muladd_38, gf8_muladd_39, gf8_muladd_3A, gf8_muladd_3B,
-    gf8_muladd_3C, gf8_muladd_3D, gf8_muladd_3E, gf8_muladd_3F,
-    gf8_muladd_40, gf8_muladd_41, gf8_muladd_42, gf8_muladd_43,
-    gf8_muladd_44, gf8_muladd_45, gf8_muladd_46, gf8_muladd_47,
-    gf8_muladd_48, gf8_muladd_49, gf8_muladd_4A, gf8_muladd_4B,
-    gf8_muladd_4C, gf8_muladd_4D, gf8_muladd_4E, gf8_muladd_4F,
-    gf8_muladd_50, gf8_muladd_51, gf8_muladd_52, gf8_muladd_53,
-    gf8_muladd_54, gf8_muladd_55, gf8_muladd_56, gf8_muladd_57,
-    gf8_muladd_58, gf8_muladd_59, gf8_muladd_5A, gf8_muladd_5B,
-    gf8_muladd_5C, gf8_muladd_5D, gf8_muladd_5E, gf8_muladd_5F,
-    gf8_muladd_60, gf8_muladd_61, gf8_muladd_62, gf8_muladd_63,
-    gf8_muladd_64, gf8_muladd_65, gf8_muladd_66, gf8_muladd_67,
-    gf8_muladd_68, gf8_muladd_69, gf8_muladd_6A, gf8_muladd_6B,
-    gf8_muladd_6C, gf8_muladd_6D, gf8_muladd_6E, gf8_muladd_6F,
-    gf8_muladd_70, gf8_muladd_71, gf8_muladd_72, gf8_muladd_73,
-    gf8_muladd_74, gf8_muladd_75, gf8_muladd_76, gf8_muladd_77,
-    gf8_muladd_78, gf8_muladd_79, gf8_muladd_7A, gf8_muladd_7B,
-    gf8_muladd_7C, gf8_muladd_7D, gf8_muladd_7E, gf8_muladd_7F,
-    gf8_muladd_80, gf8_muladd_81, gf8_muladd_82, gf8_muladd_83,
-    gf8_muladd_84, gf8_muladd_85, gf8_muladd_86, gf8_muladd_87,
-    gf8_muladd_88, gf8_muladd_89, gf8_muladd_8A, gf8_muladd_8B,
-    gf8_muladd_8C, gf8_muladd_8D, gf8_muladd_8E, gf8_muladd_8F,
-    gf8_muladd_90, gf8_muladd_91, gf8_muladd_92, gf8_muladd_93,
-    gf8_muladd_94, gf8_muladd_95, gf8_muladd_96, gf8_muladd_97,
-    gf8_muladd_98, gf8_muladd_99, gf8_muladd_9A, gf8_muladd_9B,
-    gf8_muladd_9C, gf8_muladd_9D, gf8_muladd_9E, gf8_muladd_9F,
-    gf8_muladd_A0, gf8_muladd_A1, gf8_muladd_A2, gf8_muladd_A3,
-    gf8_muladd_A4, gf8_muladd_A5, gf8_muladd_A6, gf8_muladd_A7,
-    gf8_muladd_A8, gf8_muladd_A9, gf8_muladd_AA, gf8_muladd_AB,
-    gf8_muladd_AC, gf8_muladd_AD, gf8_muladd_AE, gf8_muladd_AF,
-    gf8_muladd_B0, gf8_muladd_B1, gf8_muladd_B2, gf8_muladd_B3,
-    gf8_muladd_B4, gf8_muladd_B5, gf8_muladd_B6, gf8_muladd_B7,
-    gf8_muladd_B8, gf8_muladd_B9, gf8_muladd_BA, gf8_muladd_BB,
-    gf8_muladd_BC, gf8_muladd_BD, gf8_muladd_BE, gf8_muladd_BF,
-    gf8_muladd_C0, gf8_muladd_C1, gf8_muladd_C2, gf8_muladd_C3,
-    gf8_muladd_C4, gf8_muladd_C5, gf8_muladd_C6, gf8_muladd_C7,
-    gf8_muladd_C8, gf8_muladd_C9, gf8_muladd_CA, gf8_muladd_CB,
-    gf8_muladd_CC, gf8_muladd_CD, gf8_muladd_CE, gf8_muladd_CF,
-    gf8_muladd_D0, gf8_muladd_D1, gf8_muladd_D2, gf8_muladd_D3,
-    gf8_muladd_D4, gf8_muladd_D5, gf8_muladd_D6, gf8_muladd_D7,
-    gf8_muladd_D8, gf8_muladd_D9, gf8_muladd_DA, gf8_muladd_DB,
-    gf8_muladd_DC, gf8_muladd_DD, gf8_muladd_DE, gf8_muladd_DF,
-    gf8_muladd_E0, gf8_muladd_E1, gf8_muladd_E2, gf8_muladd_E3,
-    gf8_muladd_E4, gf8_muladd_E5, gf8_muladd_E6, gf8_muladd_E7,
-    gf8_muladd_E8, gf8_muladd_E9, gf8_muladd_EA, gf8_muladd_EB,
-    gf8_muladd_EC, gf8_muladd_ED, gf8_muladd_EE, gf8_muladd_EF,
-    gf8_muladd_F0, gf8_muladd_F1, gf8_muladd_F2, gf8_muladd_F3,
-    gf8_muladd_F4, gf8_muladd_F5, gf8_muladd_F6, gf8_muladd_F7,
-    gf8_muladd_F8, gf8_muladd_F9, gf8_muladd_FA, gf8_muladd_FB,
-    gf8_muladd_FC, gf8_muladd_FD, gf8_muladd_FE, gf8_muladd_FF
+    gf8_muladd_00, gf8_muladd_01, gf8_muladd_02, gf8_muladd_03, gf8_muladd_04,
+    gf8_muladd_05, gf8_muladd_06, gf8_muladd_07, gf8_muladd_08, gf8_muladd_09,
+    gf8_muladd_0A, gf8_muladd_0B, gf8_muladd_0C, gf8_muladd_0D, gf8_muladd_0E,
+    gf8_muladd_0F, gf8_muladd_10, gf8_muladd_11, gf8_muladd_12, gf8_muladd_13,
+    gf8_muladd_14, gf8_muladd_15, gf8_muladd_16, gf8_muladd_17, gf8_muladd_18,
+    gf8_muladd_19, gf8_muladd_1A, gf8_muladd_1B, gf8_muladd_1C, gf8_muladd_1D,
+    gf8_muladd_1E, gf8_muladd_1F, gf8_muladd_20, gf8_muladd_21, gf8_muladd_22,
+    gf8_muladd_23, gf8_muladd_24, gf8_muladd_25, gf8_muladd_26, gf8_muladd_27,
+    gf8_muladd_28, gf8_muladd_29, gf8_muladd_2A, gf8_muladd_2B, gf8_muladd_2C,
+    gf8_muladd_2D, gf8_muladd_2E, gf8_muladd_2F, gf8_muladd_30, gf8_muladd_31,
+    gf8_muladd_32, gf8_muladd_33, gf8_muladd_34, gf8_muladd_35, gf8_muladd_36,
+    gf8_muladd_37, gf8_muladd_38, gf8_muladd_39, gf8_muladd_3A, gf8_muladd_3B,
+    gf8_muladd_3C, gf8_muladd_3D, gf8_muladd_3E, gf8_muladd_3F, gf8_muladd_40,
+    gf8_muladd_41, gf8_muladd_42, gf8_muladd_43, gf8_muladd_44, gf8_muladd_45,
+    gf8_muladd_46, gf8_muladd_47, gf8_muladd_48, gf8_muladd_49, gf8_muladd_4A,
+    gf8_muladd_4B, gf8_muladd_4C, gf8_muladd_4D, gf8_muladd_4E, gf8_muladd_4F,
+    gf8_muladd_50, gf8_muladd_51, gf8_muladd_52, gf8_muladd_53, gf8_muladd_54,
+    gf8_muladd_55, gf8_muladd_56, gf8_muladd_57, gf8_muladd_58, gf8_muladd_59,
+    gf8_muladd_5A, gf8_muladd_5B, gf8_muladd_5C, gf8_muladd_5D, gf8_muladd_5E,
+    gf8_muladd_5F, gf8_muladd_60, gf8_muladd_61, gf8_muladd_62, gf8_muladd_63,
+    gf8_muladd_64, gf8_muladd_65, gf8_muladd_66, gf8_muladd_67, gf8_muladd_68,
+    gf8_muladd_69, gf8_muladd_6A, gf8_muladd_6B, gf8_muladd_6C, gf8_muladd_6D,
+    gf8_muladd_6E, gf8_muladd_6F, gf8_muladd_70, gf8_muladd_71, gf8_muladd_72,
+    gf8_muladd_73, gf8_muladd_74, gf8_muladd_75, gf8_muladd_76, gf8_muladd_77,
+    gf8_muladd_78, gf8_muladd_79, gf8_muladd_7A, gf8_muladd_7B, gf8_muladd_7C,
+    gf8_muladd_7D, gf8_muladd_7E, gf8_muladd_7F, gf8_muladd_80, gf8_muladd_81,
+    gf8_muladd_82, gf8_muladd_83, gf8_muladd_84, gf8_muladd_85, gf8_muladd_86,
+    gf8_muladd_87, gf8_muladd_88, gf8_muladd_89, gf8_muladd_8A, gf8_muladd_8B,
+    gf8_muladd_8C, gf8_muladd_8D, gf8_muladd_8E, gf8_muladd_8F, gf8_muladd_90,
+    gf8_muladd_91, gf8_muladd_92, gf8_muladd_93, gf8_muladd_94, gf8_muladd_95,
+    gf8_muladd_96, gf8_muladd_97, gf8_muladd_98, gf8_muladd_99, gf8_muladd_9A,
+    gf8_muladd_9B, gf8_muladd_9C, gf8_muladd_9D, gf8_muladd_9E, gf8_muladd_9F,
+    gf8_muladd_A0, gf8_muladd_A1, gf8_muladd_A2, gf8_muladd_A3, gf8_muladd_A4,
+    gf8_muladd_A5, gf8_muladd_A6, gf8_muladd_A7, gf8_muladd_A8, gf8_muladd_A9,
+    gf8_muladd_AA, gf8_muladd_AB, gf8_muladd_AC, gf8_muladd_AD, gf8_muladd_AE,
+    gf8_muladd_AF, gf8_muladd_B0, gf8_muladd_B1, gf8_muladd_B2, gf8_muladd_B3,
+    gf8_muladd_B4, gf8_muladd_B5, gf8_muladd_B6, gf8_muladd_B7, gf8_muladd_B8,
+    gf8_muladd_B9, gf8_muladd_BA, gf8_muladd_BB, gf8_muladd_BC, gf8_muladd_BD,
+    gf8_muladd_BE, gf8_muladd_BF, gf8_muladd_C0, gf8_muladd_C1, gf8_muladd_C2,
+    gf8_muladd_C3, gf8_muladd_C4, gf8_muladd_C5, gf8_muladd_C6, gf8_muladd_C7,
+    gf8_muladd_C8, gf8_muladd_C9, gf8_muladd_CA, gf8_muladd_CB, gf8_muladd_CC,
+    gf8_muladd_CD, gf8_muladd_CE, gf8_muladd_CF, gf8_muladd_D0, gf8_muladd_D1,
+    gf8_muladd_D2, gf8_muladd_D3, gf8_muladd_D4, gf8_muladd_D5, gf8_muladd_D6,
+    gf8_muladd_D7, gf8_muladd_D8, gf8_muladd_D9, gf8_muladd_DA, gf8_muladd_DB,
+    gf8_muladd_DC, gf8_muladd_DD, gf8_muladd_DE, gf8_muladd_DF, gf8_muladd_E0,
+    gf8_muladd_E1, gf8_muladd_E2, gf8_muladd_E3, gf8_muladd_E4, gf8_muladd_E5,
+    gf8_muladd_E6, gf8_muladd_E7, gf8_muladd_E8, gf8_muladd_E9, gf8_muladd_EA,
+    gf8_muladd_EB, gf8_muladd_EC, gf8_muladd_ED, gf8_muladd_EE, gf8_muladd_EF,
+    gf8_muladd_F0, gf8_muladd_F1, gf8_muladd_F2, gf8_muladd_F3, gf8_muladd_F4,
+    gf8_muladd_F5, gf8_muladd_F6, gf8_muladd_F7, gf8_muladd_F8, gf8_muladd_F9,
+    gf8_muladd_FA, gf8_muladd_FB, gf8_muladd_FC, gf8_muladd_FD, gf8_muladd_FE,
+    gf8_muladd_FF};
+
+static uint64_t zero[EC_METHOD_WORD_SIZE * 8] = {
+    0,
 };
 
-static uint64_t zero[EC_METHOD_WORD_SIZE * 8] = {0, };
-
-void ec_code_c_prepare(ec_gf_t *gf, uint32_t *values, uint32_t count)
+void
+ec_code_c_prepare(ec_gf_t *gf, uint32_t *values, uint32_t count)
 {
     uint32_t i, last, tmp;
 
@@ -11397,8 +11643,9 @@ void ec_code_c_prepare(ec_gf_t *gf, uint32_t *values, uint32_t count)
     }
 }
 
-void ec_code_c_linear(void *dst, void *src, uint64_t offset, uint32_t *values,
-                      uint32_t count)
+void
+ec_code_c_linear(void *dst, void *src, uint64_t offset, uint32_t *values,
+                 uint32_t count)
 {
     src += offset;
     gf8_muladd_00(dst, src);
@@ -11409,8 +11656,9 @@ void ec_code_c_linear(void *dst, void *src, uint64_t offset, uint32_t *values,
     }
 }
 
-void ec_code_c_interleaved(void *dst, void **src, uint64_t offset,
-                           uint32_t *values, uint32_t count)
+void
+ec_code_c_interleaved(void *dst, void **src, uint64_t offset, uint32_t *values,
+                      uint32_t count)
 {
     uint32_t i, last, tmp;
 
diff --git a/xlators/cluster/ec/src/ec-code-c.h b/xlators/cluster/ec/src/ec-code-c.h
index 92e8070e514..42b5a064eb8 100644
--- a/xlators/cluster/ec/src/ec-code-c.h
+++ b/xlators/cluster/ec/src/ec-code-c.h
@@ -13,12 +13,15 @@
 
 #include "ec-types.h"
 
-void ec_code_c_prepare(ec_gf_t *gf, uint32_t *values, uint32_t count);
+void
+ec_code_c_prepare(ec_gf_t *gf, uint32_t *values, uint32_t count);
 
-void ec_code_c_linear(void *dst, void *src, uint64_t offset, uint32_t *values,
-                      uint32_t count);
+void
+ec_code_c_linear(void *dst, void *src, uint64_t offset, uint32_t *values,
+                 uint32_t count);
 
-void ec_code_c_interleaved(void *dst, void **src, uint64_t offset,
-                           uint32_t *values, uint32_t count);
+void
+ec_code_c_interleaved(void *dst, void **src, uint64_t offset, uint32_t *values,
+                      uint32_t count);
 
 #endif /* __EC_CODE_C_H__ */
diff --git a/xlators/cluster/ec/src/ec-code-intel.c b/xlators/cluster/ec/src/ec-code-intel.c
index b9fdcad4421..f1c4e13e321 100644
--- a/xlators/cluster/ec/src/ec-code-intel.c
+++ b/xlators/cluster/ec/src/ec-code-intel.c
@@ -71,16 +71,15 @@ ec_code_intel_vex(ec_code_intel_t *intel, gf_boolean_t w, gf_boolean_t l,
                   uint32_t reg)
 {
     ec_code_intel_rex(intel, w);
-    if (((intel->rex.w == 1) ||
-         (intel->rex.x == 0) ||
-         (intel->rex.b == 0)) ||
+    if (((intel->rex.w == 1) || (intel->rex.x == 0) || (intel->rex.b == 0)) ||
         ((opcode != VEX_OPCODE_NONE) && (opcode != VEX_OPCODE_0F))) {
         intel->rex.present = _gf_false;
 
         intel->vex.bytes = 3;
         intel->vex.data[0] = 0xC4;
         intel->vex.data[1] = ((intel->rex.r << 7) | (intel->rex.x << 6) |
-                              (intel->rex.b << 5) | opcode) ^ 0xE0;
+                              (intel->rex.b << 5) | opcode) ^
+                             0xE0;
         intel->vex.data[2] = (intel->rex.w << 7) | ((~reg & 0x0F) << 3) |
                              (l ? 0x04 : 0x00) | prefix;
     } else {
@@ -214,22 +213,17 @@ ec_code_intel_emit(ec_code_builder_t *builder, ec_code_intel_t *intel)
         insn[count++] = intel->vex.data[i];
     }
     if (intel->rex.present) {
-        insn[count++] = 0x40 |
-                        (intel->rex.w << 3) |
-                        (intel->rex.r << 2) |
-                        (intel->rex.x << 1) |
-                        (intel->rex.b << 0);
+        insn[count++] = 0x40 | (intel->rex.w << 3) | (intel->rex.r << 2) |
+                        (intel->rex.x << 1) | (intel->rex.b << 0);
     }
     for (i = 0; i < intel->opcode.bytes; i++) {
         insn[count++] = intel->opcode.data[i];
     }
     if (intel->modrm.present) {
-        insn[count++] = (intel->modrm.mod << 6) |
-                        (intel->modrm.reg << 3) |
+        insn[count++] = (intel->modrm.mod << 6) | (intel->modrm.reg << 3) |
                         (intel->modrm.rm << 0);
         if (intel->sib.present) {
-            insn[count++] = (intel->sib.scale << 6) |
-                            (intel->sib.index << 3) |
+            insn[count++] = (intel->sib.scale << 6) | (intel->sib.index << 3) |
                             (intel->sib.base << 0);
         }
     }
@@ -467,9 +461,9 @@ ec_code_intel_op_mov_sse2m(ec_code_builder_t *builder, uint32_t src,
 }
 
 void
-ec_code_intel_op_mov_m2sse(ec_code_builder_t *builder,
-                           ec_code_intel_reg_t base, ec_code_intel_reg_t index,
-                           uint32_t scale, int32_t offset, uint32_t dst)
+ec_code_intel_op_mov_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst)
 {
     ec_code_intel_t intel;
 
@@ -500,9 +494,9 @@ ec_code_intel_op_xor_sse2sse(ec_code_builder_t *builder, uint32_t src,
 }
 
 void
-ec_code_intel_op_xor_m2sse(ec_code_builder_t *builder,
-                           ec_code_intel_reg_t base, ec_code_intel_reg_t index,
-                           uint32_t scale, int32_t offset, uint32_t dst)
+ec_code_intel_op_xor_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst)
 {
     ec_code_intel_t intel;
 
@@ -526,8 +520,8 @@ ec_code_intel_op_mov_avx2avx(ec_code_builder_t *builder, uint32_t src,
 
     ec_code_intel_modrm_reg(&intel, src, dst);
     ec_code_intel_op_1(&intel, 0x6F, 0);
-    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F,
-                      VEX_PREFIX_66, VEX_REG_NONE);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      VEX_REG_NONE);
 
     ec_code_intel_emit(builder, &intel);
 }
@@ -543,16 +537,16 @@ ec_code_intel_op_mov_avx2m(ec_code_builder_t *builder, uint32_t src,
 
     ec_code_intel_modrm_mem(&intel, src, base, index, scale, offset);
     ec_code_intel_op_1(&intel, 0x7F, 0);
-    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F,
-                      VEX_PREFIX_66, VEX_REG_NONE);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      VEX_REG_NONE);
 
     ec_code_intel_emit(builder, &intel);
 }
 
 void
-ec_code_intel_op_mov_m2avx(ec_code_builder_t *builder,
-                           ec_code_intel_reg_t base, ec_code_intel_reg_t index,
-                           uint32_t scale, int32_t offset, uint32_t dst)
+ec_code_intel_op_mov_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst)
 {
     ec_code_intel_t intel;
 
@@ -560,8 +554,8 @@ ec_code_intel_op_mov_m2avx(ec_code_builder_t *builder,
 
     ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset);
     ec_code_intel_op_1(&intel, 0x6F, 0);
-    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F,
-                      VEX_PREFIX_66, VEX_REG_NONE);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      VEX_REG_NONE);
 
     ec_code_intel_emit(builder, &intel);
 }
@@ -576,16 +570,16 @@ ec_code_intel_op_xor_avx2avx(ec_code_builder_t *builder, uint32_t src,
 
     ec_code_intel_modrm_reg(&intel, src, dst);
     ec_code_intel_op_1(&intel, 0xEF, 0);
-    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F,
-                      VEX_PREFIX_66, dst);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      dst);
 
     ec_code_intel_emit(builder, &intel);
 }
 
 void
-ec_code_intel_op_xor_m2avx(ec_code_builder_t *builder,
-                           ec_code_intel_reg_t base, ec_code_intel_reg_t index,
-                           uint32_t scale, int32_t offset, uint32_t dst)
+ec_code_intel_op_xor_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst)
 {
     ec_code_intel_t intel;
 
@@ -593,8 +587,8 @@ ec_code_intel_op_xor_m2avx(ec_code_builder_t *builder,
 
     ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset);
     ec_code_intel_op_1(&intel, 0xEF, 0);
-    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F,
-                      VEX_PREFIX_66, dst);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      dst);
 
     ec_code_intel_emit(builder, &intel);
 }
diff --git a/xlators/cluster/ec/src/ec-code-intel.h b/xlators/cluster/ec/src/ec-code-intel.h
index 903d023f962..3fa4a174765 100644
--- a/xlators/cluster/ec/src/ec-code-intel.h
+++ b/xlators/cluster/ec/src/ec-code-intel.h
@@ -76,109 +76,116 @@ enum _ec_code_vex_opcode {
 struct _ec_code_intel_buffer {
     uint32_t bytes;
     union {
-        uint8_t  data[4];
+        uint8_t data[4];
         uint32_t value;
     };
 };
 
 struct _ec_code_intel_sib {
     gf_boolean_t present;
-    uint32_t     base;
-    uint32_t     index;
-    uint32_t     scale;
+    uint32_t base;
+    uint32_t index;
+    uint32_t scale;
 };
 
 struct _ec_code_intel_modrm {
     gf_boolean_t present;
-    uint32_t     mod;
-    uint32_t     rm;
-    uint32_t     reg;
+    uint32_t mod;
+    uint32_t rm;
+    uint32_t reg;
 };
 
 struct _ec_code_intel_rex {
     gf_boolean_t present;
-    uint32_t     w;
-    uint32_t     r;
-    uint32_t     x;
-    uint32_t     b;
+    uint32_t w;
+    uint32_t r;
+    uint32_t x;
+    uint32_t b;
 };
 
 struct _ec_code_intel {
-    gf_boolean_t           invalid;
+    gf_boolean_t invalid;
     ec_code_intel_buffer_t prefix;
     ec_code_intel_buffer_t opcode;
     ec_code_intel_buffer_t offset;
     ec_code_intel_buffer_t immediate;
     ec_code_intel_buffer_t vex;
-    ec_code_intel_rex_t    rex;
-    ec_code_intel_modrm_t  modrm;
-    ec_code_intel_sib_t    sib;
-    uint32_t               reg;
+    ec_code_intel_rex_t rex;
+    ec_code_intel_modrm_t modrm;
+    ec_code_intel_sib_t sib;
+    uint32_t reg;
 };
 
-void ec_code_intel_op_push_r(ec_code_builder_t *builder,
-                             ec_code_intel_reg_t reg);
-void ec_code_intel_op_pop_r(ec_code_builder_t *builder,
-                            ec_code_intel_reg_t reg);
-void ec_code_intel_op_ret(ec_code_builder_t *builder, uint32_t size);
-
-void ec_code_intel_op_mov_r2r(ec_code_builder_t *builder,
-                              ec_code_intel_reg_t src,
-                              ec_code_intel_reg_t dst);
-void ec_code_intel_op_mov_r2m(ec_code_builder_t *builder,
-                              ec_code_intel_reg_t src,
-                              ec_code_intel_reg_t base,
-                              ec_code_intel_reg_t index, uint32_t scale,
-                              int32_t offset);
-void ec_code_intel_op_mov_m2r(ec_code_builder_t *builder,
-                              ec_code_intel_reg_t base,
-                              ec_code_intel_reg_t index, uint32_t scale,
-                              int32_t offset, ec_code_intel_reg_t dst);
-void ec_code_intel_op_xor_r2r(ec_code_builder_t *builder,
-                              ec_code_intel_reg_t src,
-                              ec_code_intel_reg_t dst);
-void ec_code_intel_op_xor_m2r(ec_code_builder_t *builder,
-                              ec_code_intel_reg_t base,
-                              ec_code_intel_reg_t index, uint32_t scale,
-                              int32_t offset, ec_code_intel_reg_t dst);
-void ec_code_intel_op_add_i2r(ec_code_builder_t *builder, int32_t value,
-                              ec_code_intel_reg_t reg);
-void ec_code_intel_op_test_i2r(ec_code_builder_t *builder, uint32_t value,
-                               ec_code_intel_reg_t reg);
-void ec_code_intel_op_jne(ec_code_builder_t *builder, uint32_t address);
-
-void ec_code_intel_op_mov_sse2sse(ec_code_builder_t *builder, uint32_t src,
-                                  uint32_t dst);
-void ec_code_intel_op_mov_sse2m(ec_code_builder_t *builder, uint32_t src,
-                                ec_code_intel_reg_t base,
-                                ec_code_intel_reg_t index, uint32_t scale,
-                                int32_t offset);
-void ec_code_intel_op_mov_m2sse(ec_code_builder_t *builder,
-                                ec_code_intel_reg_t base,
-                                ec_code_intel_reg_t index, uint32_t scale,
-                                int32_t offset, uint32_t dst);
-void ec_code_intel_op_xor_sse2sse(ec_code_builder_t *builder, uint32_t src,
-                                  uint32_t dst);
-void ec_code_intel_op_xor_m2sse(ec_code_builder_t *builder,
-                                ec_code_intel_reg_t base,
-                                ec_code_intel_reg_t index, uint32_t scale,
-                                int32_t offset, uint32_t dst);
-
-void ec_code_intel_op_mov_avx2avx(ec_code_builder_t *builder, uint32_t src,
-                                  uint32_t dst);
-void ec_code_intel_op_mov_avx2m(ec_code_builder_t *builder, uint32_t src,
-                                ec_code_intel_reg_t base,
-                                ec_code_intel_reg_t index, uint32_t scale,
-                                int32_t offset);
-void ec_code_intel_op_mov_m2avx(ec_code_builder_t *builder,
-                                ec_code_intel_reg_t base,
-                                ec_code_intel_reg_t index, uint32_t scale,
-                                int32_t offset, uint32_t dst);
-void ec_code_intel_op_xor_avx2avx(ec_code_builder_t *builder, uint32_t src,
-                                  uint32_t dst);
-void ec_code_intel_op_xor_m2avx(ec_code_builder_t *builder,
-                                ec_code_intel_reg_t base,
-                                ec_code_intel_reg_t index, uint32_t scale,
-                                int32_t offset, uint32_t dst);
+void
+ec_code_intel_op_push_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg);
+void
+ec_code_intel_op_pop_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg);
+void
+ec_code_intel_op_ret(ec_code_builder_t *builder, uint32_t size);
+
+void
+ec_code_intel_op_mov_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t dst);
+void
+ec_code_intel_op_mov_r2m(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                         uint32_t scale, int32_t offset);
+void
+ec_code_intel_op_mov_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                         ec_code_intel_reg_t index, uint32_t scale,
+                         int32_t offset, ec_code_intel_reg_t dst);
+void
+ec_code_intel_op_xor_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t dst);
+void
+ec_code_intel_op_xor_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                         ec_code_intel_reg_t index, uint32_t scale,
+                         int32_t offset, ec_code_intel_reg_t dst);
+void
+ec_code_intel_op_add_i2r(ec_code_builder_t *builder, int32_t value,
+                         ec_code_intel_reg_t reg);
+void
+ec_code_intel_op_test_i2r(ec_code_builder_t *builder, uint32_t value,
+                          ec_code_intel_reg_t reg);
+void
+ec_code_intel_op_jne(ec_code_builder_t *builder, uint32_t address);
+
+void
+ec_code_intel_op_mov_sse2sse(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst);
+void
+ec_code_intel_op_mov_sse2m(ec_code_builder_t *builder, uint32_t src,
+                           ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                           uint32_t scale, int32_t offset);
+void
+ec_code_intel_op_mov_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst);
+void
+ec_code_intel_op_xor_sse2sse(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst);
+void
+ec_code_intel_op_xor_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst);
+
+void
+ec_code_intel_op_mov_avx2avx(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst);
+void
+ec_code_intel_op_mov_avx2m(ec_code_builder_t *builder, uint32_t src,
+                           ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                           uint32_t scale, int32_t offset);
+void
+ec_code_intel_op_mov_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst);
+void
+ec_code_intel_op_xor_avx2avx(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst);
+void
+ec_code_intel_op_xor_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst);
 
 #endif /* __EC_CODE_INTEL_H__ */
diff --git a/xlators/cluster/ec/src/ec-code-sse.c b/xlators/cluster/ec/src/ec-code-sse.c
index 6f2c6fa593f..e11e7ff8400 100644
--- a/xlators/cluster/ec/src/ec-code-sse.c
+++ b/xlators/cluster/ec/src/ec-code-sse.c
@@ -34,10 +34,9 @@ ec_code_sse_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
                  uint32_t bit)
 {
     if (builder->linear) {
-        ec_code_intel_op_mov_m2sse(builder, REG_SI, REG_DX, 1,
-                                   idx * builder->width * builder->bits +
-                                   bit * builder->width,
-                                   dst);
+        ec_code_intel_op_mov_m2sse(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
     } else {
         if (builder->base != idx) {
             ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
@@ -73,10 +72,9 @@ ec_code_sse_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
                  uint32_t bit)
 {
     if (builder->linear) {
-       ec_code_intel_op_xor_m2sse(builder, REG_SI, REG_DX, 1,
-                                  idx * builder->width * builder->bits +
-                                  bit * builder->width,
-                                  dst);
+        ec_code_intel_op_xor_m2sse(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
     } else {
         if (builder->base != idx) {
             ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
@@ -88,21 +86,16 @@ ec_code_sse_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
     }
 }
 
-static char *ec_code_sse_needed_flags[] = {
-    "sse2",
-    NULL
-};
+static char *ec_code_sse_needed_flags[] = {"sse2", NULL};
 
-ec_code_gen_t ec_code_gen_sse = {
-    .name   = "sse",
-    .flags  = ec_code_sse_needed_flags,
-    .width  = 16,
-    .prolog = ec_code_sse_prolog,
-    .epilog = ec_code_sse_epilog,
-    .load   = ec_code_sse_load,
-    .store  = ec_code_sse_store,
-    .copy   = ec_code_sse_copy,
-    .xor2   = ec_code_sse_xor2,
-    .xor3   = NULL,
-    .xorm   = ec_code_sse_xorm
-};
+ec_code_gen_t ec_code_gen_sse = {.name = "sse",
+                                 .flags = ec_code_sse_needed_flags,
+                                 .width = 16,
+                                 .prolog = ec_code_sse_prolog,
+                                 .epilog = ec_code_sse_epilog,
+                                 .load = ec_code_sse_load,
+                                 .store = ec_code_sse_store,
+                                 .copy = ec_code_sse_copy,
+                                 .xor2 = ec_code_sse_xor2,
+                                 .xor3 = NULL,
+                                 .xorm = ec_code_sse_xorm};
diff --git a/xlators/cluster/ec/src/ec-code-x64.c b/xlators/cluster/ec/src/ec-code-x64.c
index e94ddd4b155..26565b4493f 100644
--- a/xlators/cluster/ec/src/ec-code-x64.c
+++ b/xlators/cluster/ec/src/ec-code-x64.c
@@ -14,8 +14,7 @@
 
 static ec_code_intel_reg_t ec_code_x64_regmap[] = {
     REG_AX, REG_CX, REG_BP, REG_8,  REG_9, REG_10,
-    REG_11, REG_12, REG_13, REG_14, REG_15
-};
+    REG_11, REG_12, REG_13, REG_14, REG_15};
 
 static void
 ec_code_x64_prolog(ec_code_builder_t *builder)
@@ -49,6 +48,7 @@ ec_code_x64_epilog(ec_code_builder_t *builder)
 
     if (builder->regs > 11) {
         ec_code_error(builder, EINVAL);
+        return;
     }
     for (i = builder->regs; i > 7; i--) {
         ec_code_intel_op_pop_r(builder, ec_code_x64_regmap[i - 1]);
@@ -67,10 +67,9 @@ ec_code_x64_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
     dst = ec_code_x64_regmap[dst];
 
     if (builder->linear) {
-        ec_code_intel_op_mov_m2r(builder, REG_SI, REG_DX, 1,
-                                 idx * builder->width * builder->bits +
-                                 bit * builder->width,
-                                 dst);
+        ec_code_intel_op_mov_m2r(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
     } else {
         if (builder->base != idx) {
             ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
@@ -116,10 +115,9 @@ ec_code_x64_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
     dst = ec_code_x64_regmap[dst];
 
     if (builder->linear) {
-        ec_code_intel_op_xor_m2r(builder, REG_SI, REG_DX, 1,
-                                 idx * builder->width * builder->bits +
-                                 bit * builder->width,
-                                 dst);
+        ec_code_intel_op_xor_m2r(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
     } else {
         if (builder->base != idx) {
             ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
@@ -131,20 +129,16 @@ ec_code_x64_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
     }
 }
 
-static char *ec_code_x64_needed_flags[] = {
-    NULL
-};
-
-ec_code_gen_t ec_code_gen_x64 = {
-    .name   = "x64",
-    .flags  = ec_code_x64_needed_flags,
-    .width  = sizeof(uint64_t),
-    .prolog = ec_code_x64_prolog,
-    .epilog = ec_code_x64_epilog,
-    .load   = ec_code_x64_load,
-    .store  = ec_code_x64_store,
-    .copy   = ec_code_x64_copy,
-    .xor2   = ec_code_x64_xor2,
-    .xor3   = NULL,
-    .xorm   = ec_code_x64_xorm
-};
+static char *ec_code_x64_needed_flags[] = {NULL};
+
+ec_code_gen_t ec_code_gen_x64 = {.name = "x64",
+                                 .flags = ec_code_x64_needed_flags,
+                                 .width = sizeof(uint64_t),
+                                 .prolog = ec_code_x64_prolog,
+                                 .epilog = ec_code_x64_epilog,
+                                 .load = ec_code_x64_load,
+                                 .store = ec_code_x64_store,
+                                 .copy = ec_code_x64_copy,
+                                 .xor2 = ec_code_x64_xor2,
+                                 .xor3 = NULL,
+                                 .xorm = ec_code_x64_xorm};
diff --git a/xlators/cluster/ec/src/ec-code.c b/xlators/cluster/ec/src/ec-code.c
index 44957dd788f..03162ae05a9 100644
--- a/xlators/cluster/ec/src/ec-code.c
+++ b/xlators/cluster/ec/src/ec-code.c
@@ -14,7 +14,7 @@
 #include <sys/stat.h>
 #include <ctype.h>
 
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 
 #include "ec-mem-types.h"
 #include "ec-code.h"
@@ -47,13 +47,13 @@ struct _ec_code_proc;
 typedef struct _ec_code_proc ec_code_proc_t;
 
 struct _ec_code_proc {
-    int32_t      fd;
+    int32_t fd;
     gf_boolean_t eof;
     gf_boolean_t error;
     gf_boolean_t skip;
-    ssize_t      size;
-    ssize_t      pos;
-    char         buffer[EC_PROC_BUFFER_SIZE];
+    ssize_t size;
+    ssize_t pos;
+    char buffer[EC_PROC_BUFFER_SIZE];
 };
 
 static ec_code_gen_t *ec_code_gen_table[] = {
@@ -66,8 +66,7 @@ static ec_code_gen_t *ec_code_gen_table[] = {
 #ifdef USE_EC_DYNAMIC_X64
     &ec_code_gen_x64,
 #endif
-    NULL
-};
+    NULL};
 
 static void
 ec_code_arg_set(ec_code_arg_t *arg, uint32_t value)
@@ -84,7 +83,6 @@ ec_code_arg_assign(ec_code_builder_t *builder, ec_code_op_t *op,
     if (builder->regs <= reg) {
         builder->regs = reg + 1;
     }
-
 }
 
 static void
@@ -202,17 +200,17 @@ static void
 ec_code_dup(ec_code_builder_t *builder, ec_gf_op_t *op)
 {
     switch (op->op) {
-    case EC_GF_OP_COPY:
-        ec_code_copy(builder, op->arg1, op->arg2);
-        break;
-    case EC_GF_OP_XOR2:
-        ec_code_xor2(builder, op->arg1, op->arg2);
-        break;
-    case EC_GF_OP_XOR3:
-        ec_code_xor3(builder, op->arg1, op->arg2, op->arg3);
-        break;
-    default:
-        break;
+        case EC_GF_OP_COPY:
+            ec_code_copy(builder, op->arg1, op->arg2);
+            break;
+        case EC_GF_OP_XOR2:
+            ec_code_xor2(builder, op->arg1, op->arg2);
+            break;
+        case EC_GF_OP_XOR3:
+            ec_code_xor3(builder, op->arg1, op->arg2, op->arg3);
+            break;
+        default:
+            break;
     }
 }
 
@@ -285,8 +283,9 @@ ec_code_prepare(ec_code_t *code, uint32_t count, uint32_t width,
 
     count *= code->gf->bits + code->gf->max_ops;
     count += code->gf->bits;
-    builder = GF_MALLOC(sizeof(ec_code_builder_t) +
-                        sizeof(ec_code_op_t) * count, ec_mt_ec_code_builder_t);
+    builder = GF_MALLOC(
+        sizeof(ec_code_builder_t) + sizeof(ec_code_op_t) * count,
+        ec_mt_ec_code_builder_t);
     if (builder == NULL) {
         return EC_ERR(ENOMEM);
     }
@@ -331,15 +330,15 @@ ec_code_chunk_from_space(ec_code_space_t *space)
 static void *
 ec_code_to_executable(ec_code_space_t *space, void *addr)
 {
-        return (void *)((uintptr_t)addr - (uintptr_t)space
-                                        + (uintptr_t)space->exec);
+    return (void *)((uintptr_t)addr - (uintptr_t)space +
+                    (uintptr_t)space->exec);
 }
 
 static void *
 ec_code_from_executable(ec_code_space_t *space, void *addr)
 {
-        return (void *)((uintptr_t)addr - (uintptr_t)space->exec
-                                        + (uintptr_t)space);
+    return (void *)((uintptr_t)addr - (uintptr_t)space->exec +
+                    (uintptr_t)space);
 }
 
 static void *
@@ -395,104 +394,105 @@ ec_code_chunk_touch(ec_code_chunk_t *prev, ec_code_chunk_t *next)
 static ec_code_space_t *
 ec_code_space_create(ec_code_t *code, size_t size)
 {
-        char path[] = GLUSTERFS_LIBEXECDIR "/ec-code-dynamic.XXXXXX";
-        ec_code_space_t *space;
-        void *exec;
-        int32_t fd, err;
-
-        /* We need to create memory areas to store the generated dynamic code.
-         * Obviously these areas need to be written to be able to create the
-         * code and they also need to be executable to execute it.
-         *
-         * However it's a bad practice to have a memory region that is both
-         * writable *and* executable. In fact, selinux forbids this and causes
-         * attempts to do so to fail (unless specifically configured).
-         *
-         * To solve the problem we'll use two distinct memory areas mapped to
-         * the same physical storage. One of the memory areas will have write
-         * permission, and the other will have execute permission. Both areas
-         * will have the same contents. The physical storage will be a regular
-         * file that will be mmapped to both areas.
-         */
-
-        /* We need to create a temporary file as the backend storage for the
-         * memory mapped areas. */
-        fd = mkstemp(path);
-        if (fd < 0) {
-                err = errno;
-                gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
-                       "Unable to create a temporary file for the ec dynamic "
-                       "code");
-                space = EC_ERR(err);
-                goto done;
-        }
-        /* Once created we don't need to keep it in the file system. It will
-         * still exist until we close the last file descriptor or unmap the
-         * memory areas bound to the file. */
-        sys_unlink(path);
-
-        size = (size + EC_CODE_ALIGN - 1) & ~(EC_CODE_ALIGN - 1);
-        if (sys_ftruncate(fd, size) < 0) {
-                err = errno;
-                gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
-                       "Unable to resize the file for the ec dynamic code");
-                space = EC_ERR(err);
-                goto done_close;
-        }
+    char path[] = GLUSTERFS_LIBEXECDIR "/ec-code-dynamic.XXXXXX";
+    ec_code_space_t *space;
+    void *exec;
+    int32_t fd, err;
+
+    /* We need to create memory areas to store the generated dynamic code.
+     * Obviously these areas need to be written to be able to create the
+     * code and they also need to be executable to execute it.
+     *
+     * However it's a bad practice to have a memory region that is both
+     * writable *and* executable. In fact, selinux forbids this and causes
+     * attempts to do so to fail (unless specifically configured).
+     *
+     * To solve the problem we'll use two distinct memory areas mapped to
+     * the same physical storage. One of the memory areas will have write
+     * permission, and the other will have execute permission. Both areas
+     * will have the same contents. The physical storage will be a regular
+     * file that will be mmapped to both areas.
+     */
+
+    /* We need to create a temporary file as the backend storage for the
+     * memory mapped areas. */
+    /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */
+    fd = mkstemp(path);
+    if (fd < 0) {
+        err = errno;
+        gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
+               "Unable to create a temporary file for the ec dynamic "
+               "code");
+        space = EC_ERR(err);
+        goto done;
+    }
+    /* Once created we don't need to keep it in the file system. It will
+     * still exist until we close the last file descriptor or unmap the
+     * memory areas bound to the file. */
+    sys_unlink(path);
+
+    size = (size + EC_CODE_ALIGN - 1) & ~(EC_CODE_ALIGN - 1);
+    if (sys_ftruncate(fd, size) < 0) {
+        err = errno;
+        gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
+               "Unable to resize the file for the ec dynamic code");
+        space = EC_ERR(err);
+        goto done_close;
+    }
 
-        /* This creates an executable memory area to be able to run the
-         * generated fragments of code. */
-        exec = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0);
-        if (exec == MAP_FAILED) {
-                err = errno;
-                gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
-                       "Unable to map the executable area for the ec dynamic "
-                       "code");
-                space = EC_ERR(err);
-                goto done_close;
-        }
-        /* It's not important to check the return value of mlock(). If it fails
-         * everything will continue to work normally. */
-        mlock(exec, size);
-
-        /* This maps a read/write memory area to be able to create the dynamici
-         * code. */
-        space = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
-        if (space == MAP_FAILED) {
-                err = errno;
-                gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
-                       "Unable to map the writable area for the ec dynamic "
-                       "code");
-                space = EC_ERR(err);
-
-                munmap(exec, size);
-
-                goto done_close;
-        }
+    /* This creates an executable memory area to be able to run the
+     * generated fragments of code. */
+    exec = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0);
+    if (exec == MAP_FAILED) {
+        err = errno;
+        gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
+               "Unable to map the executable area for the ec dynamic "
+               "code");
+        space = EC_ERR(err);
+        goto done_close;
+    }
+    /* It's not important to check the return value of mlock(). If it fails
+     * everything will continue to work normally. */
+    mlock(exec, size);
+
+    /* This maps a read/write memory area to be able to create the dynamici
+     * code. */
+    space = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    if (space == MAP_FAILED) {
+        err = errno;
+        gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
+               "Unable to map the writable area for the ec dynamic "
+               "code");
+        space = EC_ERR(err);
+
+        munmap(exec, size);
+
+        goto done_close;
+    }
 
-        space->exec = exec;
-        space->size = size;
-        space->code = code;
-        list_add_tail(&space->list, &code->spaces);
-        INIT_LIST_HEAD(&space->chunks);
+    space->exec = exec;
+    space->size = size;
+    space->code = code;
+    list_add_tail(&space->list, &code->spaces);
+    INIT_LIST_HEAD(&space->chunks);
 
 done_close:
-        /* If everything has succeeded, we already have the memory areas
-         * mapped. We don't need the file descriptor anymore because the
-         * backend storage will be there until the mmaped regions are
-         * unmapped. */
-        sys_close(fd);
+    /* If everything has succeeded, we already have the memory areas
+     * mapped. We don't need the file descriptor anymore because the
+     * backend storage will be there until the mmap()'d regions are
+     * unmapped. */
+    sys_close(fd);
 done:
-        return space;
+    return space;
 }
 
 static void
 ec_code_space_destroy(ec_code_space_t *space)
 {
-        list_del_init(&space->list);
+    list_del_init(&space->list);
 
-        munmap(space->exec, space->size);
-        munmap(space, space->size);
+    munmap(space->exec, space->size);
+    munmap(space, space->size);
 }
 
 static void
@@ -500,7 +500,8 @@ ec_code_chunk_merge(ec_code_chunk_t *chunk)
 {
     ec_code_chunk_t *item, *tmp;
 
-    list_for_each_entry_safe(item, tmp, &chunk->space->chunks, list) {
+    list_for_each_entry_safe(item, tmp, &chunk->space->chunks, list)
+    {
         if ((uintptr_t)item > (uintptr_t)chunk) {
             list_add_tail(&chunk->list, &item->list);
             if (ec_code_chunk_touch(chunk, item)) {
@@ -519,8 +520,8 @@ ec_code_chunk_merge(ec_code_chunk_t *chunk)
     list_add_tail(&chunk->list, &chunk->space->chunks);
 
 check:
-    if (chunk->size == chunk->space->size - ec_code_space_size() -
-                       ec_code_chunk_size()) {
+    if (chunk->size ==
+        chunk->space->size - ec_code_space_size() - ec_code_chunk_size()) {
         ec_code_space_destroy(chunk->space);
     }
 }
@@ -535,9 +536,12 @@ ec_code_space_alloc(ec_code_t *code, size_t size)
     /* To minimize fragmentation, we only allocate chunks of sizes multiples
      * of EC_CODE_CHUNK_MIN_SIZE. */
     size = ((size + ec_code_chunk_size() + EC_CODE_CHUNK_MIN_SIZE - 1) &
-           ~(EC_CODE_CHUNK_MIN_SIZE - 1)) - ec_code_chunk_size();
-    list_for_each_entry(space, &code->spaces, list) {
-        list_for_each_entry(chunk, &space->chunks, list) {
+            ~(EC_CODE_CHUNK_MIN_SIZE - 1)) -
+           ec_code_chunk_size();
+    list_for_each_entry(space, &code->spaces, list)
+    {
+        list_for_each_entry(chunk, &space->chunks, list)
+        {
             if (chunk->size >= size) {
                 goto out;
             }
@@ -607,26 +611,29 @@ ec_code_write(ec_code_builder_t *builder)
     for (i = 0; i < builder->count; i++) {
         op = &builder->ops[i];
         switch (op->op) {
-        case EC_GF_OP_LOAD:
-            gen->load(builder, op->arg1.value, op->arg2.value, op->arg3.value);
-            break;
-        case EC_GF_OP_STORE:
-            gen->store(builder, op->arg1.value, op->arg3.value);
-            break;
-        case EC_GF_OP_COPY:
-            gen->copy(builder, op->arg1.value, op->arg2.value);
-            break;
-        case EC_GF_OP_XOR2:
-            gen->xor2(builder, op->arg1.value, op->arg2.value);
-            break;
-        case EC_GF_OP_XOR3:
-            gen->xor3(builder, op->arg1.value, op->arg2.value, op->arg3.value);
-            break;
-        case EC_GF_OP_XORM:
-            gen->xorm(builder, op->arg1.value, op->arg2.value, op->arg3.value);
-            break;
-        default:
-            break;
+            case EC_GF_OP_LOAD:
+                gen->load(builder, op->arg1.value, op->arg2.value,
+                          op->arg3.value);
+                break;
+            case EC_GF_OP_STORE:
+                gen->store(builder, op->arg1.value, op->arg3.value);
+                break;
+            case EC_GF_OP_COPY:
+                gen->copy(builder, op->arg1.value, op->arg2.value);
+                break;
+            case EC_GF_OP_XOR2:
+                gen->xor2(builder, op->arg1.value, op->arg2.value);
+                break;
+            case EC_GF_OP_XOR3:
+                gen->xor3(builder, op->arg1.value, op->arg2.value,
+                          op->arg3.value);
+                break;
+            case EC_GF_OP_XORM:
+                gen->xorm(builder, op->arg1.value, op->arg2.value,
+                          op->arg3.value);
+                break;
+            default:
+                break;
         }
     }
     gen->epilog(builder);
@@ -715,67 +722,65 @@ static void *
 ec_code_build_dynamic(ec_code_t *code, uint32_t width, uint32_t *values,
                       uint32_t count, gf_boolean_t linear)
 {
-        ec_code_builder_t *builder;
-        uint32_t offset, val, next;
+    ec_code_builder_t *builder;
+    uint32_t offset, val, next;
 
-        builder = ec_code_prepare(code, count, width, linear);
-        if (EC_IS_ERR(builder)) {
-                return builder;
-        }
+    builder = ec_code_prepare(code, count, width, linear);
+    if (EC_IS_ERR(builder)) {
+        return builder;
+    }
 
-        offset = -1;
-        next = ec_code_value_next(values, count, &offset);
-        if (next != 0) {
-                ec_code_gf_load(builder, offset);
-                do {
-                        val = next;
-                        next = ec_code_value_next(values, count, &offset);
-                        if (next != 0) {
-                                ec_code_gf_mul(builder, ec_gf_div(code->gf,
-                                                                  val, next));
-                                ec_code_gf_load_xor(builder, offset);
-                        }
-                } while (next != 0);
-                ec_code_gf_mul(builder, val);
-                ec_code_gf_store(builder);
-        } else {
-                ec_code_gf_clear(builder);
-        }
+    offset = -1;
+    next = ec_code_value_next(values, count, &offset);
+    if (next != 0) {
+        ec_code_gf_load(builder, offset);
+        do {
+            val = next;
+            next = ec_code_value_next(values, count, &offset);
+            if (next != 0) {
+                ec_code_gf_mul(builder, ec_gf_div(code->gf, val, next));
+                ec_code_gf_load_xor(builder, offset);
+            }
+        } while (next != 0);
+        ec_code_gf_mul(builder, val);
+        ec_code_gf_store(builder);
+    } else {
+        ec_code_gf_clear(builder);
+    }
 
-        return ec_code_compile(builder);
+    return ec_code_compile(builder);
 }
 
 static void *
-ec_code_build(ec_code_t *code, uint32_t width, uint32_t *values,
-              uint32_t count, gf_boolean_t linear)
+ec_code_build(ec_code_t *code, uint32_t width, uint32_t *values, uint32_t count,
+              gf_boolean_t linear)
 {
-        void *func;
+    void *func;
 
-        if (code->gen != NULL) {
-                func = ec_code_build_dynamic(code, width, values, count,
-                                             linear);
-                if (!EC_IS_ERR(func)) {
-                        return func;
-                }
+    if (code->gen != NULL) {
+        func = ec_code_build_dynamic(code, width, values, count, linear);
+        if (!EC_IS_ERR(func)) {
+            return func;
+        }
 
-                gf_msg_debug(THIS->name, GF_LOG_DEBUG,
-                             "Unable to generate dynamic code. Falling back "
-                             "to precompiled code");
+        gf_msg_debug(THIS->name, GF_LOG_DEBUG,
+                     "Unable to generate dynamic code. Falling back "
+                     "to precompiled code");
 
-                /* The dynamic code generation shouldn't fail in normal
-                 * conditions, but if it fails at some point, it's very
-                 * probable that it will fail again, so we completely disable
-                 * dynamic code generation. */
-                code->gen = NULL;
-        }
+        /* The dynamic code generation shouldn't fail in normal
+         * conditions, but if it fails at some point, it's very
+         * probable that it will fail again, so we completely disable
+         * dynamic code generation. */
+        code->gen = NULL;
+    }
 
-        ec_code_c_prepare(code->gf, values, count);
+    ec_code_c_prepare(code->gf, values, count);
 
-        if (linear) {
-                return ec_code_c_linear;
-        }
+    if (linear) {
+        return ec_code_c_linear;
+    }
 
-        return ec_code_c_interleaved;
+    return ec_code_c_interleaved;
 }
 
 ec_code_func_linear_t
@@ -790,17 +795,17 @@ ec_code_func_interleaved_t
 ec_code_build_interleaved(ec_code_t *code, uint32_t width, uint32_t *values,
                           uint32_t count)
 {
-    return (ec_code_func_interleaved_t)ec_code_build(code, width, values,
-                                                     count, _gf_false);
+    return (ec_code_func_interleaved_t)ec_code_build(code, width, values, count,
+                                                     _gf_false);
 }
 
 void
 ec_code_release(ec_code_t *code, ec_code_func_t *func)
 {
-        if ((func->linear != ec_code_c_linear) &&
-            (func->interleaved != ec_code_c_interleaved)) {
-                ec_code_free(ec_code_chunk_from_func(func->linear));
-        }
+    if ((func->linear != ec_code_c_linear) &&
+        (func->interleaved != ec_code_c_interleaved)) {
+        ec_code_free(ec_code_chunk_from_func(func->linear));
+    }
 }
 
 void
@@ -939,7 +944,7 @@ ec_code_cpu_check(uint32_t idx, char *list, uint32_t count)
 {
     ec_code_gen_t *gen;
     char **ptr;
-    char *table[count];
+    char *table[count + 1];
     uint32_t i;
 
     for (i = 0; i < count; i++) {
@@ -1002,7 +1007,8 @@ ec_code_detect(xlator_t *xl, const char *def)
         if (ec_code_gen_table[select] == NULL) {
             gf_msg(xl->name, GF_LOG_WARNING, EINVAL, EC_MSG_EXTENSION_UNKNOWN,
                    "CPU extension '%s' is not known. Not using any cpu "
-                   "extensions", def);
+                   "extensions",
+                   def);
 
             return NULL;
         }
diff --git a/xlators/cluster/ec/src/ec-code.h b/xlators/cluster/ec/src/ec-code.h
index 355209c3944..75fb35d93e3 100644
--- a/xlators/cluster/ec/src/ec-code.h
+++ b/xlators/cluster/ec/src/ec-code.h
@@ -11,8 +11,8 @@
 #ifndef __EC_CODE_H__
 #define __EC_CODE_H__
 
-#include "xlator.h"
-#include "list.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/list.h>
 
 #include "ec-types.h"
 #include "ec-galois.h"
diff --git a/xlators/cluster/ec/src/ec-combine.c b/xlators/cluster/ec/src/ec-combine.c
index b51ec87a934..703a30e2485 100644
--- a/xlators/cluster/ec/src/ec-combine.c
+++ b/xlators/cluster/ec/src/ec-combine.c
@@ -11,14 +11,14 @@
 #include <fnmatch.h>
 
 #include "libxlator.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 
 #include "ec-types.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
 #include "ec-messages.h"
-#include "quota-common-utils.h"
+#include <glusterfs/quota-common-utils.h>
 
 #define EC_QUOTA_PREFIX "trusted.glusterfs.quota."
 
@@ -30,41 +30,38 @@ typedef struct _ec_dict_info ec_dict_info_t;
 struct _ec_dict_combine;
 typedef struct _ec_dict_combine ec_dict_combine_t;
 
-struct _ec_dict_info
-{
-    dict_t * dict;
-    int32_t  count;
+struct _ec_dict_info {
+    dict_t *dict;
+    int32_t count;
 };
 
-struct _ec_dict_combine
-{
-    ec_cbk_data_t * cbk;
-    int32_t         which;
+struct _ec_dict_combine {
+    ec_cbk_data_t *cbk;
+    int32_t which;
 };
 
 int32_t
-ec_combine_write (ec_fop_data_t *fop, ec_cbk_data_t *dst,
-                  ec_cbk_data_t *src)
+ec_combine_write(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-        int     valid = 0;
+    int valid = 0;
 
-        if (!fop || !dst || !src)
-                return 0;
+    if (!fop || !dst || !src)
+        return 0;
 
-        switch (fop->id) {
+    switch (fop->id) {
         case GF_FOP_REMOVEXATTR:
         case GF_FOP_FREMOVEXATTR:
         case GF_FOP_SETXATTR:
         case GF_FOP_FSETXATTR:
-                return 1;
+            return 1;
 
         case GF_FOP_SYMLINK:
         case GF_FOP_LINK:
         case GF_FOP_CREATE:
         case GF_FOP_MKNOD:
         case GF_FOP_MKDIR:
-                valid = 3;
-                break;
+            valid = 3;
+            break;
         case GF_FOP_UNLINK:
         case GF_FOP_RMDIR:
         case GF_FOP_SETATTR:
@@ -75,42 +72,40 @@ ec_combine_write (ec_fop_data_t *fop, ec_cbk_data_t *dst,
         case GF_FOP_FALLOCATE:
         case GF_FOP_DISCARD:
         case GF_FOP_ZEROFILL:
-                valid = 2;
-                break;
+            valid = 2;
+            break;
         case GF_FOP_RENAME:
-                valid = 5;
-                break;
+            valid = 5;
+            break;
         default:
-                gf_msg_callingfn (fop->xl->name, GF_LOG_WARNING, EINVAL,
-                                  EC_MSG_INVALID_FOP,
-                                  "Invalid fop %d", fop->id);
-                return 0;
-                break;
-        }
+            gf_msg_callingfn(fop->xl->name, GF_LOG_WARNING, EINVAL,
+                             EC_MSG_INVALID_FOP, "Invalid fop %d", fop->id);
+            return 0;
+            break;
+    }
 
-        if (!ec_iatt_combine(fop, dst->iatt, src->iatt, valid)) {
-                gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                        EC_MSG_IATT_MISMATCH,
-                        "Mismatching iatt in "
-                        "answers of '%s'", gf_fop_list[fop->id]);
-                return 0;
-        }
-        return 1;
+    if (!ec_iatt_combine(fop, dst->iatt, src->iatt, valid)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of '%s'",
+               gf_fop_list[fop->id]);
+        return 0;
+    }
+    return 1;
 }
 
-void ec_iatt_time_merge(int64_t *dst_sec, uint32_t *dst_nsec,
-                        int64_t src_sec, uint32_t src_nsec)
+void
+ec_iatt_time_merge(int64_t *dst_sec, uint32_t *dst_nsec, int64_t src_sec,
+                   uint32_t src_nsec)
 {
     if ((*dst_sec < src_sec) ||
-        ((*dst_sec == src_sec) && (*dst_nsec < src_nsec)))
-    {
+        ((*dst_sec == src_sec) && (*dst_nsec < src_nsec))) {
         *dst_sec = src_sec;
         *dst_nsec = src_nsec;
     }
 }
 
-static
-gf_boolean_t
+static gf_boolean_t
 ec_iatt_is_trusted(ec_fop_data_t *fop, struct iatt *iatt)
 {
     uint64_t ino;
@@ -141,14 +136,14 @@ ec_iatt_is_trusted(ec_fop_data_t *fop, struct iatt *iatt)
     return _gf_false;
 }
 
-int32_t ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
-                        int32_t count)
+int32_t
+ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
+                int32_t count)
 {
     int32_t i;
     gf_boolean_t failed = _gf_false;
 
-    for (i = 0; i < count; i++)
-    {
+    for (i = 0; i < count; i++) {
         /* Check for basic fields. These fields must be equal always, even if
          * the inode is not locked because in these cases the parent inode
          * will be locked and differences in these fields require changes in
@@ -172,33 +167,34 @@ int32_t ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
                  * data is returned. */
                 failed = _gf_true;
             } else {
-                gf_msg_debug (fop->xl->name, 0,
-                       "Ignoring iatt differences because inode is not "
-                       "locked");
+                gf_msg_debug(fop->xl->name, 0,
+                             "Ignoring iatt differences because inode is not "
+                             "locked");
             }
         }
         if (failed) {
-            gf_msg (fop->xl->name, GF_LOG_WARNING, 0,
-                    EC_MSG_IATT_COMBINE_FAIL,
-                    "Failed to combine iatt (inode: %lu-%lu, links: %u-%u, "
-                    "uid: %u-%u, gid: %u-%u, rdev: %lu-%lu, size: %lu-%lu, "
-                    "mode: %o-%o)",
-                    dst[i].ia_ino, src[i].ia_ino, dst[i].ia_nlink,
-                    src[i].ia_nlink, dst[i].ia_uid, src[i].ia_uid,
-                    dst[i].ia_gid, src[i].ia_gid, dst[i].ia_rdev,
-                    src[i].ia_rdev, dst[i].ia_size, src[i].ia_size,
-                    st_mode_from_ia(dst[i].ia_prot, dst[i].ia_type),
-                    st_mode_from_ia(src[i].ia_prot, dst[i].ia_type));
+            gf_msg(fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_IATT_COMBINE_FAIL,
+                   "Failed to combine iatt (inode: %" PRIu64 "-%" PRIu64
+                   ", "
+                   "links: %u-%u, uid: %u-%u, gid: %u-%u, "
+                   "rdev: %" PRIu64 "-%" PRIu64 ", size: %" PRIu64 "-%" PRIu64
+                   ", "
+                   "mode: %o-%o), %s",
+                   dst[i].ia_ino, src[i].ia_ino, dst[i].ia_nlink,
+                   src[i].ia_nlink, dst[i].ia_uid, src[i].ia_uid, dst[i].ia_gid,
+                   src[i].ia_gid, dst[i].ia_rdev, src[i].ia_rdev,
+                   dst[i].ia_size, src[i].ia_size,
+                   st_mode_from_ia(dst[i].ia_prot, dst[i].ia_type),
+                   st_mode_from_ia(src[i].ia_prot, dst[i].ia_type),
+                   ec_msg_str(fop));
 
             return 0;
         }
     }
 
-    while (count-- > 0)
-    {
+    while (count-- > 0) {
         dst[count].ia_blocks += src[count].ia_blocks;
-        if (dst[count].ia_blksize < src[count].ia_blksize)
-        {
+        if (dst[count].ia_blksize < src[count].ia_blksize) {
             dst[count].ia_blksize = src[count].ia_blksize;
         }
 
@@ -213,13 +209,12 @@ int32_t ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
     return 1;
 }
 
-void ec_iatt_rebuild(ec_t * ec, struct iatt * iatt, int32_t count,
-                     int32_t answers)
+void
+ec_iatt_rebuild(ec_t *ec, struct iatt *iatt, int32_t count, int32_t answers)
 {
     uint64_t blocks;
 
-    while (count-- > 0)
-    {
+    while (count-- > 0) {
         blocks = iatt[count].ia_blocks * ec->fragments + answers - 1;
         blocks /= answers;
         iatt[count].ia_blocks = blocks;
@@ -227,92 +222,93 @@ void ec_iatt_rebuild(ec_t * ec, struct iatt * iatt, int32_t count,
 }
 
 gf_boolean_t
-ec_xattr_match (dict_t *dict, char *key, data_t *value, void *arg)
+ec_xattr_match(dict_t *dict, char *key, data_t *value, void *arg)
 {
-        if ((fnmatch(GF_XATTR_STIME_PATTERN, key, 0) == 0) ||
-            (strcmp(key, GET_LINK_COUNT) == 0) ||
-            (strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) ||
-            (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) ||
-            (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0)) {
-                return _gf_false;
-        }
+    if ((fnmatch(GF_XATTR_STIME_PATTERN, key, 0) == 0) ||
+        (strcmp(key, GET_LINK_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0)) {
+        return _gf_false;
+    }
 
-        return _gf_true;
+    return _gf_true;
 }
 
 gf_boolean_t
-ec_value_ignore (char *key)
+ec_value_ignore(char *key)
 {
-        if ((strcmp(key, GF_CONTENT_KEY) == 0) ||
-            (strcmp(key, GF_XATTR_PATHINFO_KEY) == 0) ||
-            (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0) ||
-            (strcmp(key, GF_XATTR_LOCKINFO_KEY) == 0) ||
-            (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0) ||
-            (strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) ||
-            (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) ||
-            (strncmp(key, GF_XATTR_CLRLK_CMD,
-                     strlen (GF_XATTR_CLRLK_CMD)) == 0) ||
-            (strcmp(key, DHT_IATT_IN_XDATA_KEY) == 0) ||
-            (strncmp(key, EC_QUOTA_PREFIX, strlen(EC_QUOTA_PREFIX)) == 0) ||
-            (fnmatch(MARKER_XATTR_PREFIX ".*." XTIME, key, 0) == 0) ||
-            (fnmatch(GF_XATTR_MARKER_KEY ".*", key, 0) == 0) ||
-            (XATTR_IS_NODE_UUID(key))) {
-                return _gf_true;
-        }
-        return _gf_false;
+    if ((strcmp(key, GF_CONTENT_KEY) == 0) ||
+        (strcmp(key, GF_XATTR_PATHINFO_KEY) == 0) ||
+        (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0) ||
+        (strcmp(key, GF_XATTR_LOCKINFO_KEY) == 0) ||
+        (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) ||
+        (strncmp(key, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)) == 0) ||
+        (strcmp(key, DHT_IATT_IN_XDATA_KEY) == 0) ||
+        (strncmp(key, EC_QUOTA_PREFIX, SLEN(EC_QUOTA_PREFIX)) == 0) ||
+        (fnmatch(MARKER_XATTR_PREFIX ".*." XTIME, key, 0) == 0) ||
+        (fnmatch(GF_XATTR_MARKER_KEY ".*", key, 0) == 0) ||
+        (XATTR_IS_NODE_UUID(key))) {
+        return _gf_true;
+    }
+
+    return _gf_false;
 }
 
 int32_t
-ec_dict_compare (dict_t *dict1, dict_t *dict2)
+ec_dict_compare(dict_t *dict1, dict_t *dict2)
 {
-        if (are_dicts_equal (dict1, dict2, ec_xattr_match, ec_value_ignore))
-                return 1;
-        return 0;
+    if (are_dicts_equal(dict1, dict2, ec_xattr_match, ec_value_ignore))
+        return 1;
+    return 0;
 }
 
 static uint32_t
 ec_dict_list(data_t **list, ec_cbk_data_t *cbk, int32_t which, char *key,
              gf_boolean_t global)
 {
-        ec_t *ec = cbk->fop->xl->private;
-        ec_cbk_data_t *ans = NULL;
-        dict_t *dict = NULL;
-        data_t *data;
-        uint32_t count;
-        int32_t i;
-
-        for (i = 0; i < ec->nodes; i++) {
-                /* We initialize the list with EC_MISSING_DATA if we are
-                 * returning a global list or the current subvolume belongs
-                 * to the group of the accepted answer. Note that if some
-                 * subvolume is known to be down before issuing the request,
-                 * we won't have any answer from it, so we set here the
-                 * appropriate default value. */
-                if (global || ((cbk->mask & (1ULL << i)) != 0)) {
-                        list[i] = EC_MISSING_DATA;
-                } else {
-                        list[i] = NULL;
-                }
+    ec_t *ec = cbk->fop->xl->private;
+    ec_cbk_data_t *ans = NULL;
+    dict_t *dict = NULL;
+    data_t *data;
+    uint32_t count;
+    int32_t i;
+
+    for (i = 0; i < ec->nodes; i++) {
+        /* We initialize the list with EC_MISSING_DATA if we are
+         * returning a global list or the current subvolume belongs
+         * to the group of the accepted answer. Note that if some
+         * subvolume is known to be down before issuing the request,
+         * we won't have any answer from it, so we set here the
+         * appropriate default value. */
+        if (global || ((cbk->mask & (1ULL << i)) != 0)) {
+            list[i] = EC_MISSING_DATA;
+        } else {
+            list[i] = NULL;
         }
+    }
 
-        count = 0;
-        list_for_each_entry(ans, &cbk->fop->answer_list, answer_list) {
-                if (global || ((cbk->mask & ans->mask) != 0)) {
-                        dict = (which == EC_COMBINE_XDATA) ? ans->xdata
-                                                           : ans->dict;
-                        data = dict_get(dict, key);
-                        if (data != NULL) {
-                                list[ans->idx] = data;
-                                count++;
-                        }
-                }
+    count = 0;
+    list_for_each_entry(ans, &cbk->fop->answer_list, answer_list)
+    {
+        if (global || ((cbk->mask & ans->mask) != 0)) {
+            dict = (which == EC_COMBINE_XDATA) ? ans->xdata : ans->dict;
+            data = dict_get(dict, key);
+            if (data != NULL) {
+                list[ans->idx] = data;
+                count++;
+            }
         }
+    }
 
-        return count;
+    return count;
 }
 
-int32_t ec_concat_prepare(xlator_t *xl, char **str, char **sep, char **post,
-                          const char *fmt, va_list args)
+int32_t
+ec_concat_prepare(xlator_t *xl, char **str, char **sep, char **post,
+                  const char *fmt, va_list args)
 {
     char *tmp;
     int32_t len;
@@ -338,9 +334,8 @@ int32_t ec_concat_prepare(xlator_t *xl, char **str, char **sep, char **post,
     return 0;
 
 out:
-    gf_msg (xl->name, GF_LOG_ERROR, EINVAL,
-            EC_MSG_INVALID_FORMAT,
-            "Invalid concat format");
+    gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_FORMAT,
+           "Invalid concat format");
 
     GF_FREE(*str);
 
@@ -348,9 +343,8 @@ out:
 }
 
 static int32_t
-ec_dict_data_concat(const char *fmt, ec_cbk_data_t *cbk, int32_t which,
-                    char *key, char *new_key, const char *def,
-                    gf_boolean_t global, ...)
+ec_dict_data_concat(ec_cbk_data_t *cbk, int32_t which, char *key, char *new_key,
+                    const char *def, gf_boolean_t global, const char *fmt, ...)
 {
     ec_t *ec = cbk->fop->xl->private;
     data_t *data[ec->nodes];
@@ -362,7 +356,7 @@ ec_dict_data_concat(const char *fmt, ec_cbk_data_t *cbk, int32_t which,
 
     ec_dict_list(data, cbk, which, key, global);
 
-    va_start(args, global);
+    va_start(args, fmt);
     err = ec_concat_prepare(cbk->fop->xl, &pre, &sep, &post, fmt, args);
     va_end(args);
 
@@ -448,7 +442,8 @@ out:
     return err;
 }
 
-int32_t ec_dict_data_merge(ec_cbk_data_t *cbk, int32_t which, char *key)
+int32_t
+ec_dict_data_merge(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
     ec_t *ec = cbk->fop->xl->private;
     data_t *data[ec->nodes];
@@ -457,7 +452,6 @@ int32_t ec_dict_data_merge(ec_cbk_data_t *cbk, int32_t which, char *key)
     int32_t i, len;
     int32_t err;
 
-
     ec_dict_list(data, cbk, which, key, _gf_false);
 
     lockinfo = dict_new();
@@ -491,22 +485,12 @@ int32_t ec_dict_data_merge(ec_cbk_data_t *cbk, int32_t which, char *key)
 
     tmp = NULL;
 
-    len = dict_serialized_length(lockinfo);
-    if (len < 0) {
-        err = len;
-
-        goto out;
-    }
-    ptr = GF_MALLOC(len, gf_common_mt_char);
-    if (ptr == NULL) {
-        err = -ENOMEM;
-
-        goto out;
-    }
-    err = dict_serialize(lockinfo, ptr);
+    err = dict_allocate_and_serialize(lockinfo, (char **)&ptr,
+                                      (unsigned int *)&len);
     if (err != 0) {
         goto out;
     }
+
     dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
     err = dict_set_dynptr(dict, key, ptr, len);
     if (err != 0) {
@@ -525,11 +509,12 @@ out:
     return err;
 }
 
-int32_t ec_dict_data_uuid(ec_cbk_data_t * cbk, int32_t which, char * key)
+int32_t
+ec_dict_data_uuid(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
-    ec_cbk_data_t * ans, * min;
-    dict_t * src, * dst;
-    data_t * data;
+    ec_cbk_data_t *ans, *min;
+    dict_t *src, *dst;
+    data_t *data;
 
     min = cbk;
     for (ans = cbk->next; ans != NULL; ans = ans->next) {
@@ -554,7 +539,67 @@ int32_t ec_dict_data_uuid(ec_cbk_data_t * cbk, int32_t which, char * key)
     return 0;
 }
 
-int32_t ec_dict_data_max32(ec_cbk_data_t *cbk, int32_t which, char *key)
+int32_t
+ec_dict_data_iatt(ec_cbk_data_t *cbk, int32_t which, char *key)
+{
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
+    dict_t *dict;
+    struct iatt *stbuf, *tmp;
+    int32_t i, ret;
+
+    ec_dict_list(data, cbk, which, key, _gf_false);
+
+    stbuf = NULL;
+    for (i = 0; i < ec->nodes; i++) {
+        if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) {
+            continue;
+        }
+        tmp = data_to_iatt(data[i], key);
+        if (tmp == NULL) {
+            ret = -EINVAL;
+            goto out;
+        }
+        if (stbuf == NULL) {
+            stbuf = GF_MALLOC(sizeof(struct iatt), gf_common_mt_char);
+            if (stbuf == NULL) {
+                ret = -ENOMEM;
+                goto out;
+            }
+            *stbuf = *tmp;
+        } else {
+            if (!ec_iatt_combine(cbk->fop, stbuf, tmp, 1)) {
+                ret = -EINVAL;
+                goto out;
+            }
+        }
+    }
+
+    if ((stbuf != NULL) && (stbuf->ia_type == IA_IFREG)) {
+        ec_iatt_rebuild(ec, stbuf, 1, cbk->count);
+        /* TODO: not sure if an iatt could come in xdata from a fop that takes
+         *       no locks. */
+        if (!ec_get_inode_size(cbk->fop, cbk->fop->locks[0].lock->loc.inode,
+                               &stbuf->ia_size)) {
+            ret = -EINVAL;
+            goto out;
+        }
+    }
+
+    dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
+    ret = dict_set_iatt(dict, key, stbuf, false);
+    if (ret >= 0) {
+        stbuf = NULL;
+    }
+
+out:
+    GF_FREE(stbuf);
+
+    return ret;
+}
+
+int32_t
+ec_dict_data_max32(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
     ec_t *ec = cbk->fop->xl->private;
     data_t *data[ec->nodes];
@@ -580,7 +625,8 @@ int32_t ec_dict_data_max32(ec_cbk_data_t *cbk, int32_t which, char *key)
     return dict_set_uint32(dict, key, max);
 }
 
-int32_t ec_dict_data_max64(ec_cbk_data_t *cbk, int32_t which, char *key)
+int32_t
+ec_dict_data_max64(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
     ec_t *ec = cbk->fop->xl->private;
     data_t *data[ec->nodes];
@@ -606,14 +652,19 @@ int32_t ec_dict_data_max64(ec_cbk_data_t *cbk, int32_t which, char *key)
     return dict_set_uint64(dict, key, max);
 }
 
-int32_t ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key)
+int32_t
+ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
-    ec_t        *ec               = cbk->fop->xl->private;
-    data_t      *data[ec->nodes];
-    dict_t      *dict             = NULL;
-    int32_t      i                = 0;
-    quota_meta_t size             = {0, };
-    quota_meta_t max_size         = {0, };
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
+    dict_t *dict = NULL;
+    int32_t i = 0;
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t max_size = {
+        0,
+    };
 
     if (ec_dict_list(data, cbk, which, key, _gf_false) == 0) {
         return 0;
@@ -626,25 +677,26 @@ int32_t ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key)
      */
     for (i = 0; i < ec->nodes; i++) {
         if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA) ||
-            (quota_data_to_meta (data[i], QUOTA_SIZE_KEY, &size) < 0)) {
-                continue;
+            (quota_data_to_meta(data[i], &size) < 0)) {
+            continue;
         }
 
         if (size.size > max_size.size)
-                max_size.size = size.size;
+            max_size.size = size.size;
         if (size.file_count > max_size.file_count)
-                max_size.file_count = size.file_count;
+            max_size.file_count = size.file_count;
         if (size.dir_count > max_size.dir_count)
-                max_size.dir_count = size.dir_count;
+            max_size.dir_count = size.dir_count;
     }
 
     max_size.size *= ec->fragments;
 
     dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
-    return quota_dict_set_meta (dict, key, &max_size, IA_IFDIR);
+    return quota_dict_set_meta(dict, key, &max_size, IA_IFDIR);
 }
 
-int32_t ec_dict_data_stime(ec_cbk_data_t * cbk, int32_t which, char * key)
+int32_t
+ec_dict_data_stime(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
     ec_t *ec = cbk->fop->xl->private;
     data_t *data[ec->nodes];
@@ -660,8 +712,8 @@ int32_t ec_dict_data_stime(ec_cbk_data_t * cbk, int32_t which, char * key)
         }
         err = gf_get_max_stime(cbk->fop->xl, dict, key, data[i]);
         if (err != 0) {
-            gf_msg (cbk->fop->xl->name, GF_LOG_ERROR, -err,
-                    EC_MSG_STIME_COMBINE_FAIL, "STIME combination failed");
+            gf_msg(cbk->fop->xl->name, GF_LOG_ERROR, -err,
+                   EC_MSG_STIME_COMBINE_FAIL, "STIME combination failed");
 
             return err;
         }
@@ -670,28 +722,24 @@ int32_t ec_dict_data_stime(ec_cbk_data_t * cbk, int32_t which, char * key)
     return 0;
 }
 
-int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value,
-                             void * arg)
+int32_t
+ec_dict_data_combine(dict_t *dict, char *key, data_t *value, void *arg)
 {
-    ec_dict_combine_t * data = arg;
+    ec_dict_combine_t *data = arg;
 
     if ((strcmp(key, GF_XATTR_PATHINFO_KEY) == 0) ||
-        (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0))
-    {
-        return ec_dict_data_concat("(<EC:%s> { })", data->cbk, data->which,
-                                   key, NULL, NULL, _gf_false,
+        (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0)) {
+        return ec_dict_data_concat(data->cbk, data->which, key, NULL, NULL,
+                                   _gf_false, _gf_false, "(<EC:%s> { })",
                                    data->cbk->fop->xl->name);
     }
 
-    if (strncmp(key, GF_XATTR_CLRLK_CMD, strlen(GF_XATTR_CLRLK_CMD)) == 0)
-    {
-        return ec_dict_data_concat("{\n}", data->cbk, data->which, key, NULL,
-                                   NULL, _gf_false);
+    if (strncmp(key, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)) == 0) {
+        return ec_dict_data_concat(data->cbk, data->which, key, NULL, NULL,
+                                   _gf_false, "{\n}");
     }
 
-    if (strncmp(key, GF_XATTR_LOCKINFO_KEY,
-                strlen(GF_XATTR_LOCKINFO_KEY)) == 0)
-    {
+    if (strncmp(key, GF_XATTR_LOCKINFO_KEY, SLEN(GF_XATTR_LOCKINFO_KEY)) == 0) {
         return ec_dict_data_merge(data->cbk, data->which, key);
     }
 
@@ -699,8 +747,7 @@ int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value,
         return ec_dict_data_max32(data->cbk, data->which, key);
     }
 
-    if (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0)
-    {
+    if (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0) {
         return ec_dict_data_max32(data->cbk, data->which, key);
     }
     if ((strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) ||
@@ -712,23 +759,22 @@ int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value,
         return ec_dict_data_quota(data->cbk, data->which, key);
     }
     /* Ignore all other quota attributes */
-    if (strncmp(key, EC_QUOTA_PREFIX, strlen(EC_QUOTA_PREFIX)) == 0) {
+    if (strncmp(key, EC_QUOTA_PREFIX, SLEN(EC_QUOTA_PREFIX)) == 0) {
         return 0;
     }
 
     if (XATTR_IS_NODE_UUID(key)) {
         if (data->cbk->fop->int32) {
-                /* List of node uuid is requested */
-                return ec_dict_data_concat("{ }", data->cbk, data->which, key,
-                                           GF_XATTR_LIST_NODE_UUIDS_KEY,
-                                           UUID0_STR, _gf_true);
+            /* List of node uuid is requested */
+            return ec_dict_data_concat(data->cbk, data->which, key,
+                                       GF_XATTR_LIST_NODE_UUIDS_KEY, UUID0_STR,
+                                       _gf_true, "{ }");
         } else {
-                return ec_dict_data_uuid(data->cbk, data->which, key);
+            return ec_dict_data_uuid(data->cbk, data->which, key);
         }
     }
 
-    if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
-    {
+    if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
         return ec_dict_data_stime(data->cbk, data->which, key);
     }
 
@@ -736,10 +782,15 @@ int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value,
         return ec_dict_data_max64(data->cbk, data->which, key);
     }
 
+    if (strcmp(key, GF_PRESTAT) == 0 || strcmp(key, GF_POSTSTAT) == 0) {
+        return ec_dict_data_iatt(data->cbk, data->which, key);
+    }
+
     return 0;
 }
 
-int32_t ec_dict_combine(ec_cbk_data_t * cbk, int32_t which)
+int32_t
+ec_dict_combine(ec_cbk_data_t *cbk, int32_t which)
 {
     dict_t *dict = NULL;
     ec_dict_combine_t data;
@@ -752,9 +803,8 @@ int32_t ec_dict_combine(ec_cbk_data_t * cbk, int32_t which)
     if (dict != NULL) {
         err = dict_foreach(dict, ec_dict_data_combine, &data);
         if (err != 0) {
-            gf_msg (cbk->fop->xl->name, GF_LOG_ERROR, -err,
-                    EC_MSG_DICT_COMBINE_FAIL,
-                    "Dictionary combination failed");
+            gf_msg(cbk->fop->xl->name, GF_LOG_ERROR, -err,
+                   EC_MSG_DICT_COMBINE_FAIL, "Dictionary combination failed");
 
             return err;
         }
@@ -763,47 +813,43 @@ int32_t ec_dict_combine(ec_cbk_data_t * cbk, int32_t which)
     return 0;
 }
 
-int32_t ec_vector_compare(struct iovec * dst_vector, int32_t dst_count,
-                          struct iovec * src_vector, int32_t src_count)
+int32_t
+ec_vector_compare(struct iovec *dst_vector, int32_t dst_count,
+                  struct iovec *src_vector, int32_t src_count)
 {
     int32_t dst_size = 0, src_size = 0;
 
-    if (dst_count > 0)
-    {
+    if (dst_count > 0) {
         dst_size = iov_length(dst_vector, dst_count);
     }
-    if (src_count > 0)
-    {
+    if (src_count > 0) {
         src_size = iov_length(src_vector, src_count);
     }
 
     return (dst_size == src_size);
 }
 
-int32_t ec_flock_compare(struct gf_flock * dst, struct gf_flock * src)
+int32_t
+ec_flock_compare(struct gf_flock *dst, struct gf_flock *src)
 {
-    if ((dst->l_type != src->l_type) ||
-        (dst->l_whence != src->l_whence) ||
-        (dst->l_start != src->l_start) ||
-        (dst->l_len != src->l_len) ||
+    if ((dst->l_type != src->l_type) || (dst->l_whence != src->l_whence) ||
+        (dst->l_start != src->l_start) || (dst->l_len != src->l_len) ||
         (dst->l_pid != src->l_pid) ||
-        !is_same_lkowner(&dst->l_owner, &src->l_owner))
-    {
+        !is_same_lkowner(&dst->l_owner, &src->l_owner)) {
         return 0;
     }
 
     return 1;
 }
 
-void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src)
+void
+ec_statvfs_combine(struct statvfs *dst, struct statvfs *src)
 {
-    if (dst->f_bsize < src->f_bsize)
-    {
+    if (dst->f_bsize < src->f_bsize) {
         dst->f_bsize = src->f_bsize;
     }
 
-    if (dst->f_frsize < src->f_frsize)
-    {
+    if (dst->f_frsize < src->f_frsize) {
         dst->f_blocks *= dst->f_frsize;
         dst->f_blocks /= src->f_frsize;
 
@@ -814,9 +860,7 @@ void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src)
         dst->f_bavail /= src->f_frsize;
 
         dst->f_frsize = src->f_frsize;
-    }
-    else if (dst->f_frsize > src->f_frsize)
-    {
+    } else if (dst->f_frsize > src->f_frsize) {
         src->f_blocks *= src->f_frsize;
         src->f_blocks /= dst->f_frsize;
 
@@ -826,90 +870,80 @@ void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src)
         src->f_bavail *= src->f_frsize;
         src->f_bavail /= dst->f_frsize;
     }
-    if (dst->f_blocks > src->f_blocks)
-    {
+    if (dst->f_blocks > src->f_blocks) {
         dst->f_blocks = src->f_blocks;
     }
-    if (dst->f_bfree > src->f_bfree)
-    {
+    if (dst->f_bfree > src->f_bfree) {
         dst->f_bfree = src->f_bfree;
     }
-    if (dst->f_bavail > src->f_bavail)
-    {
+    if (dst->f_bavail > src->f_bavail) {
         dst->f_bavail = src->f_bavail;
     }
 
-    if (dst->f_files < src->f_files)
-    {
+    if (dst->f_files < src->f_files) {
         dst->f_files = src->f_files;
     }
-    if (dst->f_ffree > src->f_ffree)
-    {
+    if (dst->f_ffree > src->f_ffree) {
         dst->f_ffree = src->f_ffree;
     }
-    if (dst->f_favail > src->f_favail)
-    {
+    if (dst->f_favail > src->f_favail) {
         dst->f_favail = src->f_favail;
     }
-    if (dst->f_namemax > src->f_namemax)
-    {
+    if (dst->f_namemax > src->f_namemax) {
         dst->f_namemax = src->f_namemax;
     }
 
-    if (dst->f_flag != src->f_flag)
-    {
-        gf_msg_debug (THIS->name, 0,
-                "Mismatching file system flags "
-                "(%lX, %lX)",
-                dst->f_flag, src->f_flag);
+    if (dst->f_flag != src->f_flag) {
+        gf_msg_debug(THIS->name, 0,
+                     "Mismatching file system flags "
+                     "(%lX, %lX)",
+                     dst->f_flag, src->f_flag);
     }
     dst->f_flag &= src->f_flag;
 }
 
-int32_t ec_combine_check(ec_cbk_data_t * dst, ec_cbk_data_t * src,
-                         ec_combine_f combine)
+int32_t
+ec_combine_check(ec_cbk_data_t *dst, ec_cbk_data_t *src, ec_combine_f combine)
 {
-    ec_fop_data_t * fop = dst->fop;
+    ec_fop_data_t *fop = dst->fop;
 
-    if (dst->op_ret != src->op_ret)
-    {
-        gf_msg_debug (fop->xl->name, 0, "Mismatching return code in "
-                                            "answers of '%s': %d <-> %d",
-               ec_fop_name(fop->id), dst->op_ret, src->op_ret);
+    if (dst->op_ret != src->op_ret) {
+        gf_msg_debug(fop->xl->name, 0,
+                     "Mismatching return code in "
+                     "answers of '%s': %d <-> %d",
+                     ec_fop_name(fop->id), dst->op_ret, src->op_ret);
 
         return 0;
     }
-    if (dst->op_ret < 0)
-    {
-        if (dst->op_errno != src->op_errno)
-        {
-            gf_msg_debug (fop->xl->name, 0, "Mismatching errno code in "
-                                                "answers of '%s': %d <-> %d",
-                   ec_fop_name(fop->id), dst->op_errno, src->op_errno);
+    if (dst->op_ret < 0) {
+        if (dst->op_errno != src->op_errno) {
+            gf_msg_debug(fop->xl->name, 0,
+                         "Mismatching errno code in "
+                         "answers of '%s': %d <-> %d",
+                         ec_fop_name(fop->id), dst->op_errno, src->op_errno);
 
             return 0;
         }
     }
 
-    if (!ec_dict_compare(dst->xdata, src->xdata))
-    {
-        gf_msg (fop->xl->name, GF_LOG_DEBUG, 0,
-                EC_MSG_XDATA_MISMATCH,
-                "Mismatching xdata in answers "
-                "of '%s'", ec_fop_name(fop->id));
+    if (!ec_dict_compare(dst->xdata, src->xdata)) {
+        gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_XDATA_MISMATCH,
+               "Mismatching xdata in answers "
+               "of '%s'",
+               ec_fop_name(fop->id));
 
         return 0;
     }
 
-    if ((dst->op_ret >= 0) && (combine != NULL))
-    {
+    if ((dst->op_ret >= 0) && (combine != NULL)) {
         return combine(fop, dst, src);
     }
 
     return 1;
 }
 
-void ec_combine (ec_cbk_data_t *newcbk, ec_combine_f combine)
+void
+ec_combine(ec_cbk_data_t *newcbk, ec_combine_f combine)
 {
     ec_fop_data_t *fop = newcbk->fop;
     ec_cbk_data_t *cbk = NULL, *tmp = NULL;
@@ -924,17 +958,14 @@ void ec_combine (ec_cbk_data_t *newcbk, ec_combine_f combine)
     item = fop->cbk_list.prev;
     list_for_each_entry(cbk, &fop->cbk_list, list)
     {
-        if (ec_combine_check(newcbk, cbk, combine))
-        {
+        if (ec_combine_check(newcbk, cbk, combine)) {
             newcbk->count += cbk->count;
             newcbk->mask |= cbk->mask;
 
             item = cbk->list.prev;
-            while (item != &fop->cbk_list)
-            {
+            while (item != &fop->cbk_list) {
                 tmp = list_entry(item, ec_cbk_data_t, list);
-                if (tmp->count >= newcbk->count)
-                {
+                if (tmp->count >= newcbk->count) {
                     break;
                 }
                 item = item->prev;
diff --git a/xlators/cluster/ec/src/ec-combine.h b/xlators/cluster/ec/src/ec-combine.h
index 19a42ded706..1010cc3be26 100644
--- a/xlators/cluster/ec/src/ec-combine.h
+++ b/xlators/cluster/ec/src/ec-combine.h
@@ -11,28 +11,34 @@
 #ifndef __EC_COMBINE_H__
 #define __EC_COMBINE_H__
 
-#define EC_COMBINE_DICT  0
+#define EC_COMBINE_DICT 0
 #define EC_COMBINE_XDATA 1
 
-typedef int32_t (* ec_combine_f)(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                                 ec_cbk_data_t * src);
+typedef int32_t (*ec_combine_f)(ec_fop_data_t *fop, ec_cbk_data_t *dst,
+                                ec_cbk_data_t *src);
 
-void ec_iatt_rebuild(ec_t * ec, struct iatt * iatt, int32_t count,
-                     int32_t answers);
+void
+ec_iatt_rebuild(ec_t *ec, struct iatt *iatt, int32_t count, int32_t answers);
 
-int32_t ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
-                        int32_t count);
-int32_t ec_dict_compare(dict_t * dict1, dict_t * dict2);
-int32_t ec_vector_compare(struct iovec * dst_vector, int32_t dst_count,
-                          struct iovec * src_vector, int32_t src_count);
-int32_t ec_flock_compare(struct gf_flock * dst, struct gf_flock * src);
-void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src);
+int32_t
+ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
+                int32_t count);
+int32_t
+ec_dict_compare(dict_t *dict1, dict_t *dict2);
+int32_t
+ec_vector_compare(struct iovec *dst_vector, int32_t dst_count,
+                  struct iovec *src_vector, int32_t src_count);
+int32_t
+ec_flock_compare(struct gf_flock *dst, struct gf_flock *src);
+void
+ec_statvfs_combine(struct statvfs *dst, struct statvfs *src);
 
-int32_t ec_dict_combine(ec_cbk_data_t * cbk, int32_t which);
+int32_t
+ec_dict_combine(ec_cbk_data_t *cbk, int32_t which);
 
-void ec_combine(ec_cbk_data_t * cbk, ec_combine_f combine);
+void
+ec_combine(ec_cbk_data_t *cbk, ec_combine_f combine);
 
 int32_t
-ec_combine_write (ec_fop_data_t *fop, ec_cbk_data_t *dst,
-                  ec_cbk_data_t *src);
+ec_combine_write(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src);
 #endif /* __EC_COMBINE_H__ */
diff --git a/xlators/cluster/ec/src/ec-common.c b/xlators/cluster/ec/src/ec-common.c
index 27ea5c93d63..b955efd8c2d 100644
--- a/xlators/cluster/ec/src/ec-common.c
+++ b/xlators/cluster/ec/src/ec-common.c
@@ -8,8 +8,8 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "byte-order.h"
-#include "hashfn.h"
+#include <glusterfs/byte-order.h>
+#include <glusterfs/hashfn.h>
 
 #include "ec-mem-types.h"
 #include "ec-types.h"
@@ -22,195 +22,204 @@
 #include "ec-messages.h"
 
 #define EC_INVALID_INDEX UINT32_MAX
-#define EC_XATTROP_ALL_WAITING_FLAGS (EC_FLAG_WAITING_XATTROP |\
-                                   EC_FLAG_WAITING_DATA_DIRTY |\
-                                   EC_FLAG_WAITING_METADATA_DIRTY)
 
 void
-ec_update_fd_status (fd_t *fd, xlator_t *xl, int idx,
-                     int32_t ret_status)
+ec_update_fd_status(fd_t *fd, xlator_t *xl, int idx, int32_t ret_status)
 {
-        ec_fd_t *fd_ctx;
+    ec_fd_t *fd_ctx;
 
-        if (fd == NULL)
-                return;
+    if (fd == NULL)
+        return;
 
-        LOCK (&fd->lock);
-        {
-                fd_ctx = __ec_fd_get(fd, xl);
-                if (fd_ctx) {
-                        if (ret_status >= 0)
-                                fd_ctx->fd_status[idx] = EC_FD_OPENED;
-                        else
-                                fd_ctx->fd_status[idx] = EC_FD_NOT_OPENED;
-                }
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __ec_fd_get(fd, xl);
+        if (fd_ctx) {
+            if (ret_status >= 0)
+                fd_ctx->fd_status[idx] = EC_FD_OPENED;
+            else
+                fd_ctx->fd_status[idx] = EC_FD_NOT_OPENED;
         }
-        UNLOCK (&fd->lock);
+    }
+    UNLOCK(&fd->lock);
 }
 
-static int
-ec_fd_ctx_need_open (fd_t *fd, xlator_t *this, uintptr_t *need_open)
+static uintptr_t
+ec_fd_ctx_need_open(fd_t *fd, xlator_t *this, uintptr_t mask)
 {
     int i = 0;
     int count = 0;
     ec_t *ec = NULL;
     ec_fd_t *fd_ctx = NULL;
+    uintptr_t need_open = 0;
 
     ec = this->private;
-    *need_open = 0;
 
-    fd_ctx = ec_fd_get (fd, this);
+    fd_ctx = ec_fd_get(fd, this);
     if (!fd_ctx)
         return count;
 
-    LOCK (&fd->lock);
+    LOCK(&fd->lock);
     {
         for (i = 0; i < ec->nodes; i++) {
-                if ((fd_ctx->fd_status[i] == EC_FD_NOT_OPENED) &&
-                    (ec->xl_up & (1<<i))) {
-                        fd_ctx->fd_status[i] = EC_FD_OPENING;
-                        *need_open |= (1<<i);
-                        count++;
-                }
+            if ((fd_ctx->fd_status[i] == EC_FD_NOT_OPENED) &&
+                ((ec->xl_up & (1 << i)) != 0) && ((mask & (1 << i)) != 0)) {
+                fd_ctx->fd_status[i] = EC_FD_OPENING;
+                need_open |= (1 << i);
+                count++;
+            }
         }
     }
-    UNLOCK (&fd->lock);
+    UNLOCK(&fd->lock);
 
     /* If fd needs to open on minimum number of nodes
      * then ignore fixing the fd as it has been
      * requested from heal operation.
      */
-    if (count >= ec->fragments)
-        count = 0;
+    if (count >= ec->fragments) {
+        need_open = 0;
+    }
 
-    return count;
+    return need_open;
 }
 
 static gf_boolean_t
-ec_is_fd_fixable (fd_t *fd)
+ec_is_fd_fixable(fd_t *fd)
 {
     if (!fd || !fd->inode)
         return _gf_false;
-    else if (fd_is_anonymous (fd))
+    else if (fd_is_anonymous(fd))
         return _gf_false;
-    else if (gf_uuid_is_null (fd->inode->gfid))
+    else if (gf_uuid_is_null(fd->inode->gfid))
         return _gf_false;
 
     return _gf_true;
 }
 
 static void
-ec_fix_open (ec_fop_data_t *fop)
+ec_fix_open(ec_fop_data_t *fop, uintptr_t mask)
 {
-    int                call_count = 0;
-    uintptr_t           need_open = 0;
-    int                       ret = 0;
-    loc_t                     loc = {0, };
+    uintptr_t need_open = 0;
+    int ret = 0;
+    int32_t flags = 0;
+    loc_t loc = {
+        0,
+    };
 
-    if (!ec_is_fd_fixable (fop->fd))
+    if (!ec_is_fd_fixable(fop->fd))
         goto out;
 
     /* Evaluate how many remote fd's to be opened */
-    call_count = ec_fd_ctx_need_open (fop->fd, fop->xl, &need_open);
-    if (!call_count)
+    need_open = ec_fd_ctx_need_open(fop->fd, fop->xl, mask);
+    if (need_open == 0) {
         goto out;
+    }
 
-    loc.inode = inode_ref (fop->fd->inode);
-    gf_uuid_copy (loc.gfid, fop->fd->inode->gfid);
-    ret = loc_path (&loc, NULL);
+    loc.inode = inode_ref(fop->fd->inode);
+    gf_uuid_copy(loc.gfid, fop->fd->inode->gfid);
+    ret = loc_path(&loc, NULL);
     if (ret < 0) {
         goto out;
     }
 
+    flags = fop->fd->flags & (~(O_TRUNC | O_APPEND | O_CREAT | O_EXCL));
     if (IA_IFDIR == fop->fd->inode->ia_type) {
-        ec_opendir(fop->frame, fop->xl, need_open, EC_MINIMUM_ONE,
-                   NULL, NULL, &fop->loc[0], fop->fd, NULL);
-    } else{
-        ec_open(fop->frame, fop->xl, need_open, EC_MINIMUM_ONE,
-                NULL, NULL, &loc, fop->fd->flags, fop->fd, NULL);
+        ec_opendir(fop->frame, fop->xl, need_open,
+                   EC_MINIMUM_ONE | EC_FOP_NO_PROPAGATE_ERROR, NULL, NULL,
+                   &fop->loc[0], fop->fd, NULL);
+    } else {
+        ec_open(fop->frame, fop->xl, need_open,
+                EC_MINIMUM_ONE | EC_FOP_NO_PROPAGATE_ERROR, NULL, NULL, &loc,
+                flags, fop->fd, NULL);
     }
 
 out:
-    loc_wipe (&loc);
+    loc_wipe(&loc);
 }
 
-off_t
-ec_range_end_get (off_t fl_start, size_t fl_size)
+static off_t
+ec_range_end_get(off_t fl_start, uint64_t fl_size)
 {
-        off_t fl_end = 0;
-        switch (fl_size) {
-        case 0:
-                return fl_start;
-        case LLONG_MAX: /*Infinity*/
-                return LLONG_MAX;
-        default:
-                fl_end = fl_start + fl_size - 1;
-                if (fl_end < 0) /*over-flow*/
-                        return LLONG_MAX;
-                else
-                        return fl_end;
+    if (fl_size > 0) {
+        if (fl_size >= EC_RANGE_FULL) {
+            /* Infinity */
+            fl_start = LLONG_MAX;
+        } else {
+            fl_start += fl_size - 1;
+            if (fl_start < 0) {
+                /* Overflow */
+                fl_start = LLONG_MAX;
+            }
         }
+    }
+
+    return fl_start;
 }
 
 static gf_boolean_t
-ec_is_range_conflict (ec_lock_link_t *l1, ec_lock_link_t *l2)
+ec_is_range_conflict(ec_lock_link_t *l1, ec_lock_link_t *l2)
 {
-        return ((l1->fl_end >= l2->fl_start) && (l2->fl_end >= l1->fl_start));
+    return ((l1->fl_end >= l2->fl_start) && (l2->fl_end >= l1->fl_start));
 }
 
 static gf_boolean_t
-ec_lock_conflict (ec_lock_link_t *l1, ec_lock_link_t *l2)
+ec_lock_conflict(ec_lock_link_t *l1, ec_lock_link_t *l2)
 {
-        /* Fops like access/stat won't have to worry what the other fops are
-         * modifying as the fop is wound only to one brick. So it can be
-         * executed in parallel*/
-        if (l1->fop->minimum == EC_MINIMUM_ONE ||
-            l2->fop->minimum == EC_MINIMUM_ONE)
-                return _gf_false;
+    ec_t *ec = l1->fop->xl->private;
+
+    /* Fops like access/stat won't have to worry what the other fops are
+     * modifying as the fop is wound only to one brick. So it can be
+     * executed in parallel*/
+    if (l1->fop->minimum == EC_MINIMUM_ONE ||
+        l2->fop->minimum == EC_MINIMUM_ONE)
+        return _gf_false;
 
-        if ((l1->fop->flags & EC_FLAG_LOCK_SHARED) &&
-            (l2->fop->flags & EC_FLAG_LOCK_SHARED))
-                return _gf_false;
+    if ((l1->fop->flags & EC_FLAG_LOCK_SHARED) &&
+        (l2->fop->flags & EC_FLAG_LOCK_SHARED))
+        return _gf_false;
+
+    if (!ec->parallel_writes) {
+        return _gf_true;
+    }
 
-        return ec_is_range_conflict (l1, l2);
+    return ec_is_range_conflict(l1, l2);
 }
 
 uint32_t
-ec_select_first_by_read_policy (ec_t *ec, ec_fop_data_t *fop)
-{
-        if (ec->read_policy == EC_ROUND_ROBIN) {
-                return ec->idx;
-        } else if (ec->read_policy == EC_GFID_HASH) {
-                if (fop->use_fd) {
-                        return SuperFastHash((char *)fop->fd->inode->gfid,
-                                   sizeof(fop->fd->inode->gfid)) % ec->nodes;
-                } else {
-                        if (gf_uuid_is_null (fop->loc[0].gfid))
-                                loc_gfid (&fop->loc[0], fop->loc[0].gfid);
-                        return SuperFastHash((char *)fop->loc[0].gfid,
-                                   sizeof(fop->loc[0].gfid)) % ec->nodes;
-                }
+ec_select_first_by_read_policy(ec_t *ec, ec_fop_data_t *fop)
+{
+    if (ec->read_policy == EC_ROUND_ROBIN) {
+        return ec->idx;
+    } else if (ec->read_policy == EC_GFID_HASH) {
+        if (fop->use_fd) {
+            return SuperFastHash((char *)fop->fd->inode->gfid,
+                                 sizeof(fop->fd->inode->gfid)) %
+                   ec->nodes;
+        } else {
+            if (gf_uuid_is_null(fop->loc[0].gfid))
+                loc_gfid(&fop->loc[0], fop->loc[0].gfid);
+            return SuperFastHash((char *)fop->loc[0].gfid,
+                                 sizeof(fop->loc[0].gfid)) %
+                   ec->nodes;
         }
-        return 0;
+    }
+    return 0;
 }
 
-static
-gf_boolean_t ec_child_valid(ec_t * ec, ec_fop_data_t * fop, uint32_t idx)
+static gf_boolean_t
+ec_child_valid(ec_t *ec, ec_fop_data_t *fop, uint32_t idx)
 {
     return (idx < ec->nodes) && (((fop->remaining >> idx) & 1) == 1);
 }
 
-static
-uint32_t ec_child_next(ec_t * ec, ec_fop_data_t * fop, uint32_t idx)
+static uint32_t
+ec_child_next(ec_t *ec, ec_fop_data_t *fop, uint32_t idx)
 {
-    while (!ec_child_valid(ec, fop, idx))
-    {
-        if (++idx >= ec->nodes)
-        {
+    while (!ec_child_valid(ec, fop, idx)) {
+        if (++idx >= ec->nodes) {
             idx = 0;
         }
-        if (idx == fop->first)
-        {
+        if (idx == fop->first) {
             return EC_INVALID_INDEX;
         }
     }
@@ -218,20 +227,21 @@ uint32_t ec_child_next(ec_t * ec, ec_fop_data_t * fop, uint32_t idx)
     return idx;
 }
 
-int32_t ec_heal_report(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, uintptr_t mask,
-                       uintptr_t good, uintptr_t bad, dict_t * xdata)
+int32_t
+ec_heal_report(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, uintptr_t mask, uintptr_t good,
+               uintptr_t bad, uint32_t pending, dict_t *xdata)
 {
     if (op_ret < 0) {
-        gf_msg (this->name, GF_LOG_DEBUG, op_errno,
-                EC_MSG_HEAL_FAIL, "Heal failed");
+        gf_msg(this->name, GF_LOG_DEBUG, op_errno, EC_MSG_HEAL_FAIL,
+               "Heal failed");
     } else {
         if ((mask & ~good) != 0) {
-            gf_msg (this->name, GF_LOG_DEBUG, 0,
-                    EC_MSG_HEAL_SUCCESS, "Heal succeeded on %d/%d "
-                    "subvolumes",
-                    gf_bits_count(mask & ~(good | bad)),
-                    gf_bits_count(mask & ~good));
+            gf_msg(this->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_SUCCESS,
+                   "Heal succeeded on %d/%d "
+                   "subvolumes",
+                   gf_bits_count(mask & ~(good | bad)),
+                   gf_bits_count(mask & ~good));
         }
     }
 
@@ -239,103 +249,104 @@ int32_t ec_heal_report(call_frame_t * frame, void * cookie, xlator_t * this,
 }
 
 static uintptr_t
-ec_fop_needs_name_heal (ec_fop_data_t *fop)
+ec_fop_needs_name_heal(ec_fop_data_t *fop)
 {
-        ec_t          *ec         = NULL;
-        ec_cbk_data_t *cbk        = NULL;
-        ec_cbk_data_t *enoent_cbk = NULL;
+    ec_t *ec = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    ec_cbk_data_t *enoent_cbk = NULL;
 
-        ec = fop->xl->private;
-        if (fop->id != GF_FOP_LOOKUP)
-                return 0;
+    ec = fop->xl->private;
+    if (fop->id != GF_FOP_LOOKUP)
+        return 0;
 
-        if (!fop->loc[0].name || strlen (fop->loc[0].name) == 0)
-                return 0;
+    if (!fop->loc[0].name || strlen(fop->loc[0].name) == 0)
+        return 0;
 
-        list_for_each_entry(cbk, &fop->cbk_list, list)
-        {
-                if (cbk->op_ret < 0 && cbk->op_errno == ENOENT) {
-                        enoent_cbk = cbk;
-                        break;
-                }
+    list_for_each_entry(cbk, &fop->cbk_list, list)
+    {
+        if (cbk->op_ret < 0 && cbk->op_errno == ENOENT) {
+            enoent_cbk = cbk;
+            break;
         }
+    }
 
-        if (!enoent_cbk)
-                return 0;
+    if (!enoent_cbk)
+        return 0;
 
-        return ec->xl_up & ~enoent_cbk->mask;
+    return ec->xl_up & ~enoent_cbk->mask;
 }
 
-int32_t ec_fop_needs_heal(ec_fop_data_t *fop)
+int32_t
+ec_fop_needs_heal(ec_fop_data_t *fop)
 {
     ec_t *ec = fop->xl->private;
 
     if (fop->lock_count == 0) {
-    /*
-     * if fop->lock_count is zero that means it saw version mismatch
-     * without any locks so it can't be trusted. If we launch a heal
-     * based on this it will lead to INODELKs which will affect I/O
-     * performance. Considering self-heal-daemon and operations on
-     * the inode from client which take locks can still trigger the
-     * heal we can choose to not attempt a heal when fop->lock_count
-     * is zero.
-     */
-            return 0;
+        /*
+         * if fop->lock_count is zero that means it saw version mismatch
+         * without any locks so it can't be trusted. If we launch a heal
+         * based on this it will lead to INODELKs which will affect I/O
+         * performance. Considering self-heal-daemon and operations on
+         * the inode from client which take locks can still trigger the
+         * heal we can choose to not attempt a heal when fop->lock_count
+         * is zero.
+         */
+        return 0;
     }
     return (ec->xl_up & ~(fop->remaining | fop->good)) != 0;
 }
 
-void ec_check_status(ec_fop_data_t * fop)
+void
+ec_check_status(ec_fop_data_t *fop)
 {
-    ec_t * ec = fop->xl->private;
+    ec_t *ec = fop->xl->private;
     int32_t partial = 0;
     char str1[32], str2[32], str3[32], str4[32], str5[32];
 
-    if (!ec_fop_needs_name_heal (fop) && !ec_fop_needs_heal(fop)) {
+    if (!ec_fop_needs_name_heal(fop) && !ec_fop_needs_heal(fop)) {
         return;
     }
 
     if (fop->answer && fop->answer->op_ret >= 0) {
-        if ((fop->id == GF_FOP_LOOKUP) ||
-            (fop->id == GF_FOP_STAT) || (fop->id == GF_FOP_FSTAT)) {
+        if ((fop->id == GF_FOP_LOOKUP) || (fop->id == GF_FOP_STAT) ||
+            (fop->id == GF_FOP_FSTAT)) {
             partial = fop->answer->iatt[0].ia_type == IA_IFDIR;
         } else if (fop->id == GF_FOP_OPENDIR) {
             partial = 1;
         }
     }
 
-    gf_msg (fop->xl->name, GF_LOG_WARNING, 0,
-            EC_MSG_OP_FAIL_ON_SUBVOLS,
-            "Operation failed on %d of %d subvolumes.(up=%s, mask=%s, "
-            "remaining=%s, good=%s, bad=%s)",
-            gf_bits_count(ec->xl_up & ~(fop->remaining | fop->good)), ec->nodes,
-            ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes),
-            ec_bin(str2, sizeof(str2), fop->mask, ec->nodes),
-            ec_bin(str3, sizeof(str3), fop->remaining, ec->nodes),
-            ec_bin(str4, sizeof(str4), fop->good, ec->nodes),
-            ec_bin(str5, sizeof(str5),
-            ec->xl_up & ~(fop->remaining | fop->good), ec->nodes));
-    if (fop->use_fd)
-    {
+    gf_msg(
+        fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_OP_FAIL_ON_SUBVOLS,
+        "Operation failed on %d of %d subvolumes.(up=%s, mask=%s, "
+        "remaining=%s, good=%s, bad=%s,"
+        "(Least significant bit represents first client/brick of subvol), %s)",
+        gf_bits_count(ec->xl_up & ~(fop->remaining | fop->good)), ec->nodes,
+        ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes),
+        ec_bin(str2, sizeof(str2), fop->mask, ec->nodes),
+        ec_bin(str3, sizeof(str3), fop->remaining, ec->nodes),
+        ec_bin(str4, sizeof(str4), fop->good, ec->nodes),
+        ec_bin(str5, sizeof(str5), ec->xl_up & ~(fop->remaining | fop->good),
+               ec->nodes),
+        ec_msg_str(fop));
+    if (fop->use_fd) {
         if (fop->fd != NULL) {
             ec_fheal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL,
                      fop->fd, partial, NULL);
         }
-    }
-    else
-    {
+    } else {
         ec_heal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL,
                 &fop->loc[0], partial, NULL);
 
-        if (fop->loc[1].inode != NULL)
-        {
+        if (fop->loc[1].inode != NULL) {
             ec_heal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL,
                     &fop->loc[1], partial, NULL);
         }
     }
 }
 
-void ec_update_good(ec_fop_data_t *fop, uintptr_t good)
+void
+ec_update_good(ec_fop_data_t *fop, uintptr_t good)
 {
     fop->good = good;
 
@@ -346,7 +357,8 @@ void ec_update_good(ec_fop_data_t *fop, uintptr_t good)
     }
 }
 
-void ec_lock_update_good(ec_lock_t *lock, ec_fop_data_t *fop)
+void
+ec_lock_update_good(ec_lock_t *lock, ec_fop_data_t *fop)
 {
     /* Fops that are executed only on one brick do not have enough information
      * to update the global mask of good bricks. */
@@ -362,15 +374,16 @@ void ec_lock_update_good(ec_lock_t *lock, ec_fop_data_t *fop)
     lock->good_mask &= fop->good | fop->remaining;
 }
 
-void __ec_fop_set_error(ec_fop_data_t * fop, int32_t error)
+void
+__ec_fop_set_error(ec_fop_data_t *fop, int32_t error)
 {
-    if ((error != 0) && (fop->error == 0))
-    {
+    if ((error != 0) && (fop->error == 0)) {
         fop->error = error;
     }
 }
 
-void ec_fop_set_error(ec_fop_data_t * fop, int32_t error)
+void
+ec_fop_set_error(ec_fop_data_t *fop, int32_t error)
 {
     LOCK(&fop->lock);
 
@@ -422,18 +435,20 @@ ec_fop_prepare_answer(ec_fop_data_t *fop, gf_boolean_t ro)
     return cbk;
 }
 
-void ec_sleep(ec_fop_data_t *fop)
+void
+ec_sleep(ec_fop_data_t *fop)
 {
     LOCK(&fop->lock);
 
-    GF_ASSERT (fop->refs > 0);
+    GF_ASSERT(fop->refs > 0);
     fop->refs++;
     fop->jobs++;
 
     UNLOCK(&fop->lock);
 }
 
-int32_t ec_check_complete(ec_fop_data_t * fop, ec_resume_f resume)
+int32_t
+ec_check_complete(ec_fop_data_t *fop, ec_resume_f resume)
 {
     int32_t error = -1;
 
@@ -441,14 +456,11 @@ int32_t ec_check_complete(ec_fop_data_t * fop, ec_resume_f resume)
 
     GF_ASSERT(fop->resume == NULL);
 
-    if (--fop->jobs != 0)
-    {
+    if (--fop->jobs != 0) {
         ec_trace("WAIT", fop, "resume=%p", resume);
 
         fop->resume = resume;
-    }
-    else
-    {
+    } else {
         error = fop->error;
         fop->error = 0;
     }
@@ -458,7 +470,8 @@ int32_t ec_check_complete(ec_fop_data_t * fop, ec_resume_f resume)
     return error;
 }
 
-void ec_resume(ec_fop_data_t * fop, int32_t error)
+void
+ec_resume(ec_fop_data_t *fop, int32_t error)
 {
     ec_resume_f resume = NULL;
 
@@ -466,16 +479,13 @@ void ec_resume(ec_fop_data_t * fop, int32_t error)
 
     __ec_fop_set_error(fop, error);
 
-    if (--fop->jobs == 0)
-    {
+    if (--fop->jobs == 0) {
         resume = fop->resume;
         fop->resume = NULL;
-        if (resume != NULL)
-        {
+        if (resume != NULL) {
             ec_trace("RESUME", fop, "error=%d", error);
 
-            if (fop->error != 0)
-            {
+            if (fop->error != 0) {
                 error = fop->error;
             }
             fop->error = 0;
@@ -484,21 +494,24 @@ void ec_resume(ec_fop_data_t * fop, int32_t error)
 
     UNLOCK(&fop->lock);
 
-    if (resume != NULL)
-    {
+    if (resume != NULL) {
         resume(fop, error);
     }
 
     ec_fop_data_release(fop);
 }
 
-void ec_resume_parent(ec_fop_data_t * fop, int32_t error)
+void
+ec_resume_parent(ec_fop_data_t *fop)
 {
-    ec_fop_data_t * parent;
+    ec_fop_data_t *parent;
+    int32_t error = 0;
 
     parent = fop->parent;
-    if (parent != NULL)
-    {
+    if (parent != NULL) {
+        if ((fop->fop_flags & EC_FOP_NO_PROPAGATE_ERROR) == 0) {
+            error = fop->error;
+        }
         ec_trace("RESUME_PARENT", fop, "error=%u", error);
         fop->parent = NULL;
         ec_resume(parent, error);
@@ -506,22 +519,23 @@ void ec_resume_parent(ec_fop_data_t * fop, int32_t error)
 }
 
 gf_boolean_t
-ec_is_recoverable_error (int32_t op_errno)
+ec_is_recoverable_error(int32_t op_errno)
 {
-        switch (op_errno) {
+    switch (op_errno) {
         case ENOTCONN:
         case ESTALE:
         case ENOENT:
-        case EBADFD:/*Opened fd but brick is disconnected*/
-        case EIO:/*Backend-fs crash like XFS/ext4 etc*/
-                return _gf_true;
-        }
-        return _gf_false;
+        case EBADFD: /*Opened fd but brick is disconnected*/
+        case EIO:    /*Backend-fs crash like XFS/ext4 etc*/
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
-void ec_complete(ec_fop_data_t * fop)
+void
+ec_complete(ec_fop_data_t *fop)
 {
-    ec_cbk_data_t * cbk = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t resume = 0, update = 0;
     int healing_count = 0;
 
@@ -533,9 +547,9 @@ void ec_complete(ec_fop_data_t * fop)
         if (fop->answer == NULL) {
             if (!list_empty(&fop->cbk_list)) {
                 cbk = list_entry(fop->cbk_list.next, ec_cbk_data_t, list);
-                healing_count = gf_bits_count (cbk->mask & fop->healing);
-                    /* fop shouldn't be treated as success if it is not
-                     * successful on at least fop->minimum good copies*/
+                healing_count = gf_bits_count(cbk->mask & fop->healing);
+                /* fop shouldn't be treated as success if it is not
+                 * successful on at least fop->minimum good copies*/
                 if ((cbk->count - healing_count) >= fop->minimum) {
                     fop->answer = cbk;
 
@@ -557,8 +571,7 @@ void ec_complete(ec_fop_data_t * fop)
         ec_update_good(fop, cbk->mask);
     }
 
-    if (resume)
-    {
+    if (resume) {
         ec_resume(fop, 0);
     }
 
@@ -568,36 +581,95 @@ void ec_complete(ec_fop_data_t * fop)
 /* There could be already granted locks sitting on the bricks, unlock for which
  * must be wound at all costs*/
 static gf_boolean_t
-ec_must_wind (ec_fop_data_t *fop)
+ec_must_wind(ec_fop_data_t *fop)
 {
-        if ((fop->id == GF_FOP_INODELK) || (fop->id == GF_FOP_FINODELK) ||
-            (fop->id == GF_FOP_LK)) {
-                if (fop->flock.l_type == F_UNLCK)
-                        return _gf_true;
-        } else if ((fop->id == GF_FOP_ENTRYLK) ||
-                   (fop->id == GF_FOP_FENTRYLK)) {
-                if (fop->entrylk_cmd == ENTRYLK_UNLOCK)
-                        return _gf_true;
-        }
+    if ((fop->id == GF_FOP_INODELK) || (fop->id == GF_FOP_FINODELK) ||
+        (fop->id == GF_FOP_LK)) {
+        if (fop->flock.l_type == F_UNLCK)
+            return _gf_true;
+    } else if ((fop->id == GF_FOP_ENTRYLK) || (fop->id == GF_FOP_FENTRYLK)) {
+        if (fop->entrylk_cmd == ENTRYLK_UNLOCK)
+            return _gf_true;
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static gf_boolean_t
-ec_internal_op (ec_fop_data_t *fop)
-{
-        if (ec_must_wind (fop))
-                return _gf_true;
-        if (fop->id == GF_FOP_XATTROP)
-                return _gf_true;
-        if (fop->id == GF_FOP_FXATTROP)
-                return _gf_true;
-        return _gf_false;
+ec_internal_op(ec_fop_data_t *fop)
+{
+    if (ec_must_wind(fop))
+        return _gf_true;
+    if (fop->id == GF_FOP_XATTROP)
+        return _gf_true;
+    if (fop->id == GF_FOP_FXATTROP)
+        return _gf_true;
+    if (fop->id == GF_FOP_OPEN)
+        return _gf_true;
+    return _gf_false;
+}
+
+char *
+ec_msg_str(ec_fop_data_t *fop)
+{
+    loc_t *loc1 = NULL;
+    loc_t *loc2 = NULL;
+    char gfid1[64] = {0};
+    char gfid2[64] = {0};
+    ec_fop_data_t *parent = fop->parent;
+
+    if (fop->errstr)
+        return fop->errstr;
+    if (!fop->use_fd) {
+        loc1 = &fop->loc[0];
+        loc2 = &fop->loc[1];
+
+        if (fop->id == GF_FOP_RENAME) {
+            gf_asprintf(&fop->errstr,
+                        "FOP : '%s' failed on '%s' and '%s' with gfids "
+                        "%s and %s respectively. Parent FOP: %s",
+                        ec_fop_name(fop->id), loc1->path, loc2->path,
+                        uuid_utoa_r(loc1->gfid, gfid1),
+                        uuid_utoa_r(loc2->gfid, gfid2),
+                        parent ? ec_fop_name(parent->id) : "No Parent");
+        } else {
+            gf_asprintf(
+                &fop->errstr,
+                "FOP : '%s' failed on '%s' with gfid %s. Parent FOP: %s",
+                ec_fop_name(fop->id), loc1->path,
+                uuid_utoa_r(loc1->gfid, gfid1),
+                parent ? ec_fop_name(parent->id) : "No Parent");
+        }
+    } else {
+        gf_asprintf(
+            &fop->errstr, "FOP : '%s' failed on gfid %s. Parent FOP: %s",
+            ec_fop_name(fop->id), uuid_utoa_r(fop->fd->inode->gfid, gfid1),
+            parent ? ec_fop_name(parent->id) : "No Parent");
+    }
+    return fop->errstr;
+}
+
+static void
+ec_log_insufficient_vol(ec_fop_data_t *fop, int32_t have, uint32_t need,
+                        int32_t loglevel)
+{
+    ec_t *ec = fop->xl->private;
+    char str1[32], str2[32], str3[32];
+
+    gf_msg(ec->xl->name, loglevel, 0, EC_MSG_CHILDS_INSUFFICIENT,
+           "Insufficient available children for this request: "
+           "Have : %d, Need : %u : Child UP : %s "
+           "Mask: %s, Healing : %s : %s ",
+           have, need, ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes),
+           ec_bin(str2, sizeof(str2), fop->mask, ec->nodes),
+           ec_bin(str3, sizeof(str3), fop->healing, ec->nodes),
+           ec_msg_str(fop));
 }
 
-int32_t ec_child_select(ec_fop_data_t * fop)
+static int32_t
+ec_child_select(ec_fop_data_t *fop)
 {
-    ec_t * ec = fop->xl->private;
+    ec_t *ec = fop->xl->private;
     int32_t first = 0, num = 0;
 
     ec_fop_cleanup(fop);
@@ -606,27 +678,25 @@ int32_t ec_child_select(ec_fop_data_t * fop)
     /* Wind the fop on same subvols as parent for any internal extra fops like
      * head/tail read in case of writev fop. Unlocks shouldn't do this because
      * unlock should go on all subvols where lock is performed*/
-    if (fop->parent && !ec_internal_op (fop)) {
-            fop->mask &= (fop->parent->mask & ~fop->parent->healing);
+    if (fop->parent && !ec_internal_op(fop)) {
+        fop->mask &= (fop->parent->mask & ~fop->parent->healing);
+        if (ec_is_data_fop(fop->id)) {
+            fop->healing |= fop->parent->healing;
+        }
     }
 
-    if ((fop->mask & ~ec->xl_up) != 0)
-    {
-        gf_msg (fop->xl->name, GF_LOG_WARNING, 0,
-                EC_MSG_OP_EXEC_UNAVAIL,
-                "Executing operation with "
-                "some subvolumes unavailable "
-                "(%lX)", fop->mask & ~ec->xl_up);
-
+    if ((fop->mask & ~ec->xl_up) != 0) {
+        gf_msg(fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_OP_EXEC_UNAVAIL,
+               "Executing operation with "
+               "some subvolumes unavailable. (%" PRIXPTR "). %s ",
+               fop->mask & ~ec->xl_up, ec_msg_str(fop));
         fop->mask &= ec->xl_up;
     }
 
-    switch (fop->minimum)
-    {
+    switch (fop->minimum) {
         case EC_MINIMUM_ALL:
             fop->minimum = gf_bits_count(fop->mask);
-            if (fop->minimum >= ec->fragments)
-            {
+            if (fop->minimum >= ec->fragments) {
                 break;
             }
         case EC_MINIMUM_MIN:
@@ -637,11 +707,11 @@ int32_t ec_child_select(ec_fop_data_t * fop)
     }
 
     if (ec->read_policy == EC_ROUND_ROBIN) {
-            first = ec->idx;
-            if (++first >= ec->nodes) {
-                first = 0;
-            }
-            ec->idx = first;
+        first = ec->idx;
+        if (++first >= ec->nodes) {
+            first = 0;
+        }
+        ec->idx = first;
     }
 
     num = gf_bits_count(fop->mask);
@@ -652,26 +722,28 @@ int32_t ec_child_select(ec_fop_data_t * fop)
 
     ec_trace("SELECT", fop, "");
 
-    if ((num < fop->minimum) && (num < ec->fragments))
-    {
-        gf_msg (ec->xl->name, GF_LOG_ERROR, 0,
-                EC_MSG_CHILDS_INSUFFICIENT,
-                "Insufficient available children "
-                "for this request (have %d, need "
-                "%d)", num, fop->minimum);
-
+    if ((num < fop->minimum) && (num < ec->fragments)) {
+        ec_log_insufficient_vol(fop, num, fop->minimum, GF_LOG_ERROR);
         return 0;
     }
 
-    ec_sleep(fop);
+    if (!fop->parent && fop->lock_count &&
+        (fop->locks[0].update[EC_DATA_TXN] ||
+         fop->locks[0].update[EC_METADATA_TXN])) {
+        if (ec->quorum_count && (num < ec->quorum_count)) {
+            ec_log_insufficient_vol(fop, num, ec->quorum_count, GF_LOG_ERROR);
+            return 0;
+        }
+    }
 
     return 1;
 }
 
-void ec_dispatch_next(ec_fop_data_t * fop, uint32_t idx)
+void
+ec_dispatch_next(ec_fop_data_t *fop, uint32_t idx)
 {
     uint32_t i = EC_INVALID_INDEX;
-    ec_t * ec = fop->xl->private;
+    ec_t *ec = fop->xl->private;
 
     LOCK(&fop->lock);
 
@@ -689,15 +761,15 @@ void ec_dispatch_next(ec_fop_data_t * fop, uint32_t idx)
 
     UNLOCK(&fop->lock);
 
-    if (i < EC_MAX_NODES)
-    {
+    if (i < EC_MAX_NODES) {
         fop->wind(ec, fop, idx);
     }
 }
 
-void ec_dispatch_mask(ec_fop_data_t * fop, uintptr_t mask)
+void
+ec_dispatch_mask(ec_fop_data_t *fop, uintptr_t mask)
 {
-    ec_t * ec = fop->xl->private;
+    ec_t *ec = fop->xl->private;
     int32_t count, idx;
 
     count = gf_bits_count(mask);
@@ -714,10 +786,8 @@ void ec_dispatch_mask(ec_fop_data_t * fop, uintptr_t mask)
     UNLOCK(&fop->lock);
 
     idx = 0;
-    while (mask != 0)
-    {
-        if ((mask & 1) != 0)
-        {
+    while (mask != 0) {
+        if ((mask & 1) != 0) {
             fop->wind(ec, fop, idx);
         }
         idx++;
@@ -725,27 +795,29 @@ void ec_dispatch_mask(ec_fop_data_t * fop, uintptr_t mask)
     }
 }
 
-void ec_dispatch_start(ec_fop_data_t * fop)
+void
+ec_dispatch_start(ec_fop_data_t *fop)
 {
     fop->answer = NULL;
     fop->good = 0;
 
     INIT_LIST_HEAD(&fop->cbk_list);
 
-    if (fop->lock_count > 0)
-    {
+    if (fop->lock_count > 0) {
         ec_owner_copy(fop->frame, &fop->req_frame->root->lk_owner);
     }
 }
 
-void ec_dispatch_one(ec_fop_data_t * fop)
+void
+ec_dispatch_one(ec_fop_data_t *fop)
 {
     ec_dispatch_start(fop);
 
-    if (ec_child_select(fop))
-    {
+    if (ec_child_select(fop)) {
+        ec_sleep(fop);
+
         fop->expected = 1;
-        fop->first = ec_select_first_by_read_policy (fop->xl->private, fop);
+        fop->first = ec_select_first_by_read_policy(fop->xl->private, fop);
 
         ec_dispatch_next(fop, fop->first);
     }
@@ -761,8 +833,8 @@ ec_dispatch_one_retry(ec_fop_data_t *fop, ec_cbk_data_t **cbk)
         *cbk = tmp;
     }
     if ((tmp != NULL) && (tmp->op_ret < 0) &&
-        ec_is_recoverable_error (tmp->op_errno)) {
-        GF_ASSERT (fop->mask & (1ULL << tmp->idx));
+        ec_is_recoverable_error(tmp->op_errno)) {
+        GF_ASSERT(fop->mask & (1ULL << tmp->idx));
         fop->mask ^= (1ULL << tmp->idx);
         if (fop->mask) {
             return _gf_true;
@@ -772,12 +844,14 @@ ec_dispatch_one_retry(ec_fop_data_t *fop, ec_cbk_data_t **cbk)
     return _gf_false;
 }
 
-void ec_dispatch_inc(ec_fop_data_t * fop)
+void
+ec_dispatch_inc(ec_fop_data_t *fop)
 {
     ec_dispatch_start(fop);
 
-    if (ec_child_select(fop))
-    {
+    if (ec_child_select(fop)) {
+        ec_sleep(fop);
+
         fop->expected = gf_bits_count(fop->remaining);
         fop->first = 0;
 
@@ -786,35 +860,38 @@ void ec_dispatch_inc(ec_fop_data_t * fop)
 }
 
 void
-ec_dispatch_all (ec_fop_data_t *fop)
+ec_dispatch_all(ec_fop_data_t *fop)
 {
-        ec_dispatch_start(fop);
+    ec_dispatch_start(fop);
 
-        if (ec_child_select(fop)) {
-                fop->expected = gf_bits_count(fop->remaining);
-                fop->first = 0;
+    if (ec_child_select(fop)) {
+        ec_sleep(fop);
 
-                ec_dispatch_mask(fop, fop->remaining);
-        }
+        fop->expected = gf_bits_count(fop->remaining);
+        fop->first = 0;
+
+        ec_dispatch_mask(fop, fop->remaining);
+    }
 }
 
-void ec_dispatch_min(ec_fop_data_t * fop)
+void
+ec_dispatch_min(ec_fop_data_t *fop)
 {
-    ec_t * ec = fop->xl->private;
+    ec_t *ec = fop->xl->private;
     uintptr_t mask;
     uint32_t idx;
     int32_t count;
 
     ec_dispatch_start(fop);
 
-    if (ec_child_select(fop))
-    {
+    if (ec_child_select(fop)) {
+        ec_sleep(fop);
+
         fop->expected = count = ec->fragments;
-        fop->first = ec_select_first_by_read_policy (fop->xl->private, fop);
+        fop->first = ec_select_first_by_read_policy(fop->xl->private, fop);
         idx = fop->first - 1;
         mask = 0;
-        while (count-- > 0)
-        {
+        while (count-- > 0) {
             idx = ec_child_next(ec, fop, idx + 1);
             if (idx < EC_MAX_NODES)
                 mask |= 1ULL << idx;
@@ -824,19 +901,35 @@ void ec_dispatch_min(ec_fop_data_t * fop)
     }
 }
 
-ec_lock_t *ec_lock_allocate(ec_fop_data_t *fop, loc_t *loc)
+void
+ec_succeed_all(ec_fop_data_t *fop)
+{
+    ec_dispatch_start(fop);
+
+    if (ec_child_select(fop)) {
+        fop->expected = gf_bits_count(fop->remaining);
+        fop->first = 0;
+
+        /* Simulate a successful execution on all bricks */
+        ec_trace("SUCCEED", fop, "");
+
+        fop->good = fop->remaining;
+        fop->remaining = 0;
+    }
+}
+
+ec_lock_t *
+ec_lock_allocate(ec_fop_data_t *fop, loc_t *loc)
 {
     ec_t *ec = fop->xl->private;
-    ec_lock_t * lock;
+    ec_lock_t *lock;
     int32_t err;
 
     if ((loc->inode == NULL) ||
-        (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid)))
-    {
-        gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_INVALID_INODE,
-                "Trying to lock based on an invalid "
-                "inode");
+        (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid))) {
+        gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_INODE,
+               "Trying to lock based on an invalid "
+               "inode");
 
         __ec_fop_set_error(fop, EINVAL);
 
@@ -844,9 +937,8 @@ ec_lock_t *ec_lock_allocate(ec_fop_data_t *fop, loc_t *loc)
     }
 
     lock = mem_get0(ec->lock_pool);
-    if (lock != NULL)
-    {
-        lock->good_mask = -1ULL;
+    if (lock != NULL) {
+        lock->good_mask = UINTPTR_MAX;
         INIT_LIST_HEAD(&lock->owners);
         INIT_LIST_HEAD(&lock->waiting);
         INIT_LIST_HEAD(&lock->frozen);
@@ -862,7 +954,8 @@ ec_lock_t *ec_lock_allocate(ec_fop_data_t *fop, loc_t *loc)
     return lock;
 }
 
-void ec_lock_destroy(ec_lock_t * lock)
+void
+ec_lock_destroy(ec_lock_t *lock)
 {
     loc_wipe(&lock->loc);
     if (lock->fd != NULL) {
@@ -872,13 +965,15 @@ void ec_lock_destroy(ec_lock_t * lock)
     mem_put(lock);
 }
 
-int32_t ec_lock_compare(ec_lock_t * lock1, ec_lock_t * lock2)
+int32_t
+ec_lock_compare(ec_lock_t *lock1, ec_lock_t *lock2)
 {
     return gf_uuid_compare(lock1->loc.gfid, lock2->loc.gfid);
 }
 
-void ec_lock_insert(ec_fop_data_t *fop, ec_lock_t *lock, uint32_t flags,
-                    loc_t *base, off_t fl_start, size_t fl_size)
+static void
+ec_lock_insert(ec_fop_data_t *fop, ec_lock_t *lock, uint32_t flags, loc_t *base,
+               off_t fl_start, uint64_t fl_size)
 {
     ec_lock_link_t *link;
 
@@ -913,14 +1008,14 @@ void ec_lock_insert(ec_fop_data_t *fop, ec_lock_t *lock, uint32_t flags,
     link->update[EC_METADATA_TXN] = (flags & EC_UPDATE_META) != 0;
     link->base = base;
     link->fl_start = fl_start;
-    link->fl_end = ec_range_end_get (fl_start, fl_size);
+    link->fl_end = ec_range_end_get(fl_start, fl_size);
 
     lock->refs_pending++;
 }
 
-void ec_lock_prepare_inode_internal(ec_fop_data_t *fop, loc_t *loc,
-                                    uint32_t flags, loc_t *base,
-                                    off_t fl_start, size_t fl_size)
+static void
+ec_lock_prepare_inode_internal(ec_fop_data_t *fop, loc_t *loc, uint32_t flags,
+                               loc_t *base, off_t fl_start, uint64_t fl_size)
 {
     ec_lock_t *lock = NULL;
     ec_inode_t *ctx;
@@ -949,8 +1044,8 @@ void ec_lock_prepare_inode_internal(ec_fop_data_t *fop, loc_t *loc,
         if ((fop->lock_count > 0) && (fop->locks[0].lock == lock)) {
             /* Combine data/meta updates */
             fop->locks[0].update[EC_DATA_TXN] |= (flags & EC_UPDATE_DATA) != 0;
-            fop->locks[0].update[EC_METADATA_TXN] |=
-                                                 (flags & EC_UPDATE_META) != 0;
+            fop->locks[0].update[EC_METADATA_TXN] |= (flags & EC_UPDATE_META) !=
+                                                     0;
 
             /* Only one base inode is allowed per fop, so there shouldn't be
              * overwrites here. */
@@ -961,8 +1056,10 @@ void ec_lock_prepare_inode_internal(ec_fop_data_t *fop, loc_t *loc,
             goto update_query;
         }
 
-        ec_trace("LOCK_INODELK", fop, "lock=%p, inode=%p. Lock already "
-                                      "acquired", lock, loc->inode);
+        ec_trace("LOCK_INODELK", fop,
+                 "lock=%p, inode=%p. Lock already "
+                 "acquired",
+                 lock, loc->inode);
 
         goto insert;
     }
@@ -988,14 +1085,16 @@ unlock:
     UNLOCK(&loc->inode->lock);
 }
 
-void ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, uint32_t flags,
-                           off_t fl_start, size_t fl_size)
+void
+ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, uint32_t flags,
+                      off_t fl_start, uint64_t fl_size)
 {
     ec_lock_prepare_inode_internal(fop, loc, flags, NULL, fl_start, fl_size);
 }
 
-void ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc, loc_t *base,
-                                  uint32_t flags)
+void
+ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc, loc_t *base,
+                             uint32_t flags)
 {
     loc_t tmp;
     int32_t err;
@@ -1014,16 +1113,17 @@ void ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc, loc_t *base,
     if ((flags & EC_INODE_SIZE) != 0) {
         flags ^= EC_INODE_SIZE;
     } else {
-            base = NULL;
+        base = NULL;
     }
 
-    ec_lock_prepare_inode_internal(fop, &tmp, flags, base, 0, LLONG_MAX);
+    ec_lock_prepare_inode_internal(fop, &tmp, flags, base, 0, EC_RANGE_FULL);
 
     loc_wipe(&tmp);
 }
 
-void ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags,
-                        off_t fl_start, size_t fl_size)
+void
+ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags, off_t fl_start,
+                   uint64_t fl_size)
 {
     loc_t loc;
     int32_t err;
@@ -1045,15 +1145,14 @@ void ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags,
 }
 
 gf_boolean_t
-ec_config_check (xlator_t *xl, ec_config_t *config)
+ec_config_check(xlator_t *xl, ec_config_t *config)
 {
     ec_t *ec;
 
     ec = xl->private;
     if ((config->version != EC_CONFIG_VERSION) ||
         (config->algorithm != EC_CONFIG_ALGORITHM) ||
-        (config->gf_word_size != EC_GF_BITS) ||
-        (config->bricks != ec->nodes) ||
+        (config->gf_word_size != EC_GF_BITS) || (config->bricks != ec->nodes) ||
         (config->redundancy != ec->redundancy) ||
         (config->chunk_size != EC_METHOD_CHUNK_SIZE)) {
         uint32_t data_bricks;
@@ -1072,20 +1171,17 @@ ec_config_check (xlator_t *xl, ec_config_t *config)
         if ((config->redundancy < 1) ||
             (config->redundancy * 2 >= config->bricks) ||
             !ec_is_power_of_2(config->gf_word_size) ||
-            ((config->chunk_size * 8) % (config->gf_word_size * data_bricks)
-                                                                       != 0)) {
-            gf_msg (xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_INVALID_CONFIG,
-                    "Invalid or corrupted config");
+            ((config->chunk_size * 8) % (config->gf_word_size * data_bricks) !=
+             0)) {
+            gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_CONFIG,
+                   "Invalid or corrupted config");
         } else {
-            gf_msg (xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_INVALID_CONFIG,
-                    "Unsupported config "
-                    "(V=%u, A=%u, W=%u, "
-                    "N=%u, R=%u, S=%u)",
-                   config->version, config->algorithm,
-                   config->gf_word_size, config->bricks,
-                   config->redundancy, config->chunk_size);
+            gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_CONFIG,
+                   "Unsupported config "
+                   "(V=%u, A=%u, W=%u, "
+                   "N=%u, R=%u, S=%u)",
+                   config->version, config->algorithm, config->gf_word_size,
+                   config->bricks, config->redundancy, config->chunk_size);
         }
 
         return _gf_false;
@@ -1095,20 +1191,18 @@ ec_config_check (xlator_t *xl, ec_config_t *config)
 }
 
 gf_boolean_t
-ec_set_dirty_flag (ec_lock_link_t *link, ec_inode_t *ctx,
-                   uint64_t *dirty)
+ec_set_dirty_flag(ec_lock_link_t *link, ec_inode_t *ctx, uint64_t *dirty)
 {
-
     gf_boolean_t set_dirty = _gf_false;
 
     if (link->update[EC_DATA_TXN] && !ctx->dirty[EC_DATA_TXN]) {
-            if (!link->optimistic_changelog)
-                dirty[EC_DATA_TXN] = 1;
+        if (!link->optimistic_changelog)
+            dirty[EC_DATA_TXN] = 1;
     }
 
     if (link->update[EC_METADATA_TXN] && !ctx->dirty[EC_METADATA_TXN]) {
-            if (!link->optimistic_changelog)
-                dirty[EC_METADATA_TXN] = 1;
+        if (!link->optimistic_changelog)
+            dirty[EC_METADATA_TXN] = 1;
     }
 
     if (dirty[EC_METADATA_TXN] || dirty[EC_DATA_TXN]) {
@@ -1119,9 +1213,9 @@ ec_set_dirty_flag (ec_lock_link_t *link, ec_inode_t *ctx,
 }
 
 int32_t
-ec_prepare_update_cbk (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       dict_t *dict, dict_t *xdata)
+ec_prepare_update_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
     struct list_head list;
     ec_fop_data_t *fop = cookie, *parent, *tmp;
@@ -1130,129 +1224,123 @@ ec_prepare_update_cbk (call_frame_t *frame, void *cookie,
     ec_lock_t *lock = NULL;
     ec_inode_t *ctx;
     gf_boolean_t release = _gf_false;
-    uint64_t waiting_flags = 0;
+    uint64_t provided_flags = 0;
     uint64_t dirty[EC_VERSION_SIZE] = {0, 0};
-
     lock = parent_link->lock;
     parent = parent_link->fop;
     ctx = lock->ctx;
 
     INIT_LIST_HEAD(&list);
-    waiting_flags = parent_link->waiting_flags & EC_XATTROP_ALL_WAITING_FLAGS;
+    provided_flags = EC_PROVIDED_FLAGS(parent_link->waiting_flags);
 
     LOCK(&lock->loc.inode->lock);
 
-    list_for_each_entry(link, &lock->owners, owner_list) {
-        if ((link->waiting_flags & waiting_flags) != 0) {
-            link->waiting_flags ^= (link->waiting_flags & waiting_flags);
-            if ((link->waiting_flags & EC_XATTROP_ALL_WAITING_FLAGS) == 0)
-                    list_add_tail(&link->fop->cbk_list, &list);
+    list_for_each_entry(link, &lock->owners, owner_list)
+    {
+        if ((link->waiting_flags & provided_flags) != 0) {
+            link->waiting_flags ^= (link->waiting_flags & provided_flags);
+            if (EC_NEEDED_FLAGS(link->waiting_flags) == 0)
+                list_add_tail(&link->fop->cbk_list, &list);
         }
     }
-
     if (op_ret < 0) {
-        gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                EC_MSG_SIZE_VERS_GET_FAIL,
-                "Failed to get size and version");
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_SIZE_VERS_GET_FAIL,
+               "Failed to get size and version :  %s", ec_msg_str(fop));
 
         goto unlock;
     }
 
-    if (waiting_flags & EC_FLAG_WAITING_XATTROP) {
-            op_errno = -ec_dict_del_array(dict, EC_XATTR_VERSION,
-                                          ctx->pre_version,
-                                          EC_VERSION_SIZE);
+    if (EC_FLAGS_HAVE(provided_flags, EC_FLAG_XATTROP)) {
+        op_errno = -ec_dict_del_array(dict, EC_XATTR_VERSION, ctx->pre_version,
+                                      EC_VERSION_SIZE);
+        if (op_errno != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   EC_MSG_VER_XATTR_GET_FAIL, "Unable to get version xattr. %s",
+                   ec_msg_str(fop));
+            goto unlock;
+        }
+        ctx->post_version[0] += ctx->pre_version[0];
+        ctx->post_version[1] += ctx->pre_version[1];
+
+        ctx->have_version = _gf_true;
+
+        if (lock->loc.inode->ia_type == IA_IFREG ||
+            lock->loc.inode->ia_type == IA_INVAL) {
+            op_errno = -ec_dict_del_number(dict, EC_XATTR_SIZE, &ctx->pre_size);
             if (op_errno != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        EC_MSG_VER_XATTR_GET_FAIL,
-                        "Unable to get version xattr");
-                goto unlock;
+                if (lock->loc.inode->ia_type == IA_IFREG) {
+                    gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                           EC_MSG_SIZE_XATTR_GET_FAIL,
+                           "Unable to get size xattr. %s", ec_msg_str(fop));
+                    goto unlock;
+                }
+            } else {
+                ctx->post_size = ctx->pre_size;
+
+                ctx->have_size = _gf_true;
             }
-            ctx->post_version[0] += ctx->pre_version[0];
-            ctx->post_version[1] += ctx->pre_version[1];
 
-            ctx->have_version = _gf_true;
+            op_errno = -ec_dict_del_config(dict, EC_XATTR_CONFIG, &ctx->config);
+            if (op_errno != 0) {
+                if ((lock->loc.inode->ia_type == IA_IFREG) ||
+                    (op_errno != ENODATA)) {
+                    gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                           EC_MSG_CONFIG_XATTR_GET_FAIL,
+                           "Unable to get config xattr. %s", ec_msg_str(fop));
 
-            if (lock->loc.inode->ia_type == IA_IFREG ||
-                lock->loc.inode->ia_type == IA_INVAL) {
-                op_errno = -ec_dict_del_number(dict, EC_XATTR_SIZE,
-                                               &ctx->pre_size);
-                if (op_errno != 0) {
-                    if (lock->loc.inode->ia_type == IA_IFREG) {
-                        gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                EC_MSG_SIZE_XATTR_GET_FAIL,
-                                "Unable to get size xattr");
-                        goto unlock;
-                    }
-                } else {
-                    ctx->post_size = ctx->pre_size;
-
-                    ctx->have_size = _gf_true;
+                    goto unlock;
                 }
+            } else {
+                if (!ec_config_check(parent->xl, &ctx->config)) {
+                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                           EC_MSG_CONFIG_XATTR_INVALID, "Invalid config xattr");
 
-                op_errno = -ec_dict_del_config(dict, EC_XATTR_CONFIG,
-                                               &ctx->config);
-                if (op_errno != 0) {
-                    if ((lock->loc.inode->ia_type == IA_IFREG) ||
-                        (op_errno != ENODATA)) {
-                        gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                                EC_MSG_CONFIG_XATTR_GET_FAIL,
-                                "Unable to get config xattr");
-
-                        goto unlock;
-                    }
-                } else {
-                    if (!ec_config_check(parent->xl, &ctx->config)) {
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                EC_MSG_CONFIG_XATTR_INVALID,
-                                "Invalid config xattr");
-
-                        op_errno = EINVAL;
-
-                        goto unlock;
-                    }
-                    ctx->have_config = _gf_true;
+                    op_errno = EINVAL;
+
+                    goto unlock;
                 }
+                ctx->have_config = _gf_true;
             }
-            ctx->have_info = _gf_true;
+        }
+        ctx->have_info = _gf_true;
     }
 
-    ec_set_dirty_flag (fop->data, ctx, dirty);
+    ec_set_dirty_flag(fop->data, ctx, dirty);
     if (dirty[EC_METADATA_TXN] &&
-        (waiting_flags & EC_FLAG_WAITING_METADATA_DIRTY)) {
-            GF_ASSERT (!ctx->dirty[EC_METADATA_TXN]);
-            ctx->dirty[EC_METADATA_TXN] = 1;
+        (EC_FLAGS_HAVE(provided_flags, EC_FLAG_METADATA_DIRTY))) {
+        GF_ASSERT(!ctx->dirty[EC_METADATA_TXN]);
+        ctx->dirty[EC_METADATA_TXN] = 1;
     }
 
     if (dirty[EC_DATA_TXN] &&
-        (waiting_flags & EC_FLAG_WAITING_DATA_DIRTY)) {
-            GF_ASSERT (!ctx->dirty[EC_DATA_TXN]);
-            ctx->dirty[EC_DATA_TXN] = 1;
+        (EC_FLAGS_HAVE(provided_flags, EC_FLAG_DATA_DIRTY))) {
+        GF_ASSERT(!ctx->dirty[EC_DATA_TXN]);
+        ctx->dirty[EC_DATA_TXN] = 1;
     }
     op_errno = 0;
 unlock:
 
-    lock->waiting_flags ^= waiting_flags;
+    lock->waiting_flags ^= provided_flags;
 
     if (op_errno == 0) {
         /* If the fop fails on any of the good bricks, it is important to mark
          * it dirty and update versions right away if dirty was not set before.
          */
         if (lock->good_mask & ~(fop->good | fop->remaining)) {
-                release = _gf_true;
+            release = _gf_true;
         }
 
         if (parent_link->update[0] && !parent_link->dirty[0]) {
-                lock->release |= release;
+            lock->release |= release;
         }
 
         if (parent_link->update[1] && !parent_link->dirty[1]) {
-                lock->release |= release;
+            lock->release |= release;
         }
 
         /* We don't allow the main fop to be executed on bricks that have not
          * succeeded the initial xattrop. */
-        ec_lock_update_good (lock, fop);
+        ec_lock_update_good(lock, fop);
 
         /*As of now only data healing marks bricks as healing*/
         lock->healing |= fop->healing;
@@ -1268,7 +1356,7 @@ unlock:
             tmp->mask &= fop->good;
 
             /*As of now only data healing marks bricks as healing*/
-            if (ec_is_data_fop (tmp->id)) {
+            if (ec_is_data_fop(tmp->id)) {
                 tmp->healing |= fop->healing;
             }
         }
@@ -1279,43 +1367,56 @@ unlock:
     return 0;
 }
 
+static gf_boolean_t
+ec_set_needed_flag(ec_lock_t *lock, ec_lock_link_t *link, uint64_t flag)
+{
+    uint64_t current;
+
+    link->waiting_flags |= EC_FLAG_NEEDS(flag);
+
+    current = EC_NEEDED_FLAGS(lock->waiting_flags);
+    if (!EC_FLAGS_HAVE(current, flag)) {
+        lock->waiting_flags |= EC_FLAG_NEEDS(flag);
+        link->waiting_flags |= EC_FLAG_PROVIDES(flag);
+
+        return _gf_true;
+    }
+
+    return _gf_false;
+}
+
 static uint64_t
-ec_set_xattrop_flags_and_params (ec_lock_t *lock, ec_lock_link_t *link,
-                                 uint64_t *dirty)
+ec_set_xattrop_flags_and_params(ec_lock_t *lock, ec_lock_link_t *link,
+                                uint64_t *dirty)
 {
-        uint64_t        oldflags = 0;
-        uint64_t        newflags = 0;
-        ec_inode_t *ctx     = lock->ctx;
+    uint64_t oldflags = 0;
+    uint64_t newflags = 0;
+    ec_inode_t *ctx = lock->ctx;
 
-        oldflags = lock->waiting_flags & EC_XATTROP_ALL_WAITING_FLAGS;
+    oldflags = EC_NEEDED_FLAGS(lock->waiting_flags);
 
-        if (lock->query && !ctx->have_info) {
-                lock->waiting_flags |= EC_FLAG_WAITING_XATTROP;
-                link->waiting_flags |= EC_FLAG_WAITING_XATTROP;
-        }
+    if (lock->query && !ctx->have_info) {
+        ec_set_needed_flag(lock, link, EC_FLAG_XATTROP);
+    }
 
-        if (dirty[EC_DATA_TXN]) {
-                if (oldflags & EC_FLAG_WAITING_DATA_DIRTY) {
-                        dirty[EC_DATA_TXN] = 0;
-                } else {
-                        lock->waiting_flags |= EC_FLAG_WAITING_DATA_DIRTY;
-                }
-                link->waiting_flags |= EC_FLAG_WAITING_DATA_DIRTY;
+    if (dirty[EC_DATA_TXN]) {
+        if (!ec_set_needed_flag(lock, link, EC_FLAG_DATA_DIRTY)) {
+            dirty[EC_DATA_TXN] = 0;
         }
+    }
 
-        if (dirty[EC_METADATA_TXN]) {
-                if (oldflags & EC_FLAG_WAITING_METADATA_DIRTY) {
-                        dirty[EC_METADATA_TXN] = 0;
-                } else {
-                        lock->waiting_flags |= EC_FLAG_WAITING_METADATA_DIRTY;
-                }
-                link->waiting_flags |= EC_FLAG_WAITING_METADATA_DIRTY;
+    if (dirty[EC_METADATA_TXN]) {
+        if (!ec_set_needed_flag(lock, link, EC_FLAG_METADATA_DIRTY)) {
+            dirty[EC_METADATA_TXN] = 0;
         }
-        newflags = lock->waiting_flags & EC_XATTROP_ALL_WAITING_FLAGS;
-        return oldflags ^ newflags;
+    }
+    newflags = EC_NEEDED_FLAGS(lock->waiting_flags);
+
+    return oldflags ^ newflags;
 }
 
-void ec_get_size_version(ec_lock_link_t *link)
+void
+ec_get_size_version(ec_lock_link_t *link)
 {
     loc_t loc;
     ec_lock_t *lock;
@@ -1323,7 +1424,7 @@ void ec_get_size_version(ec_lock_link_t *link)
     ec_fop_data_t *fop;
     dict_t *dict = NULL;
     dict_t *xdata = NULL;
-    ec_t   *ec = NULL;
+    ec_t *ec = NULL;
     int32_t error = 0;
     gf_boolean_t set_dirty = _gf_false;
     uint64_t allzero[EC_VERSION_SIZE] = {0, 0};
@@ -1331,45 +1432,46 @@ void ec_get_size_version(ec_lock_link_t *link)
     lock = link->lock;
     ctx = lock->ctx;
     fop = link->fop;
-    ec  = fop->xl->private;
+    ec = fop->xl->private;
     uint64_t changed_flags = 0;
 
-    if (ec->optimistic_changelog &&
-        !(ec->node_mask & ~link->lock->good_mask) && !ec_is_data_fop (fop->id))
-            link->optimistic_changelog = _gf_true;
+    if (ec->optimistic_changelog && !(ec->node_mask & ~link->lock->good_mask) &&
+        !ec_is_data_fop(fop->id))
+        link->optimistic_changelog = _gf_true;
+
+    memset(&loc, 0, sizeof(loc));
+
+    LOCK(&lock->loc.inode->lock);
 
-    set_dirty = ec_set_dirty_flag (link, ctx, dirty);
+    set_dirty = ec_set_dirty_flag(link, ctx, dirty);
 
     /* If ec metadata has already been retrieved, do not try again. */
-    if (ctx->have_info && (!set_dirty)) {
-        if (ec_is_data_fop (fop->id)) {
+    if (ctx->have_info) {
+        if (ec_is_data_fop(fop->id)) {
             fop->healing |= lock->healing;
         }
-        return;
+        if (!set_dirty)
+            goto unlock;
     }
 
     /* Determine if there's something we need to retrieve for the current
      * operation. */
-    if (!set_dirty && !lock->query &&
-        (lock->loc.inode->ia_type != IA_IFREG) &&
+    if (!set_dirty && !lock->query && (lock->loc.inode->ia_type != IA_IFREG) &&
         (lock->loc.inode->ia_type != IA_INVAL)) {
-            return;
+        goto unlock;
     }
 
-    memset(&loc, 0, sizeof(loc));
-
-    LOCK(&lock->loc.inode->lock);
-
-    changed_flags = ec_set_xattrop_flags_and_params (lock, link, dirty);
+    changed_flags = ec_set_xattrop_flags_and_params(lock, link, dirty);
     if (link->waiting_flags) {
-            /* This fop needs to wait until all its flags are cleared which
-             * potentially can be cleared by other xattrops that are already
-             * wound*/
-            ec_sleep(fop);
+        /* This fop needs to wait until all its flags are cleared which
+         * potentially can be cleared by other xattrops that are already
+         * wound*/
+        ec_sleep(fop);
     } else {
-            GF_ASSERT (!changed_flags);
+        GF_ASSERT(!changed_flags);
     }
 
+unlock:
     UNLOCK(&lock->loc.inode->lock);
 
     if (!changed_flags)
@@ -1381,41 +1483,39 @@ void ec_get_size_version(ec_lock_link_t *link)
         goto out;
     }
 
-    if (changed_flags & EC_FLAG_WAITING_XATTROP) {
-            /* Once we know that an xattrop will be needed,
-             * we try to get all available information in a
-             * single call. */
-            error = ec_dict_set_array(dict, EC_XATTR_VERSION, allzero,
-                                      EC_VERSION_SIZE);
+    if (EC_FLAGS_HAVE(changed_flags, EC_FLAG_XATTROP)) {
+        /* Once we know that an xattrop will be needed,
+         * we try to get all available information in a
+         * single call. */
+        error = ec_dict_set_array(dict, EC_XATTR_VERSION, allzero,
+                                  EC_VERSION_SIZE);
+        if (error != 0) {
+            goto out;
+        }
+
+        if (lock->loc.inode->ia_type == IA_IFREG ||
+            lock->loc.inode->ia_type == IA_INVAL) {
+            error = ec_dict_set_number(dict, EC_XATTR_SIZE, 0);
+            if (error == 0) {
+                error = ec_dict_set_number(dict, EC_XATTR_CONFIG, 0);
+            }
             if (error != 0) {
                 goto out;
             }
 
-            if (lock->loc.inode->ia_type == IA_IFREG ||
-                lock->loc.inode->ia_type == IA_INVAL) {
-                error = ec_dict_set_number(dict, EC_XATTR_SIZE, 0);
-                if (error == 0) {
-                    error = ec_dict_set_number(dict, EC_XATTR_CONFIG, 0);
-                }
-                if (error != 0) {
-                    goto out;
-                }
-
-                xdata = dict_new();
-                if (xdata == NULL || dict_set_int32 (xdata, GF_GET_SIZE, 1)) {
-                    error = -ENOMEM;
-                    goto out;
-                }
-
+            xdata = dict_new();
+            if (xdata == NULL || dict_set_int32(xdata, GF_GET_SIZE, 1)) {
+                error = -ENOMEM;
+                goto out;
             }
+        }
     }
 
-    if (memcmp (allzero, dirty, sizeof (allzero))) {
-            error = ec_dict_set_array(dict, EC_XATTR_DIRTY, dirty,
-                                      EC_VERSION_SIZE);
-            if (error != 0) {
-                goto out;
-            }
+    if (memcmp(allzero, dirty, sizeof(allzero))) {
+        error = ec_dict_set_array(dict, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
+        if (error != 0) {
+            goto out;
+        }
     }
 
     fop->frame->root->uid = 0;
@@ -1443,13 +1543,13 @@ void ec_get_size_version(ec_lock_link_t *link)
             loc.name = NULL;
         }
 
-        ec_xattrop (fop->frame, fop->xl, fop->mask, fop->minimum,
-                    ec_prepare_update_cbk, link, &loc,
-                    GF_XATTROP_ADD_ARRAY64, dict, xdata);
+        ec_xattrop(fop->frame, fop->xl, fop->mask, fop->minimum,
+                   ec_prepare_update_cbk, link, &loc, GF_XATTROP_ADD_ARRAY64,
+                   dict, xdata);
     } else {
         ec_fxattrop(fop->frame, fop->xl, fop->mask, fop->minimum,
-                ec_prepare_update_cbk, link, lock->fd,
-                GF_XATTROP_ADD_ARRAY64, dict, xdata);
+                    ec_prepare_update_cbk, link, lock->fd,
+                    GF_XATTROP_ADD_ARRAY64, dict, xdata);
     }
 
     error = 0;
@@ -1474,8 +1574,7 @@ out:
 }
 
 gf_boolean_t
-__ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode,
-                    uint64_t *size)
+__ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size)
 {
     ec_inode_t *ctx;
     gf_boolean_t found = _gf_false;
@@ -1495,14 +1594,13 @@ out:
 }
 
 gf_boolean_t
-ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode,
-                  uint64_t *size)
+ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size)
 {
     gf_boolean_t found = _gf_false;
 
     LOCK(&inode->lock);
     {
-            found = __ec_get_inode_size (fop, inode, size);
+        found = __ec_get_inode_size(fop, inode, size);
     }
     UNLOCK(&inode->lock);
 
@@ -1510,8 +1608,7 @@ ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode,
 }
 
 gf_boolean_t
-__ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode,
-                    uint64_t size)
+__ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size)
 {
     ec_inode_t *ctx;
     gf_boolean_t found = _gf_false;
@@ -1538,38 +1635,37 @@ out:
 }
 
 gf_boolean_t
-ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode,
-                  uint64_t size)
+ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size)
 {
     gf_boolean_t found = _gf_false;
 
-    LOCK (&inode->lock);
+    LOCK(&inode->lock);
     {
-            found = __ec_set_inode_size (fop, inode, size);
+        found = __ec_set_inode_size(fop, inode, size);
     }
-    UNLOCK (&inode->lock);
+    UNLOCK(&inode->lock);
 
     return found;
 }
 
 static void
-ec_release_stripe_cache (ec_inode_t *ctx)
+ec_release_stripe_cache(ec_inode_t *ctx)
 {
-        ec_stripe_list_t *stripe_cache = NULL;
-        ec_stripe_t      *stripe = NULL;
+    ec_stripe_list_t *stripe_cache = NULL;
+    ec_stripe_t *stripe = NULL;
 
-        stripe_cache = &ctx->stripe_cache;
-        while (!list_empty (&stripe_cache->lru)) {
-                stripe = list_first_entry (&stripe_cache->lru, ec_stripe_t,
-                                           lru);
-                list_del (&stripe->lru);
-                GF_FREE (stripe);
-        }
-        stripe_cache->count = 0;
-        stripe_cache->max = 0;
+    stripe_cache = &ctx->stripe_cache;
+    while (!list_empty(&stripe_cache->lru)) {
+        stripe = list_first_entry(&stripe_cache->lru, ec_stripe_t, lru);
+        list_del(&stripe->lru);
+        GF_FREE(stripe);
+    }
+    stripe_cache->count = 0;
+    stripe_cache->max = 0;
 }
 
-void ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode)
+void
+ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode)
 {
     ec_inode_t *ctx;
 
@@ -1580,7 +1676,7 @@ void ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode)
         goto unlock;
     }
 
-    ec_release_stripe_cache (ctx);
+    ec_release_stripe_cache(ctx);
     ctx->have_info = _gf_false;
     ctx->have_config = _gf_false;
     ctx->have_version = _gf_false;
@@ -1596,10 +1692,10 @@ unlock:
     UNLOCK(&inode->lock);
 }
 
-int32_t ec_get_real_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, inode_t *inode,
-                             struct iatt *buf, dict_t *xdata,
-                             struct iatt *postparent)
+int32_t
+ec_get_real_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
     ec_fop_data_t *fop = cookie;
     ec_lock_link_t *link;
@@ -1616,14 +1712,15 @@ int32_t ec_get_real_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
 }
 
 /* This function is used to get the trusted.ec.size xattr from a file when
- * no lock is needed on the inode. This is only required to maintan iatt
+ * no lock is needed on the inode. This is only required to maintain iatt
  * structs on fops that manipulate directory entries but do not operate
  * directly on the inode, like link, rename, ...
  *
  * Any error processing this request is ignored. In the worst case, an invalid
  * or not up to date value in the iatt could cause some cache invalidation.
  */
-void ec_get_real_size(ec_lock_link_t *link)
+void
+ec_get_real_size(ec_lock_link_t *link)
 {
     ec_fop_data_t *fop;
     dict_t *xdata;
@@ -1673,28 +1770,26 @@ ec_lock_update_fd(ec_lock_t *lock, ec_fop_data_t *fop)
 }
 
 static gf_boolean_t
-ec_link_has_lock_conflict (ec_lock_link_t *link, gf_boolean_t waitlist_check)
+ec_link_has_lock_conflict(ec_lock_link_t *link, gf_boolean_t waitlist_check)
 {
-        ec_lock_link_t *trav_link = NULL;
-        ec_t           *ec = link->fop->xl->private;
-
-        if (!ec->parallel_writes)
-                return _gf_true;
+    ec_lock_link_t *trav_link = NULL;
 
-        list_for_each_entry (trav_link, &link->lock->owners, owner_list) {
-                if (ec_lock_conflict (trav_link, link))
-                        return _gf_true;
-        }
+    list_for_each_entry(trav_link, &link->lock->owners, owner_list)
+    {
+        if (ec_lock_conflict(trav_link, link))
+            return _gf_true;
+    }
 
-        if (!waitlist_check)
-                return _gf_false;
+    if (!waitlist_check)
+        return _gf_false;
 
-        list_for_each_entry (trav_link, &link->lock->waiting, wait_list) {
-                if (ec_lock_conflict (trav_link, link))
-                        return _gf_true;
-        }
+    list_for_each_entry(trav_link, &link->lock->waiting, wait_list)
+    {
+        if (ec_lock_conflict(trav_link, link))
+            return _gf_true;
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static void
@@ -1715,7 +1810,7 @@ ec_lock_wake_shared(ec_lock_t *lock, struct list_head *list)
 
         /* If the fop is not shareable, only this fop can be assigned as owner.
          * Other fops will need to wait until this one finishes. */
-        if (ec_link_has_lock_conflict (link, _gf_false)) {
+        if (ec_link_has_lock_conflict(link, _gf_false)) {
             conflict = _gf_true;
         }
 
@@ -1746,7 +1841,8 @@ ec_lock_apply(ec_lock_link_t *link)
     ec_get_real_size(link);
 }
 
-gf_boolean_t ec_lock_acquire(ec_lock_link_t *link);
+gf_boolean_t
+ec_lock_acquire(ec_lock_link_t *link);
 
 static void
 ec_lock_resume_shared(struct list_head *list)
@@ -1770,7 +1866,8 @@ ec_lock_resume_shared(struct list_head *list)
     }
 }
 
-void ec_lock_acquired(ec_lock_link_t *link)
+void
+ec_lock_acquired(ec_lock_link_t *link)
 {
     struct list_head list;
     ec_lock_t *lock;
@@ -1786,6 +1883,10 @@ void ec_lock_acquired(ec_lock_link_t *link)
     LOCK(&lock->loc.inode->lock);
 
     lock->acquired = _gf_true;
+    if (lock->contention) {
+        lock->release = _gf_true;
+        lock->contention = _gf_false;
+    }
 
     ec_lock_update_fd(lock, fop);
     ec_lock_wake_shared(lock, &list);
@@ -1796,37 +1897,44 @@ void ec_lock_acquired(ec_lock_link_t *link)
 
     if (fop->use_fd &&
         (link->update[EC_DATA_TXN] || link->update[EC_METADATA_TXN])) {
-        ec_fix_open(fop);
+        /* Try to reopen closed fd's only if lock has succeeded. */
+        ec_fix_open(fop, lock->mask);
     }
 
     ec_lock_resume_shared(&list);
 }
 
-int32_t ec_locked(call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int32_t
+ec_locked(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+          int32_t op_errno, dict_t *xdata)
 {
     ec_fop_data_t *fop = cookie;
     ec_lock_link_t *link = NULL;
     ec_lock_t *lock = NULL;
 
+    link = fop->data;
+    lock = link->lock;
     if (op_ret >= 0) {
-        link = fop->data;
-        lock = link->lock;
         lock->mask = lock->good_mask = fop->good;
         lock->healing = 0;
 
         ec_lock_acquired(link);
         ec_lock(fop->parent);
     } else {
-        gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                EC_MSG_PREOP_LOCK_FAILED,
-                "Failed to complete preop lock");
+        LOCK(&lock->loc.inode->lock);
+        {
+            lock->contention = _gf_false;
+        }
+        UNLOCK(&lock->loc.inode->lock);
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_PREOP_LOCK_FAILED,
+               "Failed to complete preop lock");
     }
 
     return 0;
 }
 
-gf_boolean_t ec_lock_acquire(ec_lock_link_t *link)
+gf_boolean_t
+ec_lock_acquire(ec_lock_link_t *link)
 {
     ec_lock_t *lock;
     ec_fop_data_t *fop;
@@ -1859,62 +1967,62 @@ gf_boolean_t ec_lock_acquire(ec_lock_link_t *link)
 static ec_lock_link_t *
 ec_lock_timer_cancel(xlator_t *xl, ec_lock_t *lock)
 {
-        ec_lock_link_t *timer_link;
-
-        /* If we don't have any timer, there's nothing to cancel. */
-        if (lock->timer == NULL) {
-                return NULL;
-        }
-
-        /* We are trying to access a lock that has an unlock timer active.
-         * This means that the lock must be idle, i.e. no fop can be in the
-         * owner, waiting or frozen lists. It also means that the lock cannot
-         * have been marked as being released (this is done without timers).
-         * There should only be one owner reference, but it's possible that
-         * some fops are being prepared to use this lock. */
-        GF_ASSERT ((lock->refs_owners == 1) &&
-                   list_empty(&lock->owners) && list_empty(&lock->waiting));
-
-        /* We take the timer_link before cancelling the timer, since a
-         * successful cancellation will destroy it. It must not be NULL
-         * because it references the fop responsible for the delayed unlock
-         * that we are currently trying to cancel. */
-        timer_link = lock->timer->data;
-        GF_ASSERT(timer_link != NULL);
-
-        if (gf_timer_call_cancel(xl->ctx, lock->timer) < 0) {
-                /* It's too late to avoid the execution of the timer callback.
-                 * Since we need to be sure that the callback has access to all
-                 * needed resources, we cannot resume the execution of the
-                 * timer fop now. This will be done in the callback. */
-                timer_link = NULL;
-        } else {
-                /* The timer has been cancelled. The fop referenced by
-                 * timer_link holds the last reference. The caller is
-                 * responsible to release it when not needed anymore. */
-                ec_trace("UNLOCK_CANCELLED", timer_link->fop, "lock=%p", lock);
-        }
+    ec_lock_link_t *timer_link;
 
-        /* We have two options here:
-         *
-         * 1. The timer has been successfully cancelled.
-         *
-         *    This is the easiest case and we can continue with the currently
-         *    acquired lock.
-         *
-         * 2. The timer callback has already been fired.
-         *
-         *    In this case we have not been able to cancel the timer before
-         *    the timer callback has been fired, but we also know that
-         *    lock->timer != NULL. This means that the timer callback is still
-         *    trying to acquire the inode mutex that we currently own. We are
-         *    safe until we release it. In this case we can safely clear
-         *    lock->timer. This will cause that the timer callback does nothing
-         *    once it acquires the mutex.
-         */
-        lock->timer = NULL;
+    /* If we don't have any timer, there's nothing to cancel. */
+    if (lock->timer == NULL) {
+        return NULL;
+    }
+
+    /* We are trying to access a lock that has an unlock timer active.
+     * This means that the lock must be idle, i.e. no fop can be in the
+     * owner, waiting or frozen lists. It also means that the lock cannot
+     * have been marked as being released (this is done without timers).
+     * There should only be one owner reference, but it's possible that
+     * some fops are being prepared to use this lock. */
+    GF_ASSERT((lock->refs_owners == 1) && list_empty(&lock->owners) &&
+              list_empty(&lock->waiting));
+
+    /* We take the timer_link before cancelling the timer, since a
+     * successful cancellation will destroy it. It must not be NULL
+     * because it references the fop responsible for the delayed unlock
+     * that we are currently trying to cancel. */
+    timer_link = lock->timer->data;
+    GF_ASSERT(timer_link != NULL);
+
+    if (gf_timer_call_cancel(xl->ctx, lock->timer) < 0) {
+        /* It's too late to avoid the execution of the timer callback.
+         * Since we need to be sure that the callback has access to all
+         * needed resources, we cannot resume the execution of the
+         * timer fop now. This will be done in the callback. */
+        timer_link = NULL;
+    } else {
+        /* The timer has been cancelled. The fop referenced by
+         * timer_link holds the last reference. The caller is
+         * responsible to release it when not needed anymore. */
+        ec_trace("UNLOCK_CANCELLED", timer_link->fop, "lock=%p", lock);
+    }
 
-        return timer_link;
+    /* We have two options here:
+     *
+     * 1. The timer has been successfully cancelled.
+     *
+     *    This is the easiest case and we can continue with the currently
+     *    acquired lock.
+     *
+     * 2. The timer callback has already been fired.
+     *
+     *    In this case we have not been able to cancel the timer before
+     *    the timer callback has been fired, but we also know that
+     *    lock->timer != NULL. This means that the timer callback is still
+     *    trying to acquire the inode mutex that we currently own. We are
+     *    safe until we release it. In this case we can safely clear
+     *    lock->timer. This will cause that the timer callback does nothing
+     *    once it acquires the mutex.
+     */
+    lock->timer = NULL;
+
+    return timer_link;
 }
 
 static gf_boolean_t
@@ -1936,7 +2044,7 @@ ec_lock_assign_owner(ec_lock_link_t *link)
 
     /* Since the link has just been prepared but it's not active yet, the
      * refs_pending must be one at least (the ref owned by this link). */
-    GF_ASSERT (lock->refs_pending > 0);
+    GF_ASSERT(lock->refs_pending > 0);
     /* The link is not pending any more. It will be assigned to the owner,
      * waiting or frozen list. */
     lock->refs_pending--;
@@ -1969,7 +2077,7 @@ ec_lock_assign_owner(ec_lock_link_t *link)
          * owners, or waiters(to prevent starvation).
          * Otherwise we need to wait.
          */
-        if (!lock->acquired || ec_link_has_lock_conflict (link, _gf_true)) {
+        if (!lock->acquired || ec_link_has_lock_conflict(link, _gf_true)) {
             ec_trace("LOCK_QUEUE_WAIT", fop, "lock=%p", lock);
 
             list_add_tail(&link->wait_list, &lock->waiting);
@@ -1984,7 +2092,7 @@ ec_lock_assign_owner(ec_lock_link_t *link)
      * reference assigned to the timer fop. In this case we simply reuse it.
      * Otherwise we need to increase the number of owners. */
     if (timer_link == NULL) {
-            lock->refs_owners++;
+        lock->refs_owners++;
     }
 
     assigned = _gf_true;
@@ -2042,14 +2150,14 @@ ec_lock_next_owner(ec_lock_link_t *link, ec_cbk_data_t *cbk,
         /* If the fop fails on any of the good bricks, it is important to mark
          * it dirty and update versions right away. */
         if (link->update[0] || link->update[1]) {
-                if (lock->good_mask & ~(fop->good | fop->remaining)) {
-                        lock->release = _gf_true;
-                }
+            if (lock->good_mask & ~(fop->good | fop->remaining)) {
+                lock->release = _gf_true;
+            }
         }
     }
 
     if (fop->healing) {
-           lock->healing = fop->healing & (fop->good | fop->remaining);
+        lock->healing = fop->healing & (fop->good | fop->remaining);
     }
     ec_lock_update_good(lock, fop);
 
@@ -2060,7 +2168,8 @@ ec_lock_next_owner(ec_lock_link_t *link, ec_cbk_data_t *cbk,
     ec_lock_resume_shared(&list);
 }
 
-void ec_lock(ec_fop_data_t *fop)
+void
+ec_lock(ec_fop_data_t *fop)
 {
     ec_lock_link_t *link;
 
@@ -2068,7 +2177,7 @@ void ec_lock(ec_fop_data_t *fop)
      * Which can result in refs == 0 for fop leading to use after free in this
      * function when it calls ec_sleep so do ec_sleep at start and ec_resume at
      * the end of this function.*/
-    ec_sleep (fop);
+    ec_sleep(fop);
 
     while (fop->locked < fop->lock_count) {
         /* Since there are only up to 2 locks per fop, this xor will change
@@ -2138,17 +2247,16 @@ ec_lock_unfreeze(ec_lock_link_t *link)
     }
 }
 
-int32_t ec_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int32_t
+ec_unlocked(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, dict_t *xdata)
 {
     ec_fop_data_t *fop = cookie;
     ec_lock_link_t *link = fop->data;
 
     if (op_ret < 0) {
-        gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                EC_MSG_UNLOCK_FAILED,
-                "entry/inode unlocking failed (%s)",
-                ec_fop_name(link->fop->id));
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_UNLOCK_FAILED,
+               "entry/inode unlocking failed :(%s)", ec_msg_str(link->fop));
     } else {
         ec_trace("UNLOCKED", link->fop, "lock=%p", link->lock);
     }
@@ -2158,7 +2266,8 @@ int32_t ec_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
     return 0;
 }
 
-void ec_unlock_lock(ec_lock_link_t *link)
+void
+ec_unlock_lock(ec_lock_link_t *link)
 {
     ec_lock_t *lock;
     ec_fop_data_t *fop;
@@ -2184,10 +2293,27 @@ void ec_unlock_lock(ec_lock_link_t *link)
     }
 }
 
-int32_t ec_update_size_version_done(call_frame_t * frame, void * cookie,
-                                    xlator_t * this, int32_t op_ret,
-                                    int32_t op_errno, dict_t * xattr,
-                                    dict_t * xdata)
+void
+ec_inode_bad_inc(inode_t *inode, xlator_t *xl)
+{
+    ec_inode_t *ctx = NULL;
+
+    LOCK(&inode->lock);
+    {
+        ctx = __ec_inode_get(inode, xl);
+        if (ctx == NULL) {
+            goto unlock;
+        }
+        ctx->bad_version++;
+    }
+unlock:
+    UNLOCK(&inode->lock);
+}
+
+int32_t
+ec_update_size_version_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                            dict_t *xdata)
 {
     ec_fop_data_t *fop = cookie;
     ec_lock_link_t *link;
@@ -2199,9 +2325,15 @@ int32_t ec_update_size_version_done(call_frame_t * frame, void * cookie,
     ctx = lock->ctx;
 
     if (op_ret < 0) {
-        gf_msg(fop->xl->name, fop_log_level (fop->id, op_errno), op_errno,
+        if (link->lock->fd == NULL) {
+            ec_inode_bad_inc(link->lock->loc.inode, this);
+        } else {
+            ec_inode_bad_inc(link->lock->fd->inode, this);
+        }
+
+        gf_msg(fop->xl->name, fop_log_level(fop->id, op_errno), op_errno,
                EC_MSG_SIZE_VERS_UPDATE_FAIL,
-               "Failed to update version and size");
+               "Failed to update version and size. %s", ec_msg_str(fop));
     } else {
         fop->parent->good &= fop->good;
 
@@ -2238,15 +2370,14 @@ int32_t ec_update_size_version_done(call_frame_t * frame, void * cookie,
 }
 
 void
-ec_update_size_version(ec_lock_link_t *link, uint64_t *version,
-                       uint64_t size, uint64_t *dirty)
+ec_update_size_version(ec_lock_link_t *link, uint64_t *version, uint64_t size,
+                       uint64_t *dirty)
 {
     ec_fop_data_t *fop;
     ec_lock_t *lock;
     ec_inode_t *ctx;
     dict_t *dict = NULL;
-    uintptr_t   update_on = 0;
-
+    uintptr_t update_on = 0;
     int32_t err = -ENOMEM;
 
     fop = link->fop;
@@ -2283,11 +2414,10 @@ ec_update_size_version(ec_lock_link_t *link, uint64_t *version,
     }
 
     if (dirty[0] || dirty[1]) {
-            err = ec_dict_set_array(dict, EC_XATTR_DIRTY,
-                                    dirty, EC_VERSION_SIZE);
-            if (err != 0) {
-                goto out;
-            }
+        err = ec_dict_set_array(dict, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
+        if (err != 0) {
+            goto out;
+        }
     }
 
     /* If config information is not known, we request it now. */
@@ -2303,13 +2433,13 @@ ec_update_size_version(ec_lock_link_t *link, uint64_t *version,
     update_on = lock->good_mask | lock->healing;
 
     if (link->lock->fd == NULL) {
-            ec_xattrop(fop->frame, fop->xl, update_on, EC_MINIMUM_MIN,
-                       ec_update_size_version_done, link, &link->lock->loc,
-                       GF_XATTROP_ADD_ARRAY64, dict, NULL);
+        ec_xattrop(fop->frame, fop->xl, update_on, EC_MINIMUM_MIN,
+                   ec_update_size_version_done, link, &link->lock->loc,
+                   GF_XATTROP_ADD_ARRAY64, dict, NULL);
     } else {
-            ec_fxattrop(fop->frame, fop->xl, update_on, EC_MINIMUM_MIN,
-                       ec_update_size_version_done, link, link->lock->fd,
-                       GF_XATTROP_ADD_ARRAY64, dict, NULL);
+        ec_fxattrop(fop->frame, fop->xl, update_on, EC_MINIMUM_MIN,
+                    ec_update_size_version_done, link, link->lock->fd,
+                    GF_XATTROP_ADD_ARRAY64, dict, NULL);
     }
 
     fop->frame->root->uid = fop->uid;
@@ -2326,13 +2456,12 @@ out:
 
     ec_fop_set_error(fop, -err);
 
-    gf_msg (fop->xl->name, GF_LOG_ERROR, -err, EC_MSG_SIZE_VERS_UPDATE_FAIL,
-            "Unable to update version and size");
+    gf_msg(fop->xl->name, GF_LOG_ERROR, -err, EC_MSG_SIZE_VERS_UPDATE_FAIL,
+           "Unable to update version and size. %s", ec_msg_str(fop));
 
     if (lock->unlock_now) {
         ec_unlock_lock(fop->data);
     }
-
 }
 
 gf_boolean_t
@@ -2344,38 +2473,57 @@ ec_update_info(ec_lock_link_t *link)
     uint64_t dirty[2] = {0, 0};
     uint64_t size;
     ec_t *ec = NULL;
+    uintptr_t mask;
 
     lock = link->lock;
     ctx = lock->ctx;
     ec = link->fop->xl->private;
 
     /* pre_version[*] will be 0 if have_version is false */
-    version[0] = ctx->post_version[0] - ctx->pre_version[0];
-    version[1] = ctx->post_version[1] - ctx->pre_version[1];
+    version[EC_DATA_TXN] = ctx->post_version[EC_DATA_TXN] -
+                           ctx->pre_version[EC_DATA_TXN];
+    version[EC_METADATA_TXN] = ctx->post_version[EC_METADATA_TXN] -
+                               ctx->pre_version[EC_METADATA_TXN];
 
     size = ctx->post_size - ctx->pre_size;
     /* If we set the dirty flag for update fop, we have to unset it.
      * If fop has failed on some bricks, leave the dirty as marked. */
+
     if (lock->unlock_now) {
-            /* Ensure that nodes are up while doing final
-             * metadata update.*/
-            if (!(ec->node_mask & ~lock->good_mask) &&
-                !(ec->node_mask & ~ec->xl_up)) {
-                    if (ctx->dirty[0] != 0) {
-                        dirty[0] = -1;
-                    }
-                    if (ctx->dirty[1] != 0) {
-                        dirty[1] = -1;
-                    }
-            } else {
-                    link->optimistic_changelog = _gf_false;
-                    ec_set_dirty_flag (link, ctx, dirty);
+        if (version[EC_DATA_TXN]) {
+            /*A data fop will have difference in post and pre version
+             *and for data fop we send writes on healing bricks also */
+            mask = lock->good_mask | lock->healing;
+        } else {
+            mask = lock->good_mask;
+        }
+        /* Ensure that nodes are up while doing final
+         * metadata update.*/
+        if (!(ec->node_mask & ~(mask)) && !(ec->node_mask & ~ec->xl_up)) {
+            if (ctx->dirty[EC_DATA_TXN] != 0) {
+                dirty[EC_DATA_TXN] = -1;
+            }
+            if (ctx->dirty[EC_METADATA_TXN] != 0) {
+                dirty[EC_METADATA_TXN] = -1;
+            }
+            /*If everything is fine and we already
+             *have version xattr set on entry, there
+             *is no need to update version again*/
+            if (ctx->pre_version[EC_DATA_TXN]) {
+                version[EC_DATA_TXN] = 0;
+            }
+            if (ctx->pre_version[EC_METADATA_TXN]) {
+                version[EC_METADATA_TXN] = 0;
             }
-            memset(ctx->dirty, 0, sizeof(ctx->dirty));
+        } else {
+            link->optimistic_changelog = _gf_false;
+            ec_set_dirty_flag(link, ctx, dirty);
+        }
+        memset(ctx->dirty, 0, sizeof(ctx->dirty));
     }
 
-    if ((version[0] != 0) || (version[1] != 0) ||
-        (dirty[0] != 0) || (dirty[1] != 0)) {
+    if ((version[EC_DATA_TXN] != 0) || (version[EC_METADATA_TXN] != 0) ||
+        (dirty[EC_DATA_TXN] != 0) || (dirty[EC_METADATA_TXN] != 0)) {
         ec_update_size_version(link, version, size, dirty);
         return _gf_true;
     }
@@ -2406,185 +2554,192 @@ ec_unlock_now(ec_lock_link_t *link)
 void
 ec_lock_release(ec_t *ec, inode_t *inode)
 {
-        ec_lock_t *lock;
-        ec_inode_t *ctx;
-        ec_lock_link_t *timer_link = NULL;
+    ec_lock_t *lock;
+    ec_inode_t *ctx;
+    ec_lock_link_t *timer_link = NULL;
 
-        LOCK(&inode->lock);
+    LOCK(&inode->lock);
 
-        ctx = __ec_inode_get(inode, ec->xl);
-        if (ctx == NULL) {
-                goto done;
-        }
-        lock = ctx->inode_lock;
-        if ((lock == NULL) || !lock->acquired || lock->release) {
-                goto done;
-        }
+    ctx = __ec_inode_get(inode, ec->xl);
+    if (ctx == NULL) {
+        goto done;
+    }
+    lock = ctx->inode_lock;
+    if ((lock == NULL) || lock->release) {
+        goto done;
+    }
 
-        gf_msg_debug(ec->xl->name, 0,
-                     "Releasing inode %p due to lock contention", inode);
+    gf_msg_debug(ec->xl->name, 0, "Releasing inode %p due to lock contention",
+                 inode);
 
-        /* The lock is not marked to be released, so the frozen list should be
-         * empty. */
-        GF_ASSERT(list_empty(&lock->frozen));
+    if (!lock->acquired) {
+        /* This happens if some bricks already got the lock while inodelk is in
+         * progress.  Set release to true after lock is acquired*/
+        lock->contention = _gf_true;
+        goto done;
+    }
 
-        timer_link = ec_lock_timer_cancel(ec->xl, lock);
+    /* The lock is not marked to be released, so the frozen list should be
+     * empty. */
+    GF_ASSERT(list_empty(&lock->frozen));
 
-        /* We mark the lock to be released as soon as possible. */
-        lock->release = _gf_true;
+    timer_link = ec_lock_timer_cancel(ec->xl, lock);
+
+    /* We mark the lock to be released as soon as possible. */
+    lock->release = _gf_true;
 
 done:
-        UNLOCK(&inode->lock);
+    UNLOCK(&inode->lock);
 
-        /* If we have cancelled the timer, we need to start the unlock of the
-         * inode. If there was a timer but we have been unable to cancel it
-         * because it was just triggered, the timer callback will take care
-         * of releasing the inode. */
-        if (timer_link != NULL) {
-                ec_unlock_now(timer_link);
-        }
+    /* If we have cancelled the timer, we need to start the unlock of the
+     * inode. If there was a timer but we have been unable to cancel it
+     * because it was just triggered, the timer callback will take care
+     * of releasing the inode. */
+    if (timer_link != NULL) {
+        ec_unlock_now(timer_link);
+    }
 }
 
-void ec_unlock_timer_add(ec_lock_link_t *link);
+void
+ec_unlock_timer_add(ec_lock_link_t *link);
 
 void
 ec_unlock_timer_del(ec_lock_link_t *link)
 {
-        ec_lock_t *lock;
-        inode_t *inode;
-        gf_boolean_t now = _gf_false;
+    ec_lock_t *lock;
+    inode_t *inode;
+    gf_boolean_t now = _gf_false;
 
-        /* If we are here, it means that the timer has expired before having
-         * been cancelled. This guarantees that 'link' is still valid because
-         * the fop that contains it must be pending (if timer cancellation in
-         * ec_lock_assign_owner() fails, the fop is left sleeping).
+    /* If we are here, it means that the timer has expired before having
+     * been cancelled. This guarantees that 'link' is still valid because
+     * the fop that contains it must be pending (if timer cancellation in
+     * ec_lock_assign_owner() fails, the fop is left sleeping).
+     *
+     * At the same time, the fop still has a reference to the lock, so
+     * it must also be valid.
+     */
+    lock = link->lock;
+
+    /* 'lock' must have a valid inode since it can only be destroyed
+     * when the lock itself is destroyed, but we have a reference to the
+     * lock to avoid this.
+     */
+    inode = lock->loc.inode;
+
+    LOCK(&inode->lock);
+
+    if (lock->timer != NULL) {
+        ec_trace("UNLOCK_DELAYED", link->fop, "lock=%p", lock);
+
+        /* The unlock timer has expired without anyone cancelling it.
+         * This means that it shouldn't have any owner, and the waiting
+         * and frozen lists should be empty.  It must have only one
+         * owner reference, but there can be fops being prepared
+         * though.
+         * */
+        GF_ASSERT(!lock->release && (lock->refs_owners == 1) &&
+                  list_empty(&lock->owners) && list_empty(&lock->waiting) &&
+                  list_empty(&lock->frozen));
+
+        gf_timer_call_cancel(link->fop->xl->ctx, lock->timer);
+        lock->timer = NULL;
+
+        /* Any fop being processed from now on, will need to wait
+         * until the next unlock/lock cycle. */
+        lock->release = now = _gf_true;
+    }
+
+    UNLOCK(&inode->lock);
+
+    if (now) {
+        ec_unlock_now(link);
+    } else {
+        /* The timer has been cancelled just after firing it but before
+         * getting here. This means that another fop has used the lock
+         * and everything should be handled as if this callback were
+         * have not been executed. However we still have an owner
+         * reference.
          *
-         * At the same time, the fop still has a reference to the lock, so
-         * it must also be valid.
+         * We need to release our reference. If this is not the last
+         * reference (the most common case because another fop has
+         * taken another ref) we only need to decrement the counter.
+         * Otherwise we have been delayed enough so that the other fop
+         * has had time to acquire the reference, do its operation and
+         * release it. At the time of releasing it, the fop did found
+         * that the ref counter was > 1 (our reference), so the delayed
+         * unlock timer wasn't started. We need to start it again if we
+         * are the last reference.
+         *
+         * ec_unlock_timer_add() handles both cases.
          */
-        lock = link->lock;
+        ec_unlock_timer_add(link);
 
-        /* 'lock' must have a valid inode since it can only be destroyed
-         * when the lock itself is destroyed, but we have a reference to the
-         * lock to avoid this.
+        /* We need to resume the fop that was waiting for the delayed
+         * unlock.
          */
-        inode = lock->loc.inode;
-
-        LOCK(&inode->lock);
-
-        if (lock->timer != NULL) {
-                ec_trace("UNLOCK_DELAYED", link->fop, "lock=%p", lock);
-
-                /* The unlock timer has expired without anyone cancelling it.
-                 * This means that it shouldn't have any owner, and the waiting
-                 * and frozen lists should be empty.  It must have only one
-                 * owner reference, but there can be fops being prepared
-                 * though.
-                 * */
-                GF_ASSERT(!lock->release &&
-                          (lock->refs_owners == 1) &&
-                          list_empty(&lock->owners) &&
-                          list_empty(&lock->waiting) &&
-                          list_empty(&lock->frozen));
-
-                gf_timer_call_cancel(link->fop->xl->ctx, lock->timer);
-                lock->timer = NULL;
-
-                /* Any fop being processed from now on, will need to wait
-                 * until the next unlock/lock cycle. */
-                lock->release = now = _gf_true;
-        }
-
-        UNLOCK(&inode->lock);
+        ec_resume(link->fop, 0);
+    }
+}
 
-        if (now) {
-                ec_unlock_now(link);
-        } else {
-                /* The timer has been cancelled just after firing it but before
-                 * getting here. This means that another fop has used the lock
-                 * and everything should be handled as if this callback were
-                 * have not been executed. However we still have an owner
-                 * reference.
-                 *
-                 * We need to release our reference. If this is not the last
-                 * reference (the most common case because another fop has
-                 * taken another ref) we only need to decrement the counter.
-                 * Otherwise we have been delayed enough so that the other fop
-                 * has had time to acquire the reference, do its operation and
-                 * release it. At the time of releasing it, the fop did found
-                 * that the ref counter was > 1 (our reference), so the delayed
-                 * unlock timer wasn't started. We need to start it again if we
-                 * are the last reference.
-                 *
-                 * ec_unlock_timer_add() handles both cases.
-                 */
-                ec_unlock_timer_add(link);
-
-                /* We need to resume the fop that was waiting for the delayed
-                 * unlock.
-                 */
-                ec_resume(link->fop, 0);
-        }
-}
-
-void ec_unlock_timer_cbk(void *data)
-{
-        ec_unlock_timer_del(data);
+void
+ec_unlock_timer_cbk(void *data)
+{
+    ec_unlock_timer_del(data);
 }
 
 static gf_boolean_t
 ec_eager_lock_used(ec_t *ec, ec_fop_data_t *fop)
 {
-        /* Fops with no locks at this point mean that they are sent as sub-fops
-         * of other higher level fops. In this case we simply assume that the
-         * parent fop will take correct care of the eager lock. */
-        if (fop->lock_count == 0) {
-                return _gf_true;
-        }
+    /* Fops with no locks at this point mean that they are sent as sub-fops
+     * of other higher level fops. In this case we simply assume that the
+     * parent fop will take correct care of the eager lock. */
+    if (fop->lock_count == 0) {
+        return _gf_true;
+    }
 
-        /* We may have more than one lock, but this only happens in the rename
-         * fop, and both locks will reference an inode of the same type (a
-         * directory in this case), so we only need to check the first lock. */
-        if (fop->locks[0].lock->loc.inode->ia_type == IA_IFREG) {
-                return ec->eager_lock;
-        }
+    /* We may have more than one lock, but this only happens in the rename
+     * fop, and both locks will reference an inode of the same type (a
+     * directory in this case), so we only need to check the first lock. */
+    if (fop->locks[0].lock->loc.inode->ia_type == IA_IFREG) {
+        return ec->eager_lock;
+    }
 
-        return ec->other_eager_lock;
+    return ec->other_eager_lock;
 }
 
 static uint32_t
 ec_eager_lock_timeout(ec_t *ec, ec_lock_t *lock)
 {
-        if (lock->loc.inode->ia_type == IA_IFREG) {
-                return ec->eager_lock_timeout;
-        }
+    if (lock->loc.inode->ia_type == IA_IFREG) {
+        return ec->eager_lock_timeout;
+    }
 
-        return ec->other_eager_lock_timeout;
+    return ec->other_eager_lock_timeout;
 }
 
 static gf_boolean_t
 ec_lock_delay_create(ec_lock_link_t *link)
 {
-        struct timespec delay;
-        ec_fop_data_t *fop = link->fop;
-        ec_lock_t *lock = link->lock;
+    struct timespec delay;
+    ec_fop_data_t *fop = link->fop;
+    ec_lock_t *lock = link->lock;
 
-        delay.tv_sec = ec_eager_lock_timeout(fop->xl->private, lock);
-        delay.tv_nsec = 0;
-        lock->timer = gf_timer_call_after(fop->xl->ctx, delay,
-                                          ec_unlock_timer_cbk, link);
-        if (lock->timer == NULL) {
-                gf_msg(fop->xl->name, GF_LOG_WARNING, ENOMEM,
-                       EC_MSG_UNLOCK_DELAY_FAILED,
-                       "Unable to delay an unlock");
+    delay.tv_sec = ec_eager_lock_timeout(fop->xl->private, lock);
+    delay.tv_nsec = 0;
+    lock->timer = gf_timer_call_after(fop->xl->ctx, delay, ec_unlock_timer_cbk,
+                                      link);
+    if (lock->timer == NULL) {
+        gf_msg(fop->xl->name, GF_LOG_WARNING, ENOMEM,
+               EC_MSG_UNLOCK_DELAY_FAILED, "Unable to delay an unlock");
 
-                return _gf_false;
-        }
+        return _gf_false;
+    }
 
-        return _gf_true;
+    return _gf_true;
 }
 
-void ec_unlock_timer_add(ec_lock_link_t *link)
+void
+ec_unlock_timer_add(ec_lock_link_t *link)
 {
     ec_fop_data_t *fop = link->fop;
     ec_lock_t *lock = link->lock;
@@ -2674,7 +2829,8 @@ void ec_unlock_timer_add(ec_lock_link_t *link)
     }
 }
 
-void ec_unlock(ec_fop_data_t *fop)
+void
+ec_unlock(ec_fop_data_t *fop)
 {
     int32_t i;
 
@@ -2683,7 +2839,8 @@ void ec_unlock(ec_fop_data_t *fop)
     }
 }
 
-void ec_flush_size_version(ec_fop_data_t * fop)
+void
+ec_flush_size_version(ec_fop_data_t *fop)
 {
     GF_ASSERT(fop->lock_count == 1);
     ec_update_info(&fop->locks[0]);
@@ -2693,99 +2850,98 @@ static void
 ec_update_stripe(ec_t *ec, ec_stripe_list_t *stripe_cache, ec_stripe_t *stripe,
                  ec_fop_data_t *fop)
 {
-        off_t base;
-
-        /* On write fops, we only update existing fragments if the write has
-         * succeeded. Otherwise, we remove them from the cache. */
-        if ((fop->id == GF_FOP_WRITE) && (fop->answer != NULL) &&
-            (fop->answer->op_ret >= 0)) {
-                base = stripe->frag_offset - fop->frag_range.first;
-                base *= ec->fragments;
-
-                /* We check if the stripe offset falls inside the real region
-                 * modified by the write fop (a write request is allowed,
-                 * though uncommon, to write less bytes than requested). The
-                 * current write fop implementation doesn't allow partial
-                 * writes of fragments, so if there's no error, we are sure
-                 * that a full stripe has been completely modified or not
-                 * touched at all. The value of op_ret may not be a multiple
-                 * of the stripe size because it depends on the requested
-                 * size by the user, so we update the stripe if the write has
-                 * modified at least one byte (meaning ec has written the full
-                 * stripe). */
-                if (base < fop->answer->op_ret + fop->head) {
-                        memcpy(stripe->data, fop->vector[0].iov_base + base,
-                               ec->stripe_size);
-                        list_move_tail(&stripe->lru, &stripe_cache->lru);
-
-                        GF_ATOMIC_INC(ec->stats.stripe_cache.updates);
-                }
-        } else {
-                stripe->frag_offset = -1;
-                list_move (&stripe->lru, &stripe_cache->lru);
-
-                GF_ATOMIC_INC(ec->stats.stripe_cache.invals);
+    off_t base;
+
+    /* On write fops, we only update existing fragments if the write has
+     * succeeded. Otherwise, we remove them from the cache. */
+    if ((fop->id == GF_FOP_WRITE) && (fop->answer != NULL) &&
+        (fop->answer->op_ret >= 0)) {
+        base = stripe->frag_offset - fop->frag_range.first;
+        base *= ec->fragments;
+
+        /* We check if the stripe offset falls inside the real region
+         * modified by the write fop (a write request is allowed,
+         * though uncommon, to write less bytes than requested). The
+         * current write fop implementation doesn't allow partial
+         * writes of fragments, so if there's no error, we are sure
+         * that a full stripe has been completely modified or not
+         * touched at all. The value of op_ret may not be a multiple
+         * of the stripe size because it depends on the requested
+         * size by the user, so we update the stripe if the write has
+         * modified at least one byte (meaning ec has written the full
+         * stripe). */
+        if (base < fop->answer->op_ret + fop->head) {
+            memcpy(stripe->data, fop->vector[0].iov_base + base,
+                   ec->stripe_size);
+            list_move_tail(&stripe->lru, &stripe_cache->lru);
+
+            GF_ATOMIC_INC(ec->stats.stripe_cache.updates);
         }
+    } else {
+        stripe->frag_offset = -1;
+        list_move(&stripe->lru, &stripe_cache->lru);
+
+        GF_ATOMIC_INC(ec->stats.stripe_cache.invals);
+    }
 }
 
 static void
-ec_update_cached_stripes (ec_fop_data_t *fop)
-{
-        uint64_t          first;
-        uint64_t          last;
-        ec_stripe_t      *stripe       = NULL;
-        ec_inode_t       *ctx          = NULL;
-        ec_stripe_list_t *stripe_cache = NULL;
-        inode_t          *inode        = NULL;
-        struct list_head *temp;
-        struct list_head  sentinel;
-
-        first = fop->frag_range.first;
-        /* 'last' represents the first stripe not touched by the operation */
-        last = fop->frag_range.last;
-
-        /* If there are no modified stripes, we don't need to do anything
-         * else. */
-        if (last <= first) {
-                return;
-        }
+ec_update_cached_stripes(ec_fop_data_t *fop)
+{
+    uint64_t first;
+    uint64_t last;
+    ec_stripe_t *stripe = NULL;
+    ec_inode_t *ctx = NULL;
+    ec_stripe_list_t *stripe_cache = NULL;
+    inode_t *inode = NULL;
+    struct list_head *temp;
+    struct list_head sentinel;
+
+    first = fop->frag_range.first;
+    /* 'last' represents the first stripe not touched by the operation */
+    last = fop->frag_range.last;
+
+    /* If there are no modified stripes, we don't need to do anything
+     * else. */
+    if (last <= first) {
+        return;
+    }
 
-        if (!fop->use_fd) {
-                inode = fop->loc[0].inode;
-        } else {
-                inode = fop->fd->inode;
-        }
+    if (!fop->use_fd) {
+        inode = fop->loc[0].inode;
+    } else {
+        inode = fop->fd->inode;
+    }
 
-        LOCK(&inode->lock);
+    LOCK(&inode->lock);
 
-        ctx = __ec_inode_get (inode, fop->xl);
-        if (ctx == NULL) {
-                goto out;
-        }
-        stripe_cache = &ctx->stripe_cache;
-
-        /* Since we'll be moving elements of the list to the tail, we might
-         * end in an infinite loop. To avoid it, we insert a sentinel element
-         * into the list, so that it will be used to detect when we have
-         * traversed all existing elements once. */
-        list_add_tail(&sentinel, &stripe_cache->lru);
-        temp = stripe_cache->lru.next;
-        while (temp != &sentinel) {
-                stripe = list_entry(temp, ec_stripe_t, lru);
-                temp = temp->next;
-                if ((first <= stripe->frag_offset) &&
-                    (stripe->frag_offset < last)) {
-                        ec_update_stripe (fop->xl->private, stripe_cache,
-                                          stripe, fop);
-                }
+    ctx = __ec_inode_get(inode, fop->xl);
+    if (ctx == NULL) {
+        goto out;
+    }
+    stripe_cache = &ctx->stripe_cache;
+
+    /* Since we'll be moving elements of the list to the tail, we might
+     * end in an infinite loop. To avoid it, we insert a sentinel element
+     * into the list, so that it will be used to detect when we have
+     * traversed all existing elements once. */
+    list_add_tail(&sentinel, &stripe_cache->lru);
+    temp = stripe_cache->lru.next;
+    while (temp != &sentinel) {
+        stripe = list_entry(temp, ec_stripe_t, lru);
+        temp = temp->next;
+        if ((first <= stripe->frag_offset) && (stripe->frag_offset < last)) {
+            ec_update_stripe(fop->xl->private, stripe_cache, stripe, fop);
         }
-        list_del(&sentinel);
+    }
+    list_del(&sentinel);
 
 out:
-        UNLOCK(&inode->lock);
+    UNLOCK(&inode->lock);
 }
 
-void ec_lock_reuse(ec_fop_data_t *fop)
+void
+ec_lock_reuse(ec_fop_data_t *fop)
 {
     ec_cbk_data_t *cbk;
     ec_t *ec = NULL;
@@ -2796,13 +2952,13 @@ void ec_lock_reuse(ec_fop_data_t *fop)
 
     if (ec_eager_lock_used(ec, fop) && cbk != NULL) {
         if (cbk->xdata != NULL) {
-            if ((dict_get_int32(cbk->xdata, GLUSTERFS_INODELK_COUNT,
-                                &count) == 0) && (count > 1)) {
+            if ((dict_get_int32(cbk->xdata, GLUSTERFS_INODELK_COUNT, &count) ==
+                 0) &&
+                (count > 1)) {
                 release = _gf_true;
             }
             if (release) {
-                gf_msg_debug (fop->xl->name, 0,
-                       "Lock contention detected");
+                gf_msg_debug(fop->xl->name, 0, "Lock contention detected");
             }
         }
     } else {
@@ -2811,24 +2967,25 @@ void ec_lock_reuse(ec_fop_data_t *fop)
          * the lock. */
         release = _gf_true;
     }
-    ec_update_cached_stripes (fop);
+    ec_update_cached_stripes(fop);
 
     for (i = 0; i < fop->lock_count; i++) {
         ec_lock_next_owner(&fop->locks[i], cbk, release);
     }
 }
 
-void __ec_manager(ec_fop_data_t * fop, int32_t error)
+void
+__ec_manager(ec_fop_data_t *fop, int32_t error)
 {
     ec_t *ec = fop->xl->private;
 
     do {
         ec_trace("MANAGER", fop, "error=%d", error);
 
-        if (!ec_must_wind (fop)) {
-                if (ec->xl_up_count < ec->fragments) {
-                    error = ENOTCONN;
-                }
+        if (!ec_must_wind(fop)) {
+            if (ec->xl_up_count < ec->fragments) {
+                error = ENOTCONN;
+            }
         }
 
         if (error != 0) {
@@ -2854,22 +3011,32 @@ void __ec_manager(ec_fop_data_t * fop, int32_t error)
         fop->jobs = 1;
 
         fop->state = fop->handler(fop, fop->state);
-        GF_ASSERT (fop->state >= 0);
+        GF_ASSERT(fop->state >= 0);
 
         error = ec_check_complete(fop, __ec_manager);
     } while (error >= 0);
 }
 
-void ec_manager(ec_fop_data_t * fop, int32_t error)
+void
+ec_manager(ec_fop_data_t *fop, int32_t error)
 {
     GF_ASSERT(fop->jobs == 0);
     GF_ASSERT(fop->winds == 0);
     GF_ASSERT(fop->error == 0);
 
-    if (fop->state == EC_STATE_START)
-    {
+    if (fop->state == EC_STATE_START) {
         fop->state = EC_STATE_INIT;
     }
 
     __ec_manager(fop, error);
 }
+
+gf_boolean_t
+__ec_is_last_fop(ec_t *ec)
+{
+    if ((list_empty(&ec->pending_fops)) &&
+        (GF_ATOMIC_GET(ec->async_fop_count) == 0)) {
+        return _gf_true;
+    }
+    return _gf_false;
+}
diff --git a/xlators/cluster/ec/src/ec-common.h b/xlators/cluster/ec/src/ec-common.h
index 99e2f0653be..51493612ac6 100644
--- a/xlators/cluster/ec/src/ec-common.h
+++ b/xlators/cluster/ec/src/ec-common.h
@@ -11,81 +11,135 @@
 #ifndef __EC_COMMON_H__
 #define __EC_COMMON_H__
 
-#include "xlator.h"
-
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
 #include "ec-data.h"
 
-typedef enum {
-        EC_DATA_TXN,
-        EC_METADATA_TXN
-} ec_txn_t;
+typedef enum { EC_DATA_TXN, EC_METADATA_TXN } ec_txn_t;
 
-#define EC_FOP_HEAL     -1
-#define EC_FOP_FHEAL    -2
+#define EC_FOP_HEAL -1
+#define EC_FOP_FHEAL -2
 
 #define EC_CONFIG_VERSION 0
 
 #define EC_CONFIG_ALGORITHM 0
 
-#define EC_FLAG_LOCK_SHARED             0x0001
-
-#define EC_FLAG_WAITING_XATTROP         0x0001
-#define EC_FLAG_WAITING_DATA_DIRTY      0x0002
-#define EC_FLAG_WAITING_METADATA_DIRTY  0x0004
+#define EC_FLAG_LOCK_SHARED 0x0001
+
+#define QUORUM_CBK(fn, fop, frame, cookie, this, op_ret, op_errno, params...)  \
+    do {                                                                       \
+        ec_t *__ec = fop->xl->private;                                         \
+        int32_t __op_ret = 0;                                                  \
+        int32_t __op_errno = 0;                                                \
+        int32_t __success_count = gf_bits_count(fop->good);                    \
+                                                                               \
+        __op_ret = op_ret;                                                     \
+        __op_errno = op_errno;                                                 \
+        if (!fop->parent && frame &&                                           \
+            (GF_CLIENT_PID_SELF_HEALD != frame->root->pid) &&                  \
+            __ec->quorum_count && (__success_count < __ec->quorum_count) &&    \
+            op_ret >= 0) {                                                     \
+            __op_ret = -1;                                                     \
+            __op_errno = EIO;                                                  \
+            gf_msg(__ec->xl->name, GF_LOG_ERROR, 0,                            \
+                   EC_MSG_CHILDS_INSUFFICIENT,                                 \
+                   "Insufficient available children for this request "         \
+                   "(have %d, need %d). %s",                                   \
+                   __success_count, __ec->quorum_count, ec_msg_str(fop));      \
+        }                                                                      \
+        fn(frame, cookie, this, __op_ret, __op_errno, params);                 \
+    } while (0)
+
+enum _ec_xattrop_flags {
+    EC_FLAG_XATTROP,
+    EC_FLAG_DATA_DIRTY,
+    EC_FLAG_METADATA_DIRTY,
+
+    /* Add any new flag here, before EC_FLAG_MAX. The maximum number of
+     * flags that can be defined is 16. */
+
+    EC_FLAG_MAX
+};
+
+/* We keep two sets of flags. One to determine what's really providing the
+ * current xattrop and the other to know what the parent fop of the xattrop
+ * needs to proceed. It might happen that a fop needs some information that
+ * is being already requested by a previous fop. The two sets are stored
+ * contiguously. */
+
+#define EC_FLAG_NEEDS(_flag) (1 << (_flag))
+#define EC_FLAG_PROVIDES(_flag) (1 << ((_flag) + EC_FLAG_MAX))
+
+#define EC_NEEDED_FLAGS(_flags) ((_flags) & ((1 << EC_FLAG_MAX) - 1))
+
+#define EC_PROVIDED_FLAGS(_flags) EC_NEEDED_FLAGS((_flags) >> EC_FLAG_MAX)
+
+#define EC_FLAGS_HAVE(_flags, _flag) (((_flags) & (1 << (_flag))) != 0)
 
 #define EC_SELFHEAL_BIT 62
 
-#define EC_MINIMUM_ONE   -1
-#define EC_MINIMUM_MIN   -2
-#define EC_MINIMUM_ALL   -3
-
-#define EC_UPDATE_DATA   1
-#define EC_UPDATE_META   2
-#define EC_QUERY_INFO    4
-#define EC_INODE_SIZE    8
-
-#define EC_STATE_START                        0
-#define EC_STATE_END                          0
-#define EC_STATE_INIT                         1
-#define EC_STATE_LOCK                         2
-#define EC_STATE_DISPATCH                     3
-#define EC_STATE_PREPARE_ANSWER               4
-#define EC_STATE_REPORT                       5
-#define EC_STATE_LOCK_REUSE                   6
-#define EC_STATE_UNLOCK                       7
-
-#define EC_STATE_DELAYED_START              100
-
-#define EC_STATE_HEAL_ENTRY_LOOKUP          200
-#define EC_STATE_HEAL_ENTRY_PREPARE         201
-#define EC_STATE_HEAL_PRE_INODELK_LOCK      202
-#define EC_STATE_HEAL_PRE_INODE_LOOKUP      203
-#define EC_STATE_HEAL_XATTRIBUTES_REMOVE    204
-#define EC_STATE_HEAL_XATTRIBUTES_SET       205
-#define EC_STATE_HEAL_ATTRIBUTES            206
-#define EC_STATE_HEAL_OPEN                  207
-#define EC_STATE_HEAL_REOPEN_FD             208
-#define EC_STATE_HEAL_UNLOCK                209
-#define EC_STATE_HEAL_UNLOCK_ENTRY          210
-#define EC_STATE_HEAL_DATA_LOCK             211
-#define EC_STATE_HEAL_DATA_COPY             212
-#define EC_STATE_HEAL_DATA_UNLOCK           213
-#define EC_STATE_HEAL_POST_INODELK_LOCK     214
-#define EC_STATE_HEAL_POST_INODE_LOOKUP     215
-#define EC_STATE_HEAL_SETATTR               216
-#define EC_STATE_HEAL_POST_INODELK_UNLOCK   217
-#define EC_STATE_HEAL_DISPATCH              218
-
-gf_boolean_t ec_dispatch_one_retry (ec_fop_data_t *fop, ec_cbk_data_t **cbk);
-void ec_dispatch_next(ec_fop_data_t * fop, uint32_t idx);
-
-void ec_complete(ec_fop_data_t *fop);
-
-void ec_update_good(ec_fop_data_t *fop, uintptr_t good);
-
-void ec_fop_set_error(ec_fop_data_t *fop, int32_t error);
-
-void __ec_fop_set_error(ec_fop_data_t *fop, int32_t error);
+#define EC_MINIMUM_ONE (1 << 6)
+#define EC_MINIMUM_MIN (2 << 6)
+#define EC_MINIMUM_ALL (3 << 6)
+#define EC_FOP_NO_PROPAGATE_ERROR (1 << 8)
+#define EC_FOP_MINIMUM(_flags) ((_flags)&255)
+#define EC_FOP_FLAGS(_flags) ((_flags) & ~255)
+
+#define EC_UPDATE_DATA 1
+#define EC_UPDATE_META 2
+#define EC_QUERY_INFO 4
+#define EC_INODE_SIZE 8
+
+#define EC_STATE_START 0
+#define EC_STATE_END 0
+#define EC_STATE_INIT 1
+#define EC_STATE_LOCK 2
+#define EC_STATE_DISPATCH 3
+#define EC_STATE_PREPARE_ANSWER 4
+#define EC_STATE_REPORT 5
+#define EC_STATE_LOCK_REUSE 6
+#define EC_STATE_UNLOCK 7
+
+#define EC_STATE_DELAYED_START 100
+
+#define EC_STATE_HEAL_ENTRY_LOOKUP 200
+#define EC_STATE_HEAL_ENTRY_PREPARE 201
+#define EC_STATE_HEAL_PRE_INODELK_LOCK 202
+#define EC_STATE_HEAL_PRE_INODE_LOOKUP 203
+#define EC_STATE_HEAL_XATTRIBUTES_REMOVE 204
+#define EC_STATE_HEAL_XATTRIBUTES_SET 205
+#define EC_STATE_HEAL_ATTRIBUTES 206
+#define EC_STATE_HEAL_OPEN 207
+#define EC_STATE_HEAL_REOPEN_FD 208
+#define EC_STATE_HEAL_UNLOCK 209
+#define EC_STATE_HEAL_UNLOCK_ENTRY 210
+#define EC_STATE_HEAL_DATA_LOCK 211
+#define EC_STATE_HEAL_DATA_COPY 212
+#define EC_STATE_HEAL_DATA_UNLOCK 213
+#define EC_STATE_HEAL_POST_INODELK_LOCK 214
+#define EC_STATE_HEAL_POST_INODE_LOOKUP 215
+#define EC_STATE_HEAL_SETATTR 216
+#define EC_STATE_HEAL_POST_INODELK_UNLOCK 217
+#define EC_STATE_HEAL_DISPATCH 218
+
+/* Value to cover the full range of a file */
+#define EC_RANGE_FULL ((uint64_t)LLONG_MAX + 1)
+
+gf_boolean_t
+ec_dispatch_one_retry(ec_fop_data_t *fop, ec_cbk_data_t **cbk);
+void
+ec_dispatch_next(ec_fop_data_t *fop, uint32_t idx);
+
+void
+ec_complete(ec_fop_data_t *fop);
+
+void
+ec_update_good(ec_fop_data_t *fop, uintptr_t good);
+
+void
+ec_fop_set_error(ec_fop_data_t *fop, int32_t error);
+
+void
+__ec_fop_set_error(ec_fop_data_t *fop, int32_t error);
 
 ec_cbk_data_t *
 ec_fop_prepare_answer(ec_fop_data_t *fop, gf_boolean_t ro);
@@ -93,55 +147,88 @@ ec_fop_prepare_answer(ec_fop_data_t *fop, gf_boolean_t ro);
 gf_boolean_t
 ec_cbk_set_error(ec_cbk_data_t *cbk, int32_t error, gf_boolean_t ro);
 
-void ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, uint32_t flags,
-                           off_t fl_start, size_t fl_size);
-void ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc, loc_t *base,
-                                  uint32_t flags);
-void ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags,
-                        off_t fl_start, size_t fl_size);
-void ec_lock(ec_fop_data_t * fop);
-void ec_lock_reuse(ec_fop_data_t *fop);
-void ec_unlock(ec_fop_data_t * fop);
-void ec_lock_release(ec_t *ec, inode_t *inode);
-
-gf_boolean_t ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode,
-                               uint64_t *size);
-gf_boolean_t __ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode,
-                                 uint64_t *size);
-gf_boolean_t ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode,
-                               uint64_t size);
-gf_boolean_t __ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode,
-                                 uint64_t size);
-void ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode);
-
-void ec_flush_size_version(ec_fop_data_t * fop);
-
-void ec_dispatch_all(ec_fop_data_t * fop);
-void ec_dispatch_inc(ec_fop_data_t * fop);
-void ec_dispatch_min(ec_fop_data_t * fop);
-void ec_dispatch_one(ec_fop_data_t * fop);
-
-void ec_sleep(ec_fop_data_t *fop);
-void ec_resume(ec_fop_data_t * fop, int32_t error);
-void ec_resume_parent(ec_fop_data_t * fop, int32_t error);
-
-void ec_manager(ec_fop_data_t * fop, int32_t error);
-gf_boolean_t ec_is_recoverable_error (int32_t op_errno);
-void ec_handle_healers_done (ec_fop_data_t *fop);
+void
+ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, uint32_t flags,
+                      off_t fl_start, uint64_t fl_size);
+void
+ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc, loc_t *base,
+                             uint32_t flags);
+void
+ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags, off_t fl_start,
+                   uint64_t fl_size);
+void
+ec_lock(ec_fop_data_t *fop);
+void
+ec_lock_reuse(ec_fop_data_t *fop);
+void
+ec_unlock(ec_fop_data_t *fop);
+void
+ec_lock_release(ec_t *ec, inode_t *inode);
+
+gf_boolean_t
+ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size);
+gf_boolean_t
+__ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size);
+gf_boolean_t
+ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size);
+gf_boolean_t
+__ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size);
+void
+ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode);
+
+void
+ec_flush_size_version(ec_fop_data_t *fop);
+
+void
+ec_dispatch_all(ec_fop_data_t *fop);
+void
+ec_dispatch_inc(ec_fop_data_t *fop);
+void
+ec_dispatch_min(ec_fop_data_t *fop);
+void
+ec_dispatch_one(ec_fop_data_t *fop);
+
+void
+ec_succeed_all(ec_fop_data_t *fop);
+
+void
+ec_sleep(ec_fop_data_t *fop);
+void
+ec_resume(ec_fop_data_t *fop, int32_t error);
+void
+ec_resume_parent(ec_fop_data_t *fop);
+
+void
+ec_manager(ec_fop_data_t *fop, int32_t error);
+gf_boolean_t
+ec_is_recoverable_error(int32_t op_errno);
+void
+ec_handle_healers_done(ec_fop_data_t *fop);
 
 int32_t
-ec_heal_inspect (call_frame_t *frame, ec_t *ec,
-                 inode_t *inode, unsigned char *locked_on,
-                 gf_boolean_t self_locked, gf_boolean_t thorough,
-                 ec_heal_need_t *need_heal);
+ec_heal_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                unsigned char *locked_on, gf_boolean_t self_locked,
+                gf_boolean_t thorough, ec_heal_need_t *need_heal);
 int32_t
-ec_get_heal_info (xlator_t *this, loc_t *loc, dict_t **dict);
+ec_get_heal_info(xlator_t *this, loc_t *loc, dict_t **dict);
 
-int32_t ec_lock_unlocked(call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno,
-                         dict_t *xdata);
+int32_t
+ec_lock_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 void
-ec_update_fd_status (fd_t *fd, xlator_t *xl,
-                     int child_index, int32_t ret_status);
+ec_update_fd_status(fd_t *fd, xlator_t *xl, int child_index,
+                    int32_t ret_status);
+gf_boolean_t
+ec_is_entry_healing(ec_fop_data_t *fop);
+void
+ec_set_entry_healing(ec_fop_data_t *fop);
+void
+ec_reset_entry_healing(ec_fop_data_t *fop);
+char *
+ec_msg_str(ec_fop_data_t *fop);
+gf_boolean_t
+__ec_is_last_fop(ec_t *ec);
+void
+ec_lock_update_good(ec_lock_t *lock, ec_fop_data_t *fop);
 #endif /* __EC_COMMON_H__ */
diff --git a/xlators/cluster/ec/src/ec-data.c b/xlators/cluster/ec/src/ec-data.c
index a869735794e..06388833546 100644
--- a/xlators/cluster/ec/src/ec-data.c
+++ b/xlators/cluster/ec/src/ec-data.c
@@ -8,53 +8,48 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "ec-mem-types.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-data.h"
 #include "ec-messages.h"
 
-ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this,
-                                     ec_fop_data_t * fop, int32_t id,
-                                     int32_t idx, int32_t op_ret,
-                                     int32_t op_errno)
+ec_cbk_data_t *
+ec_cbk_data_allocate(call_frame_t *frame, xlator_t *this, ec_fop_data_t *fop,
+                     int32_t id, int32_t idx, int32_t op_ret, int32_t op_errno)
 {
-    ec_cbk_data_t * cbk;
-    ec_t * ec = this->private;
+    ec_cbk_data_t *cbk;
+    ec_t *ec = this->private;
 
-    if (fop->xl != this)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_XLATOR_MISMATCH, "Mismatching xlators between request "
-                "and answer (req=%s, ans=%s).", fop->xl->name, this->name);
+    if (fop->xl != this) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_XLATOR_MISMATCH,
+               "Mismatching xlators between request "
+               "and answer (req=%s, ans=%s).",
+               fop->xl->name, this->name);
 
         return NULL;
     }
-    if (fop->frame != frame)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_FRAME_MISMATCH, "Mismatching frames between request "
-                                         "and answer (req=%p, ans=%p).",
-                                         fop->frame, frame);
+    if (fop->frame != frame) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_FRAME_MISMATCH,
+               "Mismatching frames between request "
+               "and answer (req=%p, ans=%p).",
+               fop->frame, frame);
 
         return NULL;
     }
-    if (fop->id != id)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_FOP_MISMATCH, "Mismatching fops between request "
-                                         "and answer (req=%d, ans=%d).",
-                                         fop->id, id);
+    if (fop->id != id) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_FOP_MISMATCH,
+               "Mismatching fops between request "
+               "and answer (req=%d, ans=%d).",
+               fop->id, id);
 
         return NULL;
     }
 
     cbk = mem_get0(ec->cbk_pool);
-    if (cbk == NULL)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                EC_MSG_NO_MEMORY, "Failed to allocate memory for an "
-                                         "answer.");
+    if (cbk == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to allocate memory for an "
+               "answer.");
         return NULL;
     }
 
@@ -64,7 +59,7 @@ ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this,
     cbk->count = 1;
     cbk->op_ret = op_ret;
     cbk->op_errno = op_errno;
-    INIT_LIST_HEAD (&cbk->entries.list);
+    INIT_LIST_HEAD(&cbk->entries.list);
 
     LOCK(&fop->lock);
 
@@ -75,50 +70,45 @@ ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this,
     return cbk;
 }
 
-void ec_cbk_data_destroy(ec_cbk_data_t * cbk)
+void
+ec_cbk_data_destroy(ec_cbk_data_t *cbk)
 {
-    if (cbk->xdata != NULL)
-    {
+    if (cbk->xdata != NULL) {
         dict_unref(cbk->xdata);
     }
-    if (cbk->dict != NULL)
-    {
+    if (cbk->dict != NULL) {
         dict_unref(cbk->dict);
     }
-    if (cbk->inode != NULL)
-    {
+    if (cbk->inode != NULL) {
         inode_unref(cbk->inode);
     }
-    if (cbk->fd != NULL)
-    {
+    if (cbk->fd != NULL) {
         fd_unref(cbk->fd);
     }
-    if (cbk->buffers != NULL)
-    {
+    if (cbk->buffers != NULL) {
         iobref_unref(cbk->buffers);
     }
     GF_FREE(cbk->vector);
-    gf_dirent_free (&cbk->entries);
-    GF_FREE (cbk->str);
+    gf_dirent_free(&cbk->entries);
+    GF_FREE(cbk->str);
 
     mem_put(cbk);
 }
 
-ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
-                                     int32_t id, uint32_t flags,
-                                     uintptr_t target, int32_t minimum,
-                                     ec_wind_f wind, ec_handler_f handler,
-                                     ec_cbk_t cbks, void * data)
+ec_fop_data_t *
+ec_fop_data_allocate(call_frame_t *frame, xlator_t *this, int32_t id,
+                     uint32_t flags, uintptr_t target, uint32_t fop_flags,
+                     ec_wind_f wind, ec_handler_f handler, ec_cbk_t cbks,
+                     void *data)
 {
-    ec_fop_data_t * fop, * parent;
-    ec_t * ec = this->private;
+    ec_fop_data_t *fop, *parent;
+    ec_t *ec = this->private;
 
     fop = mem_get0(ec->fop_pool);
-    if (fop == NULL)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                EC_MSG_NO_MEMORY, "Failed to allocate memory for a "
-                                         "request.");
+    if (fop == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to allocate memory for a "
+               "request.");
 
         return NULL;
     }
@@ -142,19 +132,15 @@ ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
      * TODO: minimize usage of private frames. Reuse req_frame as much as
      *       possible.
      */
-    if (frame != NULL)
-    {
+    if (frame != NULL) {
         fop->frame = copy_frame(frame);
-    }
-    else
-    {
+    } else {
         fop->frame = create_frame(this, this->ctx->pool);
     }
-    if (fop->frame == NULL)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                EC_MSG_NO_MEMORY, "Failed to create a private frame "
-                                         "for a request");
+    if (fop->frame == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to create a private frame "
+               "for a request");
 
         mem_put(fop);
 
@@ -164,7 +150,8 @@ ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
     fop->refs = 1;
 
     fop->flags = flags;
-    fop->minimum = minimum;
+    fop->minimum = EC_FOP_MINIMUM(fop_flags);
+    fop->fop_flags = EC_FOP_FLAGS(fop_flags);
     fop->mask = target;
 
     fop->wind = wind;
@@ -179,11 +166,9 @@ ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
 
     fop->frame->local = fop;
 
-    if (frame != NULL)
-    {
+    if (frame != NULL) {
         parent = frame->local;
-        if (parent != NULL)
-        {
+        if (parent != NULL) {
             ec_sleep(parent);
         }
 
@@ -199,7 +184,8 @@ ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
     return fop;
 }
 
-void ec_fop_data_acquire(ec_fop_data_t * fop)
+void
+ec_fop_data_acquire(ec_fop_data_t *fop)
 {
     LOCK(&fop->lock);
 
@@ -211,36 +197,40 @@ void ec_fop_data_acquire(ec_fop_data_t * fop)
 }
 
 static void
-ec_handle_last_pending_fop_completion (ec_fop_data_t *fop, gf_boolean_t *notify)
+ec_handle_last_pending_fop_completion(ec_fop_data_t *fop, gf_boolean_t *notify)
 {
-        ec_t *ec = fop->xl->private;
-
-        if (!list_empty (&fop->pending_list)) {
-                LOCK(&ec->lock);
-                {
-                        list_del_init (&fop->pending_list);
-                        *notify = list_empty (&ec->pending_fops);
-                }
-                UNLOCK(&ec->lock);
+    ec_t *ec = fop->xl->private;
+
+    *notify = _gf_false;
+
+    if (!list_empty(&fop->pending_list)) {
+        LOCK(&ec->lock);
+        {
+            list_del_init(&fop->pending_list);
+            *notify = __ec_is_last_fop(ec);
         }
+        UNLOCK(&ec->lock);
+    }
 }
 
 void
 ec_fop_cleanup(ec_fop_data_t *fop)
 {
-        ec_cbk_data_t *cbk, *tmp;
+    ec_cbk_data_t *cbk, *tmp;
 
-        list_for_each_entry_safe(cbk, tmp, &fop->answer_list, answer_list) {
-            list_del_init(&cbk->answer_list);
+    list_for_each_entry_safe(cbk, tmp, &fop->answer_list, answer_list)
+    {
+        list_del_init(&cbk->answer_list);
 
-            ec_cbk_data_destroy(cbk);
-        }
-        INIT_LIST_HEAD(&fop->cbk_list);
+        ec_cbk_data_destroy(cbk);
+    }
+    INIT_LIST_HEAD(&fop->cbk_list);
 
-        fop->answer = NULL;
+    fop->answer = NULL;
 }
 
-void ec_fop_data_release(ec_fop_data_t * fop)
+void
+ec_fop_data_release(ec_fop_data_t *fop)
 {
     ec_t *ec = NULL;
     int32_t refs;
@@ -250,36 +240,30 @@ void ec_fop_data_release(ec_fop_data_t * fop)
 
     ec_trace("RELEASE", fop, "");
 
-    GF_ASSERT (fop->refs > 0);
+    GF_ASSERT(fop->refs > 0);
     refs = --fop->refs;
 
     UNLOCK(&fop->lock);
 
-    if (refs == 0)
-    {
+    if (refs == 0) {
         fop->frame->local = NULL;
         STACK_DESTROY(fop->frame->root);
 
         LOCK_DESTROY(&fop->lock);
 
-        if (fop->xdata != NULL)
-        {
+        if (fop->xdata != NULL) {
             dict_unref(fop->xdata);
         }
-        if (fop->dict != NULL)
-        {
+        if (fop->dict != NULL) {
             dict_unref(fop->dict);
         }
-        if (fop->inode != NULL)
-        {
+        if (fop->inode != NULL) {
             inode_unref(fop->inode);
         }
-        if (fop->fd != NULL)
-        {
+        if (fop->fd != NULL) {
             fd_unref(fop->fd);
         }
-        if (fop->buffers != NULL)
-        {
+        if (fop->buffers != NULL) {
             iobref_unref(fop->buffers);
         }
         GF_FREE(fop->vector);
@@ -287,14 +271,15 @@ void ec_fop_data_release(ec_fop_data_t * fop)
         GF_FREE(fop->str[1]);
         loc_wipe(&fop->loc[0]);
         loc_wipe(&fop->loc[1]);
+        GF_FREE(fop->errstr);
 
-        ec_resume_parent(fop, fop->error);
+        ec_resume_parent(fop);
 
         ec_fop_cleanup(fop);
 
         ec = fop->xl->private;
-        ec_handle_last_pending_fop_completion (fop, &notify);
-        ec_handle_healers_done (fop);
+        ec_handle_last_pending_fop_completion(fop, &notify);
+        ec_handle_healers_done(fop);
         mem_put(fop);
         if (notify) {
             ec_pending_fops_completed(ec);
diff --git a/xlators/cluster/ec/src/ec-data.h b/xlators/cluster/ec/src/ec-data.h
index 965bc1e25ef..c8a74ffe1ed 100644
--- a/xlators/cluster/ec/src/ec-data.h
+++ b/xlators/cluster/ec/src/ec-data.h
@@ -13,20 +13,23 @@
 
 #include "ec-types.h"
 
-ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this,
-                                     ec_fop_data_t * fop, int32_t id,
-                                     int32_t idx, int32_t op_ret,
-                                     int32_t op_errno);
-ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
-                                     int32_t id, uint32_t flags,
-                                     uintptr_t target, int32_t minimum,
-                                     ec_wind_f wind, ec_handler_f handler,
-                                     ec_cbk_t cbks, void * data);
-void ec_fop_data_acquire(ec_fop_data_t * fop);
-void ec_fop_data_release(ec_fop_data_t * fop);
+ec_cbk_data_t *
+ec_cbk_data_allocate(call_frame_t *frame, xlator_t *this, ec_fop_data_t *fop,
+                     int32_t id, int32_t idx, int32_t op_ret, int32_t op_errno);
+ec_fop_data_t *
+ec_fop_data_allocate(call_frame_t *frame, xlator_t *this, int32_t id,
+                     uint32_t flags, uintptr_t target, uint32_t fop_flags,
+                     ec_wind_f wind, ec_handler_f handler, ec_cbk_t cbks,
+                     void *data);
+void
+ec_fop_data_acquire(ec_fop_data_t *fop);
+void
+ec_fop_data_release(ec_fop_data_t *fop);
 
-void ec_fop_cleanup(ec_fop_data_t *fop);
+void
+ec_fop_cleanup(ec_fop_data_t *fop);
 
-void ec_pending_fops_completed(ec_t *ec);
+void
+ec_pending_fops_completed(ec_t *ec);
 
 #endif /* __EC_DATA_H__ */
diff --git a/xlators/cluster/ec/src/ec-dir-read.c b/xlators/cluster/ec/src/ec-dir-read.c
index b44bb4239b1..f71dcfac293 100644
--- a/xlators/cluster/ec/src/ec-dir-read.c
+++ b/xlators/cluster/ec/src/ec-dir-read.c
@@ -8,15 +8,11 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
 #include "ec.h"
 #include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
-#include "ec-method.h"
 #include "ec-fops.h"
 
 /****************************************************************
@@ -25,15 +21,14 @@
  *
  ***************************************************************/
 
-int32_t ec_combine_opendir(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                           ec_cbk_data_t * src)
+int32_t
+ec_combine_opendir(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (dst->fd != src->fd)
-    {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_FD_MISMATCH, "Mismatching fd in answers "
-                "of 'GF_FOP_OPENDIR': %p <-> %p",
-                dst->fd, src->fd);
+    if (dst->fd != src->fd) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_FD_MISMATCH,
+               "Mismatching fd in answers "
+               "of 'GF_FOP_OPENDIR': %p <-> %p",
+               dst->fd, src->fd);
 
         return 0;
     }
@@ -41,12 +36,12 @@ int32_t ec_combine_opendir(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_opendir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, fd_t * fd,
-                       dict_t * xdata)
+int32_t
+ec_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -56,36 +51,31 @@ int32_t ec_opendir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_OPENDIR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (fd != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (fd != NULL) {
                 cbk->fd = fd_ref(fd);
-                if (cbk->fd == NULL)
-                {
-                    gf_msg (this->name, GF_LOG_ERROR, 0,
-                            EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                            "file descriptor.");
+                if (cbk->fd == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           EC_MSG_FILE_DESC_REF_FAIL,
+                           "Failed to reference a "
+                           "file descriptor.");
 
                     goto out;
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -93,19 +83,19 @@ int32_t ec_opendir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
         ec_combine(cbk, ec_combine_opendir);
 
-        ec_update_fd_status (fd, this, idx, op_ret);
+        ec_update_fd_status(fd, this, idx, op_ret);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_opendir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_opendir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -114,14 +104,14 @@ void ec_wind_opendir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->fd, fop->xdata);
 }
 
-int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_opendir(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
     ec_fd_t *ctx;
     int32_t err;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             LOCK(&fop->fd->lock);
 
@@ -133,13 +123,15 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
 
                 return EC_STATE_REPORT;
             }
-            err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]);
-            if (err != 0) {
-                UNLOCK(&fop->fd->lock);
+            if (!ctx->loc.inode) {
+                err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]);
+                if (err != 0) {
+                    UNLOCK(&fop->fd->lock);
 
-                fop->error = -err;
+                    fop->error = -err;
 
-                return EC_STATE_REPORT;
+                    return EC_STATE_REPORT;
+                }
             }
 
             UNLOCK(&fop->fd->lock);
@@ -148,7 +140,7 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
 
         case EC_STATE_LOCK:
             ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
-                                  LLONG_MAX);
+                                  EC_RANGE_FULL);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -182,8 +174,7 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.opendir != NULL)
-            {
+            if (fop->cbks.opendir != NULL) {
                 fop->cbks.opendir(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                   cbk->op_errno, cbk->fd, cbk->xdata);
             }
@@ -197,8 +188,7 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.opendir != NULL)
-            {
+            if (fop->cbks.opendir != NULL) {
                 fop->cbks.opendir(fop->req_frame, fop, fop->xl, -1, fop->error,
                                   NULL, NULL);
             }
@@ -218,40 +208,39 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_opendir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_opendir_cbk_t func, void * data,
-                loc_t * loc, fd_t * fd, dict_t * xdata)
+void
+ec_opendir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_opendir_cbk_t func, void *data, loc_t *loc,
+           fd_t *fd, dict_t *xdata)
 {
-    ec_cbk_t callback = { .opendir = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.opendir = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(OPENDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(OPENDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_OPENDIR,
-                               EC_FLAG_LOCK_SHARED, target, minimum,
-                               ec_wind_opendir, ec_manager_opendir, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_OPENDIR, EC_FLAG_LOCK_SHARED,
+                               target, fop_flags, ec_wind_opendir,
+                               ec_manager_opendir, callback, data);
     if (fop == NULL) {
         goto out;
     }
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -259,9 +248,9 @@ void ec_opendir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -269,9 +258,9 @@ void ec_opendir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -289,50 +278,49 @@ out:
 
 /* Returns -1 if client_id is invalid else index of child subvol in xl_list */
 int
-ec_deitransform (xlator_t *this, off_t offset)
+ec_deitransform(xlator_t *this, off_t offset)
 {
-        int  idx       = -1;
-        int  client_id = -1;
-        ec_t *ec       = this->private;
-        char id[32]    = {0};
-        int err;
-
-        client_id = gf_deitransform (this, offset);
-        sprintf (id, "%d", client_id);
-        err = dict_get_int32 (ec->leaf_to_subvolid, id, &idx);
-        if (err < 0) {
-                idx = err;
-                goto out;
-        }
+    int idx = -1;
+    int client_id = -1;
+    ec_t *ec = this->private;
+    char id[32] = {0};
+    int err;
+
+    client_id = gf_deitransform(this, offset);
+    sprintf(id, "%d", client_id);
+    err = dict_get_int32(ec->leaf_to_subvolid, id, &idx);
+    if (err < 0) {
+        idx = err;
+        goto out;
+    }
 
 out:
-        if (idx < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        EC_MSG_INVALID_REQUEST,
-                        "Invalid index %d in readdirp request", client_id);
-                idx = -EINVAL;
-        }
-        return idx;
+    if (idx < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_REQUEST,
+               "Invalid index %d in readdirp request", client_id);
+        idx = -EINVAL;
+    }
+    return idx;
 }
 
 /* FOP: readdir */
 
-void ec_adjust_readdirp (ec_t *ec, int32_t idx, gf_dirent_t *entries)
+void
+ec_adjust_readdirp(ec_t *ec, int32_t idx, gf_dirent_t *entries)
 {
-    gf_dirent_t * entry;
+    gf_dirent_t *entry;
 
     list_for_each_entry(entry, &entries->list, list)
     {
         if (!entry->inode)
-                continue;
+            continue;
 
-        if (entry->d_stat.ia_type == IA_IFREG)
-        {
+        if (entry->d_stat.ia_type == IA_IFREG) {
             if ((entry->dict == NULL) ||
                 (ec_dict_del_number(entry->dict, EC_XATTR_SIZE,
                                     &entry->d_stat.ia_size) != 0)) {
-                    inode_unref (entry->inode);
-                    entry->inode = NULL;
+                inode_unref(entry->inode);
+                entry->inode = NULL;
             } else {
                 ec_iatt_rebuild(ec, &entry->d_stat, 1, 1);
             }
@@ -341,9 +329,9 @@ void ec_adjust_readdirp (ec_t *ec, int32_t idx, gf_dirent_t *entries)
 }
 
 int32_t
-ec_common_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       gf_dirent_t *entries, dict_t *xdata)
+ec_common_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                      dict_t *xdata)
 {
     ec_fop_data_t *fop = NULL;
     ec_cbk_data_t *cbk = NULL;
@@ -356,30 +344,29 @@ ec_common_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-    cbk = ec_cbk_data_allocate (frame, this, fop, fop->id,
-                                idx, op_ret, op_errno);
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
+                               op_errno);
     if (cbk) {
         if (xdata)
-                cbk->xdata = dict_ref (xdata);
+            cbk->xdata = dict_ref(xdata);
         if (cbk->op_ret >= 0)
-                list_splice_init (&entries->list,
-                                  &cbk->entries.list);
-        ec_combine (cbk, NULL);
+            list_splice_init(&entries->list, &cbk->entries.list);
+        ec_combine(cbk, NULL);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_readdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_readdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -388,60 +375,66 @@ void ec_wind_readdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->size, fop->offset, fop->xdata);
 }
 
-int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_readdir(ec_fop_data_t *fop, int32_t state)
 {
     ec_fd_t *ctx = NULL;
     ec_cbk_data_t *cbk = NULL;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             /* Return error if opendir has not been successfully called on
              * any subvolume. */
             ctx = ec_fd_get(fop->fd, fop->xl);
-            if ((ctx == NULL) || (ctx->open == 0)) {
-                fop->error = EINVAL;
+            if (ctx == NULL) {
+                fop->error = ENOMEM;
+            } else if (ctx->open == 0) {
+                fop->error = EBADFD;
+            }
 
+            if (fop->error) {
+                gf_msg(fop->xl->name, GF_LOG_ERROR, fop->error,
+                       EC_MSG_INVALID_REQUEST, "EC is not winding readdir: %s",
+                       ec_msg_str(fop));
                 return EC_STATE_REPORT;
             }
 
             if (fop->id == GF_FOP_READDIRP) {
-                    int32_t err;
+                int32_t err;
 
+                if (fop->xdata == NULL) {
+                    fop->xdata = dict_new();
                     if (fop->xdata == NULL) {
-                        fop->xdata = dict_new();
-                        if (fop->xdata == NULL) {
-                            fop->error = ENOMEM;
+                        fop->error = ENOMEM;
 
-                            return EC_STATE_REPORT;
-                        }
+                        return EC_STATE_REPORT;
                     }
+                }
 
-                    err = dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0);
-                    if (err != 0) {
-                        fop->error = -err;
+                err = dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0);
+                if (err != 0) {
+                    fop->error = -err;
 
-                        return EC_STATE_REPORT;
-                    }
+                    return EC_STATE_REPORT;
+                }
             }
 
-            if (fop->offset != 0)
-            {
-            /* Non-zero offset is irrecoverable error as the offset may not be
-             * valid on other bricks*/
+            if (fop->offset != 0) {
+                /* Non-zero offset is irrecoverable error as the offset may not
+                 * be valid on other bricks*/
                 int32_t idx = -1;
 
-                idx = ec_deitransform (fop->xl, fop->offset);
+                idx = ec_deitransform(fop->xl, fop->offset);
 
                 if (idx < 0) {
-                        fop->error = -idx;
-                        return EC_STATE_REPORT;
+                    fop->error = -idx;
+                    return EC_STATE_REPORT;
                 }
                 fop->mask &= 1ULL << idx;
             } else {
-                    ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0,
-                                       LLONG_MAX);
-                    ec_lock(fop);
+                ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0,
+                                   EC_RANGE_FULL);
+                ec_lock(fop);
             }
 
             return EC_STATE_DISPATCH;
@@ -458,14 +451,14 @@ int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state)
 
             if ((cbk != NULL) && (cbk->op_ret > 0) &&
                 (fop->id == GF_FOP_READDIRP)) {
-                ec_adjust_readdirp (fop->xl->private, cbk->idx, &cbk->entries);
+                ec_adjust_readdirp(fop->xl->private, cbk->idx, &cbk->entries);
             }
 
             return EC_STATE_REPORT;
 
         case EC_STATE_REPORT:
             cbk = fop->answer;
-            GF_ASSERT (cbk);
+            GF_ASSERT(cbk);
             if (fop->id == GF_FOP_READDIR) {
                 if (fop->cbks.readdir != NULL) {
                     fop->cbks.readdir(fop->req_frame, fop, fop->xl, cbk->op_ret,
@@ -479,9 +472,9 @@ int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state)
                 }
             }
             if (fop->offset == 0)
-                    return EC_STATE_LOCK_REUSE;
+                return EC_STATE_LOCK_REUSE;
             else
-                    return EC_STATE_END;
+                return EC_STATE_END;
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
@@ -500,50 +493,49 @@ int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state)
                 }
             }
             if (fop->offset == 0)
-                    return EC_STATE_LOCK_REUSE;
+                return EC_STATE_LOCK_REUSE;
             else
-                    return EC_STATE_END;
+                return EC_STATE_END;
 
         case -EC_STATE_LOCK_REUSE:
         case EC_STATE_LOCK_REUSE:
-            GF_ASSERT (fop->offset == 0);
+            GF_ASSERT(fop->offset == 0);
             ec_lock_reuse(fop);
 
             return EC_STATE_UNLOCK;
 
         case -EC_STATE_UNLOCK:
         case EC_STATE_UNLOCK:
-            GF_ASSERT (fop->offset == 0);
+            GF_ASSERT(fop->offset == 0);
             ec_unlock(fop);
 
             return EC_STATE_END;
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_readdir_cbk_t func, void * data,
-                fd_t * fd, size_t size, off_t offset, dict_t * xdata)
+void
+ec_readdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_readdir_cbk_t func, void *data, fd_t *fd,
+           size_t size, off_t offset, dict_t *xdata)
 {
-    ec_cbk_t callback = { .readdir = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.readdir = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(READDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(READDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_READDIR,
-                               EC_FLAG_LOCK_SHARED, target, minimum,
-                               ec_wind_readdir, ec_manager_readdir, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_READDIR, EC_FLAG_LOCK_SHARED,
+                               target, fop_flags, ec_wind_readdir,
+                               ec_manager_readdir, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -556,9 +548,9 @@ void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -566,9 +558,9 @@ void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -586,7 +578,8 @@ out:
 
 /* FOP: readdirp */
 
-void ec_wind_readdirp(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_readdirp(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -595,24 +588,24 @@ void ec_wind_readdirp(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->size, fop->offset, fop->xdata);
 }
 
-void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_readdirp_cbk_t func, void * data,
-                 fd_t * fd, size_t size, off_t offset, dict_t * xdata)
+void
+ec_readdirp(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_readdirp_cbk_t func, void *data, fd_t *fd,
+            size_t size, off_t offset, dict_t *xdata)
 {
-    ec_cbk_t callback = { .readdirp = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.readdirp = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(READDIRP) %p", frame);
+    gf_msg_trace("ec", 0, "EC(READDIRP) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_READDIRP,
-                               EC_FLAG_LOCK_SHARED, target, minimum,
-                               ec_wind_readdirp, ec_manager_readdir, callback,
-                               data);
+    fop = ec_fop_data_allocate(
+        frame, this, GF_FOP_READDIRP, EC_FLAG_LOCK_SHARED, target, fop_flags,
+        ec_wind_readdirp, ec_manager_readdir, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -625,9 +618,9 @@ void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -635,9 +628,9 @@ void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
diff --git a/xlators/cluster/ec/src/ec-dir-write.c b/xlators/cluster/ec/src/ec-dir-write.c
index 7779d4849f3..53d27d895c3 100644
--- a/xlators/cluster/ec/src/ec-dir-write.c
+++ b/xlators/cluster/ec/src/ec-dir-write.c
@@ -8,9 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
 #include "ec.h"
 #include "ec-messages.h"
 #include "ec-helpers.h"
@@ -20,76 +17,75 @@
 #include "ec-fops.h"
 
 int
-ec_dir_write_cbk (call_frame_t *frame, xlator_t *this,
-                  void *cookie, int op_ret, int op_errno,
-                  struct iatt *poststat, struct iatt *preparent,
-                  struct iatt *postparent, struct iatt *preparent2,
-                  struct iatt *postparent2, dict_t *xdata)
+ec_dir_write_cbk(call_frame_t *frame, xlator_t *this, void *cookie, int op_ret,
+                 int op_errno, struct iatt *poststat, struct iatt *preparent,
+                 struct iatt *postparent, struct iatt *preparent2,
+                 struct iatt *postparent2, dict_t *xdata)
 {
-        ec_fop_data_t *fop = NULL;
-        ec_cbk_data_t *cbk = NULL;
-        int           i    = 0;
-        int           idx  = 0;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    int i = 0;
+    int idx = 0;
 
-        VALIDATE_OR_GOTO (this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-        fop = frame->local;
-        idx = (long) cookie;
+    fop = frame->local;
+    idx = (long)cookie;
 
-        ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-                 frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-        cbk = ec_cbk_data_allocate (frame, this, fop, fop->id, idx, op_ret,
-                                    op_errno);
-        if (!cbk)
-                goto out;
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
+                               op_errno);
+    if (!cbk)
+        goto out;
 
-        if (xdata)
-                cbk->xdata = dict_ref (xdata);
+    if (xdata)
+        cbk->xdata = dict_ref(xdata);
 
-        if (op_ret < 0)
-                goto out;
+    if (op_ret < 0)
+        goto out;
 
-        if (poststat)
-                cbk->iatt[i++] = *poststat;
+    if (poststat)
+        cbk->iatt[i++] = *poststat;
 
-        if (preparent)
-                cbk->iatt[i++] = *preparent;
+    if (preparent)
+        cbk->iatt[i++] = *preparent;
 
-        if (postparent)
-                cbk->iatt[i++] = *postparent;
+    if (postparent)
+        cbk->iatt[i++] = *postparent;
 
-        if (preparent2)
-                cbk->iatt[i++] = *preparent2;
+    if (preparent2)
+        cbk->iatt[i++] = *preparent2;
 
-        if (postparent2)
-                cbk->iatt[i++] = *postparent2;
+    if (postparent2)
+        cbk->iatt[i++] = *postparent2;
 
 out:
-        if (cbk)
-                ec_combine (cbk, ec_combine_write);
+    if (cbk)
+        ec_combine(cbk, ec_combine_write);
 
-        if (fop)
-                ec_complete (fop);
-        return 0;
+    if (fop)
+        ec_complete(fop);
+    return 0;
 }
 
 /* FOP: create */
 
-int32_t ec_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd,
-                      inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
+int32_t
+ec_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, fd_t *fd, inode_t *inode, struct iatt *buf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_create(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_create(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -99,7 +95,8 @@ void ec_wind_create(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->xdata);
 }
 
-int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_create(ec_fop_data_t *fop, int32_t state)
 {
     ec_config_t config;
     ec_t *ec;
@@ -108,8 +105,7 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
     uint64_t version[2] = {0, 0};
     int32_t err;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             LOCK(&fop->fd->lock);
 
@@ -176,7 +172,7 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
              * need to remove O_APPEND from flags (if present) */
             fop->int32 &= ~O_APPEND;
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
             ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
@@ -218,12 +214,11 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.create != NULL)
-            {
-                fop->cbks.create (fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                  cbk->op_errno, fop->fd, fop->loc[0].inode,
-                                  &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
-                                  cbk->xdata);
+            if (fop->cbks.create != NULL) {
+                QUORUM_CBK(fop->cbks.create, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->fd,
+                           fop->loc[0].inode, &cbk->iatt[0], &cbk->iatt[1],
+                           &cbk->iatt[2], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -235,8 +230,7 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.create != NULL)
-            {
+            if (fop->cbks.create != NULL) {
                 fop->cbks.create(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL, NULL, NULL, NULL);
             }
@@ -256,30 +250,29 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_create_cbk_t func, void * data,
-               loc_t * loc, int32_t flags, mode_t mode, mode_t umask,
-               fd_t * fd, dict_t * xdata)
+void
+ec_create(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_create_cbk_t func, void *data, loc_t *loc,
+          int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-    ec_cbk_t callback = { .create = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.create = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(CREATE) %p", frame);
+    gf_msg_trace("ec", 0, "EC(CREATE) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_CREATE, 0, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_CREATE, 0, target, fop_flags,
                                ec_wind_create, ec_manager_create, callback,
                                data);
     if (fop == NULL) {
@@ -292,8 +285,8 @@ void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -301,19 +294,19 @@ void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -331,16 +324,17 @@ out:
 
 /* FOP: link */
 
-int32_t ec_link_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                    int32_t op_ret, int32_t op_errno, inode_t * inode,
-                    struct iatt * buf, struct iatt * preparent,
-                    struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, inode_t *inode, struct iatt *buf,
+            struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_link(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_link(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -349,17 +343,17 @@ void ec_wind_link(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], &fop->loc[1], fop->xdata);
 }
 
-int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_link(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_parent_inode(fop, &fop->loc[1], &fop->loc[0],
-                                         EC_UPDATE_DATA | EC_UPDATE_META |
-                                         EC_INODE_SIZE);
+            ec_lock_prepare_parent_inode(
+                fop, &fop->loc[1], &fop->loc[0],
+                EC_UPDATE_DATA | EC_UPDATE_META | EC_INODE_SIZE);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -392,11 +386,11 @@ int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.link != NULL)
-            {
-                fop->cbks.link(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                               cbk->op_errno, fop->loc[0].inode, &cbk->iatt[0],
-                               &cbk->iatt[1], &cbk->iatt[2], cbk->xdata);
+            if (fop->cbks.link != NULL) {
+                QUORUM_CBK(fop->cbks.link, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->loc[0].inode,
+                           &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
+                           cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -408,8 +402,7 @@ int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.link != NULL)
-            {
+            if (fop->cbks.link != NULL) {
                 fop->cbks.link(fop->req_frame, fop, fop->xl, -1, fop->error,
                                NULL, NULL, NULL, NULL, NULL);
             }
@@ -429,29 +422,29 @@ int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_link(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_link_cbk_t func, void * data, loc_t * oldloc,
-             loc_t * newloc, dict_t * xdata)
+void
+ec_link(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_link_cbk_t func, void *data, loc_t *oldloc,
+        loc_t *newloc, dict_t *xdata)
 {
-    ec_cbk_t callback = { .link = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.link = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(LINK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(LINK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_LINK, 0, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_LINK, 0, target, fop_flags,
                                ec_wind_link, ec_manager_link, callback, data);
     if (fop == NULL) {
         goto out;
@@ -459,26 +452,26 @@ void ec_link(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (oldloc != NULL) {
         if (loc_copy(&fop->loc[0], oldloc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (newloc != NULL) {
         if (loc_copy(&fop->loc[1], newloc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -496,16 +489,17 @@ out:
 
 /* FOP: mkdir */
 
-int32_t ec_mkdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, inode_t * inode,
-                     struct iatt * buf, struct iatt * preparent,
-                     struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_mkdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_mkdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -514,14 +508,14 @@ void ec_wind_mkdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->mode[0], fop->mode[1], fop->xdata);
 }
 
-int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_mkdir(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
     uint64_t version[2] = {0, 0};
     int32_t err;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             if (fop->xdata == NULL) {
                 fop->xdata = dict_new();
@@ -539,7 +533,7 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
                 return EC_STATE_REPORT;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
             ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
@@ -572,11 +566,11 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.mkdir != NULL)
-            {
-                fop->cbks.mkdir(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                cbk->op_errno, fop->loc[0].inode, &cbk->iatt[0],
-                                &cbk->iatt[1], &cbk->iatt[2], cbk->xdata);
+            if (fop->cbks.mkdir != NULL) {
+                QUORUM_CBK(fop->cbks.mkdir, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->loc[0].inode,
+                           &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
+                           cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -589,8 +583,7 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
             cbk = fop->answer;
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.mkdir != NULL)
-            {
+            if (fop->cbks.mkdir != NULL) {
                 fop->cbks.mkdir(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL, NULL, NULL, NULL,
                                 ((cbk) ? cbk->xdata : NULL));
@@ -611,31 +604,30 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_mkdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_mkdir_cbk_t func, void * data, loc_t * loc,
-              mode_t mode, mode_t umask, dict_t * xdata)
+void
+ec_mkdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_mkdir_cbk_t func, void *data, loc_t *loc,
+         mode_t mode, mode_t umask, dict_t *xdata)
 {
-    ec_cbk_t callback = { .mkdir = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.mkdir = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(MKDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(MKDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_MKDIR, 0, target, minimum,
-                               ec_wind_mkdir, ec_manager_mkdir, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_MKDIR, 0, target, fop_flags,
+                               ec_wind_mkdir, ec_manager_mkdir, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -645,18 +637,18 @@ void ec_mkdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -674,16 +666,17 @@ out:
 
 /* FOP: mknod */
 
-int32_t ec_mknod_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, inode_t * inode,
-                     struct iatt * buf, struct iatt * preparent,
-                     struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_mknod(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_mknod(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -693,15 +686,15 @@ void ec_wind_mknod(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_mknod(ec_fop_data_t *fop, int32_t state)
 {
     ec_config_t config;
     ec_t *ec;
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
     uint64_t version[2] = {0, 0};
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             if (S_ISREG(fop->mode[0])) {
                 int32_t err;
@@ -745,7 +738,7 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
                 }
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
             ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
@@ -778,11 +771,11 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.mknod != NULL)
-            {
-                fop->cbks.mknod(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                cbk->op_errno, fop->loc[0].inode, &cbk->iatt[0],
-                                &cbk->iatt[1], &cbk->iatt[2], cbk->xdata);
+            if (fop->cbks.mknod != NULL) {
+                QUORUM_CBK(fop->cbks.mknod, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->loc[0].inode,
+                           &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
+                           cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -794,8 +787,7 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.mknod != NULL)
-            {
+            if (fop->cbks.mknod != NULL) {
                 fop->cbks.mknod(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL, NULL, NULL, NULL, NULL);
             }
@@ -815,31 +807,30 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_mknod(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_mknod_cbk_t func, void * data, loc_t * loc,
-              mode_t mode, dev_t rdev, mode_t umask, dict_t * xdata)
+void
+ec_mknod(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_mknod_cbk_t func, void *data, loc_t *loc,
+         mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
-    ec_cbk_t callback = { .mknod = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.mknod = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(MKNOD) %p", frame);
+    gf_msg_trace("ec", 0, "EC(MKNOD) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_MKNOD, 0, target, minimum,
-                               ec_wind_mknod, ec_manager_mknod, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_MKNOD, 0, target, fop_flags,
+                               ec_wind_mknod, ec_manager_mknod, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -850,18 +841,18 @@ void ec_mknod(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -879,18 +870,19 @@ out:
 
 /* FOP: rename */
 
-int32_t ec_rename_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno, struct iatt * buf,
-                      struct iatt * preoldparent, struct iatt * postoldparent,
-                      struct iatt * prenewparent, struct iatt * postnewparent,
-                      dict_t * xdata)
+int32_t
+ec_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *buf, struct iatt *preoldparent,
+              struct iatt *postoldparent, struct iatt *prenewparent,
+              struct iatt *postnewparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preoldparent, postoldparent, prenewparent,
-                                 postnewparent, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preoldparent, postoldparent, prenewparent,
+                            postnewparent, xdata);
 }
 
-void ec_wind_rename(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_rename(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -899,17 +891,17 @@ void ec_wind_rename(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], &fop->loc[1], fop->xdata);
 }
 
-int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_rename(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_parent_inode(fop, &fop->loc[0], &fop->loc[0],
-                                         EC_UPDATE_DATA | EC_UPDATE_META |
-                                         EC_INODE_SIZE);
+            ec_lock_prepare_parent_inode(
+                fop, &fop->loc[0], &fop->loc[0],
+                EC_UPDATE_DATA | EC_UPDATE_META | EC_INODE_SIZE);
             ec_lock_prepare_parent_inode(fop, &fop->loc[1], NULL,
                                          EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
@@ -938,12 +930,11 @@ int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.rename != NULL)
-            {
-                fop->cbks.rename(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                 cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
-                                 &cbk->iatt[2], &cbk->iatt[3], &cbk->iatt[4],
-                                 cbk->xdata);
+            if (fop->cbks.rename != NULL) {
+                QUORUM_CBK(fop->cbks.rename, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], &cbk->iatt[2], &cbk->iatt[3],
+                           &cbk->iatt[4], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -955,8 +946,7 @@ int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.rename != NULL)
-            {
+            if (fop->cbks.rename != NULL) {
                 fop->cbks.rename(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL, NULL, NULL, NULL);
             }
@@ -976,29 +966,29 @@ int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_rename(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_rename_cbk_t func, void * data,
-               loc_t * oldloc, loc_t * newloc, dict_t * xdata)
+void
+ec_rename(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_rename_cbk_t func, void *data, loc_t *oldloc,
+          loc_t *newloc, dict_t *xdata)
 {
-    ec_cbk_t callback = { .rename = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.rename = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(RENAME) %p", frame);
+    gf_msg_trace("ec", 0, "EC(RENAME) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_RENAME, 0, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_RENAME, 0, target, fop_flags,
                                ec_wind_rename, ec_manager_rename, callback,
                                data);
     if (fop == NULL) {
@@ -1007,26 +997,26 @@ void ec_rename(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (oldloc != NULL) {
         if (loc_copy(&fop->loc[0], oldloc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (newloc != NULL) {
         if (loc_copy(&fop->loc[1], newloc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1044,15 +1034,17 @@ out:
 
 /* FOP: rmdir */
 
-int32_t ec_rmdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, struct iatt * preparent,
-                     struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+             dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno, NULL,
-                                 preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, NULL,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_rmdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_rmdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1061,12 +1053,12 @@ void ec_wind_rmdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_rmdir(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
             ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
@@ -1090,11 +1082,10 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.rmdir != NULL)
-            {
-                fop->cbks.rmdir(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
-                                cbk->xdata);
+            if (fop->cbks.rmdir != NULL) {
+                QUORUM_CBK(fop->cbks.rmdir, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -1106,8 +1097,7 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.rmdir != NULL)
-            {
+            if (fop->cbks.rmdir != NULL) {
                 fop->cbks.rmdir(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL, NULL, NULL);
             }
@@ -1127,31 +1117,30 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_rmdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_rmdir_cbk_t func, void * data, loc_t * loc,
-              int xflags, dict_t * xdata)
+void
+ec_rmdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_rmdir_cbk_t func, void *data, loc_t *loc,
+         int xflags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .rmdir = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.rmdir = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(RMDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(RMDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_RMDIR, 0, target, minimum,
-                               ec_wind_rmdir, ec_manager_rmdir, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_RMDIR, 0, target, fop_flags,
+                               ec_wind_rmdir, ec_manager_rmdir, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -1160,18 +1149,18 @@ void ec_rmdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1189,16 +1178,18 @@ out:
 
 /* FOP: symlink */
 
-int32_t ec_symlink_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, inode_t * inode,
-                       struct iatt * buf, struct iatt * preparent,
-                       struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_symlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_symlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1207,12 +1198,12 @@ void ec_wind_symlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->str[0], &fop->loc[0], fop->mode[0], fop->xdata);
 }
 
-int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_symlink(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
             ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
@@ -1245,12 +1236,11 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.symlink != NULL)
-            {
-                fop->cbks.symlink(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                  cbk->op_errno, fop->loc[0].inode,
-                                  &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
-                                  cbk->xdata);
+            if (fop->cbks.symlink != NULL) {
+                QUORUM_CBK(fop->cbks.symlink, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->loc[0].inode,
+                           &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
+                           cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -1262,8 +1252,7 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.symlink != NULL)
-            {
+            if (fop->cbks.symlink != NULL) {
                 fop->cbks.symlink(fop->req_frame, fop, fop->xl, -1, fop->error,
                                   NULL, NULL, NULL, NULL, NULL);
             }
@@ -1283,32 +1272,31 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_symlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_symlink_cbk_t func, void * data,
-                const char * linkname, loc_t * loc, mode_t umask,
-                dict_t * xdata)
+void
+ec_symlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_symlink_cbk_t func, void *data,
+           const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata)
 {
-    ec_cbk_t callback = { .symlink = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.symlink = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(SYMLINK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(SYMLINK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_SYMLINK, 0, target, minimum,
-                               ec_wind_symlink, ec_manager_symlink, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_SYMLINK, 0, target,
+                               fop_flags, ec_wind_symlink, ec_manager_symlink,
+                               callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -1318,16 +1306,16 @@ void ec_symlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (linkname != NULL) {
         fop->str[0] = gf_strdup(linkname);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY, "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -1335,9 +1323,9 @@ void ec_symlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1355,16 +1343,17 @@ out:
 
 /* FOP: unlink */
 
-int32_t ec_unlink_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt * preparent, struct iatt * postparent,
-                      dict_t * xdata)
+int32_t
+ec_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno, NULL,
-                                 preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, NULL,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_unlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_unlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1373,12 +1362,12 @@ void ec_wind_unlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_unlink(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
             ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
@@ -1402,11 +1391,10 @@ int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.unlink != NULL)
-            {
-                fop->cbks.unlink(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                 cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
-                                 cbk->xdata);
+            if (fop->cbks.unlink != NULL) {
+                QUORUM_CBK(fop->cbks.unlink, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -1418,8 +1406,7 @@ int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.unlink != NULL)
-            {
+            if (fop->cbks.unlink != NULL) {
                 fop->cbks.unlink(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL);
             }
@@ -1439,29 +1426,29 @@ int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_unlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_unlink_cbk_t func, void * data,
-               loc_t * loc, int xflags, dict_t * xdata)
+void
+ec_unlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_unlink_cbk_t func, void *data, loc_t *loc,
+          int xflags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .unlink = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.unlink = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(UNLINK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(UNLINK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_UNLINK, 0, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_UNLINK, 0, target, fop_flags,
                                ec_wind_unlink, ec_manager_unlink, callback,
                                data);
     if (fop == NULL) {
@@ -1472,18 +1459,18 @@ void ec_unlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
diff --git a/xlators/cluster/ec/src/ec-fops.h b/xlators/cluster/ec/src/ec-fops.h
index 4a926cf4802..07edf8a7fec 100644
--- a/xlators/cluster/ec/src/ec-fops.h
+++ b/xlators/cluster/ec/src/ec-fops.h
@@ -11,204 +11,244 @@
 #ifndef __EC_FOPS_H__
 #define __EC_FOPS_H__
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
 #include "ec-types.h"
 #include "ec-common.h"
 
-void ec_access(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_access_cbk_t func, void *data, loc_t * loc,
-               int32_t mask, dict_t * xdata);
-
-void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_create_cbk_t func, void *data, loc_t * loc,
-               int32_t flags, mode_t mode, mode_t umask, fd_t * fd,
-               dict_t * xdata);
-
-void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_entrylk_cbk_t func, void *data,
-                const char * volume, loc_t * loc, const char * basename,
-                entrylk_cmd cmd, entrylk_type type, dict_t * xdata);
-
-void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fentrylk_cbk_t func, void *data,
-                 const char * volume, fd_t * fd, const char * basename,
-                 entrylk_cmd cmd, entrylk_type type, dict_t * xdata);
-
-void ec_flush(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_flush_cbk_t func, void *data, fd_t * fd,
-              dict_t * xdata);
-
-void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fsync_cbk_t func, void *data, fd_t * fd,
-              int32_t datasync, dict_t * xdata);
-
-void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fsyncdir_cbk_t func, void *data,
-                 fd_t * fd, int32_t datasync, dict_t * xdata);
-
-void ec_getxattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_getxattr_cbk_t func, void *data,
-                 loc_t * loc, const char * name, dict_t * xdata);
-
-void ec_fgetxattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                  int32_t minimum, fop_fgetxattr_cbk_t func, void *data,
-                  fd_t * fd, const char * name, dict_t * xdata);
-
-void ec_heal(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_heal_cbk_t func, void *data, loc_t * loc,
-             int32_t partial, dict_t *xdata);
-
-void ec_fheal(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fheal_cbk_t func, void *data, fd_t * fd,
-              int32_t partial, dict_t *xdata);
-
-void ec_inodelk (call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
-                 uintptr_t target, int32_t minimum, fop_inodelk_cbk_t func,
-                 void *data, const char *volume, loc_t *loc, int32_t cmd,
-                 struct gf_flock * flock, dict_t * xdata);
-
-void ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
-                 uintptr_t target, int32_t minimum, fop_finodelk_cbk_t func,
-                 void *data, const char *volume, fd_t *fd, int32_t cmd,
-                 struct gf_flock *flock, dict_t *xdata);
-
-void ec_link(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_link_cbk_t func, void *data, loc_t * oldloc,
-             loc_t * newloc, dict_t * xdata);
-
-void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-           int32_t minimum, fop_lk_cbk_t func, void *data, fd_t * fd,
-           int32_t cmd, struct gf_flock * flock, dict_t * xdata);
-
-void ec_lookup(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_lookup_cbk_t func, void *data, loc_t * loc,
-               dict_t * xdata);
-
-void ec_mkdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_mkdir_cbk_t func, void *data, loc_t * loc,
-              mode_t mode, mode_t umask, dict_t * xdata);
-
-void ec_mknod(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_mknod_cbk_t func, void *data, loc_t * loc,
-              mode_t mode, dev_t rdev, mode_t umask, dict_t * xdata);
-
-void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_open_cbk_t func, void *data, loc_t * loc,
-             int32_t flags, fd_t * fd, dict_t * xdata);
-
-void ec_opendir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_opendir_cbk_t func, void *data,
-                loc_t * loc, fd_t * fd, dict_t * xdata);
-
-void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_readdir_cbk_t func, void *data, fd_t * fd,
-                size_t size, off_t offset, dict_t * xdata);
-
-void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_readdirp_cbk_t func, void *data,
-                 fd_t * fd, size_t size, off_t offset, dict_t * xdata);
-
-void ec_readlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_readlink_cbk_t func, void *data,
-                 loc_t * loc, size_t size, dict_t * xdata);
-
-void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_readv_cbk_t func, void *data, fd_t * fd,
-              size_t size, off_t offset, uint32_t flags, dict_t * xdata);
-
-void ec_removexattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                    int32_t minimum, fop_removexattr_cbk_t func, void *data,
-                    loc_t * loc, const char * name, dict_t * xdata);
-
-void ec_fremovexattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                     int32_t minimum, fop_fremovexattr_cbk_t func, void *data,
-                     fd_t * fd, const char * name, dict_t * xdata);
-
-void ec_rename(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_rename_cbk_t func, void *data,
-               loc_t * oldloc, loc_t * newloc, dict_t * xdata);
-
-void ec_rmdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_rmdir_cbk_t func, void *data, loc_t * loc,
-              int xflags, dict_t * xdata);
-
-void ec_setattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_setattr_cbk_t func, void *data,
-                loc_t * loc, struct iatt * stbuf, int32_t valid,
-                dict_t * xdata);
-
-void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fsetattr_cbk_t func, void *data,
-                 fd_t * fd, struct iatt * stbuf, int32_t valid,
-                 dict_t * xdata);
-
-void ec_setxattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_setxattr_cbk_t func, void *data,
-                 loc_t * loc, dict_t * dict, int32_t flags, dict_t * xdata);
-
-void ec_fsetxattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                  int32_t minimum, fop_fsetxattr_cbk_t func, void *data,
-                  fd_t * fd, dict_t * dict, int32_t flags, dict_t * xdata);
-
-void ec_stat(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_stat_cbk_t func, void *data, loc_t * loc,
-             dict_t * xdata);
-
-void ec_fstat(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fstat_cbk_t func, void *data, fd_t * fd,
-              dict_t * xdata);
-
-void ec_statfs(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_statfs_cbk_t func, void *data, loc_t * loc,
-               dict_t * xdata);
-
-void ec_symlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_symlink_cbk_t func, void *data,
-                const char * linkname, loc_t * loc, mode_t umask,
-                dict_t * xdata);
-
-void ec_fallocate(call_frame_t *frame, xlator_t *this, uintptr_t target,
-              int32_t minimum, fop_fallocate_cbk_t func, void *data, fd_t *fd,
-              int32_t mode, off_t offset, size_t len, dict_t *xdata);
-
-void ec_discard(call_frame_t *frame, xlator_t *this, uintptr_t target,
-                int32_t minimum, fop_discard_cbk_t func, void *data, fd_t *fd,
-                off_t offset, size_t len, dict_t *xdata);
-
-void ec_truncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_truncate_cbk_t func, void *data,
-                 loc_t * loc, off_t offset, dict_t * xdata);
-
-void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                  int32_t minimum, fop_ftruncate_cbk_t func, void *data,
-                  fd_t * fd, off_t offset, dict_t * xdata);
-
-void ec_unlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_unlink_cbk_t func, void *data, loc_t * loc,
-               int xflags, dict_t * xdata);
-
-void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_writev_cbk_t func, void *data, fd_t * fd,
-               struct iovec * vector, int32_t count, off_t offset,
-               uint32_t flags, struct iobref * iobref, dict_t * xdata);
-
-void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_xattrop_cbk_t func, void *data,
-                loc_t * loc, gf_xattrop_flags_t optype, dict_t * xattr,
-                dict_t * xdata);
-
-void ec_fxattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fxattrop_cbk_t func, void *data,
-                 fd_t * fd, gf_xattrop_flags_t optype, dict_t * xattr,
-                 dict_t * xdata);
-
-void ec_seek(call_frame_t *frame, xlator_t *this, uintptr_t target,
-             int32_t minimum, fop_seek_cbk_t func, void *data, fd_t *fd,
-             off_t offset, gf_seek_what_t what, dict_t *xdata);
-
-void ec_ipc(call_frame_t *frame, xlator_t *this, uintptr_t target,
-            int32_t minimum, fop_ipc_cbk_t func, void *data, int32_t op,
-            dict_t *xdata);
+void
+ec_access(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_access_cbk_t func, void *data, loc_t *loc,
+          int32_t mask, dict_t *xdata);
+
+void
+ec_create(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_create_cbk_t func, void *data, loc_t *loc,
+          int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
+
+void
+ec_entrylk(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_entrylk_cbk_t func, void *data,
+           const char *volume, loc_t *loc, const char *basename,
+           entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
+
+void
+ec_fentrylk(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fentrylk_cbk_t func, void *data,
+            const char *volume, fd_t *fd, const char *basename, entrylk_cmd cmd,
+            entrylk_type type, dict_t *xdata);
+
+void
+ec_flush(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_flush_cbk_t func, void *data, fd_t *fd,
+         dict_t *xdata);
+
+void
+ec_fsync(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fsync_cbk_t func, void *data, fd_t *fd,
+         int32_t datasync, dict_t *xdata);
+
+void
+ec_fsyncdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fsyncdir_cbk_t func, void *data, fd_t *fd,
+            int32_t datasync, dict_t *xdata);
+
+void
+ec_getxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_getxattr_cbk_t func, void *data, loc_t *loc,
+            const char *name, dict_t *xdata);
+
+void
+ec_fgetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fgetxattr_cbk_t func, void *data, fd_t *fd,
+             const char *name, dict_t *xdata);
+
+void
+ec_heal(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_heal_cbk_t func, void *data, loc_t *loc,
+        int32_t partial, dict_t *xdata);
+
+void
+ec_fheal(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fheal_cbk_t func, void *data, fd_t *fd,
+         int32_t partial, dict_t *xdata);
+
+void
+ec_inodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
+           uintptr_t target, uint32_t fop_flags, fop_inodelk_cbk_t func,
+           void *data, const char *volume, loc_t *loc, int32_t cmd,
+           struct gf_flock *flock, dict_t *xdata);
+
+void
+ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
+            uintptr_t target, uint32_t fop_flags, fop_finodelk_cbk_t func,
+            void *data, const char *volume, fd_t *fd, int32_t cmd,
+            struct gf_flock *flock, dict_t *xdata);
+
+void
+ec_link(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_link_cbk_t func, void *data, loc_t *oldloc,
+        loc_t *newloc, dict_t *xdata);
+
+void
+ec_lk(call_frame_t *frame, xlator_t *this, uintptr_t target, uint32_t fop_flags,
+      fop_lk_cbk_t func, void *data, fd_t *fd, int32_t cmd,
+      struct gf_flock *flock, dict_t *xdata);
+
+void
+ec_lookup(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_lookup_cbk_t func, void *data, loc_t *loc,
+          dict_t *xdata);
+
+void
+ec_mkdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_mkdir_cbk_t func, void *data, loc_t *loc,
+         mode_t mode, mode_t umask, dict_t *xdata);
+
+void
+ec_mknod(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_mknod_cbk_t func, void *data, loc_t *loc,
+         mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
+
+void
+ec_open(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_open_cbk_t func, void *data, loc_t *loc,
+        int32_t flags, fd_t *fd, dict_t *xdata);
+
+void
+ec_opendir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_opendir_cbk_t func, void *data, loc_t *loc,
+           fd_t *fd, dict_t *xdata);
+
+void
+ec_readdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_readdir_cbk_t func, void *data, fd_t *fd,
+           size_t size, off_t offset, dict_t *xdata);
+
+void
+ec_readdirp(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_readdirp_cbk_t func, void *data, fd_t *fd,
+            size_t size, off_t offset, dict_t *xdata);
+
+void
+ec_readlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_readlink_cbk_t func, void *data, loc_t *loc,
+            size_t size, dict_t *xdata);
+
+void
+ec_readv(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_readv_cbk_t func, void *data, fd_t *fd,
+         size_t size, off_t offset, uint32_t flags, dict_t *xdata);
+
+void
+ec_removexattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+               uint32_t fop_flags, fop_removexattr_cbk_t func, void *data,
+               loc_t *loc, const char *name, dict_t *xdata);
+
+void
+ec_fremovexattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+                uint32_t fop_flags, fop_fremovexattr_cbk_t func, void *data,
+                fd_t *fd, const char *name, dict_t *xdata);
+
+void
+ec_rename(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_rename_cbk_t func, void *data, loc_t *oldloc,
+          loc_t *newloc, dict_t *xdata);
+
+void
+ec_rmdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_rmdir_cbk_t func, void *data, loc_t *loc,
+         int xflags, dict_t *xdata);
+
+void
+ec_setattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_setattr_cbk_t func, void *data, loc_t *loc,
+           struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+void
+ec_fsetattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fsetattr_cbk_t func, void *data, fd_t *fd,
+            struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+void
+ec_setxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_setxattr_cbk_t func, void *data, loc_t *loc,
+            dict_t *dict, int32_t flags, dict_t *xdata);
+
+void
+ec_fsetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fsetxattr_cbk_t func, void *data, fd_t *fd,
+             dict_t *dict, int32_t flags, dict_t *xdata);
+
+void
+ec_stat(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_stat_cbk_t func, void *data, loc_t *loc,
+        dict_t *xdata);
+
+void
+ec_fstat(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fstat_cbk_t func, void *data, fd_t *fd,
+         dict_t *xdata);
+
+void
+ec_statfs(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_statfs_cbk_t func, void *data, loc_t *loc,
+          dict_t *xdata);
+
+void
+ec_symlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_symlink_cbk_t func, void *data,
+           const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata);
+
+void
+ec_fallocate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fallocate_cbk_t func, void *data, fd_t *fd,
+             int32_t mode, off_t offset, size_t len, dict_t *xdata);
+
+void
+ec_discard(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_discard_cbk_t func, void *data, fd_t *fd,
+           off_t offset, size_t len, dict_t *xdata);
+
+void
+ec_truncate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_truncate_cbk_t func, void *data, loc_t *loc,
+            off_t offset, dict_t *xdata);
+
+void
+ec_ftruncate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_ftruncate_cbk_t func, void *data, fd_t *fd,
+             off_t offset, dict_t *xdata);
+
+void
+ec_unlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_unlink_cbk_t func, void *data, loc_t *loc,
+          int xflags, dict_t *xdata);
+
+void
+ec_writev(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_writev_cbk_t func, void *data, fd_t *fd,
+          struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+          struct iobref *iobref, dict_t *xdata);
+
+void
+ec_xattrop(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_xattrop_cbk_t func, void *data, loc_t *loc,
+           gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+void
+ec_fxattrop(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fxattrop_cbk_t func, void *data, fd_t *fd,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+void
+ec_seek(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_seek_cbk_t func, void *data, fd_t *fd,
+        off_t offset, gf_seek_what_t what, dict_t *xdata);
+
+void
+ec_ipc(call_frame_t *frame, xlator_t *this, uintptr_t target,
+       uint32_t fop_flags, fop_ipc_cbk_t func, void *data, int32_t op,
+       dict_t *xdata);
 
 #endif /* __EC_FOPS_H__ */
diff --git a/xlators/cluster/ec/src/ec-galois.c b/xlators/cluster/ec/src/ec-galois.c
index 8cb4dc2e4e3..6e4990c71f5 100644
--- a/xlators/cluster/ec/src/ec-galois.c
+++ b/xlators/cluster/ec/src/ec-galois.c
@@ -10,9 +10,6 @@
 
 #include <string.h>
 
-#include "mem-pool.h"
-#include "list.h"
-
 #include "ec-mem-types.h"
 #include "ec-gf8.h"
 #include "ec-helpers.h"
diff --git a/xlators/cluster/ec/src/ec-galois.h b/xlators/cluster/ec/src/ec-galois.h
index 02e6b6c1bc2..ed55d53e419 100644
--- a/xlators/cluster/ec/src/ec-galois.h
+++ b/xlators/cluster/ec/src/ec-galois.h
@@ -15,12 +15,18 @@
 
 #include "ec-types.h"
 
-ec_gf_t *ec_gf_prepare(uint32_t bits, uint32_t mod);
-void ec_gf_destroy(ec_gf_t *gf);
+ec_gf_t *
+ec_gf_prepare(uint32_t bits, uint32_t mod);
+void
+ec_gf_destroy(ec_gf_t *gf);
 
-uint32_t ec_gf_add(ec_gf_t *gf, uint32_t a, uint32_t b);
-uint32_t ec_gf_mul(ec_gf_t *gf, uint32_t a, uint32_t b);
-uint32_t ec_gf_div(ec_gf_t *gf, uint32_t a, uint32_t b);
-uint32_t ec_gf_exp(ec_gf_t *gf, uint32_t a, uint32_t b);
+uint32_t
+ec_gf_add(ec_gf_t *gf, uint32_t a, uint32_t b);
+uint32_t
+ec_gf_mul(ec_gf_t *gf, uint32_t a, uint32_t b);
+uint32_t
+ec_gf_div(ec_gf_t *gf, uint32_t a, uint32_t b);
+uint32_t
+ec_gf_exp(ec_gf_t *gf, uint32_t a, uint32_t b);
 
 #endif /* __EC_GALOIS_H__ */
diff --git a/xlators/cluster/ec/src/ec-generic.c b/xlators/cluster/ec/src/ec-generic.c
index a5f986e74f4..884deb93669 100644
--- a/xlators/cluster/ec/src/ec-generic.c
+++ b/xlators/cluster/ec/src/ec-generic.c
@@ -8,25 +8,23 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 
 #include "ec.h"
 #include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
-#include "ec-method.h"
 #include "ec-fops.h"
 
 /* FOP: flush */
 
-int32_t ec_flush_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -36,21 +34,18 @@ int32_t ec_flush_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FLUSH, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -60,15 +55,15 @@ int32_t ec_flush_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_flush(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_flush(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -77,15 +72,15 @@ void ec_wind_flush(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_flush(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_fd(fop, fop->fd, 0, 0, LLONG_MAX);
+            ec_lock_prepare_fd(fop, fop->fd, 0, 0, EC_RANGE_FULL);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -110,8 +105,7 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.flush != NULL)
-            {
+            if (fop->cbks.flush != NULL) {
                 fop->cbks.flush(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                 cbk->op_errno, cbk->xdata);
             }
@@ -126,8 +120,7 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.flush != NULL)
-            {
+            if (fop->cbks.flush != NULL) {
                 fop->cbks.flush(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL);
             }
@@ -147,31 +140,71 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_flush(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_flush_cbk_t func, void * data, fd_t * fd,
-              dict_t * xdata)
+static int32_t
+ec_validate_fd(fd_t *fd, xlator_t *xl)
 {
-    ec_cbk_t callback = { .flush = func };
-    ec_fop_data_t * fop = NULL;
+    uint64_t iversion = 0;
+    uint64_t fversion = 0;
+    ec_inode_t *inode_ctx = NULL;
+    ec_fd_t *fd_ctx = NULL;
+
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __ec_fd_get(fd, xl);
+        if (fd_ctx) {
+            fversion = fd_ctx->bad_version;
+        }
+    }
+    UNLOCK(&fd->lock);
+
+    LOCK(&fd->inode->lock);
+    {
+        inode_ctx = __ec_inode_get(fd->inode, xl);
+        if (inode_ctx) {
+            iversion = inode_ctx->bad_version;
+        }
+    }
+    UNLOCK(&fd->inode->lock);
+    if (fversion < iversion) {
+        return EBADF;
+    }
+    return 0;
+}
+
+void
+ec_flush(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_flush_cbk_t func, void *data, fd_t *fd,
+         dict_t *xdata)
+{
+    ec_cbk_t callback = {.flush = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FLUSH) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FLUSH) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FLUSH, 0, target, minimum,
-                               ec_wind_flush, ec_manager_flush, callback,
-                               data);
+    if (fd) {
+        error = ec_validate_fd(fd, this);
+        if (error) {
+            gf_msg(this->name, GF_LOG_ERROR, EBADF, EC_MSG_FD_BAD,
+                   "Failing %s on %s", gf_fop_list[GF_FOP_FLUSH],
+                   fd->inode ? uuid_utoa(fd->inode->gfid) : "");
+            goto out;
+        }
+    }
+
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FLUSH, 0, target, fop_flags,
+                               ec_wind_flush, ec_manager_flush, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -181,9 +214,9 @@ void ec_flush(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -191,9 +224,9 @@ void ec_flush(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -211,13 +244,13 @@ out:
 
 /* FOP: fsync */
 
-int32_t ec_combine_fsync(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                         ec_cbk_data_t * src)
+int32_t
+ec_combine_fsync(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
     if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 2)) {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_IATT_MISMATCH, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_FSYNC'");
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_FSYNC'");
 
         return 0;
     }
@@ -225,12 +258,13 @@ int32_t ec_combine_fsync(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_fsync_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, struct iatt * prebuf,
-                     struct iatt * postbuf, dict_t * xdata)
+int32_t
+ec_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+             dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -240,32 +274,26 @@ int32_t ec_fsync_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSYNC, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (prebuf != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (prebuf != NULL) {
                 cbk->iatt[0] = *prebuf;
             }
-            if (postbuf != NULL)
-            {
+            if (postbuf != NULL) {
                 cbk->iatt[1] = *postbuf;
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -275,15 +303,15 @@ int32_t ec_fsync_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fsync(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fsync(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -292,15 +320,15 @@ void ec_wind_fsync(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_fsync(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0, LLONG_MAX);
+            ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0, EC_RANGE_FULL);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -318,8 +346,7 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
         case EC_STATE_PREPARE_ANSWER:
             cbk = ec_fop_prepare_answer(fop, _gf_false);
             if (cbk != NULL) {
-                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                                cbk->count);
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
 
                 /* This shouldn't fail because we have the inode locked. */
                 GF_ASSERT(ec_get_inode_size(fop, fop->fd->inode,
@@ -334,8 +361,7 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.fsync != NULL)
-            {
+            if (fop->cbks.fsync != NULL) {
                 fop->cbks.fsync(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                 cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
                                 cbk->xdata);
@@ -351,8 +377,7 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_DELAYED_START:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.fsync != NULL)
-            {
+            if (fop->cbks.fsync != NULL) {
                 fop->cbks.fsync(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL, NULL, NULL);
             }
@@ -372,31 +397,40 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fsync_cbk_t func, void * data, fd_t * fd,
-              int32_t datasync, dict_t * xdata)
+void
+ec_fsync(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fsync_cbk_t func, void *data, fd_t *fd,
+         int32_t datasync, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fsync = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fsync = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FSYNC) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSYNC) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNC, 0, target, minimum,
-                               ec_wind_fsync, ec_manager_fsync, callback,
-                               data);
+    if (fd) {
+        error = ec_validate_fd(fd, this);
+        if (error) {
+            gf_msg(this->name, GF_LOG_ERROR, EBADF, EC_MSG_FD_BAD,
+                   "Failing %s on %s", gf_fop_list[GF_FOP_FSYNC],
+                   fd->inode ? uuid_utoa(fd->inode->gfid) : "");
+            goto out;
+        }
+    }
+
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNC, 0, target, fop_flags,
+                               ec_wind_fsync, ec_manager_fsync, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -408,9 +442,9 @@ void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -418,9 +452,9 @@ void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -438,11 +472,12 @@ out:
 
 /* FOP: fsyncdir */
 
-int32_t ec_fsyncdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -452,21 +487,18 @@ int32_t ec_fsyncdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSYNCDIR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -476,15 +508,15 @@ int32_t ec_fsyncdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fsyncdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fsyncdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -493,15 +525,15 @@ void ec_wind_fsyncdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_fsyncdir(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_fd(fop, fop->fd, 0, 0, LLONG_MAX);
+            ec_lock_prepare_fd(fop, fop->fd, 0, 0, EC_RANGE_FULL);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -526,8 +558,7 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.fsyncdir != NULL)
-            {
+            if (fop->cbks.fsyncdir != NULL) {
                 fop->cbks.fsyncdir(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                    cbk->op_errno, cbk->xdata);
             }
@@ -542,10 +573,9 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_DELAYED_START:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.fsyncdir != NULL)
-            {
-                fop->cbks.fsyncdir(fop->req_frame, fop, fop->xl, -1,
-                                   fop->error, NULL);
+            if (fop->cbks.fsyncdir != NULL) {
+                fop->cbks.fsyncdir(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                   NULL);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -563,30 +593,30 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fsyncdir_cbk_t func, void * data,
-                 fd_t * fd, int32_t datasync, dict_t * xdata)
+void
+ec_fsyncdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fsyncdir_cbk_t func, void *data, fd_t *fd,
+            int32_t datasync, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fsyncdir = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fsyncdir = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FSYNCDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSYNCDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNCDIR, 0, target,
-                               minimum, ec_wind_fsyncdir, ec_manager_fsyncdir,
+                               fop_flags, ec_wind_fsyncdir, ec_manager_fsyncdir,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -599,9 +629,9 @@ void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -609,9 +639,9 @@ void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -629,9 +659,10 @@ out:
 
 /* FOP: lookup */
 
-void ec_lookup_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk)
+void
+ec_lookup_rebuild(ec_t *ec, ec_fop_data_t *fop, ec_cbk_data_t *cbk)
 {
-    ec_inode_t * ctx = NULL;
+    ec_inode_t *ctx = NULL;
     uint64_t size = 0;
     int32_t have_size = 0, err;
 
@@ -650,8 +681,7 @@ void ec_lookup_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk)
     LOCK(&cbk->inode->lock);
 
     ctx = __ec_inode_get(cbk->inode, fop->xl);
-    if (ctx != NULL)
-    {
+    if (ctx != NULL) {
         if (ctx->have_version) {
             cbk->version[0] = ctx->post_version[0];
             cbk->version[1] = ctx->post_version[1];
@@ -664,24 +694,22 @@ void ec_lookup_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk)
 
     UNLOCK(&cbk->inode->lock);
 
-    if (cbk->iatt[0].ia_type == IA_IFREG)
-    {
+    if (cbk->iatt[0].ia_type == IA_IFREG) {
         cbk->size = cbk->iatt[0].ia_size;
         ec_dict_del_number(cbk->xdata, EC_XATTR_SIZE, &cbk->iatt[0].ia_size);
-        if (have_size)
-        {
+        if (have_size) {
             cbk->iatt[0].ia_size = size;
         }
     }
 }
 
-int32_t ec_combine_lookup(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                          ec_cbk_data_t * src)
+int32_t
+ec_combine_lookup(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
     if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 2)) {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_IATT_MISMATCH, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_LOOKUP'");
+        gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_LOOKUP'");
 
         return 0;
     }
@@ -689,15 +717,15 @@ int32_t ec_combine_lookup(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_lookup_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno, inode_t * inode,
-                      struct iatt * buf, dict_t * xdata,
-                      struct iatt * postparent)
+int32_t
+ec_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+              struct iatt *postparent)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
-    uint64_t       dirty[2] = {0};
+    uint64_t dirty[2] = {0};
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
@@ -706,63 +734,54 @@ int32_t ec_lookup_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_LOOKUP, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (inode != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (inode != NULL) {
                 cbk->inode = inode_ref(inode);
-                if (cbk->inode == NULL)
-                {
-                    gf_msg (this->name, GF_LOG_ERROR, 0,
-                            EC_MSG_INODE_REF_FAIL,
-                            "Failed to reference an inode.");
+                if (cbk->inode == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_INODE_REF_FAIL,
+                           "Failed to reference an inode.");
 
                     goto out;
                 }
             }
-            if (buf != NULL)
-            {
+            if (buf != NULL) {
                 cbk->iatt[0] = *buf;
             }
-            if (postparent != NULL)
-            {
+            if (postparent != NULL) {
                 cbk->iatt[1] = *postparent;
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
-            ec_dict_del_array (xdata, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
+            ec_dict_del_array(xdata, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
         }
 
         ec_combine(cbk, ec_combine_lookup);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_lookup(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_lookup(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -771,20 +790,21 @@ void ec_wind_lookup(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->xdata);
 }
 
-int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_lookup(ec_fop_data_t *fop, int32_t state)
 {
     ec_cbk_data_t *cbk;
     int32_t err;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             if (fop->xdata == NULL) {
                 fop->xdata = dict_new();
                 if (fop->xdata == NULL) {
-                    gf_msg (fop->xl->name, GF_LOG_ERROR, ENOMEM,
-                            EC_MSG_LOOKUP_REQ_PREP_FAIL, "Unable to prepare "
-                            "lookup request");
+                    gf_msg(fop->xl->name, GF_LOG_ERROR, ENOMEM,
+                           EC_MSG_LOOKUP_REQ_PREP_FAIL,
+                           "Unable to prepare "
+                           "lookup request");
 
                     fop->error = ENOMEM;
 
@@ -792,7 +812,7 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
                 }
             } else {
                 /*TODO: To be handled once we have 'syndromes' */
-                dict_del (fop->xdata, GF_CONTENT_KEY);
+                dict_del(fop->xdata, GF_CONTENT_KEY);
             }
             err = dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0);
             if (err == 0) {
@@ -802,16 +822,17 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
                 err = dict_set_uint64(fop->xdata, EC_XATTR_DIRTY, 0);
             }
             if (err != 0) {
-                gf_msg (fop->xl->name, GF_LOG_ERROR, -err,
-                        EC_MSG_LOOKUP_REQ_PREP_FAIL, "Unable to prepare lookup "
-                                                    "request");
+                gf_msg(fop->xl->name, GF_LOG_ERROR, -err,
+                       EC_MSG_LOOKUP_REQ_PREP_FAIL,
+                       "Unable to prepare lookup "
+                       "request");
 
                 fop->error = -err;
 
                 return EC_STATE_REPORT;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -827,8 +848,8 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
              */
 
             if (!fop->answer && !list_empty(&fop->cbk_list)) {
-                fop->answer = list_entry (fop->cbk_list.next, ec_cbk_data_t,
-                                          list);
+                fop->answer = list_entry(fop->cbk_list.next, ec_cbk_data_t,
+                                         list);
             }
 
             cbk = ec_fop_prepare_answer(fop, _gf_true);
@@ -845,8 +866,7 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.lookup != NULL)
-            {
+            if (fop->cbks.lookup != NULL) {
                 fop->cbks.lookup(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                  cbk->op_errno, cbk->inode, &cbk->iatt[0],
                                  cbk->xdata, &cbk->iatt[1]);
@@ -860,8 +880,7 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.lookup != NULL)
-            {
+            if (fop->cbks.lookup != NULL) {
                 fop->cbks.lookup(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL, NULL);
             }
@@ -869,30 +888,30 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_lookup(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_lookup_cbk_t func, void * data,
-               loc_t * loc, dict_t * xdata)
+void
+ec_lookup(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_lookup_cbk_t func, void *data, loc_t *loc,
+          dict_t *xdata)
 {
-    ec_cbk_t callback = { .lookup = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.lookup = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(LOOKUP) %p", frame);
+    gf_msg_trace("ec", 0, "EC(LOOKUP) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_LOOKUP, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_lookup,
+                               target, fop_flags, ec_wind_lookup,
                                ec_manager_lookup, callback, data);
     if (fop == NULL) {
         goto out;
@@ -900,14 +919,14 @@ void ec_lookup(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         /* Do not log failures here as a memory problem would have already
          * been logged by the corresponding alloc functions */
         if (fop->xdata == NULL)
@@ -926,20 +945,20 @@ out:
 
 /* FOP: statfs */
 
-int32_t ec_combine_statfs(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                          ec_cbk_data_t * src)
+int32_t
+ec_combine_statfs(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
     ec_statvfs_combine(&dst->statvfs, &src->statvfs);
 
     return 1;
 }
 
-int32_t ec_statfs_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno, struct statvfs * buf,
-                      dict_t * xdata)
+int32_t
+ec_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -949,28 +968,23 @@ int32_t ec_statfs_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_STATFS, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (buf != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (buf != NULL) {
                 cbk->statvfs = *buf;
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -980,15 +994,15 @@ int32_t ec_statfs_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_statfs(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_statfs(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -997,14 +1011,14 @@ void ec_wind_statfs(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->xdata);
 }
 
-int32_t ec_manager_statfs(ec_fop_data_t *fop, int32_t state)
+int32_t
+ec_manager_statfs(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t   *cbk                   = NULL;
-    gf_boolean_t     deem_statfs_enabled   = _gf_false;
-    int32_t          err                   = 0;
+    ec_cbk_data_t *cbk = NULL;
+    gf_boolean_t deem_statfs_enabled = _gf_false;
+    int32_t err = 0;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -1017,8 +1031,8 @@ int32_t ec_manager_statfs(ec_fop_data_t *fop, int32_t state)
                 ec_t *ec = fop->xl->private;
 
                 if (cbk->xdata) {
-                    err = dict_get_int8 (cbk->xdata, "quota-deem-statfs",
-                                          (int8_t *)&deem_statfs_enabled);
+                    err = dict_get_int8(cbk->xdata, "quota-deem-statfs",
+                                        (int8_t *)&deem_statfs_enabled);
                     if (err != -ENOENT) {
                         ec_cbk_set_error(cbk, -err, _gf_true);
                     }
@@ -1038,8 +1052,7 @@ int32_t ec_manager_statfs(ec_fop_data_t *fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.statfs != NULL)
-            {
+            if (fop->cbks.statfs != NULL) {
                 fop->cbks.statfs(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                  cbk->op_errno, &cbk->statvfs, cbk->xdata);
             }
@@ -1052,8 +1065,7 @@ int32_t ec_manager_statfs(ec_fop_data_t *fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.statfs != NULL)
-            {
+            if (fop->cbks.statfs != NULL) {
                 fop->cbks.statfs(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL);
             }
@@ -1061,30 +1073,30 @@ int32_t ec_manager_statfs(ec_fop_data_t *fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_statfs(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_statfs_cbk_t func, void * data,
-               loc_t * loc, dict_t * xdata)
+void
+ec_statfs(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_statfs_cbk_t func, void *data, loc_t *loc,
+          dict_t *xdata)
 {
-    ec_cbk_t callback = { .statfs = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.statfs = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(STATFS) %p", frame);
+    gf_msg_trace("ec", 0, "EC(STATFS) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_STATFS, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_statfs,
+                               target, fop_flags, ec_wind_statfs,
                                ec_manager_statfs, callback, data);
     if (fop == NULL) {
         goto out;
@@ -1092,8 +1104,8 @@ void ec_statfs(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -1101,9 +1113,9 @@ void ec_statfs(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1121,14 +1133,13 @@ out:
 
 /* FOP: xattrop */
 
-int32_t ec_combine_xattrop(ec_fop_data_t *fop, ec_cbk_data_t *dst,
-                           ec_cbk_data_t *src)
+int32_t
+ec_combine_xattrop(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_dict_compare(dst->dict, src->dict))
-    {
-        gf_msg (fop->xl->name, GF_LOG_DEBUG, 0,
-                EC_MSG_DICT_MISMATCH, "Mismatching dictionary in "
-                                             "answers of 'GF_FOP_XATTROP'");
+    if (!ec_dict_compare(dst->dict, src->dict)) {
+        gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_DICT_MISMATCH,
+               "Mismatching dictionary in "
+               "answers of 'GF_FOP_XATTROP'");
 
         return 0;
     }
@@ -1137,72 +1148,71 @@ int32_t ec_combine_xattrop(ec_fop_data_t *fop, ec_cbk_data_t *dst,
 }
 
 int32_t
-ec_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, dict_t *xattr,
-                dict_t *xdata)
+ec_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
-        ec_fop_data_t *fop = NULL;
-        ec_lock_link_t *link = NULL;
-        ec_cbk_data_t *cbk = NULL;
-        uint64_t       dirty[2] = {0};
-        data_t *data;
-        uint64_t *version;
-        int32_t idx = (int32_t)(uintptr_t)cookie;
-
-        VALIDATE_OR_GOTO (this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        fop = frame->local;
-
-        ec_trace ("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-                  frame, op_ret, op_errno);
-
-        cbk = ec_cbk_data_allocate (frame, this, fop, fop->id, idx, op_ret,
-                                    op_errno);
-        if (!cbk)
-                goto out;
+    ec_fop_data_t *fop = NULL;
+    ec_lock_link_t *link = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    uint64_t dirty[2] = {0};
+    data_t *data;
+    uint64_t *version;
+    int32_t idx = (int32_t)(uintptr_t)cookie;
 
-        if (op_ret >= 0) {
-                cbk->dict = dict_ref (xattr);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-                data = dict_get(cbk->dict, EC_XATTR_VERSION);
-                if ((data != NULL) && (data->len >= sizeof(uint64_t))) {
-                    version = (uint64_t *)data->data;
+    fop = frame->local;
 
-                    if (((ntoh64(version[0]) >> EC_SELFHEAL_BIT) & 1) != 0) {
-                            LOCK(&fop->lock);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-                            fop->healing |= 1ULL << idx;
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
+                               op_errno);
+    if (!cbk)
+        goto out;
 
-                            UNLOCK(&fop->lock);
-                    }
-                }
+    if (op_ret >= 0) {
+        cbk->dict = dict_ref(xattr);
 
-                ec_dict_del_array (xattr, EC_XATTR_DIRTY, dirty,
-                                   EC_VERSION_SIZE);
-                link = fop->data;
-                if (link) {
-                        /*Keep a note of if the dirty is already set or not*/
-                        link->dirty[0] |= (dirty[0] != 0);
-                        link->dirty[1] |= (dirty[1] != 0);
-                }
+        data = dict_get(cbk->dict, EC_XATTR_VERSION);
+        if ((data != NULL) && (data->len >= sizeof(uint64_t))) {
+            version = (uint64_t *)data->data;
+
+            if (((ntoh64(version[0]) >> EC_SELFHEAL_BIT) & 1) != 0) {
+                LOCK(&fop->lock);
+
+                fop->healing |= 1ULL << idx;
+
+                UNLOCK(&fop->lock);
+            }
         }
 
-        if (xdata)
-                cbk->xdata = dict_ref(xdata);
+        ec_dict_del_array(xattr, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
+        link = fop->data;
+        if (link) {
+            /*Keep a note of if the dirty is already set or not*/
+            link->dirty[0] |= (dirty[0] != 0);
+            link->dirty[1] |= (dirty[1] != 0);
+        }
+    }
+
+    if (xdata)
+        cbk->xdata = dict_ref(xdata);
 
-        ec_combine (cbk, ec_combine_xattrop);
+    ec_combine(cbk, ec_combine_xattrop);
 
 out:
-        if (fop)
-            ec_complete(fop);
+    if (fop)
+        ec_complete(fop);
 
-        return 0;
+    return 0;
 }
 
-void ec_wind_xattrop(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_xattrop(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1211,20 +1221,20 @@ void ec_wind_xattrop(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->xattrop_flags, fop->dict, fop->xdata);
 }
 
-int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_xattrop(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
             if (fop->fd == NULL) {
                 ec_lock_prepare_inode(fop, &fop->loc[0], EC_UPDATE_META, 0,
-                                      LLONG_MAX);
+                                      EC_RANGE_FULL);
             } else {
                 ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META, 0,
-                                   LLONG_MAX);
+                                   EC_RANGE_FULL);
             }
             ec_lock(fop);
 
@@ -1251,19 +1261,13 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_XATTROP)
-            {
-                if (fop->cbks.xattrop != NULL)
-                {
-                    fop->cbks.xattrop(fop->req_frame, fop, fop->xl,
-                                      cbk->op_ret, cbk->op_errno, cbk->dict,
-                                      cbk->xdata);
+            if (fop->id == GF_FOP_XATTROP) {
+                if (fop->cbks.xattrop != NULL) {
+                    fop->cbks.xattrop(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                      cbk->op_errno, cbk->dict, cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fxattrop != NULL)
-                {
+            } else {
+                if (fop->cbks.fxattrop != NULL) {
                     fop->cbks.fxattrop(fop->req_frame, fop, fop->xl,
                                        cbk->op_ret, cbk->op_errno, cbk->dict,
                                        cbk->xdata);
@@ -1279,18 +1283,13 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_XATTROP)
-            {
-                if (fop->cbks.xattrop != NULL)
-                {
+            if (fop->id == GF_FOP_XATTROP) {
+                if (fop->cbks.xattrop != NULL) {
                     fop->cbks.xattrop(fop->req_frame, fop, fop->xl, -1,
                                       fop->error, NULL, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fxattrop != NULL)
-                {
+            } else {
+                if (fop->cbks.fxattrop != NULL) {
                     fop->cbks.fxattrop(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL, NULL);
                 }
@@ -1311,32 +1310,31 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_xattrop_cbk_t func, void * data,
-                loc_t * loc, gf_xattrop_flags_t optype, dict_t * xattr,
-                dict_t * xdata)
+void
+ec_xattrop(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_xattrop_cbk_t func, void *data, loc_t *loc,
+           gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-    ec_cbk_t callback = { .xattrop = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.xattrop = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(XATTROP) %p", frame);
+    gf_msg_trace("ec", 0, "EC(XATTROP) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_XATTROP, 0, target, minimum,
-                               ec_wind_xattrop, ec_manager_xattrop, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_XATTROP, 0, target,
+                               fop_flags, ec_wind_xattrop, ec_manager_xattrop,
+                               callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -1345,8 +1343,8 @@ void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -1354,9 +1352,9 @@ void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xattr != NULL) {
         fop->dict = dict_ref(xattr);
         if (fop->dict == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1364,9 +1362,9 @@ void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1382,7 +1380,8 @@ out:
     }
 }
 
-void ec_wind_fxattrop(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fxattrop(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1391,23 +1390,23 @@ void ec_wind_fxattrop(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->xattrop_flags, fop->dict, fop->xdata);
 }
 
-void ec_fxattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fxattrop_cbk_t func, void * data,
-                 fd_t * fd, gf_xattrop_flags_t optype, dict_t * xattr,
-                 dict_t * xdata)
+void
+ec_fxattrop(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fxattrop_cbk_t func, void *data, fd_t *fd,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fxattrop = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fxattrop = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FXATTROP) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FXATTROP) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FXATTROP, 0, target,
-                               minimum, ec_wind_fxattrop, ec_manager_xattrop,
+                               fop_flags, ec_wind_fxattrop, ec_manager_xattrop,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -1420,9 +1419,9 @@ void ec_fxattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -1430,9 +1429,9 @@ void ec_fxattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xattr != NULL) {
         fop->dict = dict_ref(xattr);
         if (fop->dict == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1440,9 +1439,9 @@ void ec_fxattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1460,11 +1459,12 @@ out:
 
 /* FOP: IPC */
 
-int32_t ec_ipc_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                   int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+           int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -1474,16 +1474,14 @@ int32_t ec_ipc_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_IPC, idx, op_ret,
                                op_errno);
 
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
         }
 
@@ -1491,29 +1489,29 @@ int32_t ec_ipc_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_ipc(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_ipc(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
     STACK_WIND_COOKIE(fop->frame, ec_ipc_cbk, (void *)(uintptr_t)idx,
-                      ec->xl_list[idx], ec->xl_list[idx]->fops->ipc,
-                      fop->int32, fop->xdata);
+                      ec->xl_list[idx], ec->xl_list[idx]->fops->ipc, fop->int32,
+                      fop->xdata);
 }
 
-int32_t ec_manager_ipc(ec_fop_data_t *fop, int32_t state)
+int32_t
+ec_manager_ipc(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -1529,10 +1527,9 @@ int32_t ec_manager_ipc(ec_fop_data_t *fop, int32_t state)
             cbk = fop->answer;
 
             GF_ASSERT(cbk != NULL);
-            if (fop->cbks.ipc != NULL)
-            {
+            if (fop->cbks.ipc != NULL) {
                 fop->cbks.ipc(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                               cbk->op_errno, cbk->xdata);
+                              cbk->op_errno, cbk->xdata);
             }
 
             return EC_STATE_END;
@@ -1543,8 +1540,7 @@ int32_t ec_manager_ipc(ec_fop_data_t *fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.ipc != NULL)
-            {
+            if (fop->cbks.ipc != NULL) {
                 fop->cbks.ipc(fop->req_frame, fop, fop->xl, -1, fop->error,
                               NULL);
             }
@@ -1552,29 +1548,29 @@ int32_t ec_manager_ipc(ec_fop_data_t *fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_ipc(call_frame_t *frame, xlator_t *this, uintptr_t target,
-            int32_t minimum, fop_ipc_cbk_t func, void *data, int32_t op,
-            dict_t *xdata)
+void
+ec_ipc(call_frame_t *frame, xlator_t *this, uintptr_t target,
+       uint32_t fop_flags, fop_ipc_cbk_t func, void *data, int32_t op,
+       dict_t *xdata)
 {
-    ec_cbk_t callback = { .ipc = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.ipc = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(IPC) %p", frame);
+    gf_msg_trace("ec", 0, "EC(IPC) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_IPC, 0, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_IPC, 0, target, fop_flags,
                                ec_wind_ipc, ec_manager_ipc, callback, data);
     if (fop == NULL) {
         goto out;
diff --git a/xlators/cluster/ec/src/ec-gf8.c b/xlators/cluster/ec/src/ec-gf8.c
index 2665632706b..039adae5929 100644
--- a/xlators/cluster/ec/src/ec-gf8.c
+++ b/xlators/cluster/ec/src/ec-gf8.c
@@ -10,5886 +10,5810 @@
 
 #include "ec-gf8.h"
 
-static ec_gf_op_t ec_gf8_mul_00_ops[] = {
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_00 = {
-    0,
-    { 0, },
-    ec_gf8_mul_00_ops
-};
-
-static ec_gf_op_t ec_gf8_mul_01_ops[] = {
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_01 = {
-    8,
-    { 0, 1, 2, 3, 4, 5, 6, 7, },
-    ec_gf8_mul_01_ops
-};
-
-static ec_gf_op_t ec_gf8_mul_02_ops[] = {
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_02 = {
-    8,
-    { 7, 0, 1, 2, 3, 4, 5, 6, },
-    ec_gf8_mul_02_ops
-};
+static ec_gf_op_t ec_gf8_mul_00_ops[] = {{EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_00 = {0,
+                                    {
+                                        0,
+                                    },
+                                    ec_gf8_mul_00_ops};
+
+static ec_gf_op_t ec_gf8_mul_01_ops[] = {{EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_01 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_01_ops};
+
+static ec_gf_op_t ec_gf8_mul_02_ops[] = {{EC_GF_OP_XOR2, 1, 7, 0},
+                                         {EC_GF_OP_XOR2, 2, 7, 0},
+                                         {EC_GF_OP_XOR2, 3, 7, 0},
+                                         {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_02 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_02_ops};
 
 static ec_gf_op_t ec_gf8_mul_03_ops[] = {
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_COPY,   8,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_03 = {
-    9,
-    { 0, 1, 2, 3, 4, 5, 6, 7, 8, },
-    ec_gf8_mul_03_ops
-};
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_COPY, 8, 3, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_03 = {9,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_03_ops};
 
 static ec_gf_op_t ec_gf8_mul_04_ops[] = {
-    { EC_GF_OP_XOR3,   8,  6,  7 },
-    { EC_GF_OP_XOR2,   2,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_04 = {
-    9,
-    { 6, 7, 0, 1, 2, 3, 4, 5, 8, },
-    ec_gf8_mul_04_ops
-};
+    {EC_GF_OP_XOR3, 8, 6, 7}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_04 = {9,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_04_ops};
 
 static ec_gf_op_t ec_gf8_mul_05_ops[] = {
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_05 = {
-    8,
-    { 0, 1, 2, 6, 7, 3, 4, 5, },
-    ec_gf8_mul_05_ops
-};
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_05 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        6,
+                                        7,
+                                        3,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_05_ops};
 
 static ec_gf_op_t ec_gf8_mul_06_ops[] = {
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_COPY,   8,  2,  0 },
-    { EC_GF_OP_XOR2,   8,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_06 = {
-    9,
-    { 7, 0, 1, 2, 8, 3, 4, 5, 6, },
-    ec_gf8_mul_06_ops
-};
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_COPY, 8, 2, 0},
+    {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_06 = {9,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        8,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_06_ops};
 
 static ec_gf_op_t ec_gf8_mul_07_ops[] = {
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_07 = {
-    8,
-    { 6, 0, 1, 3, 2, 4, 5, 7, },
-    ec_gf8_mul_07_ops
-};
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_07 = {8,
+                                    {
+                                        6,
+                                        0,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                        7,
+                                    },
+                                    ec_gf8_mul_07_ops};
 
 static ec_gf_op_t ec_gf8_mul_08_ops[] = {
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR3,   8,  6,  7 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_08 = {
-    9,
-    { 5, 6, 7, 0, 1, 2, 3, 4, 8, },
-    ec_gf8_mul_08_ops
-};
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR3, 8, 6, 7},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_08 = {9,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        8,
+                                    },
+                                    ec_gf8_mul_08_ops};
 
 static ec_gf_op_t ec_gf8_mul_09_ops[] = {
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_09 = {
-    8,
-    { 0, 1, 2, 3, 5, 6, 7, 4, },
-    ec_gf8_mul_09_ops
-};
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_09 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        6,
+                                        7,
+                                        4,
+                                    },
+                                    ec_gf8_mul_09_ops};
 
 static ec_gf_op_t ec_gf8_mul_0A_ops[] = {
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_0A = {
-    8,
-    { 5, 0, 1, 2, 6, 7, 3, 4, },
-    ec_gf8_mul_0A_ops
-};
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0A = {8,
+                                    {
+                                        5,
+                                        0,
+                                        1,
+                                        2,
+                                        6,
+                                        7,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_0A_ops};
 
 static ec_gf_op_t ec_gf8_mul_0B_ops[] = {
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_COPY,   9,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_COPY,   8,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR3,   3,  8,  6 },
-    { EC_GF_OP_XOR2,   1,  9,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_0B = {
-    10,
-    { 7, 1, 5, 2, 4, 3, 0, 6, 8, 9, },
-    ec_gf8_mul_0B_ops
-};
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_COPY, 9, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_COPY, 8, 5, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR3, 3, 8, 6}, {EC_GF_OP_XOR2, 1, 9, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0B = {10,
+                                    {
+                                        7,
+                                        1,
+                                        5,
+                                        2,
+                                        4,
+                                        3,
+                                        0,
+                                        6,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_0B_ops};
 
 static ec_gf_op_t ec_gf8_mul_0C_ops[] = {
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_COPY,   8,  1,  0 },
-    { EC_GF_OP_XOR2,   8,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_0C = {
-    9,
-    { 5, 7, 0, 1, 8, 2, 3, 4, 6, },
-    ec_gf8_mul_0C_ops
-};
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_COPY, 8, 1, 0},
+    {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0C = {9,
+                                    {
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        8,
+                                        2,
+                                        3,
+                                        4,
+                                        6,
+                                    },
+                                    ec_gf8_mul_0C_ops};
 
 static ec_gf_op_t ec_gf8_mul_0D_ops[] = {
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR3,   8,  2,  4 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR3,   2,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_0D = {
-    9,
-    { 5, 6, 7, 3, 1, 0, 2, 4, 8, },
-    ec_gf8_mul_0D_ops
-};
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR3, 8, 2, 4}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR3, 2, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0D = {9,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                        1,
+                                        0,
+                                        2,
+                                        4,
+                                        8,
+                                    },
+                                    ec_gf8_mul_0D_ops};
 
 static ec_gf_op_t ec_gf8_mul_0E_ops[] = {
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_0E = {
-    8,
-    { 7, 0, 6, 1, 3, 2, 4, 5, },
-    ec_gf8_mul_0E_ops
-};
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0E = {8,
+                                    {
+                                        7,
+                                        0,
+                                        6,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_0E_ops};
 
 static ec_gf_op_t ec_gf8_mul_0F_ops[] = {
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_0F = {
-    8,
-    { 1, 0, 5, 6, 7, 2, 3, 4, },
-    ec_gf8_mul_0F_ops
-};
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0F = {8,
+                                    {
+                                        1,
+                                        0,
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_0F_ops};
 
 static ec_gf_op_t ec_gf8_mul_10_ops[] = {
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_10 = {
-    8,
-    { 4, 5, 6, 7, 0, 1, 2, 3, },
-    ec_gf8_mul_10_ops
-};
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_10 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_10_ops};
 
 static ec_gf_op_t ec_gf8_mul_11_ops[] = {
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_11 = {
-    8,
-    { 4, 1, 2, 6, 0, 5, 7, 3, },
-    ec_gf8_mul_11_ops
-};
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_11 = {8,
+                                    {
+                                        4,
+                                        1,
+                                        2,
+                                        6,
+                                        0,
+                                        5,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_11_ops};
 
 static ec_gf_op_t ec_gf8_mul_12_ops[] = {
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_12 = {
-    8,
-    { 7, 0, 1, 2, 3, 5, 6, 4, },
-    ec_gf8_mul_12_ops
-};
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_12 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        6,
+                                        4,
+                                    },
+                                    ec_gf8_mul_12_ops};
 
 static ec_gf_op_t ec_gf8_mul_13_ops[] = {
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR3,   8,  3,  7 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  8,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_13 = {
-    9,
-    { 4, 5, 2, 6, 0, 1, 7, 3, 8, },
-    ec_gf8_mul_13_ops
-};
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 8, 3, 7},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 8, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_13 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        2,
+                                        6,
+                                        0,
+                                        1,
+                                        7,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_13_ops};
 
 static ec_gf_op_t ec_gf8_mul_14_ops[] = {
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_14 = {
-    8,
-    { 6, 7, 0, 1, 2, 4, 5, 3, },
-    ec_gf8_mul_14_ops
-};
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_14 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        5,
+                                        3,
+                                    },
+                                    ec_gf8_mul_14_ops};
 
 static ec_gf_op_t ec_gf8_mul_15_ops[] = {
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR3,   5,  8,  7 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_15 = {
-    9,
-    { 0, 1, 2, 4, 7, 6, 5, 3, 8, },
-    ec_gf8_mul_15_ops
-};
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR3, 5, 8, 7},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_15 = {9,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        7,
+                                        6,
+                                        5,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_15_ops};
 
 static ec_gf_op_t ec_gf8_mul_16_ops[] = {
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_16 = {
-    8,
-    { 6, 7, 4, 1, 2, 3, 5, 0, },
-    ec_gf8_mul_16_ops
-};
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_16 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        4,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        0,
+                                    },
+                                    ec_gf8_mul_16_ops};
 
 static ec_gf_op_t ec_gf8_mul_17_ops[] = {
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_17 = {
-    8,
-    { 5, 7, 0, 1, 3, 2, 4, 6, },
-    ec_gf8_mul_17_ops
-};
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_17 = {8,
+                                    {
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        6,
+                                    },
+                                    ec_gf8_mul_17_ops};
 
 static ec_gf_op_t ec_gf8_mul_18_ops[] = {
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_18 = {
-    9,
-    { 4, 5, 7, 6, 0, 1, 2, 3, 8, },
-    ec_gf8_mul_18_ops
-};
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_18 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        7,
+                                        6,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_18_ops};
 
 static ec_gf_op_t ec_gf8_mul_19_ops[] = {
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_19 = {
-    8,
-    { 0, 5, 2, 6, 7, 1, 3, 4, },
-    ec_gf8_mul_19_ops
-};
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_19 = {8,
+                                    {
+                                        0,
+                                        5,
+                                        2,
+                                        6,
+                                        7,
+                                        1,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_19_ops};
 
 static ec_gf_op_t ec_gf8_mul_1A_ops[] = {
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_1A = {
-    8,
-    { 7, 0, 4, 5, 3, 1, 2, 6, },
-    ec_gf8_mul_1A_ops
-};
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1A = {8,
+                                    {
+                                        7,
+                                        0,
+                                        4,
+                                        5,
+                                        3,
+                                        1,
+                                        2,
+                                        6,
+                                    },
+                                    ec_gf8_mul_1A_ops};
 
 static ec_gf_op_t ec_gf8_mul_1B_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_1B = {
-    8,
-    { 7, 4, 5, 6, 3, 1, 2, 0, },
-    ec_gf8_mul_1B_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1B = {8,
+                                    {
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                        3,
+                                        1,
+                                        2,
+                                        0,
+                                    },
+                                    ec_gf8_mul_1B_ops};
 
 static ec_gf_op_t ec_gf8_mul_1C_ops[] = {
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_1C = {
-    8,
-    { 5, 4, 3, 0, 1, 7, 2, 6, },
-    ec_gf8_mul_1C_ops
-};
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1C = {8,
+                                    {
+                                        5,
+                                        4,
+                                        3,
+                                        0,
+                                        1,
+                                        7,
+                                        2,
+                                        6,
+                                    },
+                                    ec_gf8_mul_1C_ops};
 
 static ec_gf_op_t ec_gf8_mul_1D_ops[] = {
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR3,   8,  4,  2 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_1D = {
-    9,
-    { 0, 7, 5, 8, 2, 3, 4, 1, 6, },
-    ec_gf8_mul_1D_ops
-};
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR3, 8, 4, 2},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1D = {9,
+                                    {
+                                        0,
+                                        7,
+                                        5,
+                                        8,
+                                        2,
+                                        3,
+                                        4,
+                                        1,
+                                        6,
+                                    },
+                                    ec_gf8_mul_1D_ops};
 
 static ec_gf_op_t ec_gf8_mul_1E_ops[] = {
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_1E = {
-    8,
-    { 4, 7, 5, 1, 6, 0, 2, 3, },
-    ec_gf8_mul_1E_ops
-};
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1E = {8,
+                                    {
+                                        4,
+                                        7,
+                                        5,
+                                        1,
+                                        6,
+                                        0,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_1E_ops};
 
 static ec_gf_op_t ec_gf8_mul_1F_ops[] = {
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR3,   8,  3,  7 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_1F = {
-    9,
-    { 1, 4, 5, 6, 7, 0, 3, 2, 8, },
-    ec_gf8_mul_1F_ops
-};
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR3, 8, 3, 7},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1F = {9,
+                                    {
+                                        1,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        3,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_1F_ops};
 
 static ec_gf_op_t ec_gf8_mul_20_ops[] = {
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_20 = {
-    8,
-    { 7, 4, 5, 6, 3, 0, 1, 2, },
-    ec_gf8_mul_20_ops
-};
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_20 = {8,
+                                    {
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                        3,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_20_ops};
 
 static ec_gf_op_t ec_gf8_mul_21_ops[] = {
-    { EC_GF_OP_COPY,   9,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR3,   8,  7,  5 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_XOR2,   2,  8,  0 },
-    { EC_GF_OP_XOR2,   4,  9,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_21 = {
-    10,
-    { 0, 1, 2, 7, 5, 4, 3, 6, 8, 9, },
-    ec_gf8_mul_21_ops
-};
+    {EC_GF_OP_COPY, 9, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR3, 8, 7, 5}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 4, 9, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_21 = {10,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        7,
+                                        5,
+                                        4,
+                                        3,
+                                        6,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_21_ops};
 
 static ec_gf_op_t ec_gf8_mul_22_ops[] = {
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_22 = {
-    8,
-    { 3, 0, 5, 2, 6, 4, 1, 7, },
-    ec_gf8_mul_22_ops
-};
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_22 = {8,
+                                    {
+                                        3,
+                                        0,
+                                        5,
+                                        2,
+                                        6,
+                                        4,
+                                        1,
+                                        7,
+                                    },
+                                    ec_gf8_mul_22_ops};
 
 static ec_gf_op_t ec_gf8_mul_23_ops[] = {
-    { EC_GF_OP_COPY,   8,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_23 = {
-    9,
-    { 0, 4, 3, 2, 5, 6, 1, 8, 7, },
-    ec_gf8_mul_23_ops
-};
+    {EC_GF_OP_COPY, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_23 = {9,
+                                    {
+                                        0,
+                                        4,
+                                        3,
+                                        2,
+                                        5,
+                                        6,
+                                        1,
+                                        8,
+                                        7,
+                                    },
+                                    ec_gf8_mul_23_ops};
 
 static ec_gf_op_t ec_gf8_mul_24_ops[] = {
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_24 = {
-    8,
-    { 6, 7, 0, 1, 2, 4, 5, 3, },
-    ec_gf8_mul_24_ops
-};
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_24 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        5,
+                                        3,
+                                    },
+                                    ec_gf8_mul_24_ops};
 
 static ec_gf_op_t ec_gf8_mul_25_ops[] = {
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_25 = {
-    8,
-    { 2, 7, 0, 1, 3, 4, 5, 6, },
-    ec_gf8_mul_25_ops
-};
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_25 = {8,
+                                    {
+                                        2,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_25_ops};
 
 static ec_gf_op_t ec_gf8_mul_26_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_26 = {
-    8,
-    { 3, 4, 1, 2, 0, 5, 6, 7, },
-    ec_gf8_mul_26_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_26 = {8,
+                                    {
+                                        3,
+                                        4,
+                                        1,
+                                        2,
+                                        0,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_26_ops};
 
 static ec_gf_op_t ec_gf8_mul_27_ops[] = {
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_27 = {
-    8,
-    { 3, 0, 1, 2, 6, 7, 4, 5, },
-    ec_gf8_mul_27_ops
-};
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_27 = {8,
+                                    {
+                                        3,
+                                        0,
+                                        1,
+                                        2,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_27_ops};
 
 static ec_gf_op_t ec_gf8_mul_28_ops[] = {
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_28 = {
-    8,
-    { 5, 6, 3, 0, 1, 2, 4, 7, },
-    ec_gf8_mul_28_ops
-};
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_28 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        3,
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        7,
+                                    },
+                                    ec_gf8_mul_28_ops};
 
 static ec_gf_op_t ec_gf8_mul_29_ops[] = {
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_29 = {
-    8,
-    { 4, 6, 3, 5, 7, 0, 1, 2, },
-    ec_gf8_mul_29_ops
-};
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_29 = {8,
+                                    {
+                                        4,
+                                        6,
+                                        3,
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_29_ops};
 
 static ec_gf_op_t ec_gf8_mul_2A_ops[] = {
-    { EC_GF_OP_COPY,   8,  1,  0 },
-    { EC_GF_OP_XOR2,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR3,   6,  8,  4 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_2A = {
-    9,
-    { 3, 4, 7, 2, 6, 5, 1, 0, 8, },
-    ec_gf8_mul_2A_ops
-};
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 0, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR3, 6, 8, 4}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2A = {9,
+                                    {
+                                        3,
+                                        4,
+                                        7,
+                                        2,
+                                        6,
+                                        5,
+                                        1,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_2A_ops};
 
 static ec_gf_op_t ec_gf8_mul_2B_ops[] = {
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_2B = {
-    8,
-    { 3, 4, 7, 5, 6, 0, 1, 2, },
-    ec_gf8_mul_2B_ops
-};
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2B = {8,
+                                    {
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        6,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_2B_ops};
 
 static ec_gf_op_t ec_gf8_mul_2C_ops[] = {
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_2C = {
-    8,
-    { 5, 6, 7, 0, 2, 3, 4, 1, },
-    ec_gf8_mul_2C_ops
-};
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2C = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        2,
+                                        3,
+                                        4,
+                                        1,
+                                    },
+                                    ec_gf8_mul_2C_ops};
 
 static ec_gf_op_t ec_gf8_mul_2D_ops[] = {
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR3,   8,  4,  6 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_2D = {
-    9,
-    { 7, 0, 3, 5, 1, 4, 2, 6, 8, },
-    ec_gf8_mul_2D_ops
-};
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR3, 8, 4, 6},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2D = {9,
+                                    {
+                                        7,
+                                        0,
+                                        3,
+                                        5,
+                                        1,
+                                        4,
+                                        2,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_2D_ops};
 
 static ec_gf_op_t ec_gf8_mul_2E_ops[] = {
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_COPY,   8,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   8,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  8,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_2E = {
-    9,
-    { 5, 0, 7, 3, 2, 6, 4, 1, 8, },
-    ec_gf8_mul_2E_ops
-};
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_COPY, 8, 4, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2E = {9,
+                                    {
+                                        5,
+                                        0,
+                                        7,
+                                        3,
+                                        2,
+                                        6,
+                                        4,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_2E_ops};
 
 static ec_gf_op_t ec_gf8_mul_2F_ops[] = {
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR3,   8,  7,  6 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_XOR2,   2,  8,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_2F = {
-    9,
-    { 6, 3, 2, 5, 7, 0, 1, 4, 8, },
-    ec_gf8_mul_2F_ops
-};
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR3, 8, 7, 6}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2F = {9,
+                                    {
+                                        6,
+                                        3,
+                                        2,
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        4,
+                                        8,
+                                    },
+                                    ec_gf8_mul_2F_ops};
 
 static ec_gf_op_t ec_gf8_mul_30_ops[] = {
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   8,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR3,   6,  8,  7 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_30 = {
-    9,
-    { 3, 4, 7, 5, 0, 6, 1, 2, 8, },
-    ec_gf8_mul_30_ops
-};
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR3, 6, 8, 7},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_30 = {9,
+                                    {
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        0,
+                                        6,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_30_ops};
 
 static ec_gf_op_t ec_gf8_mul_31_ops[] = {
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_31 = {
-    8,
-    { 7, 1, 4, 5, 6, 0, 2, 3, },
-    ec_gf8_mul_31_ops
-};
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_31 = {8,
+                                    {
+                                        7,
+                                        1,
+                                        4,
+                                        5,
+                                        6,
+                                        0,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_31_ops};
 
 static ec_gf_op_t ec_gf8_mul_32_ops[] = {
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_32 = {
-    8,
-    { 3, 4, 6, 7, 5, 0, 1, 2, },
-    ec_gf8_mul_32_ops
-};
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_32 = {8,
+                                    {
+                                        3,
+                                        4,
+                                        6,
+                                        7,
+                                        5,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_32_ops};
 
 static ec_gf_op_t ec_gf8_mul_33_ops[] = {
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_33 = {
-    8,
-    { 5, 4, 3, 0, 2, 1, 6, 7, },
-    ec_gf8_mul_33_ops
-};
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_33 = {8,
+                                    {
+                                        5,
+                                        4,
+                                        3,
+                                        0,
+                                        2,
+                                        1,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_33_ops};
 
 static ec_gf_op_t ec_gf8_mul_34_ops[] = {
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_34 = {
-    8,
-    { 7, 5, 3, 0, 2, 4, 1, 6, },
-    ec_gf8_mul_34_ops
-};
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_34 = {8,
+                                    {
+                                        7,
+                                        5,
+                                        3,
+                                        0,
+                                        2,
+                                        4,
+                                        1,
+                                        6,
+                                    },
+                                    ec_gf8_mul_34_ops};
 
 static ec_gf_op_t ec_gf8_mul_35_ops[] = {
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_35 = {
-    8,
-    { 6, 7, 5, 4, 2, 0, 1, 3, },
-    ec_gf8_mul_35_ops
-};
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_35 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        5,
+                                        4,
+                                        2,
+                                        0,
+                                        1,
+                                        3,
+                                    },
+                                    ec_gf8_mul_35_ops};
 
 static ec_gf_op_t ec_gf8_mul_36_ops[] = {
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_36 = {
-    8,
-    { 6, 7, 4, 1, 2, 3, 0, 5, },
-    ec_gf8_mul_36_ops
-};
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_36 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        4,
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                        5,
+                                    },
+                                    ec_gf8_mul_36_ops};
 
 static ec_gf_op_t ec_gf8_mul_37_ops[] = {
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR3,   8,  0,  1 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_37 = {
-    9,
-    { 6, 7, 2, 1, 0, 3, 4, 5, 8, },
-    ec_gf8_mul_37_ops
-};
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR3, 8, 0, 1},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_37 = {9,
+                                    {
+                                        6,
+                                        7,
+                                        2,
+                                        1,
+                                        0,
+                                        3,
+                                        4,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_37_ops};
 
 static ec_gf_op_t ec_gf8_mul_38_ops[] = {
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR3,   8,  6,  7 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  8,  0 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_38 = {
-    9,
-    { 4, 5, 6, 3, 0, 1, 7, 2, 8, },
-    ec_gf8_mul_38_ops
-};
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR3, 8, 6, 7},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_38 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        3,
+                                        0,
+                                        1,
+                                        7,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_38_ops};
 
 static ec_gf_op_t ec_gf8_mul_39_ops[] = {
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_39 = {
-    8,
-    { 1, 6, 3, 0, 5, 2, 4, 7, },
-    ec_gf8_mul_39_ops
-};
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_39 = {8,
+                                    {
+                                        1,
+                                        6,
+                                        3,
+                                        0,
+                                        5,
+                                        2,
+                                        4,
+                                        7,
+                                    },
+                                    ec_gf8_mul_39_ops};
 
 static ec_gf_op_t ec_gf8_mul_3A_ops[] = {
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_3A = {
-    8,
-    { 3, 4, 7, 0, 5, 6, 1, 2, },
-    ec_gf8_mul_3A_ops
-};
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3A = {8,
+                                    {
+                                        3,
+                                        4,
+                                        7,
+                                        0,
+                                        5,
+                                        6,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_3A_ops};
 
 static ec_gf_op_t ec_gf8_mul_3B_ops[] = {
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR3,   8,  7,  3 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_3B = {
-    9,
-    { 3, 0, 1, 7, 6, 2, 4, 8, 5, },
-    ec_gf8_mul_3B_ops
-};
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR3, 8, 7, 3}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3B = {9,
+                                    {
+                                        3,
+                                        0,
+                                        1,
+                                        7,
+                                        6,
+                                        2,
+                                        4,
+                                        8,
+                                        5,
+                                    },
+                                    ec_gf8_mul_3B_ops};
 
 static ec_gf_op_t ec_gf8_mul_3C_ops[] = {
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_3C = {
-    8,
-    { 3, 6, 4, 1, 7, 2, 0, 5, },
-    ec_gf8_mul_3C_ops
-};
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3C = {8,
+                                    {
+                                        3,
+                                        6,
+                                        4,
+                                        1,
+                                        7,
+                                        2,
+                                        0,
+                                        5,
+                                    },
+                                    ec_gf8_mul_3C_ops};
 
 static ec_gf_op_t ec_gf8_mul_3D_ops[] = {
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_3D = {
-    8,
-    { 2, 3, 4, 5, 6, 7, 0, 1, },
-    ec_gf8_mul_3D_ops
-};
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3D = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_3D_ops};
 
 static ec_gf_op_t ec_gf8_mul_3E_ops[] = {
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_3E = {
-    8,
-    { 6, 1, 2, 7, 0, 3, 5, 4, },
-    ec_gf8_mul_3E_ops
-};
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3E = {8,
+                                    {
+                                        6,
+                                        1,
+                                        2,
+                                        7,
+                                        0,
+                                        3,
+                                        5,
+                                        4,
+                                    },
+                                    ec_gf8_mul_3E_ops};
 
 static ec_gf_op_t ec_gf8_mul_3F_ops[] = {
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_COPY,  10,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_COPY,   9,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR3,   4,  9,  7 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   3, 10,  0 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_3F = {
-    11,
-    { 1, 7, 6, 2, 4, 3, 5, 0, 8, 9, 10, },
-    ec_gf8_mul_3F_ops
-};
+    {EC_GF_OP_COPY, 8, 0, 0},  {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_COPY, 10, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0},  {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0},  {EC_GF_OP_COPY, 9, 2, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0},  {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0},  {EC_GF_OP_XOR3, 4, 9, 7},
+    {EC_GF_OP_XOR2, 3, 4, 0},  {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0},  {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 3, 10, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3F = {11,
+                                    {
+                                        1,
+                                        7,
+                                        6,
+                                        2,
+                                        4,
+                                        3,
+                                        5,
+                                        0,
+                                        8,
+                                        9,
+                                        10,
+                                    },
+                                    ec_gf8_mul_3F_ops};
 
 static ec_gf_op_t ec_gf8_mul_40_ops[] = {
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR3,   8,  7,  6 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_40 = {
-    9,
-    { 5, 7, 4, 6, 2, 3, 0, 1, 8, },
-    ec_gf8_mul_40_ops
-};
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR3, 8, 7, 6},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_40 = {9,
+                                    {
+                                        5,
+                                        7,
+                                        4,
+                                        6,
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_40_ops};
 
 static ec_gf_op_t ec_gf8_mul_41_ops[] = {
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   8,  4,  0 },
-    { EC_GF_OP_XOR2,   8,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_41 = {
-    9,
-    { 0, 7, 6, 5, 3, 4, 8, 1, 2, },
-    ec_gf8_mul_41_ops
-};
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 4, 0},
+    {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_41 = {9,
+                                    {
+                                        0,
+                                        7,
+                                        6,
+                                        5,
+                                        3,
+                                        4,
+                                        8,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_41_ops};
 
 static ec_gf_op_t ec_gf8_mul_42_ops[] = {
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   8,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_42 = {
-    9,
-    { 2, 7, 1, 6, 4, 3, 0, 5, 8, },
-    ec_gf8_mul_42_ops
-};
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_42 = {9,
+                                    {
+                                        2,
+                                        7,
+                                        1,
+                                        6,
+                                        4,
+                                        3,
+                                        0,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_42_ops};
 
 static ec_gf_op_t ec_gf8_mul_43_ops[] = {
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_43 = {
-    8,
-    { 2, 6, 4, 1, 7, 3, 0, 5, },
-    ec_gf8_mul_43_ops
-};
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_43 = {8,
+                                    {
+                                        2,
+                                        6,
+                                        4,
+                                        1,
+                                        7,
+                                        3,
+                                        0,
+                                        5,
+                                    },
+                                    ec_gf8_mul_43_ops};
 
 static ec_gf_op_t ec_gf8_mul_44_ops[] = {
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_44 = {
-    8,
-    { 2, 3, 4, 1, 6, 5, 0, 7, },
-    ec_gf8_mul_44_ops
-};
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_44 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        1,
+                                        6,
+                                        5,
+                                        0,
+                                        7,
+                                    },
+                                    ec_gf8_mul_44_ops};
 
 static ec_gf_op_t ec_gf8_mul_45_ops[] = {
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_45 = {
-    8,
-    { 2, 3, 0, 1, 7, 4, 5, 6, },
-    ec_gf8_mul_45_ops
-};
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_45 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_45_ops};
 
 static ec_gf_op_t ec_gf8_mul_46_ops[] = {
-    { EC_GF_OP_XOR3,   8,  2,  4 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_46 = {
-    9,
-    { 2, 0, 1, 3, 4, 5, 6, 7, 8, },
-    ec_gf8_mul_46_ops
-};
+    {EC_GF_OP_XOR3, 8, 2, 4}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 8, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_46 = {9,
+                                    {
+                                        2,
+                                        0,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_46_ops};
 
 static ec_gf_op_t ec_gf8_mul_47_ops[] = {
-    { EC_GF_OP_XOR3,   8,  0,  1 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_47 = {
-    9,
-    { 2, 3, 4, 5, 6, 7, 0, 1, 8, },
-    ec_gf8_mul_47_ops
-};
+    {EC_GF_OP_XOR3, 8, 0, 1}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_47 = {9,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_47_ops};
 
 static ec_gf_op_t ec_gf8_mul_48_ops[] = {
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_48 = {
-    8,
-    { 4, 5, 6, 0, 1, 3, 7, 2, },
-    ec_gf8_mul_48_ops
-};
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_48 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        0,
+                                        1,
+                                        3,
+                                        7,
+                                        2,
+                                    },
+                                    ec_gf8_mul_48_ops};
 
 static ec_gf_op_t ec_gf8_mul_49_ops[] = {
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR3,   8,  0,  6 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR3,   1,  8,  5 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_49 = {
-    9,
-    { 7, 2, 4, 0, 3, 5, 1, 6, 8, },
-    ec_gf8_mul_49_ops
-};
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR3, 8, 0, 6},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR3, 1, 8, 5},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_49 = {9,
+                                    {
+                                        7,
+                                        2,
+                                        4,
+                                        0,
+                                        3,
+                                        5,
+                                        1,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_49_ops};
 
 static ec_gf_op_t ec_gf8_mul_4A_ops[] = {
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_4A = {
-    8,
-    { 5, 6, 7, 0, 1, 3, 4, 2, },
-    ec_gf8_mul_4A_ops
-};
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4A = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        4,
+                                        2,
+                                    },
+                                    ec_gf8_mul_4A_ops};
 
 static ec_gf_op_t ec_gf8_mul_4B_ops[] = {
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR3,   8,  3,  7 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_4B = {
-    9,
-    { 5, 3, 6, 7, 0, 2, 4, 1, 8, },
-    ec_gf8_mul_4B_ops
-};
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR3, 8, 3, 7}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4B = {9,
+                                    {
+                                        5,
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        2,
+                                        4,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_4B_ops};
 
 static ec_gf_op_t ec_gf8_mul_4C_ops[] = {
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_4C = {
-    8,
-    { 5, 3, 4, 7, 0, 6, 2, 1, },
-    ec_gf8_mul_4C_ops
-};
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4C = {8,
+                                    {
+                                        5,
+                                        3,
+                                        4,
+                                        7,
+                                        0,
+                                        6,
+                                        2,
+                                        1,
+                                    },
+                                    ec_gf8_mul_4C_ops};
 
 static ec_gf_op_t ec_gf8_mul_4D_ops[] = {
-    { EC_GF_OP_COPY,   8,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR3,   9,  3,  1 },
-    { EC_GF_OP_XOR2,   5,  9,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR3,   0,  8,  2 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_4D = {
-    10,
-    { 0, 9, 3, 5, 6, 4, 7, 1, 2, 8, },
-    ec_gf8_mul_4D_ops
-};
+    {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR3, 9, 3, 1},
+    {EC_GF_OP_XOR2, 5, 9, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 0, 8, 2},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4D = {10,
+                                    {
+                                        0,
+                                        9,
+                                        3,
+                                        5,
+                                        6,
+                                        4,
+                                        7,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_4D_ops};
 
 static ec_gf_op_t ec_gf8_mul_4E_ops[] = {
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_4E = {
-    8,
-    { 2, 3, 0, 1, 5, 6, 7, 4, },
-    ec_gf8_mul_4E_ops
-};
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4E = {8,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        5,
+                                        6,
+                                        7,
+                                        4,
+                                    },
+                                    ec_gf8_mul_4E_ops};
 
 static ec_gf_op_t ec_gf8_mul_4F_ops[] = {
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_4F = {
-    8,
-    { 0, 3, 5, 6, 1, 2, 7, 4, },
-    ec_gf8_mul_4F_ops
-};
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4F = {8,
+                                    {
+                                        0,
+                                        3,
+                                        5,
+                                        6,
+                                        1,
+                                        2,
+                                        7,
+                                        4,
+                                    },
+                                    ec_gf8_mul_4F_ops};
 
 static ec_gf_op_t ec_gf8_mul_50_ops[] = {
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_50 = {
-    8,
-    { 4, 5, 7, 3, 0, 1, 2, 6, },
-    ec_gf8_mul_50_ops
-};
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_50 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        7,
+                                        3,
+                                        0,
+                                        1,
+                                        2,
+                                        6,
+                                    },
+                                    ec_gf8_mul_50_ops};
 
 static ec_gf_op_t ec_gf8_mul_51_ops[] = {
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_51 = {
-    8,
-    { 0, 1, 7, 2, 3, 4, 5, 6, },
-    ec_gf8_mul_51_ops
-};
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_51 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        7,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_51_ops};
 
 static ec_gf_op_t ec_gf8_mul_52_ops[] = {
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_COPY,   9,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR3,   3,  5,  8 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  9,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_52 = {
-    10,
-    { 2, 3, 1, 4, 6, 7, 0, 5, 8, 9, },
-    ec_gf8_mul_52_ops
-};
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_COPY, 9, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR3, 3, 5, 8},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 2, 9, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_52 = {10,
+                                    {
+                                        2,
+                                        3,
+                                        1,
+                                        4,
+                                        6,
+                                        7,
+                                        0,
+                                        5,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_52_ops};
 
 static ec_gf_op_t ec_gf8_mul_53_ops[] = {
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_53 = {
-    8,
-    { 2, 0, 1, 4, 5, 6, 7, 3, },
-    ec_gf8_mul_53_ops
-};
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_53 = {8,
+                                    {
+                                        2,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_53_ops};
 
 static ec_gf_op_t ec_gf8_mul_54_ops[] = {
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_54 = {
-    8,
-    { 7, 3, 0, 4, 2, 6, 5, 1, },
-    ec_gf8_mul_54_ops
-};
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_54 = {8,
+                                    {
+                                        7,
+                                        3,
+                                        0,
+                                        4,
+                                        2,
+                                        6,
+                                        5,
+                                        1,
+                                    },
+                                    ec_gf8_mul_54_ops};
 
 static ec_gf_op_t ec_gf8_mul_55_ops[] = {
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_55 = {
-    8,
-    { 1, 5, 6, 4, 3, 7, 2, 0, },
-    ec_gf8_mul_55_ops
-};
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_55 = {8,
+                                    {
+                                        1,
+                                        5,
+                                        6,
+                                        4,
+                                        3,
+                                        7,
+                                        2,
+                                        0,
+                                    },
+                                    ec_gf8_mul_55_ops};
 
 static ec_gf_op_t ec_gf8_mul_56_ops[] = {
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_56 = {
-    8,
-    { 2, 3, 0, 4, 5, 6, 7, 1, },
-    ec_gf8_mul_56_ops
-};
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_56 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        1,
+                                    },
+                                    ec_gf8_mul_56_ops};
 
 static ec_gf_op_t ec_gf8_mul_57_ops[] = {
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_57 = {
-    8,
-    { 2, 3, 0, 1, 4, 5, 6, 7, },
-    ec_gf8_mul_57_ops
-};
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_57 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_57_ops};
 
 static ec_gf_op_t ec_gf8_mul_58_ops[] = {
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_58 = {
-    8,
-    { 4, 3, 2, 7, 0, 1, 5, 6, },
-    ec_gf8_mul_58_ops
-};
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_58 = {8,
+                                    {
+                                        4,
+                                        3,
+                                        2,
+                                        7,
+                                        0,
+                                        1,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_58_ops};
 
 static ec_gf_op_t ec_gf8_mul_59_ops[] = {
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_59 = {
-    8,
-    { 7, 3, 5, 6, 1, 2, 0, 4, },
-    ec_gf8_mul_59_ops
-};
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_59 = {8,
+                                    {
+                                        7,
+                                        3,
+                                        5,
+                                        6,
+                                        1,
+                                        2,
+                                        0,
+                                        4,
+                                    },
+                                    ec_gf8_mul_59_ops};
 
 static ec_gf_op_t ec_gf8_mul_5A_ops[] = {
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_5A = {
-    8,
-    { 6, 7, 0, 1, 2, 3, 5, 4, },
-    ec_gf8_mul_5A_ops
-};
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5A = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        4,
+                                    },
+                                    ec_gf8_mul_5A_ops};
 
 static ec_gf_op_t ec_gf8_mul_5B_ops[] = {
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_5B = {
-    8,
-    { 6, 0, 7, 5, 2, 1, 3, 4, },
-    ec_gf8_mul_5B_ops
-};
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5B = {8,
+                                    {
+                                        6,
+                                        0,
+                                        7,
+                                        5,
+                                        2,
+                                        1,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_5B_ops};
 
 static ec_gf_op_t ec_gf8_mul_5C_ops[] = {
-    { EC_GF_OP_COPY,   8,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  8,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_5C = {
-    9,
-    { 7, 5, 2, 4, 1, 0, 6, 3, 8, },
-    ec_gf8_mul_5C_ops
-};
+    {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5C = {9,
+                                    {
+                                        7,
+                                        5,
+                                        2,
+                                        4,
+                                        1,
+                                        0,
+                                        6,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_5C_ops};
 
 static ec_gf_op_t ec_gf8_mul_5D_ops[] = {
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_5D = {
-    8,
-    { 1, 3, 5, 4, 6, 7, 2, 0, },
-    ec_gf8_mul_5D_ops
-};
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5D = {8,
+                                    {
+                                        1,
+                                        3,
+                                        5,
+                                        4,
+                                        6,
+                                        7,
+                                        2,
+                                        0,
+                                    },
+                                    ec_gf8_mul_5D_ops};
 
 static ec_gf_op_t ec_gf8_mul_5E_ops[] = {
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_5E = {
-    8,
-    { 4, 3, 6, 2, 5, 7, 0, 1, },
-    ec_gf8_mul_5E_ops
-};
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5E = {8,
+                                    {
+                                        4,
+                                        3,
+                                        6,
+                                        2,
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_5E_ops};
 
 static ec_gf_op_t ec_gf8_mul_5F_ops[] = {
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_5F = {
-    8,
-    { 6, 1, 3, 4, 5, 7, 2, 0, },
-    ec_gf8_mul_5F_ops
-};
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5F = {8,
+                                    {
+                                        6,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        7,
+                                        2,
+                                        0,
+                                    },
+                                    ec_gf8_mul_5F_ops};
 
 static ec_gf_op_t ec_gf8_mul_60_ops[] = {
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_60 = {
-    8,
-    { 2, 3, 4, 7, 5, 6, 0, 1, },
-    ec_gf8_mul_60_ops
-};
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_60 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        6,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_60_ops};
 
 static ec_gf_op_t ec_gf8_mul_61_ops[] = {
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_61 = {
-    8,
-    { 0, 5, 6, 7, 4, 2, 1, 3, },
-    ec_gf8_mul_61_ops
-};
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_61 = {8,
+                                    {
+                                        0,
+                                        5,
+                                        6,
+                                        7,
+                                        4,
+                                        2,
+                                        1,
+                                        3,
+                                    },
+                                    ec_gf8_mul_61_ops};
 
 static ec_gf_op_t ec_gf8_mul_62_ops[] = {
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_62 = {
-    8,
-    { 2, 0, 3, 4, 5, 6, 7, 1, },
-    ec_gf8_mul_62_ops
-};
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_62 = {8,
+                                    {
+                                        2,
+                                        0,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        1,
+                                    },
+                                    ec_gf8_mul_62_ops};
 
 static ec_gf_op_t ec_gf8_mul_63_ops[] = {
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_63 = {
-    8,
-    { 3, 4, 6, 5, 7, 0, 1, 2, },
-    ec_gf8_mul_63_ops
-};
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_63 = {8,
+                                    {
+                                        3,
+                                        4,
+                                        6,
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_63_ops};
 
 static ec_gf_op_t ec_gf8_mul_64_ops[] = {
-    { EC_GF_OP_COPY,   8,  1,  0 },
-    { EC_GF_OP_XOR2,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   8,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_64 = {
-    9,
-    { 2, 3, 4, 6, 5, 7, 8, 1, 0, },
-    ec_gf8_mul_64_ops
-};
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 0, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 8, 7, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_64 = {9,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        6,
+                                        5,
+                                        7,
+                                        8,
+                                        1,
+                                        0,
+                                    },
+                                    ec_gf8_mul_64_ops};
 
 static ec_gf_op_t ec_gf8_mul_65_ops[] = {
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_65 = {
-    8,
-    { 2, 5, 1, 3, 4, 0, 6, 7, },
-    ec_gf8_mul_65_ops
-};
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_65 = {8,
+                                    {
+                                        2,
+                                        5,
+                                        1,
+                                        3,
+                                        4,
+                                        0,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_65_ops};
 
 static ec_gf_op_t ec_gf8_mul_66_ops[] = {
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_66 = {
-    8,
-    { 2, 3, 1, 4, 5, 7, 0, 6, },
-    ec_gf8_mul_66_ops
-};
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_66 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        1,
+                                        4,
+                                        5,
+                                        7,
+                                        0,
+                                        6,
+                                    },
+                                    ec_gf8_mul_66_ops};
 
 static ec_gf_op_t ec_gf8_mul_67_ops[] = {
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_67 = {
-    8,
-    { 2, 4, 5, 6, 7, 3, 1, 0, },
-    ec_gf8_mul_67_ops
-};
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_67 = {8,
+                                    {
+                                        2,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                        1,
+                                        0,
+                                    },
+                                    ec_gf8_mul_67_ops};
 
 static ec_gf_op_t ec_gf8_mul_68_ops[] = {
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_68 = {
-    8,
-    { 5, 7, 2, 3, 0, 6, 4, 1, },
-    ec_gf8_mul_68_ops
-};
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_68 = {8,
+                                    {
+                                        5,
+                                        7,
+                                        2,
+                                        3,
+                                        0,
+                                        6,
+                                        4,
+                                        1,
+                                    },
+                                    ec_gf8_mul_68_ops};
 
 static ec_gf_op_t ec_gf8_mul_69_ops[] = {
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_69 = {
-    8,
-    { 0, 1, 3, 2, 4, 5, 7, 6, },
-    ec_gf8_mul_69_ops
-};
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_69 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                        7,
+                                        6,
+                                    },
+                                    ec_gf8_mul_69_ops};
 
 static ec_gf_op_t ec_gf8_mul_6A_ops[] = {
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_6A = {
-    8,
-    { 5, 7, 4, 6, 1, 2, 0, 3, },
-    ec_gf8_mul_6A_ops
-};
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6A = {8,
+                                    {
+                                        5,
+                                        7,
+                                        4,
+                                        6,
+                                        1,
+                                        2,
+                                        0,
+                                        3,
+                                    },
+                                    ec_gf8_mul_6A_ops};
 
 static ec_gf_op_t ec_gf8_mul_6B_ops[] = {
-    { EC_GF_OP_COPY,   8,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_6B = {
-    9,
-    { 6, 7, 2, 0, 3, 1, 5, 4, 8, },
-    ec_gf8_mul_6B_ops
-};
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6B = {9,
+                                    {
+                                        6,
+                                        7,
+                                        2,
+                                        0,
+                                        3,
+                                        1,
+                                        5,
+                                        4,
+                                        8,
+                                    },
+                                    ec_gf8_mul_6B_ops};
 
 static ec_gf_op_t ec_gf8_mul_6C_ops[] = {
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_6C = {
-    8,
-    { 5, 6, 7, 0, 1, 2, 3, 4, },
-    ec_gf8_mul_6C_ops
-};
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6C = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_6C_ops};
 
 static ec_gf_op_t ec_gf8_mul_6D_ops[] = {
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR3,   8,  3,  4 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_6D = {
-    9,
-    { 3, 6, 7, 0, 4, 5, 1, 2, 8, },
-    ec_gf8_mul_6D_ops
-};
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR3, 8, 3, 4}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6D = {9,
+                                    {
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        4,
+                                        5,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_6D_ops};
 
 static ec_gf_op_t ec_gf8_mul_6E_ops[] = {
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_6E = {
-    8,
-    { 5, 6, 3, 1, 7, 2, 0, 4, },
-    ec_gf8_mul_6E_ops
-};
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6E = {8,
+                                    {
+                                        5,
+                                        6,
+                                        3,
+                                        1,
+                                        7,
+                                        2,
+                                        0,
+                                        4,
+                                    },
+                                    ec_gf8_mul_6E_ops};
 
 static ec_gf_op_t ec_gf8_mul_6F_ops[] = {
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR3,   0,  8,  7 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_6F = {
-    9,
-    { 2, 6, 3, 7, 0, 1, 4, 5, 8, },
-    ec_gf8_mul_6F_ops
-};
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR3, 0, 8, 7}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6F = {9,
+                                    {
+                                        2,
+                                        6,
+                                        3,
+                                        7,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_6F_ops};
 
 static ec_gf_op_t ec_gf8_mul_70_ops[] = {
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_70 = {
-    8,
-    { 3, 4, 5, 2, 6, 0, 1, 7, },
-    ec_gf8_mul_70_ops
-};
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_70 = {8,
+                                    {
+                                        3,
+                                        4,
+                                        5,
+                                        2,
+                                        6,
+                                        0,
+                                        1,
+                                        7,
+                                    },
+                                    ec_gf8_mul_70_ops};
 
 static ec_gf_op_t ec_gf8_mul_71_ops[] = {
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_71 = {
-    8,
-    { 4, 7, 5, 3, 6, 0, 2, 1, },
-    ec_gf8_mul_71_ops
-};
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_71 = {8,
+                                    {
+                                        4,
+                                        7,
+                                        5,
+                                        3,
+                                        6,
+                                        0,
+                                        2,
+                                        1,
+                                    },
+                                    ec_gf8_mul_71_ops};
 
 static ec_gf_op_t ec_gf8_mul_72_ops[] = {
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_72 = {
-    8,
-    { 0, 5, 2, 7, 4, 1, 3, 6, },
-    ec_gf8_mul_72_ops
-};
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_72 = {8,
+                                    {
+                                        0,
+                                        5,
+                                        2,
+                                        7,
+                                        4,
+                                        1,
+                                        3,
+                                        6,
+                                    },
+                                    ec_gf8_mul_72_ops};
 
 static ec_gf_op_t ec_gf8_mul_73_ops[] = {
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_73 = {
-    8,
-    { 6, 0, 1, 7, 4, 5, 2, 3, },
-    ec_gf8_mul_73_ops
-};
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_73 = {8,
+                                    {
+                                        6,
+                                        0,
+                                        1,
+                                        7,
+                                        4,
+                                        5,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_73_ops};
 
 static ec_gf_op_t ec_gf8_mul_74_ops[] = {
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_74 = {
-    8,
-    { 3, 2, 1, 0, 4, 5, 6, 7, },
-    ec_gf8_mul_74_ops
-};
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_74 = {8,
+                                    {
+                                        3,
+                                        2,
+                                        1,
+                                        0,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_74_ops};
 
 static ec_gf_op_t ec_gf8_mul_75_ops[] = {
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_75 = {
-    8,
-    { 4, 5, 6, 7, 0, 1, 2, 3, },
-    ec_gf8_mul_75_ops
-};
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_75 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_75_ops};
 
 static ec_gf_op_t ec_gf8_mul_76_ops[] = {
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR3,   8,  6,  2 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_76 = {
-    9,
-    { 2, 3, 0, 6, 5, 1, 7, 8, 4, },
-    ec_gf8_mul_76_ops
-};
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 8, 6, 2},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 8, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_76 = {9,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        6,
+                                        5,
+                                        1,
+                                        7,
+                                        8,
+                                        4,
+                                    },
+                                    ec_gf8_mul_76_ops};
 
 static ec_gf_op_t ec_gf8_mul_77_ops[] = {
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_77 = {
-    8,
-    { 7, 4, 3, 6, 0, 1, 5, 2, },
-    ec_gf8_mul_77_ops
-};
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_77 = {8,
+                                    {
+                                        7,
+                                        4,
+                                        3,
+                                        6,
+                                        0,
+                                        1,
+                                        5,
+                                        2,
+                                    },
+                                    ec_gf8_mul_77_ops};
 
 static ec_gf_op_t ec_gf8_mul_78_ops[] = {
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR3,   8,  0,  2 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_78 = {
-    9,
-    { 4, 7, 3, 2, 5, 1, 6, 0, 8, },
-    ec_gf8_mul_78_ops
-};
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR3, 8, 0, 2},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_78 = {9,
+                                    {
+                                        4,
+                                        7,
+                                        3,
+                                        2,
+                                        5,
+                                        1,
+                                        6,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_78_ops};
 
 static ec_gf_op_t ec_gf8_mul_79_ops[] = {
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR3,   8,  4,  7 },
-    { EC_GF_OP_XOR2,   0,  8,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  8,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_79 = {
-    9,
-    { 4, 5, 7, 3, 1, 6, 2, 0, 8, },
-    ec_gf8_mul_79_ops
-};
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR3, 8, 4, 7},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_79 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        7,
+                                        3,
+                                        1,
+                                        6,
+                                        2,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_79_ops};
 
 static ec_gf_op_t ec_gf8_mul_7A_ops[] = {
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_7A = {
-    8,
-    { 1, 2, 3, 4, 5, 6, 7, 0, },
-    ec_gf8_mul_7A_ops
-};
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7A = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                    },
+                                    ec_gf8_mul_7A_ops};
 
 static ec_gf_op_t ec_gf8_mul_7B_ops[] = {
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR3,   8,  5,  3 },
-    { EC_GF_OP_XOR2,   8,  0,  0 },
-    { EC_GF_OP_COPY,   9,  4,  0 },
-    { EC_GF_OP_XOR2,   8,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR3,   4,  1,  9 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_7B = {
-    10,
-    { 1, 2, 3, 4, 8, 5, 6, 0, 7, 9, },
-    ec_gf8_mul_7B_ops
-};
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR3, 8, 5, 3},
+    {EC_GF_OP_XOR2, 8, 0, 0}, {EC_GF_OP_COPY, 9, 4, 0},
+    {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 4, 1, 9},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7B = {10,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        8,
+                                        5,
+                                        6,
+                                        0,
+                                        7,
+                                        9,
+                                    },
+                                    ec_gf8_mul_7B_ops};
 
 static ec_gf_op_t ec_gf8_mul_7C_ops[] = {
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_7C = {
-    8,
-    { 2, 4, 1, 6, 3, 5, 7, 0, },
-    ec_gf8_mul_7C_ops
-};
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7C = {8,
+                                    {
+                                        2,
+                                        4,
+                                        1,
+                                        6,
+                                        3,
+                                        5,
+                                        7,
+                                        0,
+                                    },
+                                    ec_gf8_mul_7C_ops};
 
 static ec_gf_op_t ec_gf8_mul_7D_ops[] = {
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_7D = {
-    8,
-    { 1, 0, 3, 5, 6, 7, 2, 4, },
-    ec_gf8_mul_7D_ops
-};
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7D = {8,
+                                    {
+                                        1,
+                                        0,
+                                        3,
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        4,
+                                    },
+                                    ec_gf8_mul_7D_ops};
 
 static ec_gf_op_t ec_gf8_mul_7E_ops[] = {
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR3,   6,  2,  7 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_7E = {
-    9,
-    { 5, 1, 2, 0, 7, 3, 4, 6, 8, },
-    ec_gf8_mul_7E_ops
-};
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_COPY, 8, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 6, 2, 7},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7E = {9,
+                                    {
+                                        5,
+                                        1,
+                                        2,
+                                        0,
+                                        7,
+                                        3,
+                                        4,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_7E_ops};
 
 static ec_gf_op_t ec_gf8_mul_7F_ops[] = {
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR3,   9,  7,  5 },
-    { EC_GF_OP_XOR2,   2,  9,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  9,  0 },
-    { EC_GF_OP_XOR3,   9,  6,  4 },
-    { EC_GF_OP_XOR2,   7,  9,  0 },
-    { EC_GF_OP_XOR2,   3,  9,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_7F = {
-    10,
-    { 4, 1, 0, 5, 6, 7, 2, 3, 8, 9, },
-    ec_gf8_mul_7F_ops
-};
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR3, 9, 7, 5}, {EC_GF_OP_XOR2, 2, 9, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 9, 0},
+    {EC_GF_OP_XOR3, 9, 6, 4}, {EC_GF_OP_XOR2, 7, 9, 0},
+    {EC_GF_OP_XOR2, 3, 9, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7F = {10,
+                                    {
+                                        4,
+                                        1,
+                                        0,
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        3,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_7F_ops};
 
 static ec_gf_op_t ec_gf8_mul_80_ops[] = {
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_80 = {
-    8,
-    { 7, 5, 6, 4, 1, 2, 3, 0, },
-    ec_gf8_mul_80_ops
-};
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_80 = {8,
+                                    {
+                                        7,
+                                        5,
+                                        6,
+                                        4,
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                    },
+                                    ec_gf8_mul_80_ops};
 
 static ec_gf_op_t ec_gf8_mul_81_ops[] = {
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_81 = {
-    8,
-    { 2, 7, 4, 1, 5, 6, 3, 0, },
-    ec_gf8_mul_81_ops
-};
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_81 = {8,
+                                    {
+                                        2,
+                                        7,
+                                        4,
+                                        1,
+                                        5,
+                                        6,
+                                        3,
+                                        0,
+                                    },
+                                    ec_gf8_mul_81_ops};
 
 static ec_gf_op_t ec_gf8_mul_82_ops[] = {
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_COPY,   8,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR3,   5,  8,  7 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_82 = {
-    9,
-    { 6, 2, 7, 5, 1, 3, 4, 0, 8, },
-    ec_gf8_mul_82_ops
-};
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR3, 5, 8, 7}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_82 = {9,
+                                    {
+                                        6,
+                                        2,
+                                        7,
+                                        5,
+                                        1,
+                                        3,
+                                        4,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_82_ops};
 
 static ec_gf_op_t ec_gf8_mul_83_ops[] = {
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_83 = {
-    8,
-    { 3, 5, 6, 7, 1, 2, 4, 0, },
-    ec_gf8_mul_83_ops
-};
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_83 = {8,
+                                    {
+                                        3,
+                                        5,
+                                        6,
+                                        7,
+                                        1,
+                                        2,
+                                        4,
+                                        0,
+                                    },
+                                    ec_gf8_mul_83_ops};
 
 static ec_gf_op_t ec_gf8_mul_84_ops[] = {
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_84 = {
-    8,
-    { 7, 6, 0, 4, 1, 5, 3, 2, },
-    ec_gf8_mul_84_ops
-};
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_84 = {8,
+                                    {
+                                        7,
+                                        6,
+                                        0,
+                                        4,
+                                        1,
+                                        5,
+                                        3,
+                                        2,
+                                    },
+                                    ec_gf8_mul_84_ops};
 
 static ec_gf_op_t ec_gf8_mul_85_ops[] = {
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_85 = {
-    8,
-    { 7, 6, 0, 3, 2, 4, 5, 1, },
-    ec_gf8_mul_85_ops
-};
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_85 = {8,
+                                    {
+                                        7,
+                                        6,
+                                        0,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                        1,
+                                    },
+                                    ec_gf8_mul_85_ops};
 
 static ec_gf_op_t ec_gf8_mul_86_ops[] = {
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_86 = {
-    8,
-    { 1, 2, 6, 4, 5, 7, 3, 0, },
-    ec_gf8_mul_86_ops
-};
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_86 = {8,
+                                    {
+                                        1,
+                                        2,
+                                        6,
+                                        4,
+                                        5,
+                                        7,
+                                        3,
+                                        0,
+                                    },
+                                    ec_gf8_mul_86_ops};
 
 static ec_gf_op_t ec_gf8_mul_87_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_COPY,   8,  1,  0 },
-    { EC_GF_OP_XOR2,   8,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR3,   5,  8,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_87 = {
-    9,
-    { 1, 2, 3, 4, 5, 7, 6, 0, 8, },
-    ec_gf8_mul_87_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_COPY, 8, 1, 0},
+    {EC_GF_OP_XOR2, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR3, 5, 8, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_87 = {9,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        7,
+                                        6,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_87_ops};
 
 static ec_gf_op_t ec_gf8_mul_88_ops[] = {
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_88 = {
-    8,
-    { 6, 7, 3, 1, 2, 4, 5, 0, },
-    ec_gf8_mul_88_ops
-};
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_88 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        3,
+                                        1,
+                                        2,
+                                        4,
+                                        5,
+                                        0,
+                                    },
+                                    ec_gf8_mul_88_ops};
 
 static ec_gf_op_t ec_gf8_mul_89_ops[] = {
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR3,   8,  5,  2 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_89 = {
-    9,
-    { 2, 1, 6, 5, 7, 3, 4, 0, 8, },
-    ec_gf8_mul_89_ops
-};
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR3, 8, 5, 2},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_89 = {9,
+                                    {
+                                        2,
+                                        1,
+                                        6,
+                                        5,
+                                        7,
+                                        3,
+                                        4,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_89_ops};
 
 static ec_gf_op_t ec_gf8_mul_8A_ops[] = {
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_8A = {
-    8,
-    { 1, 2, 3, 0, 6, 7, 4, 5, },
-    ec_gf8_mul_8A_ops
-};
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8A = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_8A_ops};
 
 static ec_gf_op_t ec_gf8_mul_8B_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_8B = {
-    8,
-    { 6, 1, 2, 3, 5, 7, 4, 0, },
-    ec_gf8_mul_8B_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8B = {8,
+                                    {
+                                        6,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        7,
+                                        4,
+                                        0,
+                                    },
+                                    ec_gf8_mul_8B_ops};
 
 static ec_gf_op_t ec_gf8_mul_8C_ops[] = {
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_8C = {
-    8,
-    { 1, 2, 0, 7, 3, 4, 5, 6, },
-    ec_gf8_mul_8C_ops
-};
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8C = {8,
+                                    {
+                                        1,
+                                        2,
+                                        0,
+                                        7,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_8C_ops};
 
 static ec_gf_op_t ec_gf8_mul_8D_ops[] = {
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_8D = {
-    8,
-    { 7, 1, 3, 2, 4, 5, 0, 6, },
-    ec_gf8_mul_8D_ops
-};
-
-static ec_gf_op_t ec_gf8_mul_8E_ops[] = {
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_8E = {
-    8,
-    { 1, 2, 3, 4, 5, 6, 7, 0, },
-    ec_gf8_mul_8E_ops
-};
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8D = {8,
+                                    {
+                                        7,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                        0,
+                                        6,
+                                    },
+                                    ec_gf8_mul_8D_ops};
+
+static ec_gf_op_t ec_gf8_mul_8E_ops[] = {{EC_GF_OP_XOR2, 2, 0, 0},
+                                         {EC_GF_OP_XOR2, 3, 0, 0},
+                                         {EC_GF_OP_XOR2, 4, 0, 0},
+                                         {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8E = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                    },
+                                    ec_gf8_mul_8E_ops};
 
 static ec_gf_op_t ec_gf8_mul_8F_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_8F = {
-    8,
-    { 1, 2, 3, 4, 5, 6, 7, 0, },
-    ec_gf8_mul_8F_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8F = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                    },
+                                    ec_gf8_mul_8F_ops};
 
 static ec_gf_op_t ec_gf8_mul_90_ops[] = {
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_90 = {
-    8,
-    { 4, 5, 6, 7, 0, 1, 3, 2, },
-    ec_gf8_mul_90_ops
-};
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_90 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        2,
+                                    },
+                                    ec_gf8_mul_90_ops};
 
 static ec_gf_op_t ec_gf8_mul_91_ops[] = {
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_COPY,   9,  1,  0 },
-    { EC_GF_OP_COPY,   8,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  9,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR3,   5,  8,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_91 = {
-    10,
-    { 2, 3, 1, 4, 0, 6, 7, 5, 8, 9, },
-    ec_gf8_mul_91_ops
-};
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_COPY, 9, 1, 0}, {EC_GF_OP_COPY, 8, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 9, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR3, 5, 8, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_91 = {10,
+                                    {
+                                        2,
+                                        3,
+                                        1,
+                                        4,
+                                        0,
+                                        6,
+                                        7,
+                                        5,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_91_ops};
 
 static ec_gf_op_t ec_gf8_mul_92_ops[] = {
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_92 = {
-    8,
-    { 6, 7, 0, 1, 2, 3, 5, 4, },
-    ec_gf8_mul_92_ops
-};
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_92 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        4,
+                                    },
+                                    ec_gf8_mul_92_ops};
 
 static ec_gf_op_t ec_gf8_mul_93_ops[] = {
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_93 = {
-    8,
-    { 6, 4, 5, 1, 7, 2, 3, 0, },
-    ec_gf8_mul_93_ops
-};
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_93 = {8,
+                                    {
+                                        6,
+                                        4,
+                                        5,
+                                        1,
+                                        7,
+                                        2,
+                                        3,
+                                        0,
+                                    },
+                                    ec_gf8_mul_93_ops};
 
 static ec_gf_op_t ec_gf8_mul_94_ops[] = {
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_94 = {
-    8,
-    { 7, 5, 0, 2, 6, 1, 3, 4, },
-    ec_gf8_mul_94_ops
-};
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_94 = {8,
+                                    {
+                                        7,
+                                        5,
+                                        0,
+                                        2,
+                                        6,
+                                        1,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_94_ops};
 
 static ec_gf_op_t ec_gf8_mul_95_ops[] = {
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_95 = {
-    8,
-    { 7, 6, 1, 3, 0, 4, 5, 2, },
-    ec_gf8_mul_95_ops
-};
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_95 = {8,
+                                    {
+                                        7,
+                                        6,
+                                        1,
+                                        3,
+                                        0,
+                                        4,
+                                        5,
+                                        2,
+                                    },
+                                    ec_gf8_mul_95_ops};
 
 static ec_gf_op_t ec_gf8_mul_96_ops[] = {
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR3,   8,  0,  4 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   8,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_96 = {
-    9,
-    { 4, 0, 1, 6, 7, 2, 3, 5, 8, },
-    ec_gf8_mul_96_ops
-};
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR3, 8, 0, 4}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_96 = {9,
+                                    {
+                                        4,
+                                        0,
+                                        1,
+                                        6,
+                                        7,
+                                        2,
+                                        3,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_96_ops};
 
 static ec_gf_op_t ec_gf8_mul_97_ops[] = {
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_COPY,   8,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   8,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_97 = {
-    9,
-    { 4, 5, 3, 6, 7, 1, 2, 0, 8, },
-    ec_gf8_mul_97_ops
-};
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_COPY, 8, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 8, 6, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_97 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        3,
+                                        6,
+                                        7,
+                                        1,
+                                        2,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_97_ops};
 
 static ec_gf_op_t ec_gf8_mul_98_ops[] = {
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_98 = {
-    8,
-    { 4, 2, 3, 6, 7, 5, 1, 0, },
-    ec_gf8_mul_98_ops
-};
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_98 = {8,
+                                    {
+                                        4,
+                                        2,
+                                        3,
+                                        6,
+                                        7,
+                                        5,
+                                        1,
+                                        0,
+                                    },
+                                    ec_gf8_mul_98_ops};
 
 static ec_gf_op_t ec_gf8_mul_99_ops[] = {
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_99 = {
-    8,
-    { 6, 5, 3, 7, 0, 1, 4, 2, },
-    ec_gf8_mul_99_ops
-};
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_99 = {8,
+                                    {
+                                        6,
+                                        5,
+                                        3,
+                                        7,
+                                        0,
+                                        1,
+                                        4,
+                                        2,
+                                    },
+                                    ec_gf8_mul_99_ops};
 
 static ec_gf_op_t ec_gf8_mul_9A_ops[] = {
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR3,   8,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_9A = {
-    9,
-    { 6, 3, 4, 0, 5, 1, 2, 7, 8, },
-    ec_gf8_mul_9A_ops
-};
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR3, 8, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9A = {9,
+                                    {
+                                        6,
+                                        3,
+                                        4,
+                                        0,
+                                        5,
+                                        1,
+                                        2,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_9A_ops};
 
 static ec_gf_op_t ec_gf8_mul_9B_ops[] = {
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_COPY,   9,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR3,   8,  3,  2 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  9,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_9B = {
-    10,
-    { 4, 5, 8, 6, 7, 1, 2, 0, 3, 9, },
-    ec_gf8_mul_9B_ops
-};
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_COPY, 9, 5, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR3, 8, 3, 2}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 3, 9, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9B = {10,
+                                    {
+                                        4,
+                                        5,
+                                        8,
+                                        6,
+                                        7,
+                                        1,
+                                        2,
+                                        0,
+                                        3,
+                                        9,
+                                    },
+                                    ec_gf8_mul_9B_ops};
 
 static ec_gf_op_t ec_gf8_mul_9C_ops[] = {
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_9C = {
-    8,
-    { 3, 2, 1, 0, 4, 5, 6, 7, },
-    ec_gf8_mul_9C_ops
-};
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9C = {8,
+                                    {
+                                        3,
+                                        2,
+                                        1,
+                                        0,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_9C_ops};
 
 static ec_gf_op_t ec_gf8_mul_9D_ops[] = {
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_9D = {
-    8,
-    { 0, 1, 2, 3, 7, 4, 5, 6, },
-    ec_gf8_mul_9D_ops
-};
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9D = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_9D_ops};
 
 static ec_gf_op_t ec_gf8_mul_9E_ops[] = {
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_COPY,   8,  7,  0 },
-    { EC_GF_OP_XOR2,   8,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_9E = {
-    9,
-    { 4, 5, 3, 8, 6, 0, 2, 7, 1, },
-    ec_gf8_mul_9E_ops
-};
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_COPY, 8, 7, 0},
+    {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9E = {9,
+                                    {
+                                        4,
+                                        5,
+                                        3,
+                                        8,
+                                        6,
+                                        0,
+                                        2,
+                                        7,
+                                        1,
+                                    },
+                                    ec_gf8_mul_9E_ops};
 
 static ec_gf_op_t ec_gf8_mul_9F_ops[] = {
-    { EC_GF_OP_XOR3,   8,  1,  2 },
-    { EC_GF_OP_XOR2,   8,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  8,  0 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_9F = {
-    9,
-    { 4, 5, 6, 7, 0, 1, 2, 3, 8, },
-    ec_gf8_mul_9F_ops
-};
+    {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 8, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9F = {9,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_9F_ops};
 
 static ec_gf_op_t ec_gf8_mul_A0_ops[] = {
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_A0 = {
-    8,
-    { 3, 1, 6, 7, 5, 2, 4, 0, },
-    ec_gf8_mul_A0_ops
-};
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A0 = {8,
+                                    {
+                                        3,
+                                        1,
+                                        6,
+                                        7,
+                                        5,
+                                        2,
+                                        4,
+                                        0,
+                                    },
+                                    ec_gf8_mul_A0_ops};
 
 static ec_gf_op_t ec_gf8_mul_A1_ops[] = {
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR3,   8,  0,  6 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_A1 = {
-    9,
-    { 7, 4, 1, 5, 6, 0, 2, 3, 8, },
-    ec_gf8_mul_A1_ops
-};
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR3, 8, 0, 6},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A1 = {9,
+                                    {
+                                        7,
+                                        4,
+                                        1,
+                                        5,
+                                        6,
+                                        0,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_A1_ops};
 
 static ec_gf_op_t ec_gf8_mul_A2_ops[] = {
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_A2 = {
-    8,
-    { 7, 0, 6, 3, 2, 1, 4, 5, },
-    ec_gf8_mul_A2_ops
-};
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A2 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        6,
+                                        3,
+                                        2,
+                                        1,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_A2_ops};
 
 static ec_gf_op_t ec_gf8_mul_A3_ops[] = {
-    { EC_GF_OP_COPY,   8,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_A3 = {
-    9,
-    { 3, 7, 2, 6, 1, 4, 0, 5, 8, },
-    ec_gf8_mul_A3_ops
-};
+    {EC_GF_OP_COPY, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A3 = {9,
+                                    {
+                                        3,
+                                        7,
+                                        2,
+                                        6,
+                                        1,
+                                        4,
+                                        0,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_A3_ops};
 
 static ec_gf_op_t ec_gf8_mul_A4_ops[] = {
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_A4 = {
-    8,
-    { 5, 6, 7, 2, 4, 3, 0, 1, },
-    ec_gf8_mul_A4_ops
-};
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A4 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        4,
+                                        3,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_A4_ops};
 
 static ec_gf_op_t ec_gf8_mul_A5_ops[] = {
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR3,   8,  5,  6 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_A5 = {
-    9,
-    { 1, 4, 2, 5, 6, 7, 3, 0, 8, },
-    ec_gf8_mul_A5_ops
-};
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR3, 8, 5, 6}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A5 = {9,
+                                    {
+                                        1,
+                                        4,
+                                        2,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_A5_ops};
 
 static ec_gf_op_t ec_gf8_mul_A6_ops[] = {
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_A6 = {
-    8,
-    { 1, 2, 0, 3, 4, 5, 6, 7, },
-    ec_gf8_mul_A6_ops
-};
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A6 = {8,
+                                    {
+                                        1,
+                                        2,
+                                        0,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_A6_ops};
 
 static ec_gf_op_t ec_gf8_mul_A7_ops[] = {
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_A7 = {
-    8,
-    { 0, 1, 2, 5, 6, 7, 3, 4, },
-    ec_gf8_mul_A7_ops
-};
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A7 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_A7_ops};
 
 static ec_gf_op_t ec_gf8_mul_A8_ops[] = {
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   8,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_COPY,   9,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   8,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  9,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_A8 = {
-    10,
-    { 1, 7, 5, 8, 6, 3, 4, 0, 2, 9, },
-    ec_gf8_mul_A8_ops
-};
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_COPY, 9, 4, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 8, 3, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 9, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A8 = {10,
+                                    {
+                                        1,
+                                        7,
+                                        5,
+                                        8,
+                                        6,
+                                        3,
+                                        4,
+                                        0,
+                                        2,
+                                        9,
+                                    },
+                                    ec_gf8_mul_A8_ops};
 
 static ec_gf_op_t ec_gf8_mul_A9_ops[] = {
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_A9 = {
-    8,
-    { 3, 7, 6, 1, 2, 0, 4, 5, },
-    ec_gf8_mul_A9_ops
-};
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A9 = {8,
+                                    {
+                                        3,
+                                        7,
+                                        6,
+                                        1,
+                                        2,
+                                        0,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_A9_ops};
 
 static ec_gf_op_t ec_gf8_mul_AA_ops[] = {
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_AA = {
-    8,
-    { 0, 4, 5, 3, 6, 7, 1, 2, },
-    ec_gf8_mul_AA_ops
-};
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AA = {8,
+                                    {
+                                        0,
+                                        4,
+                                        5,
+                                        3,
+                                        6,
+                                        7,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_AA_ops};
 
 static ec_gf_op_t ec_gf8_mul_AB_ops[] = {
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_COPY,   9,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   8,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR3,   3,  9,  7 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_AB = {
-    10,
-    { 2, 3, 8, 0, 5, 6, 1, 4, 7, 9, },
-    ec_gf8_mul_AB_ops
-};
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_COPY, 9, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR3, 3, 9, 7},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AB = {10,
+                                    {
+                                        2,
+                                        3,
+                                        8,
+                                        0,
+                                        5,
+                                        6,
+                                        1,
+                                        4,
+                                        7,
+                                        9,
+                                    },
+                                    ec_gf8_mul_AB_ops};
 
 static ec_gf_op_t ec_gf8_mul_AC_ops[] = {
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_AC = {
-    8,
-    { 3, 2, 1, 0, 4, 5, 6, 7, },
-    ec_gf8_mul_AC_ops
-};
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AC = {8,
+                                    {
+                                        3,
+                                        2,
+                                        1,
+                                        0,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_AC_ops};
 
 static ec_gf_op_t ec_gf8_mul_AD_ops[] = {
-    { EC_GF_OP_XOR3,   8,  1,  2 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_AD = {
-    9,
-    { 3, 4, 5, 6, 7, 0, 1, 2, 8, },
-    ec_gf8_mul_AD_ops
-};
+    {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AD = {9,
+                                    {
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_AD_ops};
 
 static ec_gf_op_t ec_gf8_mul_AE_ops[] = {
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_COPY,   8,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_AE = {
-    9,
-    { 7, 0, 5, 6, 3, 4, 1, 2, 8, },
-    ec_gf8_mul_AE_ops
-};
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_COPY, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AE = {9,
+                                    {
+                                        7,
+                                        0,
+                                        5,
+                                        6,
+                                        3,
+                                        4,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_AE_ops};
 
 static ec_gf_op_t ec_gf8_mul_AF_ops[] = {
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_AF = {
-    8,
-    { 0, 1, 2, 7, 3, 4, 5, 6, },
-    ec_gf8_mul_AF_ops
-};
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AF = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        7,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_AF_ops};
 
 static ec_gf_op_t ec_gf8_mul_B0_ops[] = {
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_B0 = {
-    8,
-    { 4, 0, 7, 2, 3, 1, 6, 5, },
-    ec_gf8_mul_B0_ops
-};
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B0 = {8,
+                                    {
+                                        4,
+                                        0,
+                                        7,
+                                        2,
+                                        3,
+                                        1,
+                                        6,
+                                        5,
+                                    },
+                                    ec_gf8_mul_B0_ops};
 
 static ec_gf_op_t ec_gf8_mul_B1_ops[] = {
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_COPY,   8,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR3,   5,  8,  1 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_B1 = {
-    9,
-    { 2, 6, 4, 7, 0, 1, 3, 5, 8, },
-    ec_gf8_mul_B1_ops
-};
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_COPY, 8, 4, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR3, 5, 8, 1}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B1 = {9,
+                                    {
+                                        2,
+                                        6,
+                                        4,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_B1_ops};
 
 static ec_gf_op_t ec_gf8_mul_B2_ops[] = {
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR3,   8,  4,  5 },
-    { EC_GF_OP_XOR2,   2,  8,  0 },
-    { EC_GF_OP_XOR2,   8,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_B2 = {
-    9,
-    { 0, 7, 4, 5, 6, 1, 2, 3, 8, },
-    ec_gf8_mul_B2_ops
-};
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 8, 4, 5},
+    {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 8, 1, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B2 = {9,
+                                    {
+                                        0,
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                        1,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_B2_ops};
 
 static ec_gf_op_t ec_gf8_mul_B3_ops[] = {
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_COPY,   9,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR3,   8,  6,  4 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   8,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_XOR2,   0,  8,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR3,   1,  9,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_B3 = {
-    10,
-    { 2, 3, 4, 5, 1, 6, 0, 7, 8, 9, },
-    ec_gf8_mul_B3_ops
-};
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_COPY, 9, 5, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR3, 8, 6, 4},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 8, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR3, 1, 9, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B3 = {10,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        1,
+                                        6,
+                                        0,
+                                        7,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_B3_ops};
 
 static ec_gf_op_t ec_gf8_mul_B4_ops[] = {
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_B4 = {
-    8,
-    { 5, 6, 7, 0, 1, 2, 3, 4, },
-    ec_gf8_mul_B4_ops
-};
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B4 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_B4_ops};
 
 static ec_gf_op_t ec_gf8_mul_B5_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_COPY,   8,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR3,   4,  8,  3 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_B5 = {
-    9,
-    { 3, 4, 0, 7, 1, 5, 6, 2, 8, },
-    ec_gf8_mul_B5_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR3, 4, 8, 3}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B5 = {9,
+                                    {
+                                        3,
+                                        4,
+                                        0,
+                                        7,
+                                        1,
+                                        5,
+                                        6,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_B5_ops};
 
 static ec_gf_op_t ec_gf8_mul_B6_ops[] = {
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_B6 = {
-    8,
-    { 5, 3, 6, 4, 7, 0, 1, 2, },
-    ec_gf8_mul_B6_ops
-};
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B6 = {8,
+                                    {
+                                        5,
+                                        3,
+                                        6,
+                                        4,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_B6_ops};
 
 static ec_gf_op_t ec_gf8_mul_B7_ops[] = {
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_B7 = {
-    8,
-    { 5, 0, 1, 4, 2, 6, 7, 3, },
-    ec_gf8_mul_B7_ops
-};
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B7 = {8,
+                                    {
+                                        5,
+                                        0,
+                                        1,
+                                        4,
+                                        2,
+                                        6,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_B7_ops};
 
 static ec_gf_op_t ec_gf8_mul_B8_ops[] = {
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_B8 = {
-    8,
-    { 6, 4, 5, 1, 2, 0, 7, 3, },
-    ec_gf8_mul_B8_ops
-};
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B8 = {8,
+                                    {
+                                        6,
+                                        4,
+                                        5,
+                                        1,
+                                        2,
+                                        0,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_B8_ops};
 
 static ec_gf_op_t ec_gf8_mul_B9_ops[] = {
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR3,   0,  8,  2 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_B9 = {
-    9,
-    { 6, 7, 0, 2, 1, 4, 5, 3, 8, },
-    ec_gf8_mul_B9_ops
-};
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR3, 0, 8, 2}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B9 = {9,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        2,
+                                        1,
+                                        4,
+                                        5,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_B9_ops};
 
 static ec_gf_op_t ec_gf8_mul_BA_ops[] = {
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_BA = {
-    8,
-    { 1, 2, 4, 3, 5, 6, 0, 7, },
-    ec_gf8_mul_BA_ops
-};
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BA = {8,
+                                    {
+                                        1,
+                                        2,
+                                        4,
+                                        3,
+                                        5,
+                                        6,
+                                        0,
+                                        7,
+                                    },
+                                    ec_gf8_mul_BA_ops};
 
 static ec_gf_op_t ec_gf8_mul_BB_ops[] = {
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_COPY,   8,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   8,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   8,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  8,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_BB = {
-    9,
-    { 7, 2, 1, 8, 3, 5, 6, 4, 0, },
-    ec_gf8_mul_BB_ops
-};
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_COPY, 8, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 8, 5, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BB = {9,
+                                    {
+                                        7,
+                                        2,
+                                        1,
+                                        8,
+                                        3,
+                                        5,
+                                        6,
+                                        4,
+                                        0,
+                                    },
+                                    ec_gf8_mul_BB_ops};
 
 static ec_gf_op_t ec_gf8_mul_BC_ops[] = {
-    { EC_GF_OP_COPY,   8,  1,  0 },
-    { EC_GF_OP_XOR2,   8,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_XOR3,   2,  8,  4 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_BC = {
-    9,
-    { 2, 6, 3, 4, 5, 1, 7, 0, 8, },
-    ec_gf8_mul_BC_ops
-};
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 2, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR3, 2, 8, 4},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BC = {9,
+                                    {
+                                        2,
+                                        6,
+                                        3,
+                                        4,
+                                        5,
+                                        1,
+                                        7,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_BC_ops};
 
 static ec_gf_op_t ec_gf8_mul_BD_ops[] = {
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_BD = {
-    8,
-    { 4, 5, 0, 2, 7, 1, 6, 3, },
-    ec_gf8_mul_BD_ops
-};
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BD = {8,
+                                    {
+                                        4,
+                                        5,
+                                        0,
+                                        2,
+                                        7,
+                                        1,
+                                        6,
+                                        3,
+                                    },
+                                    ec_gf8_mul_BD_ops};
 
 static ec_gf_op_t ec_gf8_mul_BE_ops[] = {
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_BE = {
-    8,
-    { 0, 6, 7, 4, 5, 1, 3, 2, },
-    ec_gf8_mul_BE_ops
-};
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BE = {8,
+                                    {
+                                        0,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                        1,
+                                        3,
+                                        2,
+                                    },
+                                    ec_gf8_mul_BE_ops};
 
 static ec_gf_op_t ec_gf8_mul_BF_ops[] = {
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_BF = {
-    8,
-    { 5, 6, 1, 7, 3, 0, 2, 4, },
-    ec_gf8_mul_BF_ops
-};
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BF = {8,
+                                    {
+                                        5,
+                                        6,
+                                        1,
+                                        7,
+                                        3,
+                                        0,
+                                        2,
+                                        4,
+                                    },
+                                    ec_gf8_mul_BF_ops};
 
 static ec_gf_op_t ec_gf8_mul_C0_ops[] = {
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_C0 = {
-    8,
-    { 1, 2, 3, 4, 7, 5, 6, 0, },
-    ec_gf8_mul_C0_ops
-};
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C0 = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        6,
+                                        0,
+                                    },
+                                    ec_gf8_mul_C0_ops};
 
 static ec_gf_op_t ec_gf8_mul_C1_ops[] = {
-    { EC_GF_OP_XOR3,   8,  1,  2 },
-    { EC_GF_OP_XOR2,   8,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  8,  0 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_C1 = {
-    9,
-    { 5, 6, 7, 4, 1, 2, 3, 0, 8, },
-    ec_gf8_mul_C1_ops
-};
+    {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 8, 3, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C1 = {9,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        4,
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_C1_ops};
 
 static ec_gf_op_t ec_gf8_mul_C2_ops[] = {
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_C2 = {
-    8,
-    { 7, 6, 3, 0, 1, 4, 5, 2, },
-    ec_gf8_mul_C2_ops
-};
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C2 = {8,
+                                    {
+                                        7,
+                                        6,
+                                        3,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        2,
+                                    },
+                                    ec_gf8_mul_C2_ops};
 
 static ec_gf_op_t ec_gf8_mul_C3_ops[] = {
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR3,   0,  2,  6 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR3,   9,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  9,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_C3 = {
-    10,
-    { 5, 6, 4, 7, 1, 2, 3, 0, 8, 9, },
-    ec_gf8_mul_C3_ops
-};
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR3, 0, 2, 6}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR3, 9, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 9, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C3 = {10,
+                                    {
+                                        5,
+                                        6,
+                                        4,
+                                        7,
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_C3_ops};
 
 static ec_gf_op_t ec_gf8_mul_C4_ops[] = {
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_C4 = {
-    8,
-    { 0, 2, 1, 3, 4, 5, 6, 7, },
-    ec_gf8_mul_C4_ops
-};
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C4 = {8,
+                                    {
+                                        0,
+                                        2,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_C4_ops};
 
 static ec_gf_op_t ec_gf8_mul_C5_ops[] = {
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_C5 = {
-    8,
-    { 4, 3, 5, 7, 6, 2, 0, 1, },
-    ec_gf8_mul_C5_ops
-};
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C5 = {8,
+                                    {
+                                        4,
+                                        3,
+                                        5,
+                                        7,
+                                        6,
+                                        2,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_C5_ops};
 
 static ec_gf_op_t ec_gf8_mul_C6_ops[] = {
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_COPY,   8,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR3,   9,  5,  4 },
-    { EC_GF_OP_XOR2,   6,  9,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  9,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_C6 = {
-    10,
-    { 6, 3, 0, 4, 5, 7, 2, 1, 8, 9, },
-    ec_gf8_mul_C6_ops
-};
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_COPY, 8, 4, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR3, 9, 5, 4},
+    {EC_GF_OP_XOR2, 6, 9, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 7, 9, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 8, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C6 = {10,
+                                    {
+                                        6,
+                                        3,
+                                        0,
+                                        4,
+                                        5,
+                                        7,
+                                        2,
+                                        1,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_C6_ops};
 
 static ec_gf_op_t ec_gf8_mul_C7_ops[] = {
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_C7 = {
-    8,
-    { 7, 0, 6, 2, 5, 3, 4, 1, },
-    ec_gf8_mul_C7_ops
-};
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C7 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        6,
+                                        2,
+                                        5,
+                                        3,
+                                        4,
+                                        1,
+                                    },
+                                    ec_gf8_mul_C7_ops};
 
 static ec_gf_op_t ec_gf8_mul_C8_ops[] = {
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_C8 = {
-    8,
-    { 1, 3, 2, 4, 6, 7, 5, 0, },
-    ec_gf8_mul_C8_ops
-};
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C8 = {8,
+                                    {
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        6,
+                                        7,
+                                        5,
+                                        0,
+                                    },
+                                    ec_gf8_mul_C8_ops};
 
 static ec_gf_op_t ec_gf8_mul_C9_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_C9 = {
-    8,
-    { 2, 3, 4, 5, 6, 7, 0, 1, },
-    ec_gf8_mul_C9_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C9 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_C9_ops};
 
 static ec_gf_op_t ec_gf8_mul_CA_ops[] = {
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_CA = {
-    8,
-    { 1, 2, 5, 7, 3, 4, 0, 6, },
-    ec_gf8_mul_CA_ops
-};
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CA = {8,
+                                    {
+                                        1,
+                                        2,
+                                        5,
+                                        7,
+                                        3,
+                                        4,
+                                        0,
+                                        6,
+                                    },
+                                    ec_gf8_mul_CA_ops};
 
 static ec_gf_op_t ec_gf8_mul_CB_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_CB = {
-    8,
-    { 2, 3, 4, 5, 7, 6, 0, 1, },
-    ec_gf8_mul_CB_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CB = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        7,
+                                        6,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_CB_ops};
 
 static ec_gf_op_t ec_gf8_mul_CC_ops[] = {
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_CC = {
-    8,
-    { 2, 7, 1, 0, 5, 6, 3, 4, },
-    ec_gf8_mul_CC_ops
-};
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CC = {8,
+                                    {
+                                        2,
+                                        7,
+                                        1,
+                                        0,
+                                        5,
+                                        6,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_CC_ops};
 
 static ec_gf_op_t ec_gf8_mul_CD_ops[] = {
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_CD = {
-    8,
-    { 0, 6, 1, 2, 7, 3, 4, 5, },
-    ec_gf8_mul_CD_ops
-};
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CD = {8,
+                                    {
+                                        0,
+                                        6,
+                                        1,
+                                        2,
+                                        7,
+                                        3,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_CD_ops};
 
 static ec_gf_op_t ec_gf8_mul_CE_ops[] = {
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_COPY,   8,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR3,   3,  6,  8 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR3,   8,  2,  3 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_CE = {
-    9,
-    { 5, 7, 3, 0, 2, 6, 4, 1, 8, },
-    ec_gf8_mul_CE_ops
-};
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_COPY, 8, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR3, 3, 6, 8},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR3, 8, 2, 3},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CE = {9,
+                                    {
+                                        5,
+                                        7,
+                                        3,
+                                        0,
+                                        2,
+                                        6,
+                                        4,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_CE_ops};
 
 static ec_gf_op_t ec_gf8_mul_CF_ops[] = {
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_CF = {
-    8,
-    { 3, 6, 7, 0, 2, 4, 5, 1, },
-    ec_gf8_mul_CF_ops
-};
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CF = {8,
+                                    {
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        2,
+                                        4,
+                                        5,
+                                        1,
+                                    },
+                                    ec_gf8_mul_CF_ops};
 
 static ec_gf_op_t ec_gf8_mul_D0_ops[] = {
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_D0 = {
-    8,
-    { 5, 6, 7, 2, 0, 3, 1, 4, },
-    ec_gf8_mul_D0_ops
-};
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D0 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        0,
+                                        3,
+                                        1,
+                                        4,
+                                    },
+                                    ec_gf8_mul_D0_ops};
 
 static ec_gf_op_t ec_gf8_mul_D1_ops[] = {
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR3,   8,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_D1 = {
-    9,
-    { 5, 6, 3, 2, 0, 7, 4, 1, 8, },
-    ec_gf8_mul_D1_ops
-};
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR3, 8, 6, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D1 = {9,
+                                    {
+                                        5,
+                                        6,
+                                        3,
+                                        2,
+                                        0,
+                                        7,
+                                        4,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_D1_ops};
 
 static ec_gf_op_t ec_gf8_mul_D2_ops[] = {
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_D2 = {
-    8,
-    { 7, 0, 2, 1, 3, 4, 6, 5, },
-    ec_gf8_mul_D2_ops
-};
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D2 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        2,
+                                        1,
+                                        3,
+                                        4,
+                                        6,
+                                        5,
+                                    },
+                                    ec_gf8_mul_D2_ops};
 
 static ec_gf_op_t ec_gf8_mul_D3_ops[] = {
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_COPY,   8,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   8,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_D3 = {
-    9,
-    { 0, 3, 2, 8, 4, 6, 7, 1, 5, },
-    ec_gf8_mul_D3_ops
-};
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_COPY, 8, 4, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 8, 6, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D3 = {9,
+                                    {
+                                        0,
+                                        3,
+                                        2,
+                                        8,
+                                        4,
+                                        6,
+                                        7,
+                                        1,
+                                        5,
+                                    },
+                                    ec_gf8_mul_D3_ops};
 
 static ec_gf_op_t ec_gf8_mul_D4_ops[] = {
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_COPY,   8,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR3,   1,  7,  8 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_D4 = {
-    9,
-    { 4, 1, 7, 5, 0, 6, 3, 2, 8, },
-    ec_gf8_mul_D4_ops
-};
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_COPY, 8, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR3, 1, 7, 8},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D4 = {9,
+                                    {
+                                        4,
+                                        1,
+                                        7,
+                                        5,
+                                        0,
+                                        6,
+                                        3,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_D4_ops};
 
 static ec_gf_op_t ec_gf8_mul_D5_ops[] = {
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_D5 = {
-    8,
-    { 6, 7, 4, 5, 2, 3, 1, 0, },
-    ec_gf8_mul_D5_ops
-};
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D5 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                        2,
+                                        3,
+                                        1,
+                                        0,
+                                    },
+                                    ec_gf8_mul_D5_ops};
 
 static ec_gf_op_t ec_gf8_mul_D6_ops[] = {
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_D6 = {
-    9,
-    { 0, 6, 2, 7, 1, 3, 4, 5, 8, },
-    ec_gf8_mul_D6_ops
-};
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D6 = {9,
+                                    {
+                                        0,
+                                        6,
+                                        2,
+                                        7,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_D6_ops};
 
 static ec_gf_op_t ec_gf8_mul_D7_ops[] = {
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR3,   8,  3,  5 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  8,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR3,   6,  7,  8 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_D7 = {
-    9,
-    { 3, 4, 6, 5, 0, 7, 1, 2, 8, },
-    ec_gf8_mul_D7_ops
-};
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR3, 8, 3, 5}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR3, 6, 7, 8}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D7 = {9,
+                                    {
+                                        3,
+                                        4,
+                                        6,
+                                        5,
+                                        0,
+                                        7,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_D7_ops};
 
 static ec_gf_op_t ec_gf8_mul_D8_ops[] = {
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_D8 = {
-    8,
-    { 4, 5, 6, 7, 0, 1, 2, 3, },
-    ec_gf8_mul_D8_ops
-};
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D8 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_D8_ops};
 
 static ec_gf_op_t ec_gf8_mul_D9_ops[] = {
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_D9 = {
-    8,
-    { 1, 2, 6, 7, 4, 5, 0, 3, },
-    ec_gf8_mul_D9_ops
-};
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D9 = {8,
+                                    {
+                                        1,
+                                        2,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                        0,
+                                        3,
+                                    },
+                                    ec_gf8_mul_D9_ops};
 
 static ec_gf_op_t ec_gf8_mul_DA_ops[] = {
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR3,   8,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_DA = {
-    9,
-    { 2, 5, 7, 1, 0, 4, 3, 6, 8, },
-    ec_gf8_mul_DA_ops
-};
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR3, 8, 2, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DA = {9,
+                                    {
+                                        2,
+                                        5,
+                                        7,
+                                        1,
+                                        0,
+                                        4,
+                                        3,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_DA_ops};
 
 static ec_gf_op_t ec_gf8_mul_DB_ops[] = {
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   8,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_DB = {
-    9,
-    { 7, 5, 6, 2, 3, 4, 1, 0, 8, },
-    ec_gf8_mul_DB_ops
-};
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 8, 4, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DB = {9,
+                                    {
+                                        7,
+                                        5,
+                                        6,
+                                        2,
+                                        3,
+                                        4,
+                                        1,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_DB_ops};
 
 static ec_gf_op_t ec_gf8_mul_DC_ops[] = {
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_DC = {
-    8,
-    { 4, 5, 2, 6, 7, 1, 0, 3, },
-    ec_gf8_mul_DC_ops
-};
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DC = {8,
+                                    {
+                                        4,
+                                        5,
+                                        2,
+                                        6,
+                                        7,
+                                        1,
+                                        0,
+                                        3,
+                                    },
+                                    ec_gf8_mul_DC_ops};
 
 static ec_gf_op_t ec_gf8_mul_DD_ops[] = {
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_DD = {
-    8,
-    { 1, 2, 3, 6, 7, 0, 4, 5, },
-    ec_gf8_mul_DD_ops
-};
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DD = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_DD_ops};
 
 static ec_gf_op_t ec_gf8_mul_DE_ops[] = {
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_DE = {
-    8,
-    { 0, 5, 2, 6, 7, 1, 3, 4, },
-    ec_gf8_mul_DE_ops
-};
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DE = {8,
+                                    {
+                                        0,
+                                        5,
+                                        2,
+                                        6,
+                                        7,
+                                        1,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_DE_ops};
 
 static ec_gf_op_t ec_gf8_mul_DF_ops[] = {
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   8,  3,  0 },
-    { EC_GF_OP_COPY,   9,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   8,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR3,   1,  9,  2 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_DF = {
-    10,
-    { 7, 2, 8, 4, 3, 1, 0, 6, 5, 9, },
-    ec_gf8_mul_DF_ops
-};
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 8, 3, 0},
+    {EC_GF_OP_COPY, 9, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR3, 1, 9, 2}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DF = {10,
+                                    {
+                                        7,
+                                        2,
+                                        8,
+                                        4,
+                                        3,
+                                        1,
+                                        0,
+                                        6,
+                                        5,
+                                        9,
+                                    },
+                                    ec_gf8_mul_DF_ops};
 
 static ec_gf_op_t ec_gf8_mul_E0_ops[] = {
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_E0 = {
-    8,
-    { 2, 3, 4, 7, 5, 6, 0, 1, },
-    ec_gf8_mul_E0_ops
-};
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E0 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        6,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_E0_ops};
 
 static ec_gf_op_t ec_gf8_mul_E1_ops[] = {
-    { EC_GF_OP_COPY,   8,  1,  0 },
-    { EC_GF_OP_XOR2,   8,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_XOR3,   9,  5,  3 },
-    { EC_GF_OP_XOR2,   0,  9,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  9,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  8,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_E1 = {
-    10,
-    { 0, 7, 1, 3, 4, 5, 6, 2, 8, 9, },
-    ec_gf8_mul_E1_ops
-};
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 7, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR3, 9, 5, 3},
+    {EC_GF_OP_XOR2, 0, 9, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 4, 9, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E1 = {10,
+                                    {
+                                        0,
+                                        7,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        2,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_E1_ops};
 
 static ec_gf_op_t ec_gf8_mul_E2_ops[] = {
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_E2 = {
-    8,
-    { 2, 3, 7, 1, 5, 6, 0, 4, },
-    ec_gf8_mul_E2_ops
-};
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E2 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        7,
+                                        1,
+                                        5,
+                                        6,
+                                        0,
+                                        4,
+                                    },
+                                    ec_gf8_mul_E2_ops};
 
 static ec_gf_op_t ec_gf8_mul_E3_ops[] = {
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR3,   8,  2,  7 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  8,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR3,   6,  8,  4 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_E3 = {
-    9,
-    { 5, 4, 7, 2, 1, 3, 6, 0, 8, },
-    ec_gf8_mul_E3_ops
-};
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR3, 8, 2, 7}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR3, 6, 8, 4},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E3 = {9,
+                                    {
+                                        5,
+                                        4,
+                                        7,
+                                        2,
+                                        1,
+                                        3,
+                                        6,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_E3_ops};
 
 static ec_gf_op_t ec_gf8_mul_E4_ops[] = {
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_E4 = {
-    8,
-    { 7, 0, 1, 6, 3, 4, 2, 5, },
-    ec_gf8_mul_E4_ops
-};
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E4 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        6,
+                                        3,
+                                        4,
+                                        2,
+                                        5,
+                                    },
+                                    ec_gf8_mul_E4_ops};
 
 static ec_gf_op_t ec_gf8_mul_E5_ops[] = {
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_E5 = {
-    9,
-    { 4, 5, 3, 6, 7, 1, 0, 2, 8, },
-    ec_gf8_mul_E5_ops
-};
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E5 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        3,
+                                        6,
+                                        7,
+                                        1,
+                                        0,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_E5_ops};
 
 static ec_gf_op_t ec_gf8_mul_E6_ops[] = {
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_E6 = {
-    8,
-    { 5, 4, 3, 6, 7, 0, 1, 2, },
-    ec_gf8_mul_E6_ops
-};
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E6 = {8,
+                                    {
+                                        5,
+                                        4,
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_E6_ops};
 
 static ec_gf_op_t ec_gf8_mul_E7_ops[] = {
-    { EC_GF_OP_COPY,   8,  6,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR3,   9,  0,  6 },
-    { EC_GF_OP_XOR2,   4,  9,  0 },
-    { EC_GF_OP_XOR2,   5,  9,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_E7 = {
-    10,
-    { 1, 4, 3, 6, 7, 5, 2, 0, 8, 9, },
-    ec_gf8_mul_E7_ops
-};
+    {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR3, 9, 0, 6}, {EC_GF_OP_XOR2, 4, 9, 0},
+    {EC_GF_OP_XOR2, 5, 9, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E7 = {10,
+                                    {
+                                        1,
+                                        4,
+                                        3,
+                                        6,
+                                        7,
+                                        5,
+                                        2,
+                                        0,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_E7_ops};
 
 static ec_gf_op_t ec_gf8_mul_E8_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_E8 = {
-    8,
-    { 1, 4, 2, 7, 3, 0, 5, 6, },
-    ec_gf8_mul_E8_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E8 = {8,
+                                    {
+                                        1,
+                                        4,
+                                        2,
+                                        7,
+                                        3,
+                                        0,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_E8_ops};
 
 static ec_gf_op_t ec_gf8_mul_E9_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_COPY,   8,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR3,   1,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_E9 = {
-    9,
-    { 6, 2, 0, 3, 4, 1, 5, 7, 8, },
-    ec_gf8_mul_E9_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_COPY, 8, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR3, 1, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E9 = {9,
+                                    {
+                                        6,
+                                        2,
+                                        0,
+                                        3,
+                                        4,
+                                        1,
+                                        5,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_E9_ops};
 
 static ec_gf_op_t ec_gf8_mul_EA_ops[] = {
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_EA = {
-    8,
-    { 3, 4, 5, 6, 7, 0, 1, 2, },
-    ec_gf8_mul_EA_ops
-};
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EA = {8,
+                                    {
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_EA_ops};
 
 static ec_gf_op_t ec_gf8_mul_EB_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_EB = {
-    8,
-    { 3, 4, 5, 6, 7, 0, 1, 2, },
-    ec_gf8_mul_EB_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EB = {8,
+                                    {
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_EB_ops};
 
 static ec_gf_op_t ec_gf8_mul_EC_ops[] = {
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR3,   8,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_EC = {
-    9,
-    { 7, 4, 3, 0, 2, 5, 1, 6, 8, },
-    ec_gf8_mul_EC_ops
-};
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR3, 8, 4, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EC = {9,
+                                    {
+                                        7,
+                                        4,
+                                        3,
+                                        0,
+                                        2,
+                                        5,
+                                        1,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_EC_ops};
 
 static ec_gf_op_t ec_gf8_mul_ED_ops[] = {
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_ED = {
-    8,
-    { 5, 6, 7, 0, 1, 4, 3, 2, },
-    ec_gf8_mul_ED_ops
-};
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_ED = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        4,
+                                        3,
+                                        2,
+                                    },
+                                    ec_gf8_mul_ED_ops};
 
 static ec_gf_op_t ec_gf8_mul_EE_ops[] = {
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR3,   8,  2,  3 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   8,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_EE = {
-    9,
-    { 6, 4, 5, 7, 2, 3, 0, 1, 8, },
-    ec_gf8_mul_EE_ops
-};
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR3, 8, 2, 3},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 8, 5, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EE = {9,
+                                    {
+                                        6,
+                                        4,
+                                        5,
+                                        7,
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_EE_ops};
 
 static ec_gf_op_t ec_gf8_mul_EF_ops[] = {
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_COPY,   8,  0,  0 },
-    { EC_GF_OP_XOR2,   8,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   6,  8,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_EF = {
-    9,
-    { 6, 4, 5, 7, 2, 0, 3, 1, 8, },
-    ec_gf8_mul_EF_ops
-};
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_COPY, 8, 0, 0},
+    {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 6, 8, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EF = {9,
+                                    {
+                                        6,
+                                        4,
+                                        5,
+                                        7,
+                                        2,
+                                        0,
+                                        3,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_EF_ops};
 
 static ec_gf_op_t ec_gf8_mul_F0_ops[] = {
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR3,   8,  3,  6 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_XOR2,   8,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   1,  8,  0 },
-    { EC_GF_OP_XOR2,   0,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_F0 = {
-    9,
-    { 3, 4, 6, 1, 2, 0, 5, 7, 8, },
-    ec_gf8_mul_F0_ops
-};
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR3, 8, 3, 6},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 8, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F0 = {9,
+                                    {
+                                        3,
+                                        4,
+                                        6,
+                                        1,
+                                        2,
+                                        0,
+                                        5,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_F0_ops};
 
 static ec_gf_op_t ec_gf8_mul_F1_ops[] = {
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_COPY,   8,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_COPY,   9,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   9,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  9,  0 },
-    { EC_GF_OP_XOR2,   4,  9,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR3,   9,  8,  7 },
-    { EC_GF_OP_XOR2,   1,  9,  0 },
-    { EC_GF_OP_XOR2,   5,  9,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_F1 = {
-    10,
-    { 7, 2, 6, 3, 5, 1, 4, 0, 8, 9, },
-    ec_gf8_mul_F1_ops
-};
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_COPY, 9, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 9, 0, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 7, 9, 0}, {EC_GF_OP_XOR2, 4, 9, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR3, 9, 8, 7},
+    {EC_GF_OP_XOR2, 1, 9, 0}, {EC_GF_OP_XOR2, 5, 9, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F1 = {10,
+                                    {
+                                        7,
+                                        2,
+                                        6,
+                                        3,
+                                        5,
+                                        1,
+                                        4,
+                                        0,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_F1_ops};
 
 static ec_gf_op_t ec_gf8_mul_F2_ops[] = {
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_XOR3,   8,  6,  4 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  8,  0 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_F2 = {
-    9,
-    { 1, 0, 6, 7, 4, 5, 2, 3, 8, },
-    ec_gf8_mul_F2_ops
-};
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR3, 8, 6, 4},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F2 = {9,
+                                    {
+                                        1,
+                                        0,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_F2_ops};
 
 static ec_gf_op_t ec_gf8_mul_F3_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_F3 = {
-    8,
-    { 5, 6, 7, 0, 1, 2, 3, 4, },
-    ec_gf8_mul_F3_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F3 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_F3_ops};
 
 static ec_gf_op_t ec_gf8_mul_F4_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_F4 = {
-    8,
-    { 0, 1, 2, 3, 4, 5, 6, 7, },
-    ec_gf8_mul_F4_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F4 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_F4_ops};
 
 static ec_gf_op_t ec_gf8_mul_F5_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_F5 = {
-    8,
-    { 7, 0, 1, 2, 3, 4, 5, 6, },
-    ec_gf8_mul_F5_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F5 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_F5_ops};
 
 static ec_gf_op_t ec_gf8_mul_F6_ops[] = {
-    { EC_GF_OP_XOR2,   3,  1,  0 },
-    { EC_GF_OP_COPY,   8,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_COPY,   9,  3,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   9,  4,  0 },
-    { EC_GF_OP_XOR2,   4,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  9,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR3,   7,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_F6 = {
-    10,
-    { 0, 6, 2, 7, 4, 3, 5, 9, 1, 8, },
-    ec_gf8_mul_F6_ops
-};
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_COPY, 8, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_COPY, 9, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 9, 4, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 9, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR3, 7, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F6 = {10,
+                                    {
+                                        0,
+                                        6,
+                                        2,
+                                        7,
+                                        4,
+                                        3,
+                                        5,
+                                        9,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_F6_ops};
 
 static ec_gf_op_t ec_gf8_mul_F7_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_F7 = {
-    8,
-    { 6, 7, 0, 1, 2, 3, 4, 5, },
-    ec_gf8_mul_F7_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F7 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_F7_ops};
 
 static ec_gf_op_t ec_gf8_mul_F8_ops[] = {
-    { EC_GF_OP_XOR2,   4,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  5,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_F8 = {
-    8,
-    { 6, 2, 0, 1, 4, 5, 3, 7, },
-    ec_gf8_mul_F8_ops
-};
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F8 = {8,
+                                    {
+                                        6,
+                                        2,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        3,
+                                        7,
+                                    },
+                                    ec_gf8_mul_F8_ops};
 
 static ec_gf_op_t ec_gf8_mul_F9_ops[] = {
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  4,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR3,   8,  7,  1 },
-    { EC_GF_OP_XOR2,   1,  3,  0 },
-    { EC_GF_OP_XOR2,   4,  8,  0 },
-    { EC_GF_OP_XOR2,   5,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_F9 = {
-    9,
-    { 4, 1, 7, 6, 0, 3, 5, 2, 8, },
-    ec_gf8_mul_F9_ops
-};
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR3, 8, 7, 1}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F9 = {9,
+                                    {
+                                        4,
+                                        1,
+                                        7,
+                                        6,
+                                        0,
+                                        3,
+                                        5,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_F9_ops};
 
 static ec_gf_op_t ec_gf8_mul_FA_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  2,  0 },
-    { EC_GF_OP_XOR2,   1,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_FA = {
-    8,
-    { 0, 1, 2, 4, 5, 6, 7, 3, },
-    ec_gf8_mul_FA_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FA = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_FA_ops};
 
 static ec_gf_op_t ec_gf8_mul_FB_ops[] = {
-    { EC_GF_OP_XOR2,   1,  0,  0 },
-    { EC_GF_OP_XOR2,   2,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   3,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  7,  0 },
-    { EC_GF_OP_XOR2,   2,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  6,  0 },
-    { EC_GF_OP_XOR2,   7,  6,  0 },
-    { EC_GF_OP_XOR2,   4,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  5,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_FB = {
-    8,
-    { 4, 5, 6, 7, 0, 1, 2, 3, },
-    ec_gf8_mul_FB_ops
-};
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FB = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_FB_ops};
 
 static ec_gf_op_t ec_gf8_mul_FC_ops[] = {
-    { EC_GF_OP_XOR2,   7,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  1,  0 },
-    { EC_GF_OP_COPY,   9,  3,  0 },
-    { EC_GF_OP_XOR3,   8,  5,  7 },
-    { EC_GF_OP_XOR2,   3,  6,  0 },
-    { EC_GF_OP_XOR2,   8,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  8,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   3,  4,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   6,  0,  0 },
-    { EC_GF_OP_XOR3,   0,  9,  2 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_FC = {
-    10,
-    { 5, 6, 3, 7, 1, 8, 0, 4, 2, 9, },
-    ec_gf8_mul_FC_ops
-};
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_COPY, 9, 3, 0},
+    {EC_GF_OP_XOR3, 8, 5, 7}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR3, 0, 9, 2}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FC = {10,
+                                    {
+                                        5,
+                                        6,
+                                        3,
+                                        7,
+                                        1,
+                                        8,
+                                        0,
+                                        4,
+                                        2,
+                                        9,
+                                    },
+                                    ec_gf8_mul_FC_ops};
 
 static ec_gf_op_t ec_gf8_mul_FD_ops[] = {
-    { EC_GF_OP_XOR2,   7,  1,  0 },
-    { EC_GF_OP_COPY,   8,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  0,  0 },
-    { EC_GF_OP_XOR2,   7,  5,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  5,  0 },
-    { EC_GF_OP_XOR2,   1,  2,  0 },
-    { EC_GF_OP_XOR2,   0,  1,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR3,   1,  8,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_FD = {
-    9,
-    { 5, 3, 7, 6, 1, 2, 4, 0, 8, },
-    ec_gf8_mul_FD_ops
-};
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_COPY, 8, 7, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR3, 1, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FD = {9,
+                                    {
+                                        5,
+                                        3,
+                                        7,
+                                        6,
+                                        1,
+                                        2,
+                                        4,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_FD_ops};
 
 static ec_gf_op_t ec_gf8_mul_FE_ops[] = {
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_COPY,   8,  2,  0 },
-    { EC_GF_OP_XOR2,   2,  4,  0 },
-    { EC_GF_OP_XOR2,   6,  2,  0 },
-    { EC_GF_OP_XOR2,   8,  5,  0 },
-    { EC_GF_OP_XOR2,   5,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  1,  0 },
-    { EC_GF_OP_XOR2,   0,  6,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   7,  8,  0 },
-    { EC_GF_OP_XOR2,   3,  0,  0 },
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR2,   0,  4,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_FE = {
-    9,
-    { 3, 4, 8, 2, 5, 0, 6, 1, 7, },
-    ec_gf8_mul_FE_ops
-};
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_COPY, 8, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FE = {9,
+                                    {
+                                        3,
+                                        4,
+                                        8,
+                                        2,
+                                        5,
+                                        0,
+                                        6,
+                                        1,
+                                        7,
+                                    },
+                                    ec_gf8_mul_FE_ops};
 
 static ec_gf_op_t ec_gf8_mul_FF_ops[] = {
-    { EC_GF_OP_XOR2,   4,  7,  0 },
-    { EC_GF_OP_COPY,   9,  0,  0 },
-    { EC_GF_OP_COPY,   8,  4,  0 },
-    { EC_GF_OP_XOR2,   9,  1,  0 },
-    { EC_GF_OP_XOR2,   4,  2,  0 },
-    { EC_GF_OP_XOR2,   9,  4,  0 },
-    { EC_GF_OP_XOR2,   0,  5,  0 },
-    { EC_GF_OP_XOR2,   2,  0,  0 },
-    { EC_GF_OP_XOR2,   3,  9,  0 },
-    { EC_GF_OP_XOR2,   7,  3,  0 },
-    { EC_GF_OP_XOR2,   2,  6,  0 },
-    { EC_GF_OP_XOR2,   5,  3,  0 },
-    { EC_GF_OP_XOR2,   6,  7,  0 },
-    { EC_GF_OP_XOR2,   1,  7,  0 },
-    { EC_GF_OP_XOR3,   3,  8,  5 },
-    { EC_GF_OP_XOR2,   4,  6,  0 },
-    { EC_GF_OP_END,    0,  0,  0 }
-};
-
-static ec_gf_mul_t ec_gf8_mul_FF = {
-    10,
-    { 6, 5, 0, 1, 2, 4, 9, 3, 7, 8, },
-    ec_gf8_mul_FF_ops
-};
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_COPY, 9, 0, 0},
+    {EC_GF_OP_COPY, 8, 4, 0}, {EC_GF_OP_XOR2, 9, 1, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 9, 4, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 9, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR3, 3, 8, 5}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FF = {10,
+                                    {
+                                        6,
+                                        5,
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        9,
+                                        3,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_FF_ops};
 
 ec_gf_mul_t *ec_gf8_mul[] = {
     &ec_gf8_mul_00, &ec_gf8_mul_01, &ec_gf8_mul_02, &ec_gf8_mul_03,
@@ -5955,5 +5879,4 @@ ec_gf_mul_t *ec_gf8_mul[] = {
     &ec_gf8_mul_F0, &ec_gf8_mul_F1, &ec_gf8_mul_F2, &ec_gf8_mul_F3,
     &ec_gf8_mul_F4, &ec_gf8_mul_F5, &ec_gf8_mul_F6, &ec_gf8_mul_F7,
     &ec_gf8_mul_F8, &ec_gf8_mul_F9, &ec_gf8_mul_FA, &ec_gf8_mul_FB,
-    &ec_gf8_mul_FC, &ec_gf8_mul_FD, &ec_gf8_mul_FE, &ec_gf8_mul_FF
-};
+    &ec_gf8_mul_FC, &ec_gf8_mul_FD, &ec_gf8_mul_FE, &ec_gf8_mul_FF};
diff --git a/xlators/cluster/ec/src/ec-heal.c b/xlators/cluster/ec/src/ec-heal.c
index 2ff713b7bbd..7d991f04aac 100644
--- a/xlators/cluster/ec/src/ec-heal.c
+++ b/xlators/cluster/ec/src/ec-heal.c
@@ -8,16 +8,14 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-#include "compat-errno.h"
-#include "byte-order.h"
-#include "syncop.h"
-#include "syncop-utils.h"
-#include "cluster-syncop.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/syncop-utils.h>
+#include <glusterfs/cluster-syncop.h>
 
 #include "ec.h"
-#include "ec-mem-types.h"
 #include "ec-types.h"
 #include "ec-messages.h"
 #include "ec-helpers.h"
@@ -27,85 +25,147 @@
 #include "ec-fops.h"
 #include "ec-heald.h"
 
-#define alloca0(size) ({void *__ptr; __ptr = alloca(size); memset(__ptr, 0, size); __ptr; })
-#define EC_COUNT(array, max) ({int __i; int __res = 0; for (__i = 0; __i < max; __i++) if (array[__i]) __res++; __res; })
-#define EC_INTERSECT(dst, src1, src2, max) ({int __i; for (__i = 0; __i < max; __i++) dst[__i] = src1[__i] && src2[__i]; })
-#define EC_ADJUST_SOURCE(source, sources, max) ({int __i; if (sources[source] == 0) {source = -1; for (__i = 0; __i < max; __i++) if (sources[__i]) source = __i; } })
-#define IA_EQUAL(f, s, field) (memcmp (&(f.ia_##field), &(s.ia_##field), sizeof (s.ia_##field)) == 0)
-#define EC_REPLIES_ALLOC(replies, numsubvols) do {              \
-        int __i = 0;                                            \
-        replies = alloca0(numsubvols * sizeof (*replies));      \
-        for (__i = 0; __i < numsubvols; __i++)                  \
-                INIT_LIST_HEAD (&replies[__i].entries.list);    \
-        } while (0)
-
+#define EC_COUNT(array, max)                                                   \
+    ({                                                                         \
+        int __i;                                                               \
+        int __res = 0;                                                         \
+        for (__i = 0; __i < max; __i++)                                        \
+            if (array[__i])                                                    \
+                __res++;                                                       \
+        __res;                                                                 \
+    })
+#define EC_INTERSECT(dst, src1, src2, max)                                     \
+    ({                                                                         \
+        int __i;                                                               \
+        for (__i = 0; __i < max; __i++)                                        \
+            dst[__i] = src1[__i] && src2[__i];                                 \
+    })
+#define EC_ADJUST_SOURCE(source, sources, max)                                 \
+    ({                                                                         \
+        int __i;                                                               \
+        if (sources[source] == 0) {                                            \
+            source = -1;                                                       \
+            for (__i = 0; __i < max; __i++)                                    \
+                if (sources[__i])                                              \
+                    source = __i;                                              \
+        }                                                                      \
+    })
+#define IA_EQUAL(f, s, field)                                                  \
+    (memcmp(&(f.ia_##field), &(s.ia_##field), sizeof(s.ia_##field)) == 0)
+#define EC_REPLIES_ALLOC(replies, numsubvols)                                  \
+    do {                                                                       \
+        int __i = 0;                                                           \
+        replies = alloca0(numsubvols * sizeof(*replies));                      \
+        for (__i = 0; __i < numsubvols; __i++)                                 \
+            INIT_LIST_HEAD(&replies[__i].entries.list);                        \
+    } while (0)
 
 struct ec_name_data {
-        call_frame_t *frame;
-        unsigned char *participants;
-        unsigned char *failed_on;
-        unsigned char *gfidless;
-        unsigned char *enoent;
-        unsigned char *same;
-        char *name;
-        inode_t *parent;
-        default_args_cbk_t *replies;
+    call_frame_t *frame;
+    unsigned char *participants;
+    unsigned char *failed_on;
+    unsigned char *gfidless;
+    unsigned char *enoent;
+    unsigned char *same;
+    char *name;
+    inode_t *parent;
+    default_args_cbk_t *replies;
+    uint32_t heal_pending;
 };
 
-static char *ec_ignore_xattrs[] = {
-        GF_SELINUX_XATTR_KEY,
-        QUOTA_SIZE_KEY,
-        NULL
-};
+static char *ec_ignore_xattrs[] = {GF_SELINUX_XATTR_KEY, QUOTA_SIZE_KEY, NULL};
 
 static gf_boolean_t
-ec_ignorable_key_match (dict_t *dict, char *key, data_t *val, void *mdata)
+ec_ignorable_key_match(dict_t *dict, char *key, data_t *val, void *mdata)
 {
-        int i = 0;
+    int i = 0;
 
-        if (!key)
-                goto out;
+    if (!key)
+        goto out;
 
-        if (strncmp (key, EC_XATTR_PREFIX, strlen (EC_XATTR_PREFIX)) == 0)
-                        return _gf_true;
+    if (strncmp(key, EC_XATTR_PREFIX, SLEN(EC_XATTR_PREFIX)) == 0)
+        return _gf_true;
 
-        for (i = 0; ec_ignore_xattrs[i]; i++) {
-                if (!strcmp (key, ec_ignore_xattrs[i]))
-                       return _gf_true;
-        }
+    for (i = 0; ec_ignore_xattrs[i]; i++) {
+        if (!strcmp(key, ec_ignore_xattrs[i]))
+            return _gf_true;
+    }
 
 out:
-        return _gf_false;
+    return _gf_false;
 }
 
 static gf_boolean_t
-ec_sh_key_match (dict_t *dict, char *key, data_t *val, void *mdata)
+ec_sh_key_match(dict_t *dict, char *key, data_t *val, void *mdata)
 {
-        return !ec_ignorable_key_match (dict, key, val, mdata);
+    return !ec_ignorable_key_match(dict, key, val, mdata);
 }
 /* FOP: heal */
 
-uintptr_t ec_heal_check(ec_fop_data_t * fop, uintptr_t * pgood)
+void
+ec_set_entry_healing(ec_fop_data_t *fop)
+{
+    ec_inode_t *ctx = NULL;
+    loc_t *loc = NULL;
+
+    if (!fop)
+        return;
+
+    loc = &fop->loc[0];
+    LOCK(&loc->inode->lock);
+    {
+        ctx = __ec_inode_get(loc->inode, fop->xl);
+        if (ctx) {
+            ctx->heal_count += 1;
+        }
+    }
+    UNLOCK(&loc->inode->lock);
+}
+
+void
+ec_reset_entry_healing(ec_fop_data_t *fop)
+{
+    ec_inode_t *ctx = NULL;
+    loc_t *loc = NULL;
+    int32_t heal_count = 0;
+    if (!fop)
+        return;
+
+    loc = &fop->loc[0];
+    LOCK(&loc->inode->lock);
+    {
+        ctx = __ec_inode_get(loc->inode, fop->xl);
+        if (ctx) {
+            ctx->heal_count += -1;
+            heal_count = ctx->heal_count;
+        }
+    }
+    UNLOCK(&loc->inode->lock);
+    GF_ASSERT(heal_count >= 0);
+}
+
+uintptr_t
+ec_heal_check(ec_fop_data_t *fop, uintptr_t *pgood)
 {
-    ec_cbk_data_t * cbk;
-    uintptr_t mask[2] = { 0, 0 };
+    ec_cbk_data_t *cbk;
+    uintptr_t mask[2] = {0, 0};
 
     list_for_each_entry(cbk, &fop->cbk_list, list)
     {
         mask[cbk->op_ret >= 0] |= cbk->mask;
     }
 
-    if (pgood != NULL)
-    {
+    if (pgood != NULL) {
         *pgood = mask[1];
     }
 
     return mask[0];
 }
 
-void ec_heal_update(ec_fop_data_t * fop, int32_t is_open)
+void
+ec_heal_update(ec_fop_data_t *fop, int32_t is_open)
 {
-    ec_heal_t * heal = fop->data;
+    ec_heal_t *heal = fop->data;
     uintptr_t good, bad;
 
     bad = ec_heal_check(fop, &good);
@@ -113,8 +173,7 @@ void ec_heal_update(ec_fop_data_t * fop, int32_t is_open)
     LOCK(&heal->lock);
 
     heal->bad &= ~bad;
-    if (is_open)
-    {
+    if (is_open) {
         heal->open |= good;
     }
 
@@ -123,9 +182,10 @@ void ec_heal_update(ec_fop_data_t * fop, int32_t is_open)
     fop->error = 0;
 }
 
-void ec_heal_avoid(ec_fop_data_t * fop)
+void
+ec_heal_avoid(ec_fop_data_t *fop)
 {
-    ec_heal_t * heal = fop->data;
+    ec_heal_t *heal = fop->data;
     uintptr_t bad;
 
     bad = ec_heal_check(fop, NULL);
@@ -137,22 +197,24 @@ void ec_heal_avoid(ec_fop_data_t * fop)
     UNLOCK(&heal->lock);
 }
 
-int32_t ec_heal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int32_t
+ec_heal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
     ec_fop_data_t *fop = cookie;
     ec_heal_t *heal = fop->data;
 
     if (op_ret >= 0) {
-        GF_ASSERT(ec_set_inode_size(heal->fop, heal->fd->inode,
-                                    heal->total_size));
+        GF_ASSERT(
+            ec_set_inode_size(heal->fop, heal->fd->inode, heal->total_size));
     }
 
     return 0;
 }
 
-void ec_heal_lock(ec_heal_t *heal, int32_t type, fd_t *fd, loc_t *loc,
-                  off_t offset, size_t size)
+void
+ec_heal_lock(ec_heal_t *heal, int32_t type, fd_t *fd, loc_t *loc, off_t offset,
+             size_t size)
 {
     struct gf_flock flock;
     fop_inodelk_cbk_t cbk = NULL;
@@ -177,15 +239,12 @@ void ec_heal_lock(ec_heal_t *heal, int32_t type, fd_t *fd, loc_t *loc,
         cbk = ec_heal_lock_cbk;
     }
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         ec_finodelk(heal->fop->frame, heal->xl,
                     &heal->fop->frame->root->lk_owner, heal->fop->mask,
                     EC_MINIMUM_ALL, cbk, heal, heal->xl->name, fd, F_SETLKW,
                     &flock, NULL);
-    }
-    else
-    {
+    } else {
         ec_inodelk(heal->fop->frame, heal->xl,
                    &heal->fop->frame->root->lk_owner, heal->fop->mask,
                    EC_MINIMUM_ALL, cbk, heal, heal->xl->name, loc, F_SETLKW,
@@ -193,28 +252,28 @@ void ec_heal_lock(ec_heal_t *heal, int32_t type, fd_t *fd, loc_t *loc,
     }
 }
 
-void ec_heal_inodelk(ec_heal_t *heal, int32_t type, int32_t use_fd,
-                     off_t offset, size_t size)
+void
+ec_heal_inodelk(ec_heal_t *heal, int32_t type, int32_t use_fd, off_t offset,
+                size_t size)
 {
     ec_heal_lock(heal, type, use_fd ? heal->fd : NULL, &heal->loc, offset,
                  size);
 }
 
 int32_t
-ec_heal_xattr_clean (dict_t *dict, char *key, data_t *data,
-                     void *arg)
+ec_heal_xattr_clean(dict_t *dict, char *key, data_t *data, void *arg)
 {
-        dict_t *base = arg;
+    dict_t *base = arg;
 
-        if (ec_ignorable_key_match (NULL, key, NULL, NULL)) {
-                dict_del (dict, key);
-                return 0;
-        }
+    if (ec_ignorable_key_match(NULL, key, NULL, NULL)) {
+        dict_del(dict, key);
+        return 0;
+    }
 
-        if (dict_get (base, key) != NULL)
-                dict_del (dict, key);
+    if (dict_get(base, key) != NULL)
+        dict_del(dict, key);
 
-        return 0;
+    return 0;
 }
 
 /********************************************************************
@@ -224,68 +283,66 @@ ec_heal_xattr_clean (dict_t *dict, char *key, data_t *data,
  *
  *******************************************************************/
 void
-ec_wind_xattrop_parallel (call_frame_t *frame, xlator_t *subvol,
-                          int child_index, loc_t *loc,
-                          gf_xattrop_flags_t flags, dict_t **dict,
-                          dict_t *xdata)
+ec_wind_xattrop_parallel(call_frame_t *frame, xlator_t *subvol, int child_index,
+                         loc_t *loc, gf_xattrop_flags_t flags, dict_t **dict,
+                         dict_t *xdata)
 {
-        gf_msg_debug ("EC", 0, "WIND: on child %d ", child_index);
-        STACK_WIND_COOKIE (frame, cluster_xattrop_cbk,
-                           (void *)(uintptr_t) child_index,
-                           subvol, subvol->fops->xattrop, loc,
-                           flags, dict[child_index], xdata);
+    gf_msg_debug("EC", 0, "WIND: on child %d ", child_index);
+    STACK_WIND_COOKIE(
+        frame, cluster_xattrop_cbk, (void *)(uintptr_t)child_index, subvol,
+        subvol->fops->xattrop, loc, flags, dict[child_index], xdata);
 }
 
 int32_t
-ec_heal_writev_cbk (call_frame_t *frame, void *cookie,
-                    xlator_t *this, int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf,
-                    dict_t *xdata)
+ec_heal_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
     ec_fop_data_t *fop = cookie;
     ec_heal_t *heal = fop->data;
 
     ec_trace("WRITE_CBK", cookie, "ret=%d, errno=%d", op_ret, op_errno);
 
-    gf_msg_debug (fop->xl->name, 0, "%s: write op_ret %d, op_errno %s"
-            " at %"PRIu64, uuid_utoa (heal->fd->inode->gfid), op_ret,
-            strerror (op_errno), heal->offset);
+    gf_msg_debug(fop->xl->name, 0,
+                 "%s: write op_ret %d, op_errno %s"
+                 " at %" PRIu64,
+                 uuid_utoa(heal->fd->inode->gfid), op_ret, strerror(op_errno),
+                 heal->offset);
 
     ec_heal_update(cookie, 0);
 
     return 0;
 }
 
-int32_t ec_heal_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iovec * vector, int32_t count,
-                          struct iatt * stbuf, struct iobref * iobref,
-                          dict_t * xdata)
+int32_t
+ec_heal_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                  dict_t *xdata)
 {
-    ec_fop_data_t * fop = cookie;
-    ec_heal_t * heal = fop->data;
+    ec_fop_data_t *fop = cookie;
+    ec_heal_t *heal = fop->data;
 
     ec_trace("READ_CBK", fop, "ret=%d, errno=%d", op_ret, op_errno);
 
     ec_heal_avoid(fop);
 
-    if (op_ret > 0)
-    {
-        gf_msg_debug (fop->xl->name, 0, "%s: read succeeded, proceeding "
-                "to write at %"PRIu64, uuid_utoa (heal->fd->inode->gfid),
-                heal->offset);
+    if (op_ret > 0) {
+        gf_msg_debug(fop->xl->name, 0,
+                     "%s: read succeeded, proceeding "
+                     "to write at %" PRIu64,
+                     uuid_utoa(heal->fd->inode->gfid), heal->offset);
         ec_writev(heal->fop->frame, heal->xl, heal->bad, EC_MINIMUM_ONE,
                   ec_heal_writev_cbk, heal, heal->fd, vector, count,
                   heal->offset, 0, iobref, NULL);
-    }
-    else
-    {
+    } else {
         if (op_ret < 0) {
-                gf_msg_debug (fop->xl->name, 0, "%s: read failed %s, failing "
-                        "to heal block at %"PRIu64,
-                        uuid_utoa (heal->fd->inode->gfid), strerror (op_errno),
-                        heal->offset);
-                heal->bad = 0;
+            gf_msg_debug(fop->xl->name, 0,
+                         "%s: read failed %s, failing "
+                         "to heal block at %" PRIu64,
+                         uuid_utoa(heal->fd->inode->gfid), strerror(op_errno),
+                         heal->offset);
+            heal->bad = 0;
         }
         heal->done = 1;
     }
@@ -293,1652 +350,1657 @@ int32_t ec_heal_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     return 0;
 }
 
-void ec_heal_data_block(ec_heal_t *heal)
+void
+ec_heal_data_block(ec_heal_t *heal)
 {
     ec_trace("DATA", heal->fop, "good=%lX, bad=%lX", heal->good, heal->bad);
 
     if ((heal->good != 0) && (heal->bad != 0) &&
-        (heal->iatt.ia_type == IA_IFREG))
-    {
+        (heal->iatt.ia_type == IA_IFREG)) {
         ec_readv(heal->fop->frame, heal->xl, heal->good, EC_MINIMUM_MIN,
-                 ec_heal_readv_cbk, heal, heal->fd, heal->size, heal->offset,
-                 0, NULL);
+                 ec_heal_readv_cbk, heal, heal->fd, heal->size, heal->offset, 0,
+                 NULL);
     }
 }
 
 /* FOP: fheal */
 
-void ec_fheal(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fheal_cbk_t func, void * data, fd_t * fd,
-              int32_t partial, dict_t *xdata)
+void
+ec_fheal(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fheal_cbk_t func, void *data, fd_t *fd,
+         int32_t partial, dict_t *xdata)
 {
-    ec_fd_t * ctx = ec_fd_get(fd, this);
+    ec_fd_t *ctx = ec_fd_get(fd, this);
 
-    if (ctx != NULL)
-    {
-        gf_msg_trace ("ec", 0, "FHEAL ctx: flags=%X, open=%lX", ctx->flags,
-                      ctx->open);
-        ec_heal(frame, this, target, minimum, func, data, &ctx->loc, partial,
+    if (ctx != NULL) {
+        gf_msg_trace("ec", 0, "FHEAL ctx: flags=%X, open=%" PRIXPTR, ctx->flags,
+                     ctx->open);
+        ec_heal(frame, this, target, fop_flags, func, data, &ctx->loc, partial,
                 xdata);
     }
 }
 
 /* Common heal code */
 void
-ec_mask_to_char_array (uintptr_t mask, unsigned char *array, int numsubvols)
+ec_mask_to_char_array(uintptr_t mask, unsigned char *array, int numsubvols)
 {
-        int     i = 0;
+    int i = 0;
 
-        for (i = 0; i < numsubvols; i++)
-                array[i] = ((mask >> i) & 1);
+    for (i = 0; i < numsubvols; i++)
+        array[i] = ((mask >> i) & 1);
 }
 
 uintptr_t
-ec_char_array_to_mask (unsigned char *array, int numsubvols)
+ec_char_array_to_mask(unsigned char *array, int numsubvols)
 {
-        int       i    = 0;
-        uintptr_t mask = 0;
+    int i = 0;
+    uintptr_t mask = 0;
 
-        for (i = 0; i < numsubvols; i++)
-                if (array[i])
-                        mask |= (1ULL<<i);
-        return mask;
+    if (array == NULL)
+        goto out;
+
+    for (i = 0; i < numsubvols; i++)
+        if (array[i])
+            mask |= (1ULL << i);
+out:
+    return mask;
 }
 
 int
-ec_heal_entry_find_direction (ec_t *ec, default_args_cbk_t *replies,
-                        uint64_t *versions, uint64_t *dirty,
-                        unsigned char *sources, unsigned char *healed_sinks)
+ec_heal_entry_find_direction(ec_t *ec, default_args_cbk_t *replies,
+                             uint64_t *versions, uint64_t *dirty,
+                             unsigned char *sources,
+                             unsigned char *healed_sinks)
 {
-        uint64_t    xattr[EC_VERSION_SIZE] = {0};
-        int         source      = -1;
-        uint64_t    max_version = 0;
-        int         ret         = 0;
-        int         i           = 0;
+    uint64_t xattr[EC_VERSION_SIZE] = {0};
+    int source = -1;
+    uint64_t max_version = 0;
+    int ret = 0;
+    int i = 0;
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!replies[i].valid)
-                        continue;
-
-                if (replies[i].op_ret == -1)
-                        continue;
-
-                if (source == -1)
-                        source = i;
-
-                ret = ec_dict_get_array (replies[i].xdata, EC_XATTR_VERSION,
-                                         xattr, EC_VERSION_SIZE);
-                if (ret == 0) {
-                        versions[i] = xattr[EC_DATA_TXN];
-                        if (max_version < versions[i]) {
-                                max_version = versions[i];
-                                source = i;
-                        }
-                }
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                memset (xattr, 0, sizeof(xattr));
-                ret = ec_dict_get_array (replies[i].xdata, EC_XATTR_DIRTY,
-                                         xattr, EC_VERSION_SIZE);
-                if (ret == 0) {
-                        dirty[i] = xattr[EC_DATA_TXN];
-                }
+        if (replies[i].op_ret == -1)
+            continue;
+
+        if (source == -1)
+            source = i;
+
+        ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_VERSION, xattr,
+                                EC_VERSION_SIZE);
+        if (ret == 0) {
+            versions[i] = xattr[EC_DATA_TXN];
+            if (max_version < versions[i]) {
+                max_version = versions[i];
+                source = i;
+            }
         }
 
-        if (source < 0)
-                goto out;
+        memset(xattr, 0, sizeof(xattr));
+        ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_DIRTY, xattr,
+                                EC_VERSION_SIZE);
+        if (ret == 0) {
+            dirty[i] = xattr[EC_DATA_TXN];
+        }
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!replies[i].valid)
-                        continue;
+    if (source < 0)
+        goto out;
 
-                if (replies[i].op_ret == -1)
-                        continue;
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                if (versions[i] == versions[source])
-                        sources[i] = 1;
-                else
-                        healed_sinks[i] = 1;
-        }
+        if (replies[i].op_ret == -1)
+            continue;
+
+        if (versions[i] == versions[source])
+            sources[i] = 1;
+        else
+            healed_sinks[i] = 1;
+    }
 
 out:
-        return source;
+    return source;
 }
 
 int
-ec_adjust_versions (call_frame_t *frame, ec_t *ec, ec_txn_t type,
-                    inode_t *inode, int source, unsigned char *sources,
-                    unsigned char *healed_sinks, uint64_t *versions,
-                    uint64_t *dirty)
-{
-        int                        i                 = 0;
-        int                        ret               = 0;
-        int                        call_count        = 0;
-        dict_t                     **xattr           = NULL;
-        int                        op_ret            = 0;
-        loc_t                      loc               = {0};
-        gf_boolean_t               erase_dirty       = _gf_false;
-        uint64_t                   *versions_xattr   = NULL;
-        uint64_t                   *dirty_xattr      = NULL;
-        uint64_t                   allzero[2]        = {0};
-        unsigned char              *on               = NULL;
-        unsigned char              *output           = NULL;
-        default_args_cbk_t         *replies          = NULL;
-
-        /* Allocate the required memory */
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        on = alloca0 (ec->nodes);
-        output = alloca0 (ec->nodes);
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        xattr = GF_CALLOC (ec->nodes, sizeof (*xattr), gf_common_mt_pointer);
-        if (!xattr) {
-                op_ret = -ENOMEM;
-                goto out;
-        }
-        for (i = 0; i < ec->nodes; i++) {
-                xattr[i] = dict_new ();
-                if (!xattr[i]) {
-                        op_ret = -ENOMEM;
-                        goto out;
-                }
+ec_adjust_versions(call_frame_t *frame, ec_t *ec, ec_txn_t type, inode_t *inode,
+                   int source, unsigned char *sources,
+                   unsigned char *healed_sinks, uint64_t *versions,
+                   uint64_t *dirty)
+{
+    int i = 0;
+    int ret = 0;
+    int call_count = 0;
+    dict_t **xattr = NULL;
+    int op_ret = 0;
+    loc_t loc = {0};
+    gf_boolean_t erase_dirty = _gf_false;
+    uint64_t *versions_xattr = NULL;
+    uint64_t *dirty_xattr = NULL;
+    uint64_t allzero[2] = {0};
+    unsigned char *on = NULL;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+
+    /* Allocate the required memory */
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    xattr = GF_CALLOC(ec->nodes, sizeof(*xattr), gf_common_mt_pointer);
+    if (!xattr) {
+        op_ret = -ENOMEM;
+        goto out;
+    }
+    for (i = 0; i < ec->nodes; i++) {
+        xattr[i] = dict_new();
+        if (!xattr[i]) {
+            op_ret = -ENOMEM;
+            goto out;
         }
+    }
 
-        /* dirty xattr represents if the file/dir needs heal. Unless all the
-         * copies are healed, don't erase it */
-        if (EC_COUNT (sources, ec->nodes) +
-            EC_COUNT (healed_sinks, ec->nodes) == ec->nodes)
-                erase_dirty = _gf_true;
-        else
-                op_ret = -ENOTCONN;
-
-        /* Populate the xattr array */
-        for (i = 0; i < ec->nodes; i++) {
-                if (!sources[i] && !healed_sinks[i])
-                        continue;
-                versions_xattr = GF_CALLOC (EC_VERSION_SIZE,
-                                            sizeof(*versions_xattr),
-                                            gf_common_mt_pointer);
-                if (!versions_xattr) {
-                        op_ret = -ENOMEM;
-                        continue;
-                }
-
-                versions_xattr[type] = hton64(versions[source] - versions[i]);
-                ret = dict_set_bin (xattr[i], EC_XATTR_VERSION,
-                                    versions_xattr,
-                                    (sizeof (*versions_xattr) * EC_VERSION_SIZE)
-                                   );
-                if (ret < 0) {
-                        op_ret = -ENOMEM;
-                        continue;
-                }
-
-                if (erase_dirty) {
-                        dirty_xattr = GF_CALLOC (EC_VERSION_SIZE,
-                                                 sizeof(*dirty_xattr),
-                                                 gf_common_mt_pointer);
-                        if (!dirty_xattr) {
-                                op_ret = -ENOMEM;
-                                continue;
-                        }
-
-                        dirty_xattr[type] = hton64(-dirty[i]);
-                        ret = dict_set_bin (xattr[i], EC_XATTR_DIRTY,
-                                            dirty_xattr,
-                                            (sizeof(*dirty_xattr) *
-                                            EC_VERSION_SIZE)
-                                           );
-                        if (ret < 0) {
-                                op_ret = -ENOMEM;
-                                continue;
-                        }
-                }
+    /* dirty xattr represents if the file/dir needs heal. Unless all the
+     * copies are healed, don't erase it */
+    if (EC_COUNT(sources, ec->nodes) + EC_COUNT(healed_sinks, ec->nodes) ==
+        ec->nodes)
+        erase_dirty = _gf_true;
+    else
+        op_ret = -ENOTCONN;
+
+    /* Populate the xattr array */
+    for (i = 0; i < ec->nodes; i++) {
+        if (!sources[i] && !healed_sinks[i])
+            continue;
+        versions_xattr = GF_CALLOC(EC_VERSION_SIZE, sizeof(*versions_xattr),
+                                   gf_common_mt_pointer);
+        if (!versions_xattr) {
+            op_ret = -ENOMEM;
+            continue;
+        }
+
+        versions_xattr[type] = hton64(versions[source] - versions[i]);
+        ret = dict_set_bin(xattr[i], EC_XATTR_VERSION, versions_xattr,
+                           (sizeof(*versions_xattr) * EC_VERSION_SIZE));
+        if (ret < 0) {
+            op_ret = -ENOMEM;
+            continue;
+        }
 
-                if (memcmp (versions_xattr, allzero,
-                            (sizeof(*versions_xattr) * EC_VERSION_SIZE)) == 0) {
+        if (erase_dirty) {
+            dirty_xattr = GF_CALLOC(EC_VERSION_SIZE, sizeof(*dirty_xattr),
+                                    gf_common_mt_pointer);
+            if (!dirty_xattr) {
+                op_ret = -ENOMEM;
+                continue;
+            }
 
-                        if (!erase_dirty) {
-                                continue;
-                        }
+            dirty_xattr[type] = hton64(-dirty[i]);
+            ret = dict_set_bin(xattr[i], EC_XATTR_DIRTY, dirty_xattr,
+                               (sizeof(*dirty_xattr) * EC_VERSION_SIZE));
+            if (ret < 0) {
+                op_ret = -ENOMEM;
+                continue;
+            }
+        }
 
-                        if (memcmp (dirty_xattr, allzero, (sizeof (*dirty_xattr)
-                                    * EC_VERSION_SIZE)) == 0) {
-                                continue;
-                        }
-                }
+        if (memcmp(versions_xattr, allzero,
+                   (sizeof(*versions_xattr) * EC_VERSION_SIZE)) == 0) {
+            if (!erase_dirty) {
+                continue;
+            }
 
-                on[i] = 1;
-                call_count++;
+            if (memcmp(dirty_xattr, allzero,
+                       (sizeof(*dirty_xattr) * EC_VERSION_SIZE)) == 0) {
+                continue;
+            }
         }
 
-        /* Update the bricks with xattr */
-        if (call_count) {
-                PARALLEL_FOP_ONLIST (ec->xl_list, on, ec->nodes, replies,
-                                     frame, ec_wind_xattrop_parallel,
-                                     &loc, GF_XATTROP_ADD_ARRAY64, xattr, NULL);
-                ret = cluster_fop_success_fill (replies, ec->nodes, output);
-        }
+        on[i] = 1;
+        call_count++;
+    }
 
-        if (ret < call_count) {
-                op_ret = -ENOTCONN;
-                goto out;
-        }
+    /* Update the bricks with xattr */
+    if (call_count) {
+        PARALLEL_FOP_ONLIST(ec->xl_list, on, ec->nodes, replies, frame,
+                            ec_wind_xattrop_parallel, &loc,
+                            GF_XATTROP_ADD_ARRAY64, xattr, NULL);
+        ret = cluster_fop_success_fill(replies, ec->nodes, output);
+    }
+
+    if (ret < call_count) {
+        op_ret = -ENOTCONN;
+        goto out;
+    }
 
 out:
-        /* Cleanup */
-        if (xattr) {
-                for (i = 0; i < ec->nodes; i++) {
-                        if (xattr[i])
-                                dict_unref (xattr[i]);
-                }
-                GF_FREE (xattr);
+    /* Cleanup */
+    if (xattr) {
+        for (i = 0; i < ec->nodes; i++) {
+            if (xattr[i])
+                dict_unref(xattr[i]);
         }
-        cluster_replies_wipe (replies, ec->nodes);
-        loc_wipe (&loc);
-        return op_ret;
+        GF_FREE(xattr);
+    }
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return op_ret;
 }
 
 int
-ec_heal_metadata_find_direction (ec_t *ec, default_args_cbk_t *replies,
-                                 uint64_t *versions, uint64_t *dirty,
-                            unsigned char *sources, unsigned char *healed_sinks)
-{
-        uint64_t xattr[EC_VERSION_SIZE] = {0};
-        uint64_t max_version    = 0;
-        int      same_count     = 0;
-        int      max_same_count = 0;
-        int      same_source    = -1;
-        int      ret            = 0;
-        int      i              = 0;
-        int      j              = 0;
-        int      *groups        = NULL;
-        struct iatt source_ia   = {0};
-        struct iatt child_ia    = {0};
-
-        groups = alloca0 (ec->nodes * sizeof(*groups));
-        for (i = 0; i < ec->nodes; i++)
-                groups[i] = -1;
-
-        for (i = 0; i < ec->nodes; i++) {
-                if (!replies[i].valid)
-                        continue;
-                if (replies[i].op_ret < 0)
-                        continue;
-                ret = ec_dict_get_array (replies[i].xdata, EC_XATTR_VERSION,
-                                         xattr, EC_VERSION_SIZE);
-                if (ret == 0) {
-                        versions[i] = xattr[EC_METADATA_TXN];
-                }
-
-                memset (xattr, 0, sizeof (xattr));
-                ret = ec_dict_get_array (replies[i].xdata, EC_XATTR_DIRTY,
-                                         xattr, EC_VERSION_SIZE);
-                if (ret == 0) {
-                        dirty[i] = xattr[EC_METADATA_TXN];
-                }
-                if (groups[i] >= 0) /*Already part of group*/
-                        continue;
-                groups[i] = i;
-                same_count = 1;
-                source_ia = replies[i].stat;
-                for (j = i + 1; j < ec->nodes; j++) {
-                        if (!replies[j].valid || replies[j].op_ret < 0)
-                                continue;
-                        child_ia = replies[j].stat;
-                        if (!IA_EQUAL(source_ia, child_ia, gfid) ||
-                            !IA_EQUAL(source_ia, child_ia, type) ||
-                            !IA_EQUAL(source_ia, child_ia, prot) ||
-                            !IA_EQUAL(source_ia, child_ia, uid) ||
-                            !IA_EQUAL(source_ia, child_ia, gid))
-                                continue;
-                        if (!are_dicts_equal(replies[i].xdata, replies[j].xdata,
-                                             ec_sh_key_match, NULL))
-                                continue;
-                        groups[j] = i;
-                        same_count++;
-                }
-
-                if (max_same_count < same_count) {
-                        max_same_count = same_count;
-                        same_source = i;
-                }
+ec_heal_metadata_find_direction(ec_t *ec, default_args_cbk_t *replies,
+                                uint64_t *versions, uint64_t *dirty,
+                                unsigned char *sources,
+                                unsigned char *healed_sinks)
+{
+    uint64_t xattr[EC_VERSION_SIZE] = {0};
+    uint64_t max_version = 0;
+    int same_count = 0;
+    int max_same_count = 0;
+    int same_source = -1;
+    int ret = 0;
+    int i = 0;
+    int j = 0;
+    int *groups = NULL;
+    struct iatt source_ia = {0};
+    struct iatt child_ia = {0};
+
+    groups = alloca0(ec->nodes * sizeof(*groups));
+    for (i = 0; i < ec->nodes; i++)
+        groups[i] = -1;
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret < 0)
+            continue;
+        ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_VERSION, xattr,
+                                EC_VERSION_SIZE);
+        if (ret == 0) {
+            versions[i] = xattr[EC_METADATA_TXN];
         }
 
-        if (max_same_count < ec->fragments) {
-                ret = -EIO;
-                goto out;
+        memset(xattr, 0, sizeof(xattr));
+        ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_DIRTY, xattr,
+                                EC_VERSION_SIZE);
+        if (ret == 0) {
+            dirty[i] = xattr[EC_METADATA_TXN];
+        }
+        if (groups[i] >= 0) /*Already part of group*/
+            continue;
+        groups[i] = i;
+        same_count = 1;
+        source_ia = replies[i].stat;
+        for (j = i + 1; j < ec->nodes; j++) {
+            if (!replies[j].valid || replies[j].op_ret < 0)
+                continue;
+            child_ia = replies[j].stat;
+            if (!IA_EQUAL(source_ia, child_ia, gfid) ||
+                !IA_EQUAL(source_ia, child_ia, type) ||
+                !IA_EQUAL(source_ia, child_ia, prot) ||
+                !IA_EQUAL(source_ia, child_ia, uid) ||
+                !IA_EQUAL(source_ia, child_ia, gid))
+                continue;
+            if (!are_dicts_equal(replies[i].xdata, replies[j].xdata,
+                                 ec_sh_key_match, NULL))
+                continue;
+            groups[j] = i;
+            same_count++;
+        }
+
+        if (max_same_count < same_count) {
+            max_same_count = same_count;
+            same_source = i;
         }
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (groups[i] == groups[same_source])
-                        sources[i] = 1;
-                else if (replies[i].valid && replies[i].op_ret >= 0)
-                        healed_sinks[i] = 1;
-        }
-        for (i = 0; i < ec->nodes; i++) {
-                if (sources[i] && (versions[i] > max_version)) {
-                         same_source = i;
-                         max_version = versions[i];
-                }
+    if (max_same_count < ec->fragments) {
+        ret = -EIO;
+        goto out;
+    }
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (groups[i] == groups[same_source])
+            sources[i] = 1;
+        else if (replies[i].valid && replies[i].op_ret >= 0)
+            healed_sinks[i] = 1;
+    }
+    for (i = 0; i < ec->nodes; i++) {
+        if (sources[i] && (versions[i] > max_version)) {
+            same_source = i;
+            max_version = versions[i];
         }
-        ret = same_source;
+    }
+    ret = same_source;
 out:
-        return ret;
+    return ret;
 }
 
 int
-__ec_heal_metadata_prepare (call_frame_t *frame, ec_t *ec, inode_t *inode,
-                   unsigned char *locked_on, default_args_cbk_t *replies,
-                   uint64_t *versions, uint64_t *dirty, unsigned char *sources,
-                   unsigned char *healed_sinks)
-{
-        loc_t              loc        = {0};
-        unsigned char      *output    = NULL;
-        unsigned char      *lookup_on = NULL;
-        int                ret        = 0;
-        int                source     = 0;
-        default_args_cbk_t *greplies  = NULL;
-        int                i          = 0;
-        EC_REPLIES_ALLOC (greplies, ec->nodes);
-
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        output = alloca0 (ec->nodes);
-        lookup_on = alloca0 (ec->nodes);
-        ret = cluster_lookup (ec->xl_list, locked_on, ec->nodes, replies,
-                              output, frame, ec->xl, &loc, NULL);
-        if (ret <= ec->fragments) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+__ec_heal_metadata_prepare(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                           unsigned char *locked_on,
+                           default_args_cbk_t *replies, uint64_t *versions,
+                           uint64_t *dirty, unsigned char *sources,
+                           unsigned char *healed_sinks)
+{
+    loc_t loc = {0};
+    unsigned char *output = NULL;
+    unsigned char *lookup_on = NULL;
+    int ret = 0;
+    int source = 0;
+    default_args_cbk_t *greplies = NULL;
+    int i = 0;
+    EC_REPLIES_ALLOC(greplies, ec->nodes);
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    output = alloca0(ec->nodes);
+    lookup_on = alloca0(ec->nodes);
+    ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output,
+                         frame, ec->xl, &loc, NULL);
+    if (ret <= ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        memcpy (lookup_on, output, ec->nodes);
-        /*Use getxattr to get the filtered xattrs which filter internal xattrs*/
-        ret = cluster_getxattr (ec->xl_list, lookup_on, ec->nodes, greplies,
-                                output, frame, ec->xl, &loc, NULL, NULL);
-        for (i = 0; i < ec->nodes; i++) {
-                if (lookup_on[i] && !output[i]) {
-                        replies[i].valid = 0;
-                        continue;
-                }
-                if (replies[i].xdata) {
-                        dict_unref (replies[i].xdata);
-                        replies[i].xdata = NULL;
-                        if (greplies[i].xattr)
-                                replies[i].xdata = dict_ref (greplies[i].xattr);
-                }
+    memcpy(lookup_on, output, ec->nodes);
+    /*Use getxattr to get the filtered xattrs which filter internal xattrs*/
+    ret = cluster_getxattr(ec->xl_list, lookup_on, ec->nodes, greplies, output,
+                           frame, ec->xl, &loc, NULL, NULL);
+    for (i = 0; i < ec->nodes; i++) {
+        if (lookup_on[i] && !output[i]) {
+            replies[i].valid = 0;
+            continue;
+        }
+        if (replies[i].xdata) {
+            dict_unref(replies[i].xdata);
+            replies[i].xdata = NULL;
+            if (greplies[i].xattr)
+                replies[i].xdata = dict_ref(greplies[i].xattr);
         }
+    }
 
-        source = ec_heal_metadata_find_direction (ec, replies, versions,
-                                         dirty, sources, healed_sinks);
-        if (source < 0) {
-                ret = -EIO;
-                goto out;
-        }
-        ret = source;
+    source = ec_heal_metadata_find_direction(ec, replies, versions, dirty,
+                                             sources, healed_sinks);
+    if (source < 0) {
+        ret = -EIO;
+        goto out;
+    }
+    ret = source;
 out:
-        cluster_replies_wipe (greplies, ec->nodes);
-        loc_wipe (&loc);
-        return ret;
+    cluster_replies_wipe(greplies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
 }
 
 /* Metadata heal */
 int
-__ec_removexattr_sinks (call_frame_t *frame, ec_t *ec, inode_t *inode,
-                        int source, unsigned char *sources,
-                        unsigned char *healed_sinks,
-                        default_args_cbk_t *replies)
-{
-        int   i   = 0;
-        int   ret = 0;
-        loc_t loc = {0};
-
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-
-        for (i = 0; i < ec->nodes; i++) {
-                if (i == source)
-                        continue;
-                if (!sources[i] && !healed_sinks[i])
-                        continue;
-                ret = dict_foreach (replies[i].xdata, ec_heal_xattr_clean,
-                                    replies[source].xdata);
-                if (ret < 0) {
-                        sources[i] = 0;
-                        healed_sinks[i] = 0;
-                        continue;
-                }
-
-                if (replies[i].xdata->count == 0) {
-                        continue;
-                } else if (sources[i]) {
-                        /* This can happen if setxattr/removexattr succeeds on
-                         * the bricks but fails to update the version. This
-                         * will make sure that the xattrs are made equal after
-                         * heal*/
-                        sources[i] = 0;
-                        healed_sinks[i] = 1;
-                }
+__ec_removexattr_sinks(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                       int source, unsigned char *sources,
+                       unsigned char *healed_sinks, default_args_cbk_t *replies)
+{
+    int i = 0;
+    int ret = 0;
+    loc_t loc = {0};
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (i == source)
+            continue;
+        if (!sources[i] && !healed_sinks[i])
+            continue;
+        ret = dict_foreach(replies[i].xdata, ec_heal_xattr_clean,
+                           replies[source].xdata);
+        if (ret < 0) {
+            sources[i] = 0;
+            healed_sinks[i] = 0;
+            continue;
+        }
 
-                ret = syncop_removexattr (ec->xl_list[i], &loc, "",
-                                          replies[i].xdata, NULL);
-                if (ret < 0)
-                        healed_sinks[i] = 0;
+        if (replies[i].xdata->count == 0) {
+            continue;
+        } else if (sources[i]) {
+            /* This can happen if setxattr/removexattr succeeds on
+             * the bricks but fails to update the version. This
+             * will make sure that the xattrs are made equal after
+             * heal*/
+            sources[i] = 0;
+            healed_sinks[i] = 1;
         }
 
-        loc_wipe (&loc);
-        if (EC_COUNT (healed_sinks, ec->nodes) == 0)
-                return -ENOTCONN;
-        return 0;
+        ret = syncop_removexattr(ec->xl_list[i], &loc, "", replies[i].xdata,
+                                 NULL);
+        if (ret < 0)
+            healed_sinks[i] = 0;
+    }
+
+    loc_wipe(&loc);
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0)
+        return -ENOTCONN;
+    return 0;
 }
 
 int
-__ec_heal_metadata (call_frame_t *frame, ec_t *ec, inode_t *inode,
-                    unsigned char *locked_on, unsigned char *sources,
-                    unsigned char *healed_sinks)
-{
-        loc_t              loc           = {0};
-        int                ret           = 0;
-        int                source        = 0;
-        default_args_cbk_t *replies      = NULL;
-        default_args_cbk_t *sreplies     = NULL;
-        uint64_t           *versions     = NULL;
-        uint64_t           *dirty        = NULL;
-        unsigned char      *output       = NULL;
-        dict_t             *source_dict  = NULL;
-        struct iatt        source_buf    = {0};
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        EC_REPLIES_ALLOC (sreplies, ec->nodes);
-
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        output = alloca0 (ec->nodes);
-        versions = alloca0 (ec->nodes * sizeof (*versions));
-        dirty = alloca0 (ec->nodes * sizeof (*dirty));
-        source = __ec_heal_metadata_prepare (frame, ec, inode, locked_on, replies,
-                                    versions, dirty, sources, healed_sinks);
-        if (source < 0) {
-                ret = -EIO;
-                goto out;
-        }
+__ec_heal_metadata(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                   unsigned char *locked_on, unsigned char *sources,
+                   unsigned char *healed_sinks)
+{
+    loc_t loc = {0};
+    int ret = 0;
+    int source = 0;
+    default_args_cbk_t *replies = NULL;
+    default_args_cbk_t *sreplies = NULL;
+    uint64_t *versions = NULL;
+    uint64_t *dirty = NULL;
+    unsigned char *output = NULL;
+    dict_t *source_dict = NULL;
+    struct iatt source_buf = {0};
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    EC_REPLIES_ALLOC(sreplies, ec->nodes);
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    output = alloca0(ec->nodes);
+    versions = alloca0(ec->nodes * sizeof(*versions));
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    source = __ec_heal_metadata_prepare(frame, ec, inode, locked_on, replies,
+                                        versions, dirty, sources, healed_sinks);
+    if (source < 0) {
+        ret = -EIO;
+        goto out;
+    }
 
-        if ((EC_COUNT (sources, ec->nodes) == ec->nodes) ||
-            (EC_COUNT (healed_sinks, ec->nodes) == 0)) {
-                ret = 0;
-                goto erase_dirty;
-        }
+    if ((EC_COUNT(sources, ec->nodes) == ec->nodes) ||
+        (EC_COUNT(healed_sinks, ec->nodes) == 0)) {
+        ret = 0;
+        goto erase_dirty;
+    }
 
-        source_buf = replies[source].stat;
-        ret = cluster_setattr (ec->xl_list, healed_sinks, ec->nodes, sreplies,
-                               output, frame, ec->xl, &loc,
-                               &source_buf, GF_SET_ATTR_MODE |
-                               GF_SET_ATTR_UID | GF_SET_ATTR_GID, NULL);
-        /*In case the operation fails on some of the subvols*/
-        memcpy (healed_sinks, output, ec->nodes);
-        if (EC_COUNT (healed_sinks, ec->nodes) == 0) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    source_buf = replies[source].stat;
+    ret = cluster_setattr(ec->xl_list, healed_sinks, ec->nodes, sreplies,
+                          output, frame, ec->xl, &loc, &source_buf,
+                          GF_SET_ATTR_MODE | GF_SET_ATTR_UID | GF_SET_ATTR_GID,
+                          NULL);
+    /*In case the operation fails on some of the subvols*/
+    memcpy(healed_sinks, output, ec->nodes);
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        ret = __ec_removexattr_sinks (frame, ec, inode, source, sources,
-                                      healed_sinks, replies);
-        if (ret < 0)
-                goto out;
+    ret = __ec_removexattr_sinks(frame, ec, inode, source, sources,
+                                 healed_sinks, replies);
+    if (ret < 0)
+        goto out;
 
-        source_dict = dict_ref (replies[source].xdata);
-        if (dict_foreach_match (source_dict, ec_ignorable_key_match, NULL,
-                                dict_remove_foreach_fn, NULL) == -1) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    source_dict = dict_ref(replies[source].xdata);
+    if (dict_foreach_match(source_dict, ec_ignorable_key_match, NULL,
+                           dict_remove_foreach_fn, NULL) == -1) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        ret = cluster_setxattr (ec->xl_list, healed_sinks, ec->nodes,
-                                replies, output, frame, ec->xl, &loc,
-                                source_dict, 0, NULL);
+    ret = cluster_setxattr(ec->xl_list, healed_sinks, ec->nodes, replies,
+                           output, frame, ec->xl, &loc, source_dict, 0, NULL);
 
-        EC_INTERSECT (healed_sinks, healed_sinks, output, ec->nodes);
-        if (EC_COUNT (healed_sinks, ec->nodes) == 0) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    EC_INTERSECT(healed_sinks, healed_sinks, output, ec->nodes);
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
 erase_dirty:
-        ret = ec_adjust_versions (frame, ec, EC_METADATA_TXN, inode, source,
-                                  sources, healed_sinks, versions, dirty);
+    ret = ec_adjust_versions(frame, ec, EC_METADATA_TXN, inode, source, sources,
+                             healed_sinks, versions, dirty);
 out:
-        if (source_dict)
-                dict_unref (source_dict);
+    if (source_dict)
+        dict_unref(source_dict);
 
-        loc_wipe (&loc);
-        cluster_replies_wipe (replies, ec->nodes);
-        cluster_replies_wipe (sreplies, ec->nodes);
-        return ret;
+    loc_wipe(&loc);
+    cluster_replies_wipe(replies, ec->nodes);
+    cluster_replies_wipe(sreplies, ec->nodes);
+    return ret;
 }
 
 int
-ec_heal_metadata (call_frame_t *frame, ec_t *ec, inode_t *inode,
-                  unsigned char *sources, unsigned char *healed_sinks)
-{
-        unsigned char      *locked_on  = NULL;
-        unsigned char      *up_subvols = NULL;
-        unsigned char      *output     = NULL;
-        int                ret         = 0;
-        default_args_cbk_t *replies    = NULL;
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        locked_on = alloca0(ec->nodes);
-        output = alloca0(ec->nodes);
-        up_subvols = alloca0(ec->nodes);
-        ec_mask_to_char_array (ec->xl_up, up_subvols, ec->nodes);
-        ret = cluster_inodelk (ec->xl_list, up_subvols, ec->nodes, replies,
-                               locked_on, frame, ec->xl, ec->xl->name, inode, 0,
-                               0);
-        {
-                if (ret <= ec->fragments) {
-                        gf_msg_debug (ec->xl->name, 0, "%s: Skipping heal "
-                                "as only %d number of subvolumes could "
-                                "be locked", uuid_utoa (inode->gfid), ret);
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
-                ret = __ec_heal_metadata (frame, ec, inode, locked_on, sources,
-                                          healed_sinks);
-        }
+ec_heal_metadata(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                 unsigned char *sources, unsigned char *healed_sinks)
+{
+    unsigned char *locked_on = NULL;
+    unsigned char *up_subvols = NULL;
+    unsigned char *output = NULL;
+    int ret = 0;
+    default_args_cbk_t *replies = NULL;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    up_subvols = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, up_subvols, ec->nodes, replies,
+                          locked_on, frame, ec->xl, ec->xl->name, inode, 0, 0);
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        ret = __ec_heal_metadata(frame, ec, inode, locked_on, sources,
+                                 healed_sinks);
+    }
 unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes, replies, output,
-                           frame, ec->xl, ec->xl->name, inode, 0, 0);
-        cluster_replies_wipe (replies, ec->nodes);
-        return ret;
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, inode, 0, 0);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 
 /*entry heal*/
 int
-__ec_heal_entry_prepare (call_frame_t *frame, ec_t *ec, inode_t *inode,
-                         unsigned char *locked_on, uint64_t *versions,
-                         uint64_t *dirty, unsigned char *sources,
-                         unsigned char *healed_sinks)
-{
-        loc_t              loc      = {0};
-        int                source   = 0;
-        int                ret      = 0;
-        default_args_cbk_t *replies = NULL;
-        unsigned char      *output  = NULL;
-        dict_t             *xdata   = NULL;
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        xdata = dict_new ();
-        if (!xdata) {
-                ret = -ENOMEM;
-                goto out;
-        }
+__ec_heal_entry_prepare(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                        unsigned char *locked_on, uint64_t *versions,
+                        uint64_t *dirty, unsigned char *sources,
+                        unsigned char *healed_sinks)
+{
+    loc_t loc = {0};
+    int source = 0;
+    int ret = 0;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    dict_t *xdata = NULL;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    xdata = dict_new();
+    if (!xdata) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        if (dict_set_uint64(xdata, EC_XATTR_VERSION, 0) ||
-            dict_set_uint64(xdata, EC_XATTR_DIRTY, 0)) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    if (dict_set_uint64(xdata, EC_XATTR_VERSION, 0) ||
+        dict_set_uint64(xdata, EC_XATTR_DIRTY, 0)) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        output = alloca0 (ec->nodes);
-        ret = cluster_lookup (ec->xl_list, locked_on, ec->nodes, replies,
-                              output, frame, ec->xl, &loc, xdata);
-        if (ret <= ec->fragments) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    output = alloca0(ec->nodes);
+    ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output,
+                         frame, ec->xl, &loc, xdata);
+    if (ret <= ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        source = ec_heal_entry_find_direction (ec, replies, versions,
-                                         dirty, sources, healed_sinks);
-        if (source < 0) {
-                ret = -EIO;
-                goto out;
-        }
-        ret = source;
+    source = ec_heal_entry_find_direction(ec, replies, versions, dirty, sources,
+                                          healed_sinks);
+    if (source < 0) {
+        ret = -EIO;
+        goto out;
+    }
+    ret = source;
 out:
-        if (xdata)
-                dict_unref (xdata);
-        loc_wipe (&loc);
-        cluster_replies_wipe (replies, ec->nodes);
-        return ret;
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 int32_t
-ec_set_new_entry_dirty (ec_t *ec, loc_t *loc, struct iatt *ia,
-                        call_frame_t *frame, xlator_t *this, unsigned char *on)
-{
-        dict_t              *xattr = NULL;
-        int32_t             ret    = -1;
-        default_args_cbk_t  *replies = NULL;
-        unsigned char       *output  = NULL;
-        uint64_t            dirty[EC_VERSION_SIZE] = {1, 1};
-        loc_t               newloc = {0};
-
-        /*Symlinks don't have any data to be healed*/
-        if (ia->ia_type == IA_IFLNK)
-                dirty[EC_DATA_TXN] = 0;
-
-        newloc.inode = inode_ref (loc->inode);
-        gf_uuid_copy (newloc.gfid, ia->ia_gfid);
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        output = alloca0 (ec->nodes);
-        xattr = dict_new();
-        if (!xattr) {
-                ret = -ENOMEM;
-                goto out;
-        }
+ec_set_new_entry_dirty(ec_t *ec, loc_t *loc, struct iatt *ia,
+                       call_frame_t *frame, xlator_t *this, unsigned char *on)
+{
+    dict_t *xattr = NULL;
+    int32_t ret = -1;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    uint64_t dirty[EC_VERSION_SIZE] = {1, 1};
+    loc_t newloc = {0};
+
+    /*Symlinks don't have any data to be healed*/
+    if (ia->ia_type == IA_IFLNK)
+        dirty[EC_DATA_TXN] = 0;
+
+    newloc.inode = inode_ref(loc->inode);
+    gf_uuid_copy(newloc.gfid, ia->ia_gfid);
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    output = alloca0(ec->nodes);
+    xattr = dict_new();
+    if (!xattr) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        ret = ec_dict_set_array (xattr, EC_XATTR_DIRTY, dirty,
-                                 EC_VERSION_SIZE);
-        if (ret)
-                goto out;
+    ret = ec_dict_set_array(xattr, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
+    if (ret)
+        goto out;
 
-        ret = cluster_xattrop (ec->xl_list, on, ec->nodes, replies, output,
-                               frame, ec->xl, &newloc,
-                               GF_XATTROP_ADD_ARRAY64, xattr, NULL);
+    ret = cluster_xattrop(ec->xl_list, on, ec->nodes, replies, output, frame,
+                          ec->xl, &newloc, GF_XATTROP_ADD_ARRAY64, xattr, NULL);
+
+    if (ret < ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        if (ret < ec->fragments) {
-                ret = -ENOTCONN;
-                goto out;
-        }
 out:
-        if (xattr)
-                dict_unref (xattr);
-        cluster_replies_wipe (replies, ec->nodes);
-        loc_wipe (&newloc);
-        return ret;
+    if (xattr)
+        dict_unref(xattr);
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&newloc);
+    return ret;
 }
 
 /*Name heal*/
 int
-ec_delete_stale_name (dict_t *gfid_db, char *key, data_t *d, void *data)
-{
-        struct ec_name_data *name_data   = data;
-        struct iatt         *ia          = NULL;
-        ec_t                *ec          = NULL;
-        loc_t               loc          = {0};
-        unsigned char       *same        = data_to_bin (d);
-        default_args_cbk_t  *replies     = NULL;
-        unsigned char       *output      = NULL;
-        int                 ret          = 0;
-        int                 estale_count = 0;
-        int                 i            = 0;
-        call_frame_t        *frame       = name_data->frame;
-
-        ec = name_data->frame->this->private;
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        if (EC_COUNT (same, ec->nodes) >= ec->fragments) {
-                ret = 0;
-                goto out;
-        }
+ec_delete_stale_name(dict_t *gfid_db, char *key, data_t *d, void *data)
+{
+    struct ec_name_data *name_data = data;
+    struct iatt *ia = NULL;
+    ec_t *ec = NULL;
+    loc_t loc = {0};
+    unsigned char *same = data_to_bin(d);
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    int ret = 0;
+    int estale_count = 0;
+    int i = 0;
+    call_frame_t *frame = name_data->frame;
+    uuid_t gfid;
+
+    ec = name_data->frame->this->private;
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    if (EC_COUNT(same, ec->nodes) >= ec->fragments) {
+        ret = 0;
+        goto out;
+    }
 
-        loc.inode = inode_new (name_data->parent->table);
-        if (!loc.inode) {
-                ret = -ENOMEM;
-                goto out;
-        }
-        gf_uuid_parse (key, loc.gfid);
-        output = alloca0(ec->nodes);
-        ret = cluster_lookup (ec->xl_list, name_data->participants, ec->nodes,
-                              replies, output, name_data->frame, ec->xl, &loc,
-                              NULL);
+    loc.parent = inode_ref(name_data->parent);
+    loc.inode = inode_new(name_data->parent->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!replies[i].valid)
-                        continue;
-                if (replies[i].op_ret == -1) {
-                        if (replies[i].op_errno == ESTALE ||
-                            replies[i].op_errno == ENOENT)
-                                estale_count++;
-                        else
-                                name_data->participants[i] = 0;
-                }
+    gf_uuid_parse(key, gfid);
+    gf_uuid_copy(loc.pargfid, name_data->parent->gfid);
+    loc.name = name_data->name;
+    output = alloca0(ec->nodes);
+    ret = cluster_lookup(ec->xl_list, name_data->participants, ec->nodes,
+                         replies, output, name_data->frame, ec->xl, &loc, NULL);
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret == -1) {
+            if (replies[i].op_errno == ESTALE || replies[i].op_errno == ENOENT)
+                estale_count++;
+            else
+                name_data->participants[i] = 0;
+        } else if (gf_uuid_compare(gfid, replies[i].stat.ia_gfid)) {
+            estale_count++;
+            gf_msg_debug(ec->xl->name, 0, "%s/%s: different gfid as %s",
+                         uuid_utoa(name_data->parent->gfid), name_data->name,
+                         key);
         }
+    }
 
-        if (estale_count <= ec->redundancy) {
-                /* We have at least ec->fragments number of fragments, so the
-                 * file is recoverable, so don't delete it*/
+    if (estale_count <= ec->redundancy) {
+        /* We have at least ec->fragments number of fragments, so the
+         * file is recoverable, so don't delete it*/
 
-                /* Please note that the lookup call above could fail with
-                 * ENOTCONN on all subvoumes and still this branch will be
-                 * true, but in those cases conservatively we decide to not
-                 * delete the file until we are sure*/
-                ret = 0;
-                goto out;
-        }
+        /* Please note that the lookup call above could fail with
+         * ENOTCONN on all subvoumes and still this branch will be
+         * true, but in those cases conservatively we decide to not
+         * delete the file until we are sure*/
+        ret = 0;
+        goto out;
+    }
 
-        /*Noway to recover, delete the name*/
-        loc_wipe (&loc);
-        loc.parent = inode_ref (name_data->parent);
-        gf_uuid_copy (loc.pargfid, loc.parent->gfid);
-        loc.name = name_data->name;
-        for (i = 0; i < ec->nodes; i++) {
-                if (same[i] && replies[i].valid && (replies[i].op_ret == 0)) {
-                        ia = &replies[i].stat;
-                        break;
-                }
+    /*Noway to recover, delete the name*/
+    loc_wipe(&loc);
+    loc.parent = inode_ref(name_data->parent);
+    gf_uuid_copy(loc.pargfid, loc.parent->gfid);
+    loc.name = name_data->name;
+    for (i = 0; i < ec->nodes; i++) {
+        if (same[i] && replies[i].valid && (replies[i].op_ret == 0)) {
+            ia = &replies[i].stat;
+            break;
         }
+    }
 
-        if (!ia) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    if (!ia) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        if (IA_ISDIR (ia->ia_type)) {
-                ret = cluster_rmdir (ec->xl_list, same, ec->nodes, replies,
-                                     output, frame, ec->xl, &loc, 1, NULL);
-                gf_msg_debug (ec->xl->name, 0, "cluster rmdir succeeded on %d "
-                              "nodes", ret);
-        } else {
-                ret = cluster_unlink (ec->xl_list, same, ec->nodes, replies,
-                                      output, frame, ec->xl, &loc, 0, NULL);
-                gf_msg_debug (ec->xl->name, 0, "cluster unlink succeeded on %d "
-                              "nodes", ret);
-        }
+    if (IA_ISDIR(ia->ia_type)) {
+        ret = cluster_rmdir(ec->xl_list, same, ec->nodes, replies, output,
+                            frame, ec->xl, &loc, 1, NULL);
+        gf_msg_debug(ec->xl->name, 0,
+                     "cluster rmdir succeeded on %d "
+                     "nodes",
+                     ret);
+    } else {
+        ret = cluster_unlink(ec->xl_list, same, ec->nodes, replies, output,
+                             frame, ec->xl, &loc, 0, NULL);
+        gf_msg_debug(ec->xl->name, 0,
+                     "cluster unlink succeeded on %d "
+                     "nodes",
+                     ret);
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (output[i]) {
-                        same[i] = 0;
-                        name_data->enoent[i] = 1;
-                } else {
-                        /*op failed*/
-                        if (same[i])
-                                name_data->participants[i] = 0;
-                }
+    for (i = 0; i < ec->nodes; i++) {
+        if (output[i]) {
+            same[i] = 0;
+            name_data->enoent[i] = 1;
+        } else {
+            /*op failed*/
+            if (same[i])
+                name_data->participants[i] = 0;
         }
-        ret = 0;
-        /*This will help in making decisions about creating names*/
-        dict_del (gfid_db, key);
+    }
+    ret = 0;
+    /*This will help in making decisions about creating names*/
+    dict_del(gfid_db, key);
 out:
-        if (ret < 0) {
-                gf_msg_debug (ec->xl->name, 0, "%s/%s: heal failed %s",
-                        uuid_utoa (name_data->parent->gfid), name_data->name,
-                        strerror (-ret));
-        }
-        cluster_replies_wipe (replies, ec->nodes);
-        loc_wipe (&loc);
-        return ret;
+    if (ret < 0) {
+        gf_msg_debug(ec->xl->name, 0, "%s/%s: heal failed %s",
+                     uuid_utoa(name_data->parent->gfid), name_data->name,
+                     strerror(-ret));
+    }
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-ec_delete_stale_names (call_frame_t *frame, ec_t *ec, inode_t *parent,
-                       char *name, default_args_cbk_t *replies, dict_t *gfid_db,
-                       unsigned char *enoent, unsigned char *gfidless,
-                       unsigned char *participants)
+ec_delete_stale_names(call_frame_t *frame, ec_t *ec, inode_t *parent,
+                      char *name, default_args_cbk_t *replies, dict_t *gfid_db,
+                      unsigned char *enoent, unsigned char *gfidless,
+                      unsigned char *participants)
 {
-        struct ec_name_data name_data = {0};
+    struct ec_name_data name_data = {0};
 
-        name_data.enoent = enoent;
-        name_data.gfidless = gfidless;
-        name_data.participants = participants;
-        name_data.name = name;
-        name_data.parent = parent;
-        name_data.frame = frame;
-        name_data.replies = replies;
-        return dict_foreach (gfid_db, ec_delete_stale_name, &name_data);
+    name_data.enoent = enoent;
+    name_data.gfidless = gfidless;
+    name_data.participants = participants;
+    name_data.name = name;
+    name_data.parent = parent;
+    name_data.frame = frame;
+    name_data.replies = replies;
+    return dict_foreach(gfid_db, ec_delete_stale_name, &name_data);
 }
 
 int
-_assign_same (dict_t *dict, char *key, data_t *value, void *data)
+_assign_same(dict_t *dict, char *key, data_t *value, void *data)
 {
-        struct ec_name_data *name_data = data;
+    struct ec_name_data *name_data = data;
 
-        name_data->same = data_to_bin (value);
-        return 0;
+    name_data->same = data_to_bin(value);
+    return 0;
 }
 
 int
-ec_create_name (call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
-                default_args_cbk_t *lookup_replies, dict_t *gfid_db,
-                unsigned char *enoent, unsigned char *participants)
-{
-        int                 ret       = 0;
-        int                 i         = 0;
-        struct ec_name_data name_data = {0};
-        struct iatt         *ia       = NULL;
-        unsigned char       *output   = 0;
-        unsigned char       *output1  = 0;
-        unsigned char       *on       = NULL;
-        default_args_cbk_t  *replies  = NULL;
-        loc_t               loc       = {0};
-        loc_t               srcloc    = {0};
-        unsigned char       *link     = NULL;
-        unsigned char       *create   = NULL;
-        dict_t              *xdata    = NULL;
-        char                *linkname = NULL;
-        ec_config_t         config;
-        /* There should be just one gfid key */
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        if (gfid_db->count != 1) {
-                ret = -EINVAL;
-                goto out;
-        }
-
-        ret = dict_foreach (gfid_db, _assign_same, &name_data);
-        if (ret < 0)
-                goto out;
-        /*There should at least be one valid success reply with gfid*/
-        for (i = 0; i < ec->nodes; i++)
-                if (name_data.same[i])
-                        break;
+ec_create_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
+               default_args_cbk_t *lookup_replies, dict_t *gfid_db,
+               unsigned char *enoent, unsigned char *participants)
+{
+    int ret = 0;
+    int i = 0;
+    struct ec_name_data name_data = {0};
+    struct iatt *ia = NULL;
+    unsigned char *output = 0;
+    unsigned char *output1 = 0;
+    unsigned char *on = NULL;
+    default_args_cbk_t *replies = NULL;
+    loc_t loc = {0};
+    loc_t srcloc = {0};
+    unsigned char *link = NULL;
+    unsigned char *create = NULL;
+    dict_t *xdata = NULL;
+    char *linkname = NULL;
+    ec_config_t config;
+
+    /* There should be just one gfid key */
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    if (gfid_db->count != 1) {
+        ret = -EINVAL;
+        goto out;
+    }
 
-        if (i == ec->nodes) {
-                ret = -EINVAL;
-                goto out;
-        }
+    ret = dict_foreach(gfid_db, _assign_same, &name_data);
+    if (ret < 0)
+        goto out;
+    /*There should at least be one valid success reply with gfid*/
+    for (i = 0; i < ec->nodes; i++)
+        if (name_data.same[i])
+            break;
 
-        ia = &lookup_replies[i].stat;
-        xdata = dict_new ();
-        loc.parent = inode_ref (parent);
-        gf_uuid_copy (loc.pargfid, parent->gfid);
-        loc.inode = inode_new (parent->table);
-        if (loc.inode)
-                srcloc.inode = inode_ref (loc.inode);
-        gf_uuid_copy (srcloc.gfid, ia->ia_gfid);
-        if (!loc.inode || !xdata || dict_set_static_bin (xdata, "gfid-req",
-                                                         ia->ia_gfid,
-                                                        sizeof (ia->ia_gfid))) {
-                ret = -ENOMEM;
-                goto out;
-        }
-        loc.name = name;
-        link = alloca0 (ec->nodes);
-        create = alloca0 (ec->nodes);
-        on = alloca0 (ec->nodes);
-        output = alloca0 (ec->nodes);
-        output1 = alloca0 (ec->nodes);
+    if (i == ec->nodes) {
+        ret = -EINVAL;
+        goto out;
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!lookup_replies[i].valid)
-                        continue;
-                if (lookup_replies[i].op_ret)
-                        continue;
-                on[i] = 1;
-        }
-        switch (ia->ia_type) {
+    ia = &lookup_replies[i].stat;
+    xdata = dict_new();
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.inode = inode_new(parent->table);
+    if (loc.inode)
+        srcloc.inode = inode_ref(loc.inode);
+    gf_uuid_copy(srcloc.gfid, ia->ia_gfid);
+    if (!loc.inode || !xdata ||
+        dict_set_static_bin(xdata, "gfid-req", ia->ia_gfid,
+                            sizeof(ia->ia_gfid))) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    loc.name = name;
+    link = alloca0(ec->nodes);
+    create = alloca0(ec->nodes);
+    on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    output1 = alloca0(ec->nodes);
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!lookup_replies[i].valid)
+            continue;
+        if (lookup_replies[i].op_ret)
+            continue;
+        on[i] = 1;
+    }
+    switch (ia->ia_type) {
         case IA_IFDIR:
-                ec_set_new_entry_dirty (ec, &loc, ia, frame, ec->xl, on);
-                (void) cluster_mkdir (ec->xl_list, enoent, ec->nodes,
-                                   replies, output, frame, ec->xl, &loc,
-                                   st_mode_from_ia (ia->ia_prot,
-                                                ia->ia_type), 0, xdata);
-                break;
+            ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on);
+            (void)cluster_mkdir(
+                ec->xl_list, enoent, ec->nodes, replies, output, frame, ec->xl,
+                &loc, st_mode_from_ia(ia->ia_prot, ia->ia_type), 0, xdata);
+            break;
 
         case IA_IFLNK:
-                /*Check for hard links and create/link*/
-                ret = cluster_lookup (ec->xl_list, enoent, ec->nodes,
-                                      replies, output, frame, ec->xl,
-                                      &srcloc, NULL);
-                for (i = 0; i < ec->nodes; i++) {
-                        if (output[i]) {
-                                link[i] = 1;
-                        } else {
-                                if (replies[i].op_errno == ENOENT ||
-                                    replies[i].op_errno == ESTALE) {
-                                        create[i] = 1;
-                                }
-                        }
+            /*Check for hard links and create/link*/
+            ret = cluster_lookup(ec->xl_list, enoent, ec->nodes, replies,
+                                 output, frame, ec->xl, &srcloc, NULL);
+            for (i = 0; i < ec->nodes; i++) {
+                if (output[i]) {
+                    link[i] = 1;
+                } else {
+                    if (replies[i].op_errno == ENOENT ||
+                        replies[i].op_errno == ESTALE) {
+                        create[i] = 1;
+                    }
                 }
-
-                if (EC_COUNT (link, ec->nodes)) {
-                        cluster_link (ec->xl_list, link, ec->nodes,
-                                      replies, output1, frame, ec->xl,
-                                      &srcloc, &loc, NULL);
+            }
+
+            if (EC_COUNT(link, ec->nodes)) {
+                cluster_link(ec->xl_list, link, ec->nodes, replies, output1,
+                             frame, ec->xl, &srcloc, &loc, NULL);
+            }
+
+            if (EC_COUNT(create, ec->nodes)) {
+                cluster_readlink(ec->xl_list, name_data.same, ec->nodes,
+                                 replies, output, frame, ec->xl, &srcloc, 4096,
+                                 NULL);
+                if (EC_COUNT(output, ec->nodes) == 0) {
+                    ret = -ENOTCONN;
+                    goto out;
                 }
 
-                if (EC_COUNT (create, ec->nodes)) {
-                        cluster_readlink (ec->xl_list, name_data.same,
-                                          ec->nodes, replies, output,
-                                          frame, ec->xl, &srcloc, 4096,
-                                          NULL);
-                        if (EC_COUNT (output, ec->nodes) == 0) {
-                                ret = -ENOTCONN;
-                                goto out;
-                        }
-
-                        for (i = 0; i < ec->nodes; i++) {
-                                if (output[i])
-                                        break;
-                        }
-                        linkname = alloca0 (strlen(replies[i].buf) + 1);
-                        strcpy (linkname, replies[i].buf);
-                        ec_set_new_entry_dirty (ec, &loc, ia, frame,
-                                                ec->xl, on);
-                        cluster_symlink (ec->xl_list, create, ec->nodes,
-                                         replies, output, frame, ec->xl,
-                                         linkname, &loc, 0, xdata);
+                for (i = 0; i < ec->nodes; i++) {
+                    if (output[i])
+                        break;
                 }
-                for (i = 0; i < ec->nodes; i++)
-                        if (output1[i])
-                                output[i] = 1;
-                break;
+                linkname = alloca0(strlen(replies[i].buf) + 1);
+                strcpy(linkname, replies[i].buf);
+                ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on);
+                cluster_symlink(ec->xl_list, create, ec->nodes, replies, output,
+                                frame, ec->xl, linkname, &loc, 0, xdata);
+            }
+            for (i = 0; i < ec->nodes; i++)
+                if (output1[i])
+                    output[i] = 1;
+            break;
         case IA_IFREG:
-                ec_set_new_entry_dirty (ec, &loc, ia,
-                                        frame, ec->xl, on);
-                config.version = EC_CONFIG_VERSION;
-                config.algorithm = EC_CONFIG_ALGORITHM;
-                config.gf_word_size = EC_GF_BITS;
-                config.bricks = ec->nodes;
-                config.redundancy = ec->redundancy;
-                config.chunk_size = EC_METHOD_CHUNK_SIZE;
-
-                ret = ec_dict_set_config(xdata, EC_XATTR_CONFIG, &config);
-                if (ret != 0) {
-                        goto out;
-                }
+            ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on);
+            config.version = EC_CONFIG_VERSION;
+            config.algorithm = EC_CONFIG_ALGORITHM;
+            config.gf_word_size = EC_GF_BITS;
+            config.bricks = ec->nodes;
+            config.redundancy = ec->redundancy;
+            config.chunk_size = EC_METHOD_CHUNK_SIZE;
 
-        /* Fall through */
+            ret = ec_dict_set_config(xdata, EC_XATTR_CONFIG, &config);
+            if (ret != 0) {
+                goto out;
+            }
+
+            /* Fall through */
 
         default:
-                ret = dict_set_int32 (xdata, GLUSTERFS_INTERNAL_FOP_KEY,
-                                      1);
-                if (ret)
-                        goto out;
-                ret = cluster_mknod (ec->xl_list, enoent, ec->nodes,
-                                     replies, output, frame, ec->xl,
-                                     &loc, st_mode_from_ia (ia->ia_prot,
-                                                           ia->ia_type),
-                         makedev(ia_major(ia->ia_rdev), ia_minor(ia->ia_rdev)),
-                         0, xdata);
-                break;
-        }
+            ret = dict_set_int32(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+            if (ret)
+                goto out;
+            ret = cluster_mknod(
+                ec->xl_list, enoent, ec->nodes, replies, output, frame, ec->xl,
+                &loc, st_mode_from_ia(ia->ia_prot, ia->ia_type),
+                makedev(ia_major(ia->ia_rdev), ia_minor(ia->ia_rdev)), 0,
+                xdata);
+            break;
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (enoent[i] && !output[i])
-                        participants[i] = 0;
-        }
+    for (i = 0; i < ec->nodes; i++) {
+        if (enoent[i] && !output[i])
+            participants[i] = 0;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret < 0)
-                gf_msg_debug (ec->xl->name, 0, "%s/%s: heal failed %s",
-                        uuid_utoa (parent->gfid), name, strerror (-ret));
-        cluster_replies_wipe (replies, ec->nodes);
-        loc_wipe (&loc);
-        loc_wipe (&srcloc);
-        if (xdata)
-                dict_unref (xdata);
-        return ret;
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "%s/%s: heal failed %s",
+                     uuid_utoa(parent->gfid), name, strerror(-ret));
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    loc_wipe(&srcloc);
+    if (xdata)
+        dict_unref(xdata);
+    return ret;
 }
 
 int
-__ec_heal_name (call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
-                unsigned char *participants)
-{
-        unsigned char      *output   = NULL;
-        unsigned char      *enoent   = NULL;
-        default_args_cbk_t *replies  = NULL;
-        dict_t             *xdata    = NULL;
-        dict_t             *gfid_db  = NULL;
-        int                ret       = 0;
-        loc_t              loc       = {0};
-        int                i         = 0;
-        struct iatt        *ia       = NULL;
-        char               gfid[64]  = {0};
-        unsigned char      *same     = NULL;
-        unsigned char      *gfidless = NULL;
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        loc.parent = inode_ref (parent);
-        loc.inode = inode_new (parent->table);
-        gf_uuid_copy (loc.pargfid, parent->gfid);
-        loc.name = name;
-        xdata = dict_new ();
-        gfid_db = dict_new ();
-        if (!xdata || !gfid_db || !loc.inode) {
-                ret = -ENOMEM;
-                goto out;
-        }
+__ec_heal_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
+               unsigned char *participants)
+{
+    unsigned char *output = NULL;
+    unsigned char *enoent = NULL;
+    default_args_cbk_t *replies = NULL;
+    dict_t *xdata = NULL;
+    dict_t *gfid_db = NULL;
+    int ret = 0;
+    loc_t loc = {0};
+    int i = 0;
+    struct iatt *ia = NULL;
+    char gfid[64] = {0};
+    unsigned char *same = NULL;
+    unsigned char *gfidless = NULL;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    loc.parent = inode_ref(parent);
+    loc.inode = inode_new(parent->table);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = name;
+    xdata = dict_new();
+    gfid_db = dict_new();
+    if (!xdata || !gfid_db || !loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        ret = dict_set_int32 (xdata, GF_GFIDLESS_LOOKUP, 1);
-        if (ret) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    ret = dict_set_int32(xdata, GF_GFIDLESS_LOOKUP, 1);
+    if (ret) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        output = alloca0 (ec->nodes);
-        gfidless = alloca0 (ec->nodes);
-        enoent = alloca0 (ec->nodes);
-        ret = cluster_lookup (ec->xl_list, participants, ec->nodes, replies,
-                              output, frame, ec->xl, &loc, NULL);
-        for (i = 0; i < ec->nodes; i++) {
-                if (!replies[i].valid)
-                        continue;
-
-                if (replies[i].op_ret == -1) {
-                        /*If ESTALE comes here, that means parent dir is not
-                         * present, nothing to do there, so reset participants
-                         * for that brick*/
-                        if (replies[i].op_errno == ENOENT)
-                                enoent[i] = 1;
-                        else
-                                participants[i] = 0;
-                        continue;
-                }
-                ia = &replies[i].stat;
-                if (gf_uuid_is_null (ia->ia_gfid)) {
-                        if (IA_ISDIR (ia->ia_type) || ia->ia_size == 0)
-                                gfidless[i] = 1;
-                        else
-                                participants[i] = 0;
-                } else {
-                        uuid_utoa_r (ia->ia_gfid, gfid);
-                        ret = dict_get_bin (gfid_db, gfid, (void **)&same);
-                        if (ret < 0) {
-                                same = alloca0(ec->nodes);
-                        }
-                        same[i] = 1;
-                        if (ret < 0) {
-                                ret = dict_set_static_bin (gfid_db, gfid, same,
-                                                           ec->nodes);
-                        }
-                        if (ret < 0)
-                                goto out;
-                }
+    output = alloca0(ec->nodes);
+    gfidless = alloca0(ec->nodes);
+    enoent = alloca0(ec->nodes);
+    ret = cluster_lookup(ec->xl_list, participants, ec->nodes, replies, output,
+                         frame, ec->xl, &loc, NULL);
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret == -1) {
+            /*If ESTALE comes here, that means parent dir is not
+             * present, nothing to do there, so reset participants
+             * for that brick*/
+            if (replies[i].op_errno == ENOENT)
+                enoent[i] = 1;
+            else
+                participants[i] = 0;
+            continue;
+        }
+        ia = &replies[i].stat;
+        if (gf_uuid_is_null(ia->ia_gfid)) {
+            if (IA_ISDIR(ia->ia_type) || ia->ia_size == 0)
+                gfidless[i] = 1;
+            else
+                participants[i] = 0;
+        } else {
+            uuid_utoa_r(ia->ia_gfid, gfid);
+            ret = dict_get_bin(gfid_db, gfid, (void **)&same);
+            if (ret < 0) {
+                same = alloca0(ec->nodes);
+            }
+            same[i] = 1;
+            if (ret < 0) {
+                ret = dict_set_static_bin(gfid_db, gfid, same, ec->nodes);
+            }
+            if (ret < 0)
+                goto out;
         }
+    }
 
-        ret = ec_delete_stale_names (frame, ec, parent, name, replies, gfid_db,
-                                     enoent, gfidless, participants);
+    ret = ec_delete_stale_names(frame, ec, parent, name, replies, gfid_db,
+                                enoent, gfidless, participants);
 
-        if (gfid_db->count == 0) {
-                /* All entries seem to be stale entries and deleted,
-                 * nothing more to do.*/
-                goto out;
-        }
+    if (gfid_db->count == 0) {
+        /* All entries seem to be stale entries and deleted,
+         * nothing more to do.*/
+        goto out;
+    }
 
-        if (gfid_db->count > 1) {
-                gf_msg (ec->xl->name, GF_LOG_INFO, 0,
-                        EC_MSG_HEAL_FAIL, "%s/%s: Not able to heal",
-                        uuid_utoa (parent->gfid), name);
-                memset (participants, 0, ec->nodes);
-                goto out;
-        }
+    if (gfid_db->count > 1) {
+        gf_msg(ec->xl->name, GF_LOG_INFO, 0, EC_MSG_HEAL_FAIL,
+               "%s/%s: Not able to heal", uuid_utoa(parent->gfid), name);
+        memset(participants, 0, ec->nodes);
+        goto out;
+    }
 
-        EC_INTERSECT (enoent, enoent, participants, ec->nodes);
-        if (EC_COUNT (enoent, ec->nodes) == 0) {
-                ret = 0;
-                goto out;
-        }
+    EC_INTERSECT(enoent, enoent, participants, ec->nodes);
+    if (EC_COUNT(enoent, ec->nodes) == 0) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = ec_create_name (frame, ec, parent, name, replies, gfid_db, enoent,
-                              participants);
+    ret = ec_create_name(frame, ec, parent, name, replies, gfid_db, enoent,
+                         participants);
+    if (ret >= 0) {
+        /* If ec_create_name() succeeded we return 1 to indicate that a new
+         * file has been created and it will need to be healed. */
+        ret = 1;
+    }
 out:
-        cluster_replies_wipe (replies, ec->nodes);
-        loc_wipe (&loc);
-        if (xdata)
-                dict_unref (xdata);
-        if (gfid_db)
-                dict_unref (gfid_db);
-        return ret;
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    if (xdata)
+        dict_unref(xdata);
+    if (gfid_db)
+        dict_unref(gfid_db);
+    return ret;
 }
 
 int
-ec_heal_name (call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
-              unsigned char *participants)
-{
-        int                ret        = 0;
-        default_args_cbk_t *replies   = NULL;
-        unsigned char      *output    = NULL;
-        unsigned char      *locked_on = NULL;
-        loc_t              loc        = {0};
-
-        loc.parent = inode_ref (parent);
-        loc.name = name;
-        loc.inode = inode_new (parent->table);
-        if (!loc.inode) {
-                ret = -ENOMEM;
-                goto out;
-        }
+ec_heal_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
+             unsigned char *participants)
+{
+    int ret = 0;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    unsigned char *locked_on = NULL;
+    loc_t loc = {0};
+
+    loc.parent = inode_ref(parent);
+    loc.name = name;
+    loc.inode = inode_new(parent->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        output = alloca0 (ec->nodes);
-        locked_on = alloca0 (ec->nodes);
-        ret = cluster_inodelk (ec->xl_list, participants, ec->nodes, replies,
-                               locked_on, frame, ec->xl, ec->xl->name, parent,
-                               0, 0);
-        {
-                if (ret <= ec->fragments) {
-                        gf_msg_debug (ec->xl->name, 0, "%s/%s: Skipping "
-                                "heal as only %d number of subvolumes could "
-                                "be locked", uuid_utoa (parent->gfid), name,
-                                ret);
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
-                EC_INTERSECT (participants, participants, locked_on, ec->nodes);
-                ret = __ec_heal_name (frame, ec, parent, name, participants);
-        }
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    output = alloca0(ec->nodes);
+    locked_on = alloca0(ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, participants, ec->nodes, replies,
+                          locked_on, frame, ec->xl, ec->xl->name, parent, 0, 0);
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s/%s: Skipping "
+                         "heal as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(parent->gfid), name, ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        EC_INTERSECT(participants, participants, locked_on, ec->nodes);
+        ret = __ec_heal_name(frame, ec, parent, name, participants);
+    }
 unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes, replies, output,
-                           frame, ec->xl, ec->xl->name, parent, 0, 0);
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, parent, 0, 0);
 out:
-        cluster_replies_wipe (replies, ec->nodes);
-        loc_wipe (&loc);
-        return ret;
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-ec_name_heal_handler (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                      void *data)
-{
-        struct ec_name_data *name_data = data;
-        xlator_t            *this      = THIS;
-        ec_t                *ec        = this->private;
-        unsigned char       *name_on   = alloca0 (ec->nodes);
-        int                 i          = 0;
-        int                 ret        = 0;
+ec_name_heal_handler(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                     void *data)
+{
+    struct ec_name_data *name_data = data;
+    xlator_t *this = THIS;
+    ec_t *ec = this->private;
+    unsigned char *name_on = alloca0(ec->nodes);
+    int i = 0;
+    int ret = 0;
+
+    if (ec->shutdown) {
+        gf_msg_debug(this->name, 0,
+                     "Cancelling directory heal "
+                     "because EC is stopping.");
+        return -ENOTCONN;
+    }
 
-        if (ec->shutdown) {
-                gf_msg_debug(this->name, 0, "Cancelling directory heal "
-                                            "because EC is stopping.");
-                return -ENOTCONN;
-        }
+    memcpy(name_on, name_data->participants, ec->nodes);
+    ret = ec_heal_name(name_data->frame, ec, parent->inode, entry->d_name,
+                       name_on);
 
-        memcpy (name_on, name_data->participants, ec->nodes);
-        ret = ec_heal_name (name_data->frame, ec, parent->inode,
-                            entry->d_name, name_on);
+    if (ret < 0) {
+        memset(name_on, 0, ec->nodes);
+    } else {
+        name_data->heal_pending += ret;
+    }
 
-        if (ret < 0)
-                memset (name_on, 0, ec->nodes);
+    for (i = 0; i < ec->nodes; i++)
+        if (name_data->participants[i] && !name_on[i])
+            name_data->failed_on[i] = 1;
 
-        for (i = 0; i < ec->nodes; i++)
-                if (name_data->participants[i] && !name_on[i])
-                        name_data->failed_on[i] = 1;
-        return 0;
+    return 0;
 }
 
 int
-ec_heal_names (call_frame_t *frame, ec_t *ec, inode_t *inode,
-               unsigned char *participants)
-{
-        int i = 0;
-        int j = 0;
-        loc_t loc = {0};
-        struct ec_name_data name_data = {0};
-        int ret = 0;
-
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        name_data.frame = frame;
-        name_data.participants = participants;
-        name_data.failed_on = alloca0(ec->nodes);;
-
-        for (i = 0; i < ec->nodes; i++) {
-                if (!participants[i])
-                        continue;
-                ret = syncop_dir_scan (ec->xl_list[i], &loc,
-                                       GF_CLIENT_PID_SELF_HEALD, &name_data,
-                                       ec_name_heal_handler);
-                if (ret < 0) {
-                        break;
-                }
-                for (j = 0; j < ec->nodes; j++)
-                        if (name_data.failed_on[j])
-                                participants[j] = 0;
+ec_heal_names(call_frame_t *frame, ec_t *ec, inode_t *inode,
+              unsigned char *participants, uint32_t *pending)
+{
+    int i = 0;
+    int j = 0;
+    loc_t loc = {0};
+    struct ec_name_data name_data = {0};
+    int ret = 0;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    name_data.frame = frame;
+    name_data.participants = participants;
+    name_data.failed_on = alloca0(ec->nodes);
+    name_data.heal_pending = 0;
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!participants[i])
+            continue;
+        ret = syncop_dir_scan(ec->xl_list[i], &loc, GF_CLIENT_PID_SELF_HEALD,
+                              &name_data, ec_name_heal_handler);
+        if (ret < 0) {
+            break;
+        }
+        for (j = 0; j < ec->nodes; j++)
+            if (name_data.failed_on[j])
+                participants[j] = 0;
 
-                if (EC_COUNT (participants, ec->nodes) <= ec->fragments) {
-                        ret = -ENOTCONN;
-                        break;
-                }
+        if (EC_COUNT(participants, ec->nodes) <= ec->fragments) {
+            ret = -ENOTCONN;
+            break;
         }
-        loc_wipe (&loc);
-        return ret;
+    }
+    *pending += name_data.heal_pending;
+
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-__ec_heal_entry (call_frame_t *frame, ec_t *ec, inode_t *inode,
-                 unsigned char *heal_on, unsigned char *sources,
-                 unsigned char *healed_sinks)
-{
-        unsigned char      *locked_on    = NULL;
-        unsigned char      *output       = NULL;
-        uint64_t           *versions     = NULL;
-        uint64_t           *dirty        = NULL;
-        unsigned char      *participants = NULL;
-        default_args_cbk_t *replies      = NULL;
-        int                ret           = 0;
-        int                source        = 0;
-        int                i             = 0;
-
-        locked_on = alloca0(ec->nodes);
-        output = alloca0(ec->nodes);
-        versions = alloca0 (ec->nodes * sizeof (*versions));
-        dirty = alloca0 (ec->nodes * sizeof (*dirty));
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        ret = cluster_inodelk (ec->xl_list, heal_on, ec->nodes, replies,
-                               locked_on, frame, ec->xl, ec->xl->name, inode,
-                               0, 0);
-        {
-                if (ret <= ec->fragments) {
-                        gf_msg_debug (ec->xl->name, 0, "%s: Skipping heal "
-                                "as only %d number of subvolumes could "
-                                "be locked", uuid_utoa (inode->gfid), ret);
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
-                ret = __ec_heal_entry_prepare (frame, ec, inode, locked_on,
-                                               versions, dirty, sources,
-                                               healed_sinks);
-                source = ret;
-        }
+__ec_heal_entry(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                unsigned char *heal_on, unsigned char *sources,
+                unsigned char *healed_sinks, uint32_t *pending)
+{
+    unsigned char *locked_on = NULL;
+    unsigned char *output = NULL;
+    uint64_t *versions = NULL;
+    uint64_t *dirty = NULL;
+    unsigned char *participants = NULL;
+    default_args_cbk_t *replies = NULL;
+    int ret = 0;
+    int source = 0;
+    int i = 0;
+
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    versions = alloca0(ec->nodes * sizeof(*versions));
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, heal_on, ec->nodes, replies, locked_on,
+                          frame, ec->xl, ec->xl->name, inode, 0, 0);
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        ret = __ec_heal_entry_prepare(frame, ec, inode, locked_on, versions,
+                                      dirty, sources, healed_sinks);
+        source = ret;
+    }
 unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes, replies, output,
-                           frame, ec->xl, ec->xl->name, inode, 0, 0);
-        if (ret < 0)
-                goto out;
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, inode, 0, 0);
+    if (ret < 0)
+        goto out;
 
-        participants = alloca0 (ec->nodes);
-        for (i = 0; i < ec->nodes; i++) {
-                if (sources[i] || healed_sinks[i])
-                        participants[i] = 1;
-        }
-        ret = ec_heal_names (frame, ec, inode, participants);
+    participants = alloca0(ec->nodes);
+    for (i = 0; i < ec->nodes; i++) {
+        if (sources[i] || healed_sinks[i])
+            participants[i] = 1;
+    }
+    ret = ec_heal_names(frame, ec, inode, participants, pending);
 
-        if (EC_COUNT (participants, ec->nodes) <= ec->fragments)
-                goto out;
+    if (EC_COUNT(participants, ec->nodes) <= ec->fragments)
+        goto out;
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!participants[i]) {
-                        sources[i] = 0;
-                        healed_sinks[i] = 0;
-                }
+    for (i = 0; i < ec->nodes; i++) {
+        if (!participants[i]) {
+            sources[i] = 0;
+            healed_sinks[i] = 0;
         }
+    }
 
-        ec_adjust_versions (frame, ec, EC_DATA_TXN, inode, source,
-                            sources, healed_sinks, versions, dirty);
+    ec_adjust_versions(frame, ec, EC_DATA_TXN, inode, source, sources,
+                       healed_sinks, versions, dirty);
 out:
-        cluster_replies_wipe (replies, ec->nodes);
-        return ret;
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 
 int
-ec_heal_entry (call_frame_t *frame, ec_t *ec, inode_t *inode,
-               unsigned char *sources, unsigned char *healed_sinks)
-{
-        unsigned char      *locked_on            = NULL;
-        unsigned char      *up_subvols           = NULL;
-        unsigned char      *output               = NULL;
-        char               selfheal_domain[1024] = {0};
-        int                ret                   = 0;
-        default_args_cbk_t *replies              = NULL;
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        locked_on = alloca0(ec->nodes);
-        output = alloca0(ec->nodes);
-        up_subvols = alloca0(ec->nodes);
-
-        sprintf (selfheal_domain, "%s:self-heal", ec->xl->name);
-        ec_mask_to_char_array (ec->xl_up, up_subvols, ec->nodes);
-        /*If other processes are already doing the heal, don't block*/
-        ret = cluster_tiebreaker_inodelk (ec->xl_list, up_subvols, ec->nodes,
-                                          replies, locked_on, frame, ec->xl,
-                                          selfheal_domain, inode, 0, 0);
-        {
-                if (ret <= ec->fragments) {
-                        gf_msg_debug (ec->xl->name, 0, "%s: Skipping heal "
-                                "as only %d number of subvolumes could "
-                                "be locked", uuid_utoa (inode->gfid), ret);
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
-                ret = __ec_heal_entry (frame, ec, inode, locked_on,
-                                       sources, healed_sinks);
-        }
+ec_heal_entry(call_frame_t *frame, ec_t *ec, inode_t *inode,
+              unsigned char *sources, unsigned char *healed_sinks,
+              uint32_t *pending)
+{
+    unsigned char *locked_on = NULL;
+    unsigned char *up_subvols = NULL;
+    unsigned char *output = NULL;
+    char selfheal_domain[1024] = {0};
+    int ret = 0;
+    default_args_cbk_t *replies = NULL;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    up_subvols = alloca0(ec->nodes);
+
+    sprintf(selfheal_domain, "%s:self-heal", ec->xl->name);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
+    /*If other processes are already doing the heal, don't block*/
+    ret = cluster_tiebreaker_inodelk(ec->xl_list, up_subvols, ec->nodes,
+                                     replies, locked_on, frame, ec->xl,
+                                     selfheal_domain, inode, 0, 0);
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        ret = __ec_heal_entry(frame, ec, inode, locked_on, sources,
+                              healed_sinks, pending);
+    }
 unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes, replies, output,
-                           frame, ec->xl, selfheal_domain, inode, 0, 0);
-        cluster_replies_wipe (replies, ec->nodes);
-        return ret;
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, selfheal_domain, inode, 0, 0);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 
 /*Find direction for data heal and heal info*/
 int
-ec_heal_data_find_direction (ec_t *ec, default_args_cbk_t *replies,
-                       uint64_t *data_versions,
-                       uint64_t *dirty, uint64_t *size, unsigned char *sources,
-                       unsigned char *healed_sinks,
-                       gf_boolean_t check_ondisksize, int which)
-{
-        uint64_t        xattr[EC_VERSION_SIZE] = {0};
-        char            version_size[128] = {0};
-        dict_t          *version_size_db = NULL;
-        unsigned char   *same            = NULL;
-        int             max_same_count   = 0;
-        int             source           = 0;
-        int             i                = 0;
-        int             ret              = 0;
-        dict_t          *dict            = NULL;
-        uint64_t        source_size      = 0;
-
-        version_size_db = dict_new ();
-        if (!version_size_db) {
-                ret = -ENOMEM;
-                goto out;
+ec_heal_data_find_direction(ec_t *ec, default_args_cbk_t *replies,
+                            uint64_t *data_versions, uint64_t *dirty,
+                            uint64_t *size, unsigned char *sources,
+                            unsigned char *healed_sinks,
+                            gf_boolean_t check_ondisksize, int which)
+{
+    uint64_t xattr[EC_VERSION_SIZE] = {0};
+    char version_size[128] = {0};
+    dict_t *version_size_db = NULL;
+    unsigned char *same = NULL;
+    int max_same_count = 0;
+    int source = 0;
+    int i = 0;
+    int ret = 0;
+    dict_t *dict = NULL;
+    uint64_t source_size = 0;
+
+    version_size_db = dict_new();
+    if (!version_size_db) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret < 0)
+            continue;
+        dict = (which == EC_COMBINE_XDATA) ? replies[i].xdata
+                                           : replies[i].xattr;
+
+        ret = ec_dict_get_array(dict, EC_XATTR_VERSION, xattr, EC_VERSION_SIZE);
+        if (ret == 0) {
+            data_versions[i] = xattr[EC_DATA_TXN];
         }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!replies[i].valid)
-                        continue;
-                if (replies[i].op_ret < 0)
-                        continue;
-                dict = (which == EC_COMBINE_XDATA) ? replies[i].xdata :
-                                                     replies[i].xattr;
-
-                ret = ec_dict_get_array (dict, EC_XATTR_VERSION,
-                                         xattr, EC_VERSION_SIZE);
-                if (ret == 0) {
-                        data_versions[i] = xattr[EC_DATA_TXN];
-                }
+        memset(xattr, 0, sizeof(xattr));
+        ret = ec_dict_get_array(dict, EC_XATTR_DIRTY, xattr, EC_VERSION_SIZE);
+        if (ret == 0) {
+            dirty[i] = xattr[EC_DATA_TXN];
+        }
+        ret = ec_dict_del_number(dict, EC_XATTR_SIZE, &size[i]);
+        /*Build a db of same metadata and data version and size*/
+        snprintf(version_size, sizeof(version_size), "%" PRIu64 "-%" PRIu64,
+                 data_versions[i], size[i]);
 
-                memset (xattr, 0, sizeof (xattr));
-                ret = ec_dict_get_array (dict, EC_XATTR_DIRTY,
-                                         xattr, EC_VERSION_SIZE);
-                if (ret == 0) {
-                        dirty[i] = xattr[EC_DATA_TXN];
-                }
-                ret = ec_dict_del_number (dict, EC_XATTR_SIZE,
-                                          &size[i]);
-                /*Build a db of same metadata and data version and size*/
-                snprintf (version_size, sizeof (version_size),
-                          "%"PRIu64"-%"PRIu64, data_versions[i], size[i]);
-
-                ret = dict_get_bin (version_size_db, version_size,
-                                    (void **)&same);
-                if (ret < 0) {
-                        same = alloca0 (ec->nodes);
-                }
+        ret = dict_get_bin(version_size_db, version_size, (void **)&same);
+        if (ret < 0) {
+            same = alloca0(ec->nodes);
+        }
 
-                same[i] = 1;
-                if (max_same_count < EC_COUNT (same, ec->nodes)) {
-                        max_same_count = EC_COUNT (same, ec->nodes);
-                        source = i;
-                }
+        same[i] = 1;
+        if (max_same_count < EC_COUNT(same, ec->nodes)) {
+            max_same_count = EC_COUNT(same, ec->nodes);
+            source = i;
+        }
 
-                if (ret < 0) {
-                        ret = dict_set_static_bin (version_size_db,
-                                                version_size, same, ec->nodes);
-                }
+        if (ret < 0) {
+            ret = dict_set_static_bin(version_size_db, version_size, same,
+                                      ec->nodes);
+        }
 
-                if (ret < 0) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
+        if (ret < 0) {
+            ret = -ENOMEM;
+            goto out;
         }
-        /* If we don't have ec->fragments number of same version,size it is not
-         * recoverable*/
-        if (max_same_count < ec->fragments) {
-                ret = -EIO;
-                goto out;
-        } else {
-                snprintf (version_size, sizeof (version_size),
-                          "%"PRIu64"-%"PRIu64, data_versions[source],
-                          size[source]);
-
-                ret = dict_get_bin (version_size_db, version_size,
-                                    (void **)&same);
-                if (ret < 0)
-                        goto out;
-                memcpy (sources, same, ec->nodes);
-                for (i = 0; i < ec->nodes; i++) {
-                        if (replies[i].valid && (replies[i].op_ret == 0) &&
-                            !sources[i])
-                                healed_sinks[i] = 1;
-                }
+    }
+    /* If we don't have ec->fragments number of same version,size it is not
+     * recoverable*/
+    if (max_same_count < ec->fragments) {
+        ret = -EIO;
+        goto out;
+    } else {
+        snprintf(version_size, sizeof(version_size), "%" PRIu64 "-%" PRIu64,
+                 data_versions[source], size[source]);
+
+        ret = dict_get_bin(version_size_db, version_size, (void **)&same);
+        if (ret < 0)
+            goto out;
+        memcpy(sources, same, ec->nodes);
+        for (i = 0; i < ec->nodes; i++) {
+            if (replies[i].valid && (replies[i].op_ret == 0) && !sources[i])
+                healed_sinks[i] = 1;
         }
+    }
 
-        /* There could be files with versions, size same but on disk ia_size
-         * could be different because of disk crashes, mark them as sinks as
-         * well*/
+    /* There could be files with versions, size same but on disk ia_size
+     * could be different because of disk crashes, mark them as sinks as
+     * well*/
 
-        if (check_ondisksize) {
-                source_size = size[source];
-                ec_adjust_size_up (ec, &source_size, _gf_true);
+    if (check_ondisksize) {
+        source_size = size[source];
+        ec_adjust_size_up(ec, &source_size, _gf_true);
 
-                for (i = 0; i < ec->nodes; i++) {
-                        if (sources[i]) {
-                                if (replies[i].stat.ia_size != source_size) {
-                                        sources[i] = 0;
-                                        healed_sinks[i] = 1;
-                                        max_same_count--;
-                                } else {
-                                        source = i;
-                                }
-                        }
-                }
-                if (max_same_count < ec->fragments) {
-                        ret = -EIO;
-                        goto out;
+        for (i = 0; i < ec->nodes; i++) {
+            if (sources[i]) {
+                if (replies[i].stat.ia_size != source_size) {
+                    sources[i] = 0;
+                    healed_sinks[i] = 1;
+                    max_same_count--;
+                } else {
+                    source = i;
                 }
+            }
+        }
+        if (max_same_count < ec->fragments) {
+            ret = -EIO;
+            goto out;
         }
+    }
 
-        ret = source;
+    ret = source;
 out:
-        if (version_size_db)
-                dict_unref (version_size_db);
-        return ret;
+    if (version_size_db)
+        dict_unref(version_size_db);
+    return ret;
 }
 
 int
-__ec_heal_data_prepare (call_frame_t *frame, ec_t *ec, fd_t *fd,
-                        unsigned char *locked_on, uint64_t *versions,
-                        uint64_t *dirty, uint64_t *size, unsigned char *sources,
-                        unsigned char *healed_sinks, unsigned char *trim,
-                        struct iatt *stbuf)
-{
-        default_args_cbk_t *replies = NULL;
-        default_args_cbk_t *fstat_replies = NULL;
-        unsigned char      *output  = NULL;
-        unsigned char      *fstat_output  = NULL;
-        dict_t             *xattrs  = NULL;
-        uint64_t           zero_array[2] = {0};
-        int                source   = 0;
-        int                ret      = 0;
-        uint64_t           zero_value = 0;
-        int                i = 0;
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        EC_REPLIES_ALLOC (fstat_replies, ec->nodes);
-        output = alloca0(ec->nodes);
-        fstat_output = alloca0(ec->nodes);
-        xattrs = dict_new ();
-        if (!xattrs ||
-            dict_set_static_bin (xattrs, EC_XATTR_VERSION, zero_array,
-                                 sizeof (zero_array)) ||
-            dict_set_static_bin (xattrs, EC_XATTR_DIRTY, zero_array,
-                                 sizeof (zero_array)) ||
-            dict_set_static_bin (xattrs, EC_XATTR_SIZE, &zero_value,
-                                 sizeof (zero_value))) {
-                ret = -ENOMEM;
-                goto out;
-        }
+__ec_heal_data_prepare(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                       unsigned char *locked_on, uint64_t *versions,
+                       uint64_t *dirty, uint64_t *size, unsigned char *sources,
+                       unsigned char *healed_sinks, unsigned char *trim,
+                       struct iatt *stbuf)
+{
+    default_args_cbk_t *replies = NULL;
+    default_args_cbk_t *fstat_replies = NULL;
+    unsigned char *output = NULL;
+    unsigned char *fstat_output = NULL;
+    dict_t *xattrs = NULL;
+    uint64_t zero_array[2] = {0};
+    int source = 0;
+    int ret = 0;
+    uint64_t zero_value = 0;
+    int i = 0;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    EC_REPLIES_ALLOC(fstat_replies, ec->nodes);
+    output = alloca0(ec->nodes);
+    fstat_output = alloca0(ec->nodes);
+    xattrs = dict_new();
+    if (!xattrs ||
+        dict_set_static_bin(xattrs, EC_XATTR_VERSION, zero_array,
+                            sizeof(zero_array)) ||
+        dict_set_static_bin(xattrs, EC_XATTR_DIRTY, zero_array,
+                            sizeof(zero_array)) ||
+        dict_set_static_bin(xattrs, EC_XATTR_SIZE, &zero_value,
+                            sizeof(zero_value))) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        ret = cluster_fxattrop (ec->xl_list, locked_on, ec->nodes,
-                                replies, output, frame, ec->xl, fd,
-                                GF_XATTROP_ADD_ARRAY64, xattrs, NULL);
+    ret = cluster_fxattrop(ec->xl_list, locked_on, ec->nodes, replies, output,
+                           frame, ec->xl, fd, GF_XATTROP_ADD_ARRAY64, xattrs,
+                           NULL);
 
-        ret = cluster_fstat (ec->xl_list, locked_on, ec->nodes, fstat_replies,
-                             fstat_output, frame, ec->xl, fd, NULL);
+    ret = cluster_fstat(ec->xl_list, locked_on, ec->nodes, fstat_replies,
+                        fstat_output, frame, ec->xl, fd, NULL);
 
-        for (i = 0; i < ec->nodes; i++) {
-                output[i] = output[i] && fstat_output[i];
-                replies[i].valid = output[i];
-                if (output[i])
-                        replies[i].stat = fstat_replies[i].stat;
-        }
+    for (i = 0; i < ec->nodes; i++) {
+        output[i] = output[i] && fstat_output[i];
+        replies[i].valid = output[i];
+        if (output[i])
+            replies[i].stat = fstat_replies[i].stat;
+    }
 
-        if (EC_COUNT (output, ec->nodes) <= ec->fragments) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    if (EC_COUNT(output, ec->nodes) <= ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        source = ec_heal_data_find_direction (ec, replies, versions,
-                                              dirty, size, sources,
-                                              healed_sinks, _gf_true,
-                                              EC_COMBINE_DICT);
-        ret = source;
-        if (ret < 0)
-                goto out;
+    source = ec_heal_data_find_direction(ec, replies, versions, dirty, size,
+                                         sources, healed_sinks, _gf_true,
+                                         EC_COMBINE_DICT);
+    ret = source;
+    if (ret < 0)
+        goto out;
 
-        if (stbuf)
-                *stbuf = replies[source].stat;
+    if (stbuf)
+        *stbuf = replies[source].stat;
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (healed_sinks[i]) {
-                        if (replies[i].stat.ia_size)
-                                trim[i] = 1;
-                }
+    for (i = 0; i < ec->nodes; i++) {
+        if (healed_sinks[i]) {
+            if (replies[i].stat.ia_size)
+                trim[i] = 1;
         }
+    }
 
-        if (EC_COUNT(sources, ec->nodes) < ec->fragments) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    if (EC_COUNT(sources, ec->nodes) < ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        ret = source;
+    ret = source;
 out:
-        if (xattrs)
-                dict_unref (xattrs);
-        cluster_replies_wipe (replies, ec->nodes);
-        cluster_replies_wipe (fstat_replies, ec->nodes);
-        if (ret < 0) {
-                gf_msg_debug (ec->xl->name, 0, "%s: heal failed %s",
-                        uuid_utoa (fd->inode->gfid), strerror (-ret));
-        } else {
-                gf_msg_debug (ec->xl->name, 0, "%s: sources: %d, sinks: "
-                        "%d", uuid_utoa (fd->inode->gfid),
-                        EC_COUNT (sources, ec->nodes),
-                        EC_COUNT (healed_sinks, ec->nodes));
-        }
-        return ret;
+    if (xattrs)
+        dict_unref(xattrs);
+    cluster_replies_wipe(replies, ec->nodes);
+    cluster_replies_wipe(fstat_replies, ec->nodes);
+    if (ret < 0) {
+        gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s",
+                     uuid_utoa(fd->inode->gfid), strerror(-ret));
+    } else {
+        gf_msg_debug(ec->xl->name, 0,
+                     "%s: sources: %d, sinks: "
+                     "%d",
+                     uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes),
+                     EC_COUNT(healed_sinks, ec->nodes));
+    }
+    return ret;
 }
 
 int
-__ec_heal_mark_sinks (call_frame_t *frame, ec_t *ec, fd_t *fd,
-                      uint64_t *versions, unsigned char *healed_sinks)
-{
-        int                i        = 0;
-        int                ret      = 0;
-        unsigned char      *mark    = NULL;
-        dict_t             *xattrs  = NULL;
-        default_args_cbk_t *replies = NULL;
-        unsigned char      *output  = NULL;
-        uint64_t versions_xattr[2]  = {0};
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        xattrs = dict_new ();
-        if (!xattrs) {
-                ret = -ENOMEM;
-                goto out;
-        }
+__ec_heal_mark_sinks(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                     uint64_t *versions, unsigned char *healed_sinks)
+{
+    int i = 0;
+    int ret = 0;
+    unsigned char *mark = NULL;
+    dict_t *xattrs = NULL;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    uint64_t versions_xattr[2] = {0};
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    xattrs = dict_new();
+    if (!xattrs) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        mark = alloca0 (ec->nodes);
-        for (i = 0; i < ec->nodes; i++) {
-                if (!healed_sinks[i])
-                        continue;
-                if ((versions[i] >> EC_SELFHEAL_BIT) & 1)
-                        continue;
-                mark[i] = 1;
-        }
+    mark = alloca0(ec->nodes);
+    for (i = 0; i < ec->nodes; i++) {
+        if (!healed_sinks[i])
+            continue;
+        if ((versions[i] >> EC_SELFHEAL_BIT) & 1)
+            continue;
+        mark[i] = 1;
+    }
 
-        if (EC_COUNT (mark, ec->nodes) == 0)
-                return 0;
+    if (EC_COUNT(mark, ec->nodes) == 0)
+        return 0;
 
-        versions_xattr[EC_DATA_TXN] = hton64(1ULL<<EC_SELFHEAL_BIT);
-        if (dict_set_static_bin (xattrs, EC_XATTR_VERSION, versions_xattr,
-                                 sizeof (versions_xattr))) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    versions_xattr[EC_DATA_TXN] = hton64(1ULL << EC_SELFHEAL_BIT);
+    if (dict_set_static_bin(xattrs, EC_XATTR_VERSION, versions_xattr,
+                            sizeof(versions_xattr))) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        output = alloca0 (ec->nodes);
-        ret = cluster_fxattrop (ec->xl_list, mark, ec->nodes,
-                                replies, output, frame, ec->xl, fd,
-                                GF_XATTROP_ADD_ARRAY64, xattrs, NULL);
-        for (i = 0; i < ec->nodes; i++) {
-                if (!output[i]) {
-                        if (mark[i])
-                                healed_sinks[i] = 0;
-                        continue;
-                }
-                versions[i] |= (1ULL<<EC_SELFHEAL_BIT);
+    output = alloca0(ec->nodes);
+    ret = cluster_fxattrop(ec->xl_list, mark, ec->nodes, replies, output, frame,
+                           ec->xl, fd, GF_XATTROP_ADD_ARRAY64, xattrs, NULL);
+    for (i = 0; i < ec->nodes; i++) {
+        if (!output[i]) {
+            if (mark[i])
+                healed_sinks[i] = 0;
+            continue;
         }
+        versions[i] |= (1ULL << EC_SELFHEAL_BIT);
+    }
 
-        if (EC_COUNT (healed_sinks, ec->nodes) == 0) {
-                ret = -ENOTCONN;
-                goto out;
-        }
-        ret = 0;
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+        ret = -ENOTCONN;
+        goto out;
+    }
+    ret = 0;
 
 out:
-        cluster_replies_wipe (replies, ec->nodes);
-        if (xattrs)
-                dict_unref (xattrs);
-        if (ret < 0)
-                gf_msg_debug (ec->xl->name, 0, "%s: heal failed %s",
-                        uuid_utoa (fd->inode->gfid), strerror (-ret));
-        return ret;
+    cluster_replies_wipe(replies, ec->nodes);
+    if (xattrs)
+        dict_unref(xattrs);
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s",
+                     uuid_utoa(fd->inode->gfid), strerror(-ret));
+    return ret;
 }
 
 int32_t
-ec_manager_heal_block (ec_fop_data_t *fop, int32_t state)
+ec_manager_heal_block(ec_fop_data_t *fop, int32_t state)
 {
     ec_heal_t *heal = fop->data;
     heal->fop = fop;
 
     switch (state) {
-    case EC_STATE_INIT:
-        ec_owner_set(fop->frame, fop->frame->root);
+        case EC_STATE_INIT:
+            ec_owner_set(fop->frame, fop->frame->root);
 
-        ec_heal_inodelk(heal, F_WRLCK, 1, 0, 0);
+            ec_heal_inodelk(heal, F_WRLCK, 1, 0, 0);
 
-        return EC_STATE_HEAL_DATA_COPY;
+            return EC_STATE_HEAL_DATA_COPY;
 
-    case EC_STATE_HEAL_DATA_COPY:
-        gf_msg_debug (fop->xl->name, 0, "%s: read/write starting",
-                uuid_utoa (heal->fd->inode->gfid));
-        ec_heal_data_block (heal);
+        case EC_STATE_HEAL_DATA_COPY:
+            gf_msg_debug(fop->xl->name, 0, "%s: read/write starting",
+                         uuid_utoa(heal->fd->inode->gfid));
+            ec_heal_data_block(heal);
 
-        return EC_STATE_HEAL_DATA_UNLOCK;
+            return EC_STATE_HEAL_DATA_UNLOCK;
 
-    case -EC_STATE_HEAL_DATA_COPY:
-    case -EC_STATE_HEAL_DATA_UNLOCK:
-    case EC_STATE_HEAL_DATA_UNLOCK:
-        ec_heal_inodelk(heal, F_UNLCK, 1, 0, 0);
+        case -EC_STATE_HEAL_DATA_COPY:
+        case -EC_STATE_HEAL_DATA_UNLOCK:
+        case EC_STATE_HEAL_DATA_UNLOCK:
+            ec_heal_inodelk(heal, F_UNLCK, 1, 0, 0);
 
-         return EC_STATE_REPORT;
+            return EC_STATE_REPORT;
 
-    case EC_STATE_REPORT:
-        if (fop->cbks.heal) {
-            fop->cbks.heal (fop->req_frame, fop, fop->xl, 0,
-                            0, (heal->good | heal->bad),
-                            heal->good, heal->bad, NULL);
-        }
+        case EC_STATE_REPORT:
+            if (fop->cbks.heal) {
+                fop->cbks.heal(fop->req_frame, fop->data, fop->xl, 0, 0,
+                               (heal->good | heal->bad), heal->good, heal->bad,
+                               0, NULL);
+            }
 
-        return EC_STATE_END;
-    case -EC_STATE_REPORT:
-        if (fop->cbks.heal) {
-            fop->cbks.heal (fop->req_frame, fop, fop->xl, -1,
-                            fop->error, 0, 0, 0, NULL);
-        }
+            return EC_STATE_END;
+        case -EC_STATE_REPORT:
+            if (fop->cbks.heal) {
+                fop->cbks.heal(fop->req_frame, fop->data, fop->xl, -1,
+                               fop->error, 0, 0, 0, 0, NULL);
+            }
 
-        return EC_STATE_END;
-    default:
-        gf_msg (fop->xl->name, GF_LOG_ERROR, 0,
-                EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                state, ec_fop_name(fop->id));
+            return EC_STATE_END;
+        default:
+            gf_msg(fop->xl->name, GF_LOG_ERROR, 0, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
-        return EC_STATE_END;
+            return EC_STATE_END;
     }
 }
 
 /*Takes lock */
 void
-ec_heal_block (call_frame_t *frame, xlator_t *this, uintptr_t target,
-              int32_t minimum, fop_heal_cbk_t func, ec_heal_t *heal)
+ec_heal_block(call_frame_t *frame, xlator_t *this, uintptr_t target,
+              uint32_t fop_flags, fop_heal_cbk_t func, ec_heal_t *heal)
 {
-    ec_cbk_t callback = { .heal = func };
+    ec_cbk_t callback = {.heal = func};
     ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
@@ -1947,9 +2009,8 @@ ec_heal_block (call_frame_t *frame, xlator_t *this, uintptr_t target,
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate (frame, this, EC_FOP_HEAL, 0, target, minimum,
-                                NULL, ec_manager_heal_block, callback,
-                                heal);
+    fop = ec_fop_data_allocate(frame, this, EC_FOP_HEAL, 0, target, fop_flags,
+                               NULL, ec_manager_heal_block, callback, heal);
     if (fop == NULL)
         goto out;
 
@@ -1959,766 +2020,878 @@ out:
     if (fop != NULL) {
         ec_manager(fop, error);
     } else {
-        func(frame, NULL, this, -1, error, 0, 0, 0, NULL);
+        func(frame, heal, this, -1, error, 0, 0, 0, 0, NULL);
     }
 }
 
 int32_t
-ec_heal_block_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, uintptr_t mask,
-                    uintptr_t good, uintptr_t bad, dict_t *xdata)
+ec_heal_block_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, uintptr_t mask,
+                   uintptr_t good, uintptr_t bad, uint32_t pending,
+                   dict_t *xdata)
 {
-        ec_fop_data_t *fop = cookie;
-        ec_heal_t *heal = fop->data;
+    ec_heal_t *heal = cookie;
 
-        fop->heal = NULL;
-        heal->fop = NULL;
-        heal->error = op_ret < 0 ? op_errno : 0;
-        syncbarrier_wake (heal->data);
-        return 0;
+    if (heal->fop) {
+        heal->fop->heal = NULL;
+    }
+    heal->fop = NULL;
+    heal->error = op_ret < 0 ? op_errno : 0;
+    syncbarrier_wake(heal->data);
+    return 0;
 }
 
 int
-ec_sync_heal_block (call_frame_t *frame, xlator_t *this, ec_heal_t *heal)
+ec_sync_heal_block(call_frame_t *frame, xlator_t *this, ec_heal_t *heal)
 {
-        ec_heal_block (frame, this, heal->bad|heal->good, EC_MINIMUM_ONE,
-                       ec_heal_block_done, heal);
-        syncbarrier_wait (heal->data, 1);
-        if (heal->error != 0) {
-                return -heal->error;
-        }
-        if (heal->bad == 0)
-                return -ENOTCONN;
-        return 0;
+    ec_heal_block(frame, this, heal->bad | heal->good, EC_MINIMUM_ONE,
+                  ec_heal_block_done, heal);
+    syncbarrier_wait(heal->data, 1);
+    if (heal->error != 0) {
+        return -heal->error;
+    }
+    if (heal->bad == 0)
+        return -ENOTCONN;
+    return 0;
 }
 
 int
-ec_rebuild_data (call_frame_t *frame, ec_t *ec, fd_t *fd, uint64_t size,
-                 unsigned char *sources, unsigned char *healed_sinks)
+ec_rebuild_data(call_frame_t *frame, ec_t *ec, fd_t *fd, uint64_t size,
+                unsigned char *sources, unsigned char *healed_sinks)
 {
-        ec_heal_t        *heal = NULL;
-        int              ret = 0;
-        syncbarrier_t    barrier;
-
-        if (syncbarrier_init (&barrier))
-                return -ENOMEM;
-
-        heal = alloca0(sizeof (*heal));
-        heal->fd = fd_ref (fd);
-        heal->xl = ec->xl;
-        heal->data = &barrier;
-        ec_adjust_size_up (ec, &size, _gf_false);
-        heal->total_size = size;
-        heal->size = (128 * GF_UNIT_KB * (ec->self_heal_window_size));
-        /* We need to adjust the size to a multiple of the stripe size of the
-         * volume. Otherwise writes would need to fill gaps (head and/or tail)
-         * with existent data from the bad bricks. This could be garbage on a
-         * damaged file or it could fail if there aren't enough bricks. */
-        heal->size -= heal->size % ec->stripe_size;
-        heal->bad       = ec_char_array_to_mask (healed_sinks, ec->nodes);
-        heal->good      = ec_char_array_to_mask (sources, ec->nodes);
-        heal->iatt.ia_type = IA_IFREG;
-        LOCK_INIT(&heal->lock);
-
-        for (heal->offset = 0; (heal->offset < size) && !heal->done;
-                                                   heal->offset += heal->size) {
-                /* We immediately abort any heal if a shutdown request has been
-                 * received to avoid delays. The healing of this file will be
-                 * restarted by another SHD or other client that accesses the
-                 * file. */
-                if (ec->shutdown) {
-                        gf_msg_debug(ec->xl->name, 0, "Cancelling heal because "
-                                                      "EC is stopping.");
-                        ret = -ENOTCONN;
-                        break;
-                }
-
-                gf_msg_debug (ec->xl->name, 0, "%s: sources: %d, sinks: "
-                        "%d, offset: %"PRIu64" bsize: %"PRIu64,
-                        uuid_utoa (fd->inode->gfid),
-                        EC_COUNT (sources, ec->nodes),
-                        EC_COUNT (healed_sinks, ec->nodes), heal->offset,
-                        heal->size);
-                ret = ec_sync_heal_block (frame, ec->xl, heal);
-                if (ret < 0)
-                        break;
-
-        }
-        memset (healed_sinks, 0, ec->nodes);
-        ec_mask_to_char_array (heal->bad, healed_sinks, ec->nodes);
-        fd_unref (heal->fd);
-        LOCK_DESTROY (&heal->lock);
-        syncbarrier_destroy (heal->data);
+    ec_heal_t *heal = NULL;
+    int ret = 0;
+    syncbarrier_t barrier;
+
+    if (syncbarrier_init(&barrier))
+        return -ENOMEM;
+
+    heal = alloca0(sizeof(*heal));
+    heal->fd = fd_ref(fd);
+    heal->xl = ec->xl;
+    heal->data = &barrier;
+    ec_adjust_size_up(ec, &size, _gf_false);
+    heal->total_size = size;
+    heal->size = (128 * GF_UNIT_KB * (ec->self_heal_window_size));
+    /* We need to adjust the size to a multiple of the stripe size of the
+     * volume. Otherwise writes would need to fill gaps (head and/or tail)
+     * with existent data from the bad bricks. This could be garbage on a
+     * damaged file or it could fail if there aren't enough bricks. */
+    heal->size -= heal->size % ec->stripe_size;
+    heal->bad = ec_char_array_to_mask(healed_sinks, ec->nodes);
+    heal->good = ec_char_array_to_mask(sources, ec->nodes);
+    heal->iatt.ia_type = IA_IFREG;
+    LOCK_INIT(&heal->lock);
+
+    for (heal->offset = 0; (heal->offset < size) && !heal->done;
+         heal->offset += heal->size) {
+        /* We immediately abort any heal if a shutdown request has been
+         * received to avoid delays. The healing of this file will be
+         * restarted by another SHD or other client that accesses the
+         * file. */
+        if (ec->shutdown) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "Cancelling heal because "
+                         "EC is stopping.");
+            ret = -ENOTCONN;
+            break;
+        }
+
+        gf_msg_debug(ec->xl->name, 0,
+                     "%s: sources: %d, sinks: "
+                     "%d, offset: %" PRIu64 " bsize: %" PRIu64,
+                     uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes),
+                     EC_COUNT(healed_sinks, ec->nodes), heal->offset,
+                     heal->size);
+        ret = ec_sync_heal_block(frame, ec->xl, heal);
         if (ret < 0)
-                gf_msg_debug (ec->xl->name, 0, "%s: heal failed %s",
-                        uuid_utoa (fd->inode->gfid), strerror (-ret));
-        return ret;
+            break;
+    }
+    memset(healed_sinks, 0, ec->nodes);
+    ec_mask_to_char_array(heal->bad, healed_sinks, ec->nodes);
+    fd_unref(heal->fd);
+    LOCK_DESTROY(&heal->lock);
+    syncbarrier_destroy(heal->data);
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s",
+                     uuid_utoa(fd->inode->gfid), strerror(-ret));
+    return ret;
 }
 
 int
-__ec_heal_trim_sinks (call_frame_t *frame, ec_t *ec,
-                      fd_t *fd, unsigned char *healed_sinks,
-                      unsigned char *trim, uint64_t size)
+__ec_heal_trim_sinks(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                     unsigned char *healed_sinks, unsigned char *trim,
+                     uint64_t size)
 {
-        default_args_cbk_t *replies = NULL;
-        unsigned char      *output  = NULL;
-        int                ret      = 0;
-        int                i        = 0;
-        off_t        trim_offset    = 0;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    int ret = 0;
+    int i = 0;
+    off_t trim_offset = 0;
 
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        output       = alloca0 (ec->nodes);
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    output = alloca0(ec->nodes);
 
-        if (EC_COUNT (trim, ec->nodes) == 0) {
-                ret = 0;
-                goto out;
-        }
-        trim_offset = size;
-        ec_adjust_offset_up (ec, &trim_offset, _gf_true);
-        ret = cluster_ftruncate (ec->xl_list, trim, ec->nodes, replies, output,
-                                 frame, ec->xl, fd, trim_offset, NULL);
-        for (i = 0; i < ec->nodes; i++) {
-                if (!output[i] && trim[i])
-                        healed_sinks[i] = 0;
-        }
+    if (EC_COUNT(trim, ec->nodes) == 0) {
+        ret = 0;
+        goto out;
+    }
+    trim_offset = size;
+    ec_adjust_offset_up(ec, &trim_offset, _gf_true);
+    ret = cluster_ftruncate(ec->xl_list, trim, ec->nodes, replies, output,
+                            frame, ec->xl, fd, trim_offset, NULL);
+    for (i = 0; i < ec->nodes; i++) {
+        if (!output[i] && trim[i])
+            healed_sinks[i] = 0;
+    }
 
-        if (EC_COUNT (healed_sinks, ec->nodes) == 0) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
 out:
-        cluster_replies_wipe (replies, ec->nodes);
-        if (ret < 0)
-                gf_msg_debug (ec->xl->name, 0, "%s: heal failed %s",
-                        uuid_utoa (fd->inode->gfid), strerror (-ret));
-        return ret;
+    cluster_replies_wipe(replies, ec->nodes);
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s",
+                     uuid_utoa(fd->inode->gfid), strerror(-ret));
+    return ret;
 }
 
 int
-ec_data_undo_pending (call_frame_t *frame, ec_t *ec, fd_t *fd, dict_t *xattr,
-                      uint64_t *versions, uint64_t *dirty, uint64_t *size,
-                      int source, gf_boolean_t erase_dirty, int idx)
-{
-        uint64_t versions_xattr[2] = {0};
-        uint64_t dirty_xattr[2]    = {0};
-        uint64_t allzero[2]        = {0};
-        uint64_t size_xattr        = 0;
-        int      ret               = 0;
-
-        versions_xattr[EC_DATA_TXN] = hton64(versions[source] - versions[idx]);
-        ret = dict_set_static_bin (xattr, EC_XATTR_VERSION,
-                                   versions_xattr,
-                                   sizeof (versions_xattr));
-        if (ret < 0)
-                goto out;
+ec_data_undo_pending(call_frame_t *frame, ec_t *ec, fd_t *fd, dict_t *xattr,
+                     uint64_t *versions, uint64_t *dirty, uint64_t *size,
+                     int source, gf_boolean_t erase_dirty, int idx)
+{
+    uint64_t versions_xattr[2] = {0};
+    uint64_t dirty_xattr[2] = {0};
+    uint64_t allzero[2] = {0};
+    uint64_t size_xattr = 0;
+    int ret = 0;
+
+    versions_xattr[EC_DATA_TXN] = hton64(versions[source] - versions[idx]);
+    ret = dict_set_static_bin(xattr, EC_XATTR_VERSION, versions_xattr,
+                              sizeof(versions_xattr));
+    if (ret < 0)
+        goto out;
 
-        size_xattr = hton64(size[source] - size[idx]);
-        ret = dict_set_static_bin (xattr, EC_XATTR_SIZE,
-                                   &size_xattr, sizeof (size_xattr));
+    size_xattr = hton64(size[source] - size[idx]);
+    ret = dict_set_static_bin(xattr, EC_XATTR_SIZE, &size_xattr,
+                              sizeof(size_xattr));
+    if (ret < 0)
+        goto out;
+
+    if (erase_dirty) {
+        dirty_xattr[EC_DATA_TXN] = hton64(-dirty[idx]);
+        ret = dict_set_static_bin(xattr, EC_XATTR_DIRTY, dirty_xattr,
+                                  sizeof(dirty_xattr));
         if (ret < 0)
-                goto out;
+            goto out;
+    }
 
-        if (erase_dirty) {
-                dirty_xattr[EC_DATA_TXN] = hton64(-dirty[idx]);
-                ret = dict_set_static_bin (xattr, EC_XATTR_DIRTY,
-                                           dirty_xattr,
-                                           sizeof (dirty_xattr));
-                if (ret < 0)
-                        goto out;
-        }
-
-        if ((memcmp (versions_xattr, allzero, sizeof (allzero)) == 0) &&
-            (memcmp (dirty_xattr, allzero, sizeof (allzero)) == 0) &&
-             (size_xattr == 0)) {
-                ret = 0;
-                goto out;
-        }
+    if ((memcmp(versions_xattr, allzero, sizeof(allzero)) == 0) &&
+        (memcmp(dirty_xattr, allzero, sizeof(allzero)) == 0) &&
+        (size_xattr == 0)) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = syncop_fxattrop (ec->xl_list[idx], fd,
-                               GF_XATTROP_ADD_ARRAY64, xattr, NULL, NULL, NULL);
+    ret = syncop_fxattrop(ec->xl_list[idx], fd, GF_XATTROP_ADD_ARRAY64, xattr,
+                          NULL, NULL, NULL);
 out:
-        return ret;
+    return ret;
 }
 
 int
-__ec_fd_data_adjust_versions (call_frame_t *frame, ec_t *ec, fd_t *fd,
-                            unsigned char *sources, unsigned char *healed_sinks,
-                            uint64_t *versions, uint64_t *dirty, uint64_t *size)
-{
-        dict_t                     *xattr            = NULL;
-        int                        i                 = 0;
-        int                        ret               = 0;
-        int                        op_ret            = 0;
-        int                        source            = -1;
-        gf_boolean_t               erase_dirty       = _gf_false;
-
-        xattr = dict_new ();
-        if (!xattr) {
-                op_ret = -ENOMEM;
-                goto out;
-        }
+__ec_fd_data_adjust_versions(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                             unsigned char *sources,
+                             unsigned char *healed_sinks, uint64_t *versions,
+                             uint64_t *dirty, uint64_t *size)
+{
+    dict_t *xattr = NULL;
+    int i = 0;
+    int ret = 0;
+    int op_ret = 0;
+    int source = -1;
+    gf_boolean_t erase_dirty = _gf_false;
+
+    xattr = dict_new();
+    if (!xattr) {
+        op_ret = -ENOMEM;
+        goto out;
+    }
 
-        /* dirty xattr represents if the file needs heal. Unless all the
-         * copies are healed, don't erase it */
-        if (EC_COUNT (sources, ec->nodes) +
-            EC_COUNT (healed_sinks, ec->nodes) == ec->nodes)
-                erase_dirty = _gf_true;
+    /* dirty xattr represents if the file needs heal. Unless all the
+     * copies are healed, don't erase it */
+    if (EC_COUNT(sources, ec->nodes) + EC_COUNT(healed_sinks, ec->nodes) ==
+        ec->nodes)
+        erase_dirty = _gf_true;
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (sources[i]) {
-                        source = i;
-                        break;
-                }
-        }
-
-        if (source == -1) {
-                op_ret = -ENOTCONN;
-                goto out;
+    for (i = 0; i < ec->nodes; i++) {
+        if (sources[i]) {
+            source = i;
+            break;
         }
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (healed_sinks[i]) {
-                        ret = ec_data_undo_pending (frame, ec, fd, xattr,
-                                                    versions, dirty, size,
-                                                    source, erase_dirty, i);
-                        if (ret < 0)
-                                goto out;
-                }
-
-        }
+    if (source == -1) {
+        op_ret = -ENOTCONN;
+        goto out;
+    }
 
-        if (!erase_dirty)
+    for (i = 0; i < ec->nodes; i++) {
+        if (healed_sinks[i]) {
+            ret = ec_data_undo_pending(frame, ec, fd, xattr, versions, dirty,
+                                       size, source, erase_dirty, i);
+            if (ret < 0)
                 goto out;
+        }
+    }
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (sources[i]) {
-                        ret = ec_data_undo_pending (frame, ec, fd, xattr,
-                                                    versions, dirty, size,
-                                                    source, erase_dirty, i);
-                        if (ret < 0)
-                                continue;
-                }
+    if (!erase_dirty)
+        goto out;
 
+    for (i = 0; i < ec->nodes; i++) {
+        if (sources[i]) {
+            ret = ec_data_undo_pending(frame, ec, fd, xattr, versions, dirty,
+                                       size, source, erase_dirty, i);
+            if (ret < 0)
+                continue;
         }
+    }
 out:
-        if (xattr)
-                dict_unref (xattr);
-        return op_ret;
+    if (xattr)
+        dict_unref(xattr);
+    return op_ret;
 }
 
 int
-ec_restore_time_and_adjust_versions (call_frame_t *frame, ec_t *ec, fd_t *fd,
-                                     unsigned char *sources,
-                                     unsigned char *healed_sinks,
-                                     uint64_t *versions, uint64_t *dirty,
-                                     uint64_t *size)
-{
-        unsigned char      *locked_on           = NULL;
-        unsigned char      *participants        = NULL;
-        unsigned char      *output              = NULL;
-        default_args_cbk_t *replies             = NULL;
-        unsigned char      *postsh_sources      = NULL;
-        unsigned char      *postsh_healed_sinks = NULL;
-        unsigned char      *postsh_trim         = NULL;
-        uint64_t           *postsh_versions     = NULL;
-        uint64_t           *postsh_dirty        = NULL;
-        uint64_t           *postsh_size         = NULL;
-        int                ret                  = 0;
-        int                i                    = 0;
-        struct iatt        source_buf           = {0};
-        loc_t              loc                  = {0};
-
-        locked_on           = alloca0(ec->nodes);
-        output              = alloca0(ec->nodes);
-        participants        = alloca0(ec->nodes);
-        postsh_sources      = alloca0(ec->nodes);
-        postsh_healed_sinks = alloca0(ec->nodes);
-        postsh_trim         = alloca0(ec->nodes);
-        postsh_versions     = alloca0(ec->nodes * sizeof (*postsh_versions));
-        postsh_dirty        = alloca0(ec->nodes * sizeof (*postsh_dirty));
-        postsh_size         = alloca0(ec->nodes * sizeof (*postsh_size));
-
-        for (i = 0; i < ec->nodes; i++) {
-                if (healed_sinks[i] || sources[i])
-                        participants[i] = 1;
-        }
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        ret = cluster_inodelk (ec->xl_list, participants, ec->nodes, replies,
-                               locked_on, frame, ec->xl, ec->xl->name,
-                               fd->inode, 0, 0);
-        {
-                if (ret <= ec->fragments) {
-                        gf_msg_debug (ec->xl->name, 0, "%s: Skipping heal "
-                                "as only %d number of subvolumes could "
-                                "be locked", uuid_utoa (fd->inode->gfid), ret);
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
+ec_restore_time_and_adjust_versions(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                                    unsigned char *sources,
+                                    unsigned char *healed_sinks,
+                                    uint64_t *versions, uint64_t *dirty,
+                                    uint64_t *size)
+{
+    unsigned char *locked_on = NULL;
+    unsigned char *participants = NULL;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *postsh_sources = NULL;
+    unsigned char *postsh_healed_sinks = NULL;
+    unsigned char *postsh_trim = NULL;
+    uint64_t *postsh_versions = NULL;
+    uint64_t *postsh_dirty = NULL;
+    uint64_t *postsh_size = NULL;
+    int ret = 0;
+    int i = 0;
+    struct iatt source_buf = {0};
+    loc_t loc = {0};
+
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    participants = alloca0(ec->nodes);
+    postsh_sources = alloca0(ec->nodes);
+    postsh_healed_sinks = alloca0(ec->nodes);
+    postsh_trim = alloca0(ec->nodes);
+    postsh_versions = alloca0(ec->nodes * sizeof(*postsh_versions));
+    postsh_dirty = alloca0(ec->nodes * sizeof(*postsh_dirty));
+    postsh_size = alloca0(ec->nodes * sizeof(*postsh_size));
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (healed_sinks[i] || sources[i])
+            participants[i] = 1;
+    }
 
-                ret = __ec_heal_data_prepare (frame, ec, fd, locked_on,
-                                              postsh_versions, postsh_dirty,
-                                              postsh_size, postsh_sources,
-                                              postsh_healed_sinks, postsh_trim,
-                                              &source_buf);
-                if (ret < 0)
-                        goto unlock;
-
-                loc.inode = inode_ref (fd->inode);
-                gf_uuid_copy (loc.gfid, fd->inode->gfid);
-                ret = cluster_setattr (ec->xl_list, healed_sinks, ec->nodes,
-                                       replies, output, frame, ec->xl, &loc,
-                                       &source_buf,
-                                       GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME,
-                                       NULL);
-                EC_INTERSECT (healed_sinks, healed_sinks, output, ec->nodes);
-                if (EC_COUNT (healed_sinks, ec->nodes) == 0) {
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
-                ret = __ec_fd_data_adjust_versions (frame, ec, fd, sources,
-                                           healed_sinks, versions, dirty, size);
-        }
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, participants, ec->nodes, replies,
+                          locked_on, frame, ec->xl, ec->xl->name, fd->inode, 0,
+                          0);
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(fd->inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __ec_heal_data_prepare(frame, ec, fd, locked_on, postsh_versions,
+                                     postsh_dirty, postsh_size, postsh_sources,
+                                     postsh_healed_sinks, postsh_trim,
+                                     &source_buf);
+        if (ret < 0)
+            goto unlock;
+
+        loc.inode = inode_ref(fd->inode);
+        gf_uuid_copy(loc.gfid, fd->inode->gfid);
+        ret = cluster_setattr(
+            ec->xl_list, healed_sinks, ec->nodes, replies, output, frame,
+            ec->xl, &loc, &source_buf,
+            GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME, NULL);
+        EC_INTERSECT(healed_sinks, healed_sinks, output, ec->nodes);
+        if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        ret = __ec_fd_data_adjust_versions(frame, ec, fd, sources, healed_sinks,
+                                           versions, dirty, size);
+    }
 unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes, replies, output,
-                           frame, ec->xl, ec->xl->name, fd->inode, 0, 0);
-        cluster_replies_wipe (replies, ec->nodes);
-        loc_wipe (&loc);
-        return ret;
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, fd->inode, 0, 0);
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-__ec_heal_data (call_frame_t *frame, ec_t *ec, fd_t *fd, unsigned char *heal_on,
-                unsigned char *sources, unsigned char *healed_sinks)
+__ec_heal_data(call_frame_t *frame, ec_t *ec, fd_t *fd, unsigned char *heal_on,
+               unsigned char *sources, unsigned char *healed_sinks)
 {
-        unsigned char      *locked_on    = NULL;
-        unsigned char      *output       = NULL;
-        uint64_t           *versions     = NULL;
-        uint64_t           *dirty        = NULL;
-        uint64_t           *size         = NULL;
-        unsigned char      *trim         = NULL;
-        default_args_cbk_t *replies      = NULL;
-        int                ret           = 0;
-        int                source        = 0;
-
-        locked_on    = alloca0(ec->nodes);
-        output       = alloca0(ec->nodes);
-        trim         = alloca0 (ec->nodes);
-        versions     = alloca0 (ec->nodes * sizeof (*versions));
-        dirty        = alloca0 (ec->nodes * sizeof (*dirty));
-        size         = alloca0 (ec->nodes * sizeof (*size));
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        ret = cluster_inodelk (ec->xl_list, heal_on, ec->nodes, replies,
-                               locked_on, frame, ec->xl, ec->xl->name,
-                               fd->inode, 0, 0);
-        {
-                if (ret <= ec->fragments) {
-                        gf_msg_debug (ec->xl->name, 0, "%s: Skipping heal "
-                                "as only %d number of subvolumes could "
-                                "be locked", uuid_utoa (fd->inode->gfid), ret);
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
-
-                ret = __ec_heal_data_prepare (frame, ec, fd, locked_on,
-                                              versions, dirty, size, sources,
-                                              healed_sinks, trim, NULL);
-                if (ret < 0)
-                        goto unlock;
-
-                if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
-                        ret = __ec_fd_data_adjust_versions (frame, ec, fd,
-                                                            sources,
-                                        healed_sinks, versions, dirty, size);
-                        goto unlock;
-                }
+    unsigned char *locked_on = NULL;
+    unsigned char *output = NULL;
+    uint64_t *versions = NULL;
+    uint64_t *dirty = NULL;
+    uint64_t *size = NULL;
+    unsigned char *trim = NULL;
+    default_args_cbk_t *replies = NULL;
+    int ret = 0;
+    int source = 0;
+
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    trim = alloca0(ec->nodes);
+    versions = alloca0(ec->nodes * sizeof(*versions));
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    size = alloca0(ec->nodes * sizeof(*size));
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, heal_on, ec->nodes, replies, locked_on,
+                          frame, ec->xl, ec->xl->name, fd->inode, 0, 0);
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(fd->inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
 
-                source = ret;
-                ret = __ec_heal_mark_sinks (frame, ec, fd, versions,
-                                            healed_sinks);
-                if (ret < 0)
-                        goto unlock;
+        ret = __ec_heal_data_prepare(frame, ec, fd, locked_on, versions, dirty,
+                                     size, sources, healed_sinks, trim, NULL);
+        if (ret < 0)
+            goto unlock;
 
-                ret = __ec_heal_trim_sinks (frame, ec, fd, healed_sinks,
-                                            trim, size[source]);
+        if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+            ret = __ec_fd_data_adjust_versions(
+                frame, ec, fd, sources, healed_sinks, versions, dirty, size);
+            goto unlock;
         }
-unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes, replies, output,
-                           frame, ec->xl, ec->xl->name, fd->inode, 0, 0);
+
+        source = ret;
+        ret = __ec_heal_mark_sinks(frame, ec, fd, versions, healed_sinks);
         if (ret < 0)
-                goto out;
+            goto unlock;
 
-        if (EC_COUNT(healed_sinks, ec->nodes) == 0)
-                goto out;
+        ret = __ec_heal_trim_sinks(frame, ec, fd, healed_sinks, trim,
+                                   size[source]);
+    }
+unlock:
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, fd->inode, 0, 0);
+    if (ret < 0)
+        goto out;
 
-        gf_msg_debug (ec->xl->name, 0, "%s: sources: %d, sinks: "
-                "%d", uuid_utoa (fd->inode->gfid),
-                EC_COUNT (sources, ec->nodes),
-                EC_COUNT (healed_sinks, ec->nodes));
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0)
+        goto out;
 
-        ret = ec_rebuild_data (frame, ec, fd, size[source], sources,
-                               healed_sinks);
-        if (ret < 0)
-                goto out;
+    gf_msg_debug(ec->xl->name, 0,
+                 "%s: sources: %d, sinks: "
+                 "%d",
+                 uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes),
+                 EC_COUNT(healed_sinks, ec->nodes));
+
+    ret = ec_rebuild_data(frame, ec, fd, size[source], sources, healed_sinks);
+    if (ret < 0)
+        goto out;
 
-        ret = ec_restore_time_and_adjust_versions (frame, ec, fd, sources,
-                                                   healed_sinks, versions,
-                                                   dirty, size);
+    ret = ec_restore_time_and_adjust_versions(
+        frame, ec, fd, sources, healed_sinks, versions, dirty, size);
 out:
-        cluster_replies_wipe (replies, ec->nodes);
-        return ret;
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 
 int
-ec_heal_data (call_frame_t *frame, ec_t *ec, gf_boolean_t block, inode_t *inode,
-              unsigned char *sources, unsigned char *healed_sinks)
-{
-        unsigned char      *locked_on            = NULL;
-        unsigned char      *up_subvols           = NULL;
-        unsigned char      *output               = NULL;
-        default_args_cbk_t *replies              = NULL;
-        fd_t               *fd                   = NULL;
-        loc_t               loc                  = {0};
-        char               selfheal_domain[1024] = {0};
-        int                ret                   = 0;
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-
-        locked_on  = alloca0(ec->nodes);
-        output     = alloca0(ec->nodes);
-        up_subvols = alloca0(ec->nodes);
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-
-        fd = fd_create (inode, 0);
-        if (!fd) {
-                ret = -ENOMEM;
-                goto out;
-        }
+ec_heal_data(call_frame_t *frame, ec_t *ec, gf_boolean_t block, inode_t *inode,
+             unsigned char *sources, unsigned char *healed_sinks)
+{
+    unsigned char *locked_on = NULL;
+    unsigned char *up_subvols = NULL;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+    fd_t *fd = NULL;
+    loc_t loc = {0};
+    char selfheal_domain[1024] = {0};
+    int ret = 0;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    up_subvols = alloca0(ec->nodes);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    fd = fd_create(inode, 0);
+    if (!fd) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        ec_mask_to_char_array (ec->xl_up, up_subvols, ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
 
-        ret = cluster_open (ec->xl_list, up_subvols, ec->nodes, replies, output,
-                            frame, ec->xl, &loc, O_RDWR|O_LARGEFILE, fd, NULL);
-        if (ret <= ec->fragments) {
-                ret = -ENOTCONN;
-                goto out;
-        }
+    ret = cluster_open(ec->xl_list, up_subvols, ec->nodes, replies, output,
+                       frame, ec->xl, &loc, O_RDWR | O_LARGEFILE, fd, NULL);
+    if (ret <= ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-        fd_bind (fd);
-        sprintf (selfheal_domain, "%s:self-heal", ec->xl->name);
-        /*If other processes are already doing the heal, don't block*/
-        if (block) {
-                ret = cluster_inodelk (ec->xl_list, output, ec->nodes, replies,
-                                       locked_on, frame, ec->xl,
-                                       selfheal_domain, inode, 0, 0);
-        } else {
-                ret = cluster_tiebreaker_inodelk (ec->xl_list, output,
-                                                  ec->nodes, replies, locked_on,
-                                                  frame, ec->xl,
-                                                  selfheal_domain, inode, 0, 0);
-        }
-        {
-                if (ret <= ec->fragments) {
-                        gf_msg_debug (ec->xl->name, 0, "%s: Skipping heal "
-                                "as only %d number of subvolumes could "
-                                "be locked", uuid_utoa (inode->gfid), ret);
-                        ret = -ENOTCONN;
-                        goto unlock;
-                }
-                ret = __ec_heal_data (frame, ec, fd, locked_on, sources,
-                                      healed_sinks);
-        }
+    fd_bind(fd);
+    sprintf(selfheal_domain, "%s:self-heal", ec->xl->name);
+    /*If other processes are already doing the heal, don't block*/
+    if (block) {
+        ret = cluster_inodelk(ec->xl_list, output, ec->nodes, replies,
+                              locked_on, frame, ec->xl, selfheal_domain, inode,
+                              0, 0);
+    } else {
+        ret = cluster_tiebreaker_inodelk(ec->xl_list, output, ec->nodes,
+                                         replies, locked_on, frame, ec->xl,
+                                         selfheal_domain, inode, 0, 0);
+    }
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        ret = __ec_heal_data(frame, ec, fd, locked_on, sources, healed_sinks);
+    }
 unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes, replies, output,
-                           frame, ec->xl, selfheal_domain, inode, 0, 0);
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, selfheal_domain, inode, 0, 0);
 out:
-        if (fd)
-                fd_unref (fd);
-        loc_wipe (&loc);
-        cluster_replies_wipe (replies, ec->nodes);
-        return ret;
+    if (fd)
+        fd_unref(fd);
+    loc_wipe(&loc);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
+}
+
+int
+ec_heal_purge_stale_index(call_frame_t *frame, ec_t *ec, inode_t *inode)
+{
+    int i = 0;
+    int ret = 0;
+    dict_t **xattr = NULL;
+    loc_t loc = {0};
+    uint64_t dirty_xattr[EC_VERSION_SIZE] = {0};
+    unsigned char *on = NULL;
+    default_args_cbk_t *replies = NULL;
+    dict_t *dict = NULL;
+
+    /* Allocate the required memory */
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    on = alloca0(ec->nodes);
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    xattr = GF_CALLOC(ec->nodes, sizeof(*xattr), gf_common_mt_pointer);
+    if (!xattr) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    dict = dict_new();
+    if (!dict) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    for (i = 0; i < ec->nodes; i++) {
+        xattr[i] = dict;
+        on[i] = 1;
+    }
+    ret = dict_set_static_bin(dict, EC_XATTR_DIRTY, dirty_xattr,
+                              (sizeof(*dirty_xattr) * EC_VERSION_SIZE));
+    if (ret < 0) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    PARALLEL_FOP_ONLIST(ec->xl_list, on, ec->nodes, replies, frame,
+                        ec_wind_xattrop_parallel, &loc, GF_XATTROP_ADD_ARRAY64,
+                        xattr, NULL);
+out:
+    if (dict) {
+        dict_unref(dict);
+    }
+    if (xattr) {
+        GF_FREE(xattr);
+    }
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
 }
 
 void
-ec_heal_do (xlator_t *this, void *data, loc_t *loc, int32_t partial)
-{
-        call_frame_t  *frame         = NULL;
-        unsigned char *participants  = NULL;
-        unsigned char *msources      = NULL;
-        unsigned char *mhealed_sinks = NULL;
-        unsigned char *sources       = NULL;
-        unsigned char *healed_sinks  = NULL;
-        ec_t          *ec            = NULL;
-        int           ret            = 0;
-        int           op_ret         = 0;
-        int           op_errno       = 0;
-        intptr_t      mgood          = 0;
-        intptr_t      mbad           = 0;
-        intptr_t      good           = 0;
-        intptr_t      bad            = 0;
-        ec_fop_data_t *fop           = data;
-        gf_boolean_t  blocking       = _gf_false;
-        ec_heal_need_t  need_heal    = EC_HEAL_NONEED;
-        unsigned char *up_subvols    = NULL;
-        char up_bricks[32];
-
-        ec = this->private;
-
-        /* If it is heal request from getxattr, complete the heal and then
-         * unwind, if it is ec_heal with NULL as frame then no need to block
-         * the heal as the caller doesn't care about its completion. In case
-         * of heald whichever gets tiebreaking inodelk will take care of the
-         * heal, so no need to block*/
-        if (fop->req_frame && !ec->shd.iamshd)
-                blocking = _gf_true;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
+ec_heal_do(xlator_t *this, void *data, loc_t *loc, int32_t partial)
+{
+    call_frame_t *frame = NULL;
+    unsigned char *participants = NULL;
+    unsigned char *msources = NULL;
+    unsigned char *mhealed_sinks = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *healed_sinks = NULL;
+    ec_t *ec = NULL;
+    int ret = 0;
+    int op_ret = 0;
+    int op_errno = 0;
+    intptr_t mgood = 0;
+    intptr_t mbad = 0;
+    intptr_t good = 0;
+    intptr_t bad = 0;
+    uint32_t pending = 0;
+    ec_fop_data_t *fop = data;
+    gf_boolean_t blocking = _gf_false;
+    ec_heal_need_t need_heal = EC_HEAL_NONEED;
+    unsigned char *up_subvols = NULL;
+    char up_bricks[32];
+
+    ec = this->private;
+
+    /* If it is heal request from getxattr, complete the heal and then
+     * unwind, if it is ec_heal with NULL as frame then no need to block
+     * the heal as the caller doesn't care about its completion. In case
+     * of heald whichever gets tiebreaking inodelk will take care of the
+     * heal, so no need to block*/
+    if (fop->req_frame && !ec->shd.iamshd)
+        blocking = _gf_true;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
 
-        ec_owner_set(frame, frame->root);
-        /*Do heal as root*/
-        frame->root->uid = 0;
-        frame->root->gid = 0;
-        /*Mark the fops as internal*/
-        frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
-        participants = alloca0(ec->nodes);
-        ec_mask_to_char_array (ec->xl_up, participants, ec->nodes);
-
-        up_subvols = alloca0(ec->nodes);
-        ec_mask_to_char_array (ec->xl_up, up_subvols, ec->nodes);
-
-        if (loc->name && strlen (loc->name)) {
-                ret = ec_heal_name (frame, ec, loc->parent, (char *)loc->name,
-                                    participants);
-                if (ret == 0) {
-                        gf_msg_debug (this->name, 0, "%s: name heal "
-                                "successful on %lX", loc->path,
-                                ec_char_array_to_mask (participants,
-                                    ec->nodes));
-                } else {
-                        gf_msg_debug (this->name, 0, "%s: name heal "
-                                "failed. ret = %d, subvolumes up = %s",
-                                loc->path, ret,
-                                ec_bin(up_bricks, sizeof(up_bricks), ec->xl_up,
-                                ec->nodes));
-                }
+    ec_owner_set(frame, frame->root);
+    /*Do heal as root*/
+    frame->root->uid = 0;
+    frame->root->gid = 0;
+    /*Mark the fops as internal*/
+    frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
+    participants = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, participants, ec->nodes);
+
+    up_subvols = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
+
+    if (loc->name && strlen(loc->name)) {
+        ret = ec_heal_name(frame, ec, loc->parent, (char *)loc->name,
+                           participants);
+        if (ret >= 0) {
+            gf_msg_debug(this->name, 0,
+                         "%s: name heal "
+                         "successful on %" PRIXPTR,
+                         loc->path,
+                         ec_char_array_to_mask(participants, ec->nodes));
+        } else {
+            gf_msg_debug(
+                this->name, 0,
+                "%s: name heal "
+                "failed. ret = %d, subvolumes up = %s",
+                loc->path, ret,
+                ec_bin(up_bricks, sizeof(up_bricks), ec->xl_up, ec->nodes));
         }
+    }
 
-        /* Mount triggers heal only when it detects that it must need heal, shd
-         * triggers heals periodically which need not be thorough*/
-        ec_heal_inspect (frame, ec, loc->inode, up_subvols, _gf_false,
-                         !ec->shd.iamshd, &need_heal);
-
-        if (need_heal == EC_HEAL_NONEED)  {
-                gf_msg (ec->xl->name, GF_LOG_DEBUG, 0,
-                        EC_MSG_HEAL_FAIL, "Heal is not required for : %s ",
-                        uuid_utoa(loc->gfid));
-                goto out;
+    /* Mount triggers heal only when it detects that it must need heal, shd
+     * triggers heals periodically which need not be thorough*/
+    if (ec->shd.iamshd && (ret <= 0)) {
+        ec_heal_inspect(frame, ec, loc->inode, up_subvols, _gf_false, _gf_false,
+                        &need_heal);
+
+        if (need_heal == EC_HEAL_PURGE_INDEX) {
+            gf_msg(ec->xl->name, GF_LOG_INFO, 0, EC_MSG_HEAL_FAIL,
+                   "Index entry needs to be purged for: %s ",
+                   uuid_utoa(loc->gfid));
+            /* We need to send zero-xattrop so that stale index entry could be
+             * removed. We need not take lock on this entry to do so as
+             * xattrop on a brick is atomic. */
+            ec_heal_purge_stale_index(frame, ec, loc->inode);
+            goto out;
+        } else if (need_heal == EC_HEAL_NONEED) {
+            gf_msg(ec->xl->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_FAIL,
+                   "Heal is not required for : %s ", uuid_utoa(loc->gfid));
+            goto out;
         }
+    }
 
-        msources = alloca0(ec->nodes);
-        mhealed_sinks = alloca0(ec->nodes);
-        ret = ec_heal_metadata (frame, ec, loc->inode, msources, mhealed_sinks);
-        if (ret == 0) {
-                mgood = ec_char_array_to_mask (msources, ec->nodes);
-                mbad  = ec_char_array_to_mask (mhealed_sinks, ec->nodes);
-        } else {
-                op_ret = -1;
-                op_errno = -ret;
-        }
-        sources = alloca0(ec->nodes);
-        healed_sinks = alloca0(ec->nodes);
-        if (IA_ISREG (loc->inode->ia_type)) {
-                ret = ec_heal_data (frame, ec, blocking, loc->inode, sources,
-                                    healed_sinks);
-        } else if (IA_ISDIR (loc->inode->ia_type) && !partial) {
-                ret = ec_heal_entry (frame, ec, loc->inode, sources,
-                                     healed_sinks);
-        } else {
-                ret = 0;
-                memcpy (sources, participants, ec->nodes);
-                memcpy (healed_sinks, participants, ec->nodes);
-        }
+    sources = alloca0(ec->nodes);
+    healed_sinks = alloca0(ec->nodes);
+    if (IA_ISREG(loc->inode->ia_type)) {
+        ret = ec_heal_data(frame, ec, blocking, loc->inode, sources,
+                           healed_sinks);
+    } else if (IA_ISDIR(loc->inode->ia_type) && !partial) {
+        ret = ec_heal_entry(frame, ec, loc->inode, sources, healed_sinks,
+                            &pending);
+    } else {
+        ret = 0;
+        memcpy(sources, participants, ec->nodes);
+        memcpy(healed_sinks, participants, ec->nodes);
+    }
 
-        if (ret == 0) {
-                good = ec_char_array_to_mask (sources, ec->nodes);
-                bad  = ec_char_array_to_mask (healed_sinks, ec->nodes);
-        } else {
-                op_ret = -1;
-                op_errno = -ret;
-        }
+    if (ret == 0) {
+        good = ec_char_array_to_mask(sources, ec->nodes);
+        bad = ec_char_array_to_mask(healed_sinks, ec->nodes);
+    } else {
+        op_ret = -1;
+        op_errno = -ret;
+    }
+    msources = alloca0(ec->nodes);
+    mhealed_sinks = alloca0(ec->nodes);
+    ret = ec_heal_metadata(frame, ec, loc->inode, msources, mhealed_sinks);
+    if (ret == 0) {
+        mgood = ec_char_array_to_mask(msources, ec->nodes);
+        mbad = ec_char_array_to_mask(mhealed_sinks, ec->nodes);
+    } else {
+        op_ret = -1;
+        op_errno = -ret;
+    }
 
 out:
-        if (fop->cbks.heal) {
-                fop->cbks.heal (fop->req_frame, fop, fop->xl, op_ret,
-                                op_errno, ec_char_array_to_mask (participants,
-                                                                 ec->nodes),
-                                mgood & good, mbad & bad, NULL);
-        }
-        if (frame)
-                STACK_DESTROY (frame->root);
-        return;
+    ec_reset_entry_healing(fop);
+    if (fop->cbks.heal) {
+        fop->cbks.heal(fop->req_frame, fop->data, fop->xl, op_ret, op_errno,
+                       ec_char_array_to_mask(participants, ec->nodes),
+                       mgood & good, mbad & bad, pending, NULL);
+    }
+    if (frame)
+        STACK_DESTROY(frame->root);
+
+    return;
 }
 
 int
-ec_synctask_heal_wrap (void *opaque)
+ec_synctask_heal_wrap(void *opaque)
 {
-        ec_fop_data_t *fop = opaque;
-        ec_heal_do (fop->xl, fop, &fop->loc[0], fop->int32);
-        return 0;
+    ec_fop_data_t *fop = opaque;
+    ec_heal_do(fop->xl, fop, &fop->loc[0], fop->int32);
+    return 0;
 }
 
 int
-ec_heal_done (int ret, call_frame_t *heal, void *opaque)
+ec_heal_done(int ret, call_frame_t *heal, void *opaque)
 {
-        if (opaque)
-                ec_fop_data_release (opaque);
-        return 0;
+    if (opaque)
+        ec_fop_data_release(opaque);
+    return 0;
 }
 
-ec_fop_data_t*
-__ec_dequeue_heals (ec_t *ec)
+ec_fop_data_t *
+__ec_dequeue_heals(ec_t *ec)
 {
-        ec_fop_data_t *fop = NULL;
+    ec_fop_data_t *fop = NULL;
 
-        if (list_empty (&ec->heal_waiting))
-                goto none;
+    if (list_empty(&ec->heal_waiting))
+        goto none;
 
-        if ((ec->background_heals > 0) && (ec->healers >= ec->background_heals))
-                goto none;
+    if ((ec->background_heals > 0) && (ec->healers >= ec->background_heals))
+        goto none;
 
-        fop = list_entry(ec->heal_waiting.next, ec_fop_data_t, healer);
-        ec->heal_waiters--;
-        list_del_init(&fop->healer);
-        list_add(&fop->healer, &ec->healing);
-        ec->healers++;
-        return fop;
+    fop = list_entry(ec->heal_waiting.next, ec_fop_data_t, healer);
+    ec->heal_waiters--;
+    list_del_init(&fop->healer);
+    list_add(&fop->healer, &ec->healing);
+    ec->healers++;
+    return fop;
 none:
-        gf_msg_debug (ec->xl->name, 0, "Num healers: %d, Num Waiters: %d",
-                      ec->healers, ec->heal_waiters);
-        return NULL;
+    gf_msg_debug(ec->xl->name, 0, "Num healers: %d, Num Waiters: %d",
+                 ec->healers, ec->heal_waiters);
+    return NULL;
 }
 
 void
-ec_heal_fail (ec_t *ec, ec_fop_data_t *fop)
+ec_heal_fail(ec_t *ec, ec_fop_data_t *fop)
 {
-        if (fop->cbks.heal) {
-            fop->cbks.heal (fop->req_frame, NULL, ec->xl, -1, fop->error, 0, 0,
-                            0, NULL);
-        }
-        ec_fop_data_release (fop);
+    if (fop->cbks.heal) {
+        fop->cbks.heal(fop->req_frame, fop->data, ec->xl, -1, fop->error, 0, 0,
+                       0, 0, NULL);
+    }
+    ec_fop_data_release(fop);
 }
 
 void
-ec_launch_heal (ec_t *ec, ec_fop_data_t *fop)
+ec_launch_heal(ec_t *ec, ec_fop_data_t *fop)
 {
-        int     ret = 0;
+    int ret = 0;
+    call_frame_t *frame = NULL;
 
-        ret = synctask_new (ec->xl->ctx->env, ec_synctask_heal_wrap,
-                            ec_heal_done, NULL, fop);
-        if (ret < 0) {
-                ec_fop_set_error(fop, ENOMEM);
-                ec_heal_fail (ec, fop);
-        }
+    frame = create_frame(ec->xl, ec->xl->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    ec_owner_set(frame, frame->root);
+    /*Do heal as root*/
+    frame->root->uid = 0;
+    frame->root->gid = 0;
+    /*Mark the fops as internal*/
+    frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
+
+    ret = synctask_new(ec->xl->ctx->env, ec_synctask_heal_wrap, ec_heal_done,
+                       frame, fop);
+out:
+    if (ret < 0) {
+        ec_fop_set_error(fop, ENOMEM);
+        ec_heal_fail(ec, fop);
+    }
+
+    if (frame)
+        STACK_DESTROY(frame->root);
 }
 
 void
-ec_handle_healers_done (ec_fop_data_t *fop)
+ec_handle_healers_done(ec_fop_data_t *fop)
 {
-        ec_t *ec = fop->xl->private;
-        ec_fop_data_t *heal_fop = NULL;
+    ec_t *ec = fop->xl->private;
+    ec_fop_data_t *heal_fop = NULL;
 
-        if (list_empty (&fop->healer))
-                return;
+    if (list_empty(&fop->healer))
+        return;
 
-        LOCK (&ec->lock);
+    LOCK(&ec->lock);
 
-        list_del_init (&fop->healer);
+    list_del_init(&fop->healer);
 
-        do {
-                ec->healers--;
-                heal_fop = __ec_dequeue_heals (ec);
+    do {
+        ec->healers--;
+        heal_fop = __ec_dequeue_heals(ec);
 
-                if ((heal_fop != NULL) && ec->shutdown) {
-                        /* This will prevent ec_handle_healers_done() to be
-                         * called recursively. That would be problematic if
-                         * the queue is too big. */
-                        list_del_init(&heal_fop->healer);
+        if ((heal_fop != NULL) && ec->shutdown) {
+            /* This will prevent ec_handle_healers_done() to be
+             * called recursively. That would be problematic if
+             * the queue is too big. */
+            list_del_init(&heal_fop->healer);
 
-                        UNLOCK(&ec->lock);
+            UNLOCK(&ec->lock);
 
-                        ec_fop_set_error(fop, ENOTCONN);
-                        ec_heal_fail(ec, heal_fop);
+            ec_fop_set_error(fop, ENOTCONN);
+            ec_heal_fail(ec, heal_fop);
 
-                        LOCK(&ec->lock);
-                }
-        } while ((heal_fop != NULL) && ec->shutdown);
+            LOCK(&ec->lock);
+        }
+    } while ((heal_fop != NULL) && ec->shutdown);
 
-        UNLOCK (&ec->lock);
+    UNLOCK(&ec->lock);
 
-        if (heal_fop)
-                ec_launch_heal (ec, heal_fop);
+    if (heal_fop)
+        ec_launch_heal(ec, heal_fop);
+}
+
+gf_boolean_t
+ec_is_entry_healing(ec_fop_data_t *fop)
+{
+    ec_inode_t *ctx = NULL;
+    int32_t heal_count = 0;
+    loc_t *loc = NULL;
+
+    loc = &fop->loc[0];
+
+    LOCK(&loc->inode->lock);
+    {
+        ctx = __ec_inode_get(loc->inode, fop->xl);
+        if (ctx) {
+            heal_count = ctx->heal_count;
+        }
+    }
+    UNLOCK(&loc->inode->lock);
+    GF_ASSERT(heal_count >= 0);
+    return heal_count;
 }
 
 void
-ec_heal_throttle (xlator_t *this, ec_fop_data_t *fop)
-{
-        gf_boolean_t can_heal = _gf_true;
-        ec_t         *ec      = this->private;
-
-        if (fop->req_frame == NULL) {
-
-                LOCK (&ec->lock);
-                {
-                        if ((ec->background_heals > 0) &&
-                            (ec->heal_wait_qlen + ec->background_heals) >
-                                             (ec->heal_waiters + ec->healers)) {
-                                list_add_tail(&fop->healer, &ec->heal_waiting);
-                                ec->heal_waiters++;
-                                fop = __ec_dequeue_heals (ec);
-                        } else {
-                                can_heal = _gf_false;
-                        }
+ec_heal_throttle(xlator_t *this, ec_fop_data_t *fop)
+{
+    gf_boolean_t can_heal = _gf_true;
+    ec_t *ec = this->private;
+    ec_fop_data_t *fop_rel = NULL;
+
+    if (fop->req_frame == NULL) {
+        LOCK(&ec->lock);
+        {
+            if ((ec->background_heals > 0) &&
+                (ec->heal_wait_qlen + ec->background_heals) >
+                    (ec->heal_waiters + ec->healers)) {
+                if (!ec_is_entry_healing(fop)) {
+                    list_add_tail(&fop->healer, &ec->heal_waiting);
+                    ec->heal_waiters++;
+                    ec_set_entry_healing(fop);
+                } else {
+                    fop_rel = fop;
                 }
-                UNLOCK (&ec->lock);
+                fop = __ec_dequeue_heals(ec);
+            } else {
+                can_heal = _gf_false;
+            }
         }
+        UNLOCK(&ec->lock);
+    }
 
-        if (can_heal) {
-                if (fop)
-                        ec_launch_heal (ec, fop);
-        } else {
-                gf_msg_debug (this->name, 0, "Max number of heals are "
-                              "pending, background self-heal rejected");
-                ec_fop_set_error(fop, EBUSY);
-                ec_heal_fail (ec, fop);
+    if (can_heal) {
+        if (fop) {
+            if (fop->req_frame != NULL) {
+                ec_set_entry_healing(fop);
+            }
+            ec_launch_heal(ec, fop);
         }
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Max number of heals are "
+                     "pending, background self-heal rejected");
+        ec_fop_set_error(fop, EBUSY);
+        ec_heal_fail(ec, fop);
+    }
+    if (fop_rel) {
+        ec_heal_done(0, NULL, fop_rel);
+    }
 }
 
 void
-ec_heal (call_frame_t *frame, xlator_t *this, uintptr_t target,
-         int32_t minimum, fop_heal_cbk_t func, void *data, loc_t *loc,
-         int32_t partial, dict_t *xdata)
+ec_heal(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_heal_cbk_t func, void *data, loc_t *loc,
+        int32_t partial, dict_t *xdata)
 {
-    ec_cbk_t callback = { .heal = func };
+    ec_cbk_t callback = {.heal = func};
     ec_fop_data_t *fop = NULL;
     int32_t err = EINVAL;
 
-    gf_msg_trace ("ec", 0, "EC(HEAL) %p", frame);
+    gf_msg_trace("ec", 0, "EC(HEAL) %p", frame);
 
     VALIDATE_OR_GOTO(this, fail);
     GF_VALIDATE_OR_GOTO(this->name, this->private, fail);
 
-    if (!loc || !loc->inode || gf_uuid_is_null (loc->inode->gfid))
-            goto fail;
+    if (!loc || !loc->inode || gf_uuid_is_null(loc->inode->gfid))
+        goto fail;
 
     if (frame && frame->local)
-            goto fail;
-    fop = ec_fop_data_allocate (frame, this, EC_FOP_HEAL, 0, target, minimum,
-                                NULL, NULL, callback, data);
+        goto fail;
+    fop = ec_fop_data_allocate(frame, this, EC_FOP_HEAL, 0, target, fop_flags,
+                               NULL, NULL, callback, data);
 
     err = ENOMEM;
 
@@ -2735,454 +2908,460 @@ ec_heal (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (xdata)
         fop->xdata = dict_ref(xdata);
 
-    ec_heal_throttle (this, fop);
+    ec_heal_throttle(this, fop);
 
     return;
 
 fail:
     if (fop)
-            ec_fop_data_release (fop);
+        ec_fop_data_release(fop);
     if (func)
-            func (frame, NULL, this, -1, err, 0, 0, 0, NULL);
+        func(frame, data, this, -1, err, 0, 0, 0, 0, NULL);
 }
 
 int
-ec_replace_heal_done (int ret, call_frame_t *heal, void *opaque)
+ec_replace_heal_done(int ret, call_frame_t *heal, void *opaque)
 {
-        ec_t *ec = opaque;
+    ec_t *ec = opaque;
+    gf_boolean_t last_fop = _gf_false;
 
-        gf_msg_debug (ec->xl->name, 0,
-                "getxattr on bricks is done ret %d", ret);
-        return 0;
+    if (GF_ATOMIC_DEC(ec->async_fop_count) == 0) {
+        LOCK(&ec->lock);
+        {
+            last_fop = __ec_is_last_fop(ec);
+        }
+        UNLOCK(&ec->lock);
+    }
+    gf_msg_debug(ec->xl->name, 0, "getxattr on bricks is done ret %d", ret);
+
+    if (last_fop)
+        ec_pending_fops_completed(ec);
+
+    return 0;
 }
 
 int32_t
-ec_replace_heal (ec_t *ec, inode_t *inode)
+ec_replace_heal(ec_t *ec, inode_t *inode)
 {
-        loc_t        loc = {0};
-        int          ret = 0;
+    loc_t loc = {0};
+    int ret = 0;
 
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        ret = syncop_getxattr (ec->xl, &loc, NULL, EC_XATTR_HEAL,
-                               NULL, NULL);
-        if (ret < 0)
-                gf_msg_debug (ec->xl->name, 0,
-                        "Heal failed for replace brick ret = %d", ret);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    ret = syncop_getxattr(ec->xl, &loc, NULL, EC_XATTR_HEAL, NULL, NULL);
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "Heal failed for replace brick ret = %d",
+                     ret);
 
-        /* Once the root inode has been checked, it might have triggered a
-         * self-heal on it after a replace brick command or for some other
-         * reason. It can also happen that the volume already had damaged
-         * files in the index, even if the heal on the root directory failed.
-         * In both cases we need to wake all index healers to continue
-         * healing remaining entries that are marked as dirty. */
-        ec_shd_index_healer_wake(ec);
+    /* Once the root inode has been checked, it might have triggered a
+     * self-heal on it after a replace brick command or for some other
+     * reason. It can also happen that the volume already had damaged
+     * files in the index, even if the heal on the root directory failed.
+     * In both cases we need to wake all index healers to continue
+     * healing remaining entries that are marked as dirty. */
+    ec_shd_index_healer_wake(ec);
 
-        loc_wipe (&loc);
-        return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
 int32_t
-ec_replace_brick_heal_wrap (void  *opaque)
+ec_replace_brick_heal_wrap(void *opaque)
 {
-         ec_t *ec = opaque;
-         inode_table_t   *itable = NULL;
-         int32_t         ret     = -1;
+    ec_t *ec = opaque;
+    inode_table_t *itable = NULL;
+    int32_t ret = -1;
 
-         if (ec->xl->itable)
-                 itable = ec->xl->itable;
-         else
-                 goto out;
-         ret = ec_replace_heal (ec, itable->root);
+    if (ec->xl->itable)
+        itable = ec->xl->itable;
+    else
+        goto out;
+
+    if (xlator_is_cleanup_starting(ec->xl))
+        goto out;
+
+    ret = ec_replace_heal(ec, itable->root);
 out:
-         return ret;
+    return ret;
 }
 
 int32_t
-ec_launch_replace_heal (ec_t *ec)
+ec_launch_replace_heal(ec_t *ec)
 {
-        int ret = -1;
+    int ret = -1;
 
-        if (!ec)
-                return ret;
-        ret = synctask_new (ec->xl->ctx->env, ec_replace_brick_heal_wrap,
-                            ec_replace_heal_done, NULL, ec);
-        if (ret < 0) {
-                gf_msg_debug (ec->xl->name, 0,
-                        "Heal failed for replace brick ret = %d", ret);
-        }
-        return ret;
+    ret = synctask_new(ec->xl->ctx->env, ec_replace_brick_heal_wrap,
+                       ec_replace_heal_done, NULL, ec);
+
+    if (ret < 0) {
+        gf_msg_debug(ec->xl->name, 0, "Heal failed for replace brick ret = %d",
+                     ret);
+        ec_replace_heal_done(-1, NULL, ec);
+    }
+
+    return ret;
 }
 
 int32_t
 ec_set_heal_info(dict_t **dict_rsp, char *status)
 {
-        dict_t *dict = NULL;
-        int    ret   = 0;
+    dict_t *dict = NULL;
+    int ret = 0;
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -ENOMEM;
-                goto out;
-        }
-        ret = dict_set_str (dict, "heal-info", status);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_WARNING, -ret,
-                        EC_MSG_HEAL_FAIL,
-                        "Failed to set heal-info key to "
-                        "%s", status);
-                dict_unref(dict);
-                dict = NULL;
-        }
-        *dict_rsp = dict;
+    dict = dict_new();
+    if (!dict) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    ret = dict_set_str(dict, "heal-info", status);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_WARNING, -ret, EC_MSG_HEAL_FAIL,
+               "Failed to set heal-info key to "
+               "%s",
+               status);
+        dict_unref(dict);
+        dict = NULL;
+    }
+    *dict_rsp = dict;
 out:
-        return ret;
+    return ret;
 }
 
 static int32_t
-_need_heal_calculate (ec_t *ec, uint64_t *dirty, unsigned char *sources,
-                      gf_boolean_t self_locked, int32_t lock_count,
-                      ec_heal_need_t *need_heal)
-{
-        int i = 0;
-        int source_count = 0;
-
-        source_count = EC_COUNT (sources, ec->nodes);
-        if (source_count == ec->nodes) {
-                *need_heal = EC_HEAL_NONEED;
-                if (self_locked || lock_count == 0) {
-                        for (i = 0; i < ec->nodes; i++) {
-                                if (dirty[i]) {
-                                        *need_heal = EC_HEAL_MUST;
-                                        goto out;
-                                }
-                        }
-                } else {
-                        for (i = 0; i < ec->nodes; i++) {
-                                /* Since each lock can only increment the dirty
-                                 * count once, if dirty is > 1 it means that
-                                 * another operation has left the dirty count
-                                 * set and this indicates a problem in the
-                                 * inode.*/
-                                if (dirty[i] > 1) {
-                                        *need_heal = EC_HEAL_MUST;
-                                        goto out;
-                                }
-                        }
+_need_heal_calculate(ec_t *ec, uint64_t *dirty, unsigned char *sources,
+                     gf_boolean_t self_locked, int32_t lock_count,
+                     ec_heal_need_t *need_heal, uint64_t *versions)
+{
+    int i = 0;
+    int source_count = 0;
+
+    source_count = EC_COUNT(sources, ec->nodes);
+    if (source_count == ec->nodes) {
+        *need_heal = EC_HEAL_NONEED;
+        if (self_locked || lock_count == 0) {
+            for (i = 0; i < ec->nodes; i++) {
+                if (dirty[i] || (versions[i] != versions[0])) {
+                    *need_heal = EC_HEAL_MUST;
+                    goto out;
                 }
+            }
+            /* If lock count is 0, all dirty flags are 0 and all the
+             * versions are macthing then why are we here. It looks
+             * like something went wrong while removing the index entries
+             * after completing a successful heal or fop. In this case
+             * we need to remove this index entry to avoid triggering heal
+             * in a loop and causing lookups again and again*/
+            *need_heal = EC_HEAL_PURGE_INDEX;
         } else {
-                *need_heal = EC_HEAL_MUST;
+            for (i = 0; i < ec->nodes; i++) {
+                /* Since each lock can only increment the dirty
+                 * count once, if dirty is > 1 it means that
+                 * another operation has left the dirty count
+                 * set and this indicates a problem in the
+                 * inode.*/
+                if (dirty[i] > 1) {
+                    *need_heal = EC_HEAL_MUST;
+                    goto out;
+                }
+                if (dirty[i] != dirty[0] || (versions[i] != versions[0])) {
+                    *need_heal = EC_HEAL_MAYBE;
+                }
+            }
         }
+    } else {
+        *need_heal = EC_HEAL_MUST;
+    }
 
 out:
-        return source_count;
+    return source_count;
 }
 
 static int32_t
-ec_need_metadata_heal (ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
-                       int32_t lock_count, gf_boolean_t self_locked,
-                       gf_boolean_t thorough, ec_heal_need_t *need_heal)
-{
-        uint64_t           *dirty         = NULL;
-        unsigned char      *sources       = NULL;
-        unsigned char      *healed_sinks  = NULL;
-        uint64_t           *meta_versions = NULL;
-        int                ret            = 0;
-        int                i              = 0;
-
-        sources = alloca0(ec->nodes);
-        healed_sinks = alloca0(ec->nodes);
-        dirty = alloca0 (ec->nodes * sizeof (*dirty));
-        meta_versions = alloca0 (ec->nodes * sizeof (*meta_versions));
-        ret = ec_heal_metadata_find_direction (ec, replies, meta_versions,
-                                               dirty, sources, healed_sinks);
-        if (ret < 0 && ret != -EIO) {
-                goto out;
-        }
+ec_need_metadata_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
+                      int32_t lock_count, gf_boolean_t self_locked,
+                      gf_boolean_t thorough, ec_heal_need_t *need_heal)
+{
+    uint64_t *dirty = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *healed_sinks = NULL;
+    uint64_t *meta_versions = NULL;
+    int ret = 0;
+
+    sources = alloca0(ec->nodes);
+    healed_sinks = alloca0(ec->nodes);
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    meta_versions = alloca0(ec->nodes * sizeof(*meta_versions));
+    ret = ec_heal_metadata_find_direction(ec, replies, meta_versions, dirty,
+                                          sources, healed_sinks);
+    if (ret < 0 && ret != -EIO) {
+        goto out;
+    }
 
-        ret = _need_heal_calculate (ec, dirty, sources, self_locked, lock_count,
-                                    need_heal);
-        if (ret == ec->nodes && *need_heal == EC_HEAL_NONEED) {
-                for (i = 1; i < ec->nodes; i++) {
-                        if (meta_versions[i] != meta_versions[0]) {
-                                *need_heal = EC_HEAL_MUST;
-                                goto out;
-                        }
-                }
-        }
+    ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count,
+                               need_heal, meta_versions);
 out:
-        return ret;
+    return ret;
 }
 
 static int32_t
-ec_need_data_heal (ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
-                   int32_t lock_count, gf_boolean_t self_locked,
-                   gf_boolean_t thorough, ec_heal_need_t *need_heal)
-{
-        uint64_t           *dirty         = NULL;
-        unsigned char      *sources       = NULL;
-        unsigned char      *healed_sinks  = NULL;
-        uint64_t           *data_versions = NULL;
-        uint64_t           *size          = NULL;
-        int                ret            = 0;
-
-        sources = alloca0(ec->nodes);
-        healed_sinks = alloca0(ec->nodes);
-        dirty = alloca0 (ec->nodes * sizeof (*dirty));
-        data_versions = alloca0 (ec->nodes * sizeof (*data_versions));
-        size = alloca0 (ec->nodes * sizeof (*size));
-
-        /* When dd is going on and heal info is called there is a very good
-         * chance for on disk sizes to mismatch eventhough nothing is wrong
-         * we don't need ondisk size check there. But if the file is either
-         * self-locked or the caller wants a thorough check then make sure to
-         * perform on disk check also. */
-        ret = ec_heal_data_find_direction (ec, replies, data_versions,
-                                           dirty, size, sources, healed_sinks,
-                                           self_locked || thorough,
-                                           EC_COMBINE_XDATA);
-        if (ret < 0 && ret != -EIO) {
-                goto out;
-        }
+ec_need_data_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
+                  int32_t lock_count, gf_boolean_t self_locked,
+                  gf_boolean_t thorough, ec_heal_need_t *need_heal)
+{
+    uint64_t *dirty = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *healed_sinks = NULL;
+    uint64_t *data_versions = NULL;
+    uint64_t *size = NULL;
+    int ret = 0;
+
+    sources = alloca0(ec->nodes);
+    healed_sinks = alloca0(ec->nodes);
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    data_versions = alloca0(ec->nodes * sizeof(*data_versions));
+    size = alloca0(ec->nodes * sizeof(*size));
+
+    /* When dd is going on and heal info is called there is a very good
+     * chance for on disk sizes to mismatch even though nothing is wrong
+     * we don't need ondisk size check there. But if the file is either
+     * self-locked or the caller wants a thorough check then make sure to
+     * perform on disk check also. */
+    ret = ec_heal_data_find_direction(
+        ec, replies, data_versions, dirty, size, sources, healed_sinks,
+        self_locked || thorough, EC_COMBINE_XDATA);
+    if (ret < 0 && ret != -EIO) {
+        goto out;
+    }
 
-        ret = _need_heal_calculate (ec, dirty, sources, self_locked, lock_count,
-                                    need_heal);
+    ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count,
+                               need_heal, data_versions);
 out:
-        return ret;
+    return ret;
 }
 
 static int32_t
-ec_need_entry_heal (ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
-                    int32_t lock_count, gf_boolean_t self_locked,
-                    gf_boolean_t thorough, ec_heal_need_t *need_heal)
-{
-        uint64_t           *dirty         = NULL;
-        unsigned char      *sources       = NULL;
-        unsigned char      *healed_sinks  = NULL;
-        uint64_t           *data_versions = NULL;
-        int                ret            = 0;
-
-        sources = alloca0(ec->nodes);
-        healed_sinks = alloca0(ec->nodes);
-        dirty = alloca0 (ec->nodes * sizeof (*dirty));
-        data_versions = alloca0 (ec->nodes * sizeof (*data_versions));
-
-        ret = ec_heal_entry_find_direction (ec, replies, data_versions,
-                                           dirty, sources, healed_sinks);
-        if (ret < 0 && ret != -EIO) {
-                goto out;
-        }
+ec_need_entry_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
+                   int32_t lock_count, gf_boolean_t self_locked,
+                   gf_boolean_t thorough, ec_heal_need_t *need_heal)
+{
+    uint64_t *dirty = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *healed_sinks = NULL;
+    uint64_t *data_versions = NULL;
+    int ret = 0;
+
+    sources = alloca0(ec->nodes);
+    healed_sinks = alloca0(ec->nodes);
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    data_versions = alloca0(ec->nodes * sizeof(*data_versions));
+
+    ret = ec_heal_entry_find_direction(ec, replies, data_versions, dirty,
+                                       sources, healed_sinks);
+    if (ret < 0 && ret != -EIO) {
+        goto out;
+    }
 
-        ret = _need_heal_calculate (ec, dirty, sources, self_locked, lock_count,
-                                    need_heal);
+    ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count,
+                               need_heal, data_versions);
 out:
-        return ret;
+    return ret;
 }
 
 static int32_t
-ec_need_heal (ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
-              int32_t lock_count, gf_boolean_t self_locked,
-              gf_boolean_t thorough, ec_heal_need_t *need_heal)
+ec_need_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
+             int32_t lock_count, gf_boolean_t self_locked,
+             gf_boolean_t thorough, ec_heal_need_t *need_heal)
 {
-        int                ret            = 0;
+    int ret = 0;
 
+    ret = ec_need_metadata_heal(ec, inode, replies, lock_count, self_locked,
+                                thorough, need_heal);
+    if (ret < 0)
+        goto out;
 
-        ret = ec_need_metadata_heal (ec, inode, replies, lock_count,
-                                     self_locked, thorough, need_heal);
-        if (ret < 0)
-                goto out;
-
-        if (*need_heal == EC_HEAL_MUST)
-                goto out;
+    if (*need_heal == EC_HEAL_MUST)
+        goto out;
 
-        if (inode->ia_type == IA_IFREG) {
-                ret = ec_need_data_heal (ec, inode, replies, lock_count,
-                                         self_locked, thorough, need_heal);
-        } else if (inode->ia_type == IA_IFDIR) {
-                ret = ec_need_entry_heal (ec, inode, replies, lock_count,
-                                          self_locked, thorough, need_heal);
-        }
+    if (inode->ia_type == IA_IFREG) {
+        ret = ec_need_data_heal(ec, inode, replies, lock_count, self_locked,
+                                thorough, need_heal);
+    } else if (inode->ia_type == IA_IFDIR) {
+        ret = ec_need_entry_heal(ec, inode, replies, lock_count, self_locked,
+                                 thorough, need_heal);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-ec_heal_inspect (call_frame_t *frame, ec_t *ec,
-                 inode_t *inode, unsigned char *locked_on,
-                 gf_boolean_t self_locked, gf_boolean_t thorough,
-                 ec_heal_need_t *need_heal)
-{
-        loc_t              loc           = {0};
-        int                i             = 0;
-        int                ret           = 0;
-        dict_t             *xdata        = NULL;
-        uint64_t           zero_array[2] = {0};
-        uint64_t           zero_value    = 0;
-        unsigned char      *output       = NULL;
-        default_args_cbk_t *replies      = NULL;
-        int32_t            lock_count    = 0;
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        output = alloca0 (ec->nodes);
-
-        loc.inode = inode_ref (inode);
-        gf_uuid_copy (loc.gfid, inode->gfid);
-
-        xdata = dict_new ();
-        if (!xdata ||
-            dict_set_static_bin (xdata, EC_XATTR_VERSION, zero_array,
-                                 sizeof (zero_array)) ||
-            dict_set_static_bin (xdata, EC_XATTR_DIRTY, zero_array,
-                                 sizeof (zero_array)) ||
-            dict_set_static_bin (xdata, EC_XATTR_SIZE, &zero_value,
-                                 sizeof (zero_value))) {
-                ret = -ENOMEM;
-                goto out;
-        }
+ec_heal_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                unsigned char *locked_on, gf_boolean_t self_locked,
+                gf_boolean_t thorough, ec_heal_need_t *need_heal)
+{
+    loc_t loc = {0};
+    int i = 0;
+    int ret = 0;
+    dict_t *xdata = NULL;
+    uint64_t zero_array[2] = {0};
+    uint64_t zero_value = 0;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+    int32_t lock_count = 0;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    output = alloca0(ec->nodes);
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    xdata = dict_new();
+    if (!xdata ||
+        dict_set_static_bin(xdata, EC_XATTR_VERSION, zero_array,
+                            sizeof(zero_array)) ||
+        dict_set_static_bin(xdata, EC_XATTR_DIRTY, zero_array,
+                            sizeof(zero_array)) ||
+        dict_set_static_bin(xdata, EC_XATTR_SIZE, &zero_value,
+                            sizeof(zero_value))) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        if (!self_locked) {
-                ret = dict_set_str(xdata, GLUSTERFS_INODELK_DOM_COUNT,
-                             ec->xl->name);
-                if (ret) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
+    if (!self_locked) {
+        ret = dict_set_str(xdata, GLUSTERFS_INODELK_DOM_COUNT, ec->xl->name);
+        if (ret) {
+            ret = -ENOMEM;
+            goto out;
         }
+    }
 
-        ret = cluster_lookup (ec->xl_list, locked_on, ec->nodes, replies,
-                              output, frame, ec->xl, &loc, xdata);
+    ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output,
+                         frame, ec->xl, &loc, xdata);
 
-        if (ret != ec->nodes) {
-                ret = ec->nodes;
-                *need_heal = EC_HEAL_MUST;
-                goto out;
-        }
+    if (ret != ec->nodes) {
+        ret = ec->nodes;
+        *need_heal = EC_HEAL_MUST;
+        goto out;
+    }
 
-        if (self_locked)
-                goto need_heal;
+    if (self_locked)
+        goto need_heal;
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (!output[i] || !replies[i].xdata) {
-                        continue;
-                }
-                if ((dict_get_int32 (replies[i].xdata, GLUSTERFS_INODELK_COUNT,
-                                     &lock_count) == 0) && lock_count > 0) {
-                        break;
-                }
+    for (i = 0; i < ec->nodes; i++) {
+        if (!output[i] || !replies[i].xdata) {
+            continue;
         }
-need_heal:
-        ret = ec_need_heal (ec, inode, replies, lock_count,
-                            self_locked, thorough, need_heal);
-
-        if (!self_locked && *need_heal == EC_HEAL_MUST) {
-                *need_heal = EC_HEAL_MAYBE;
+        if ((dict_get_int32(replies[i].xdata, GLUSTERFS_INODELK_COUNT,
+                            &lock_count) == 0) &&
+            lock_count > 0) {
+            break;
         }
+    }
+need_heal:
+    ret = ec_need_heal(ec, inode, replies, lock_count, self_locked, thorough,
+                       need_heal);
 out:
-        cluster_replies_wipe (replies, ec->nodes);
-        loc_wipe (&loc);
-        if (xdata) {
-                dict_unref(xdata);
-        }
-        return ret;
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    if (xdata) {
+        dict_unref(xdata);
+    }
+    return ret;
 }
 
 int32_t
-ec_heal_locked_inspect (call_frame_t *frame, ec_t *ec, inode_t *inode,
-                        ec_heal_need_t *need_heal)
-{
-        unsigned char      *locked_on  = NULL;
-        unsigned char      *up_subvols = NULL;
-        unsigned char      *output     = NULL;
-        default_args_cbk_t *replies    = NULL;
-        int                ret         = 0;
-
-        EC_REPLIES_ALLOC (replies, ec->nodes);
-        locked_on = alloca0(ec->nodes);
-        output = alloca0(ec->nodes);
-        up_subvols = alloca0(ec->nodes);
-        ec_mask_to_char_array (ec->xl_up, up_subvols, ec->nodes);
-
-        ret = cluster_inodelk (ec->xl_list, up_subvols, ec->nodes,
-                               replies, locked_on, frame, ec->xl,
-                               ec->xl->name, inode, 0, 0);
-        if (ret != ec->nodes) {
-                *need_heal = EC_HEAL_MUST;
-                goto unlock;
-        }
-        ret = ec_heal_inspect (frame, ec, inode, locked_on, _gf_true, _gf_true,
-                               need_heal);
+ec_heal_locked_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                       ec_heal_need_t *need_heal)
+{
+    unsigned char *locked_on = NULL;
+    unsigned char *up_subvols = NULL;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+    int ret = 0;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    up_subvols = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
+
+    ret = cluster_inodelk(ec->xl_list, up_subvols, ec->nodes, replies,
+                          locked_on, frame, ec->xl, ec->xl->name, inode, 0, 0);
+    if (ret != ec->nodes) {
+        *need_heal = EC_HEAL_MUST;
+        goto unlock;
+    }
+    ret = ec_heal_inspect(frame, ec, inode, locked_on, _gf_true, _gf_true,
+                          need_heal);
 unlock:
-        cluster_uninodelk (ec->xl_list, locked_on, ec->nodes,
-                           replies, output, frame, ec->xl,
-                           ec->xl->name, inode, 0, 0);
-        cluster_replies_wipe (replies, ec->nodes);
-        return ret;
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, inode, 0, 0);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 
 int32_t
-ec_get_heal_info (xlator_t *this, loc_t *entry_loc, dict_t **dict_rsp)
+ec_get_heal_info(xlator_t *this, loc_t *entry_loc, dict_t **dict_rsp)
 {
-        int             ret             = -ENOMEM;
-        ec_heal_need_t  need_heal       = EC_HEAL_NONEED;
-        call_frame_t    *frame          = NULL;
-        ec_t            *ec             = NULL;
-        unsigned char   *up_subvols     = NULL;
-        loc_t           loc             = {0, };
-
-        VALIDATE_OR_GOTO(this, out);
-        GF_VALIDATE_OR_GOTO(this->name, entry_loc, out);
+    int ret = -ENOMEM;
+    ec_heal_need_t need_heal = EC_HEAL_NONEED;
+    call_frame_t *frame = NULL;
+    ec_t *ec = NULL;
+    unsigned char *up_subvols = NULL;
+    loc_t loc = {
+        0,
+    };
 
-        ec = this->private;
-        up_subvols = alloca0(ec->nodes);
-        ec_mask_to_char_array (ec->xl_up, up_subvols, ec->nodes);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, entry_loc, out);
 
-        if (EC_COUNT (up_subvols, ec->nodes) != ec->nodes) {
-                need_heal = EC_HEAL_MUST;
-                goto set_heal;
-        }
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                goto out;
-        }
-        ec_owner_set(frame, frame->root);
-        frame->root->uid = 0;
-        frame->root->gid = 0;
-        frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
+    ec = this->private;
+    up_subvols = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
 
-        if (loc_copy(&loc, entry_loc) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR,
-                        ENOMEM, EC_MSG_LOC_COPY_FAIL,
-                        "Failed to copy a location.");
-                goto out;
-        }
-        if (!loc.inode) {
-                ret = syncop_inode_find (this, this, loc.gfid,
-                                         &loc.inode, NULL, NULL);
-                if (ret < 0)
-                        goto out;
-        }
-
-        ret = ec_heal_inspect (frame, ec, loc.inode, up_subvols, _gf_false,
-                               _gf_false, &need_heal);
-        if (ret == ec->nodes && need_heal == EC_HEAL_NONEED) {
-                goto set_heal;
-        }
-        need_heal = EC_HEAL_NONEED;
-        ret = ec_heal_locked_inspect (frame, ec, loc.inode,
-                                      &need_heal);
+    if (EC_COUNT(up_subvols, ec->nodes) != ec->nodes) {
+        need_heal = EC_HEAL_MUST;
+        goto set_heal;
+    }
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        goto out;
+    }
+    ec_owner_set(frame, frame->root);
+    frame->root->uid = 0;
+    frame->root->gid = 0;
+    frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
+
+    if (loc_copy(&loc, entry_loc) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+               "Failed to copy a location.");
+        goto out;
+    }
+    if (!loc.inode) {
+        ret = syncop_inode_find(this, this, loc.gfid, &loc.inode, NULL, NULL);
         if (ret < 0)
-                goto out;
+            goto out;
+    }
+
+    ret = ec_heal_inspect(frame, ec, loc.inode, up_subvols, _gf_false,
+                          _gf_false, &need_heal);
+    if (ret == ec->nodes && need_heal != EC_HEAL_MAYBE) {
+        goto set_heal;
+    }
+    need_heal = EC_HEAL_NONEED;
+    ret = ec_heal_locked_inspect(frame, ec, loc.inode, &need_heal);
+    if (ret < 0)
+        goto out;
 set_heal:
-        if (need_heal == EC_HEAL_MUST) {
-                ret =  ec_set_heal_info (dict_rsp, "heal");
-        } else {
-                ret =  ec_set_heal_info (dict_rsp, "no-heal");
-        }
+    if (need_heal == EC_HEAL_MUST) {
+        ret = ec_set_heal_info(dict_rsp, "heal");
+    } else {
+        ret = ec_set_heal_info(dict_rsp, "no-heal");
+    }
 out:
-        if (frame) {
-                STACK_DESTROY (frame->root);
-        }
-        loc_wipe (&loc);
-        return ret;
+    if (frame) {
+        STACK_DESTROY(frame->root);
+    }
+    loc_wipe(&loc);
+    return ret;
 }
diff --git a/xlators/cluster/ec/src/ec-heald.c b/xlators/cluster/ec/src/ec-heald.c
index 65d8b5c4284..5c1586bc9c5 100644
--- a/xlators/cluster/ec/src/ec-heald.c
+++ b/xlators/cluster/ec/src/ec-heald.c
@@ -8,611 +8,674 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-#include "compat-errno.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat-errno.h>
 #include "ec.h"
 #include "ec-messages.h"
 #include "ec-heald.h"
 #include "ec-mem-types.h"
-#include "syncop.h"
-#include "syncop-utils.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/syncop-utils.h>
 #include "protocol-common.h"
 
-#define ASSERT_LOCAL(this, healer)				        \
-        do {                                                            \
-                if (!ec_shd_is_subvol_local (this, healer->subvol)) {	\
-                        healer->local = _gf_false;			\
-                        if (safe_break (healer)) {			\
-                                break;					\
-                        } else {					\
-                                continue;				\
-                        }						\
-                } else {						\
-                        healer->local = _gf_true;			\
-                }                                                       \
-        } while (0);
-
-
-#define NTH_INDEX_HEALER(this, n) (&((((ec_t *)this->private))->shd.index_healers[n]))
-#define NTH_FULL_HEALER(this, n) (&((((ec_t *)this->private))->shd.full_healers[n]))
+#define NTH_INDEX_HEALER(this, n)                                              \
+    (&((((ec_t *)this->private))->shd.index_healers[n]))
+#define NTH_FULL_HEALER(this, n)                                               \
+    (&((((ec_t *)this->private))->shd.full_healers[n]))
 
 gf_boolean_t
-ec_shd_is_subvol_local (xlator_t *this, int subvol)
+ec_shd_is_subvol_local(xlator_t *this, int subvol)
 {
-        ec_t *ec = NULL;
-        gf_boolean_t is_local = _gf_false;
-        loc_t loc = {0, };
-
-        ec = this->private;
-        loc.inode = this->itable->root;
-        syncop_is_subvol_local (ec->xl_list[subvol], &loc, &is_local);
-        return is_local;
+    ec_t *ec = NULL;
+    gf_boolean_t is_local = _gf_false;
+    loc_t loc = {
+        0,
+    };
+
+    ec = this->private;
+    loc.inode = this->itable->root;
+    syncop_is_subvol_local(ec->xl_list[subvol], &loc, &is_local);
+    return is_local;
 }
 
 char *
-ec_subvol_name (xlator_t *this, int subvol)
+ec_subvol_name(xlator_t *this, int subvol)
 {
-        ec_t *ec = NULL;
+    ec_t *ec = NULL;
 
-        ec = this->private;
-        if (subvol < 0 || subvol > ec->nodes)
-                return NULL;
+    ec = this->private;
+    if (subvol < 0 || subvol > ec->nodes)
+        return NULL;
 
-        return ec->xl_list[subvol]->name;
+    return ec->xl_list[subvol]->name;
 }
 
 int
-__ec_shd_healer_wait (struct subvol_healer *healer)
+__ec_shd_healer_wait(struct subvol_healer *healer)
 {
-        ec_t *ec = NULL;
-        struct timespec wait_till = {0, };
-        int ret = 0;
+    ec_t *ec = NULL;
+    struct timespec wait_till = {
+        0,
+    };
+    int ret = 0;
 
-        ec = healer->this->private;
+    ec = healer->this->private;
 
 disabled_loop:
-        wait_till.tv_sec = time (NULL) + 60;
-
-        while (!healer->rerun) {
-                ret = pthread_cond_timedwait (&healer->cond,
-                                              &healer->mutex,
-                                              &wait_till);
-                if (ret == ETIMEDOUT)
-                        break;
-        }
-
-        ret = healer->rerun;
-        healer->rerun = 0;
-
-        if (!ec->shd.enabled || !ec->up)
-                goto disabled_loop;
+    wait_till.tv_sec = gf_time() + ec->shd.timeout;
 
-        return ret;
-}
+    while (!healer->rerun) {
+        ret = pthread_cond_timedwait(&healer->cond, &healer->mutex, &wait_till);
+        if (ret == ETIMEDOUT)
+            break;
+    }
 
+    if (ec->shutdown) {
+        healer->running = _gf_false;
+        return -1;
+    }
 
-int
-ec_shd_healer_wait (struct subvol_healer *healer)
-{
-        int ret = 0;
+    ret = healer->rerun;
+    healer->rerun = 0;
 
-        pthread_mutex_lock (&healer->mutex);
-        {
-                ret = __ec_shd_healer_wait (healer);
-        }
-        pthread_mutex_unlock (&healer->mutex);
+    if (!ec->shd.enabled || !ec->up)
+        goto disabled_loop;
 
-        return ret;
+    return ret;
 }
 
-
-gf_boolean_t
-safe_break (struct subvol_healer *healer)
+int
+ec_shd_healer_wait(struct subvol_healer *healer)
 {
-        gf_boolean_t ret = _gf_false;
-
-        pthread_mutex_lock (&healer->mutex);
-        {
-                if (healer->rerun)
-                        goto unlock;
+    int ret = 0;
 
-                healer->running = _gf_false;
-                ret = _gf_true;
-        }
-unlock:
-        pthread_mutex_unlock (&healer->mutex);
+    pthread_mutex_lock(&healer->mutex);
+    {
+        ret = __ec_shd_healer_wait(healer);
+    }
+    pthread_mutex_unlock(&healer->mutex);
 
-        return ret;
+    return ret;
 }
 
 int
-ec_shd_index_inode (xlator_t *this, xlator_t *subvol, inode_t **inode)
+ec_shd_index_inode(xlator_t *this, xlator_t *subvol, inode_t **inode)
 {
-        loc_t   rootloc     = {0, };
-        int     ret         = 0;
-        dict_t  *xattr      = NULL;
-        void    *index_gfid = NULL;
-
-        *inode = NULL;
-        rootloc.inode = inode_ref (this->itable->root);
-        gf_uuid_copy (rootloc.gfid, rootloc.inode->gfid);
-
-        ret = syncop_getxattr (subvol, &rootloc, &xattr,
-                               GF_XATTROP_INDEX_GFID, NULL, NULL);
-        if (ret < 0)
-                goto out;
-        if (!xattr) {
-                ret = -EINVAL;
-                goto out;
-        }
-
-        ret = dict_get_ptr (xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
-        if (ret)
-                goto out;
-
-        gf_msg_debug (this->name, 0, "index-dir gfid for %s: %s",
-                subvol->name, uuid_utoa (index_gfid));
-
-        ret = syncop_inode_find (this, subvol, index_gfid,
-                                 inode, NULL, NULL);
+    loc_t rootloc = {
+        0,
+    };
+    int ret = 0;
+    dict_t *xattr = NULL;
+    void *index_gfid = NULL;
+
+    *inode = NULL;
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
+
+    ret = syncop_getxattr(subvol, &rootloc, &xattr, GF_XATTROP_INDEX_GFID, NULL,
+                          NULL);
+    if (ret < 0)
+        goto out;
+    if (!xattr) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    ret = dict_get_ptr(xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
+    if (ret)
+        goto out;
+
+    gf_msg_debug(this->name, 0, "index-dir gfid for %s: %s", subvol->name,
+                 uuid_utoa(index_gfid));
+
+    ret = syncop_inode_find(this, subvol, index_gfid, inode, NULL, NULL);
 
 out:
-        loc_wipe (&rootloc);
+    loc_wipe(&rootloc);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-        return ret;
+    return ret;
 }
 
 int
-ec_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name)
+ec_shd_index_purge(xlator_t *subvol, inode_t *inode, char *name)
 {
-        loc_t loc = {0, };
-        int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
 
-        loc.parent = inode_ref (inode);
-        loc.name = name;
+    loc.parent = inode_ref(inode);
+    loc.name = name;
 
-        ret = syncop_unlink (subvol, &loc, NULL, NULL);
+    ret = syncop_unlink(subvol, &loc, NULL, NULL);
 
-        loc_wipe (&loc);
-        return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
-int
-ec_shd_selfheal (struct subvol_healer *healer, int child, loc_t *loc)
+static gf_boolean_t
+ec_is_heal_completed(char *status)
 {
-        int32_t ret;
-
-        ret = syncop_getxattr (healer->this, loc, NULL, EC_XATTR_HEAL, NULL,
-                               NULL);
-        if ((ret >= 0) && (loc->inode->ia_type == IA_IFDIR)) {
-                /* If we have just healed a directory, it's possible that
-                 * other index entries have appeared to be healed. We put a
-                 * mark so that we can check it later and restart a scan
-                 * without delay. */
-                healer->rerun = _gf_true;
-        }
-
-        return ret;
+    char *bad_pos = NULL;
+    char *zero_pos = NULL;
+
+    if (!status) {
+        return _gf_false;
+    }
+
+    /*Logic:
+     * Status will be of the form Good: <binary>, Bad: <binary>
+     * If heal completes, if we do strchr for '0' it should be present after
+     * 'Bad:' i.e. strRchr for ':'
+     * */
+
+    zero_pos = strchr(status, '0');
+    bad_pos = strrchr(status, ':');
+    if (!zero_pos || !bad_pos) {
+        /*malformed status*/
+        return _gf_false;
+    }
+
+    if (zero_pos > bad_pos) {
+        return _gf_true;
+    }
+
+    return _gf_false;
 }
 
-
 int
-ec_shd_index_heal (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                   void *data)
+ec_shd_selfheal(struct subvol_healer *healer, int child, loc_t *loc,
+                gf_boolean_t full)
 {
-        struct subvol_healer *healer = data;
-        ec_t                 *ec     = NULL;
-        loc_t                loc     = {0};
-        int                  ret     = 0;
-
-        ec = healer->this->private;
-        if (ec->xl_up_count <= ec->fragments) {
-                return -ENOTCONN;
+    dict_t *xdata = NULL;
+    dict_t *dict = NULL;
+    uint32_t count;
+    int32_t ret;
+    char *heal_status = NULL;
+    ec_t *ec = healer->this->private;
+
+    GF_ATOMIC_INC(ec->stats.shd.attempted);
+    ret = syncop_getxattr(healer->this, loc, &dict, EC_XATTR_HEAL, NULL,
+                          &xdata);
+    if (ret == 0) {
+        if (dict && (dict_get_str(dict, EC_XATTR_HEAL, &heal_status) == 0)) {
+            if (ec_is_heal_completed(heal_status)) {
+                GF_ATOMIC_INC(ec->stats.shd.completed);
+            }
         }
-        if (!ec->shd.enabled)
-                return -EBUSY;
+    }
+
+    if (!full && (loc->inode->ia_type == IA_IFDIR)) {
+        /* If we have just healed a directory, it's possible that
+         * other index entries have appeared to be healed. */
+        if ((xdata != NULL) &&
+            (dict_get_uint32(xdata, EC_XATTR_HEAL_NEW, &count) == 0) &&
+            (count > 0)) {
+            /* Force a rerun of the index healer. */
+            gf_msg_debug(healer->this->name, 0, "%d more entries to heal",
+                         count);
+
+            healer->rerun = _gf_true;
+        }
+    }
 
-        gf_msg_debug (healer->this->name, 0, "got entry: %s",
-                entry->d_name);
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
 
-        ret = gf_uuid_parse (entry->d_name, loc.gfid);
-        if (ret)
-                return 0;
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        /* If this fails with ENOENT/ESTALE index is stale */
-        ret = syncop_gfid_to_path (healer->this->itable, subvol, loc.gfid,
-                                   (char **)&loc.path);
-        if (ret < 0)
-                goto out;
+    return ret;
+}
 
-        ret = syncop_inode_find (healer->this, healer->this, loc.gfid,
-                                 &loc.inode, NULL, NULL);
-        if (ret < 0)
-                goto out;
+int
+ec_shd_index_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                  void *data)
+{
+    struct subvol_healer *healer = data;
+    ec_t *ec = NULL;
+    loc_t loc = {0};
+    int ret = 0;
+
+    ec = healer->this->private;
+    if (ec->xl_up_count <= ec->fragments) {
+        return -ENOTCONN;
+    }
+    if (!ec->shd.enabled)
+        return -EBUSY;
+
+    gf_msg_debug(healer->this->name, 0, "got entry: %s", entry->d_name);
+
+    ret = gf_uuid_parse(entry->d_name, loc.gfid);
+    if (ret)
+        return 0;
 
-        ec_shd_selfheal (healer, healer->subvol, &loc);
-out:
-        if (ret == -ENOENT || ret == -ESTALE) {
-                gf_msg (healer->this->name, GF_LOG_DEBUG, 0,
-                        EC_MSG_HEAL_FAIL, "Purging index for gfid %s:",
-                        uuid_utoa(loc.gfid));
-                ec_shd_index_purge (subvol, parent->inode, entry->d_name);
-        }
-        loc_wipe (&loc);
+    /* If this fails with ENOENT/ESTALE index is stale */
+    ret = syncop_gfid_to_path(healer->this->itable, subvol, loc.gfid,
+                              (char **)&loc.path);
+    if (ret < 0)
+        goto out;
 
-        return 0;
+    ret = syncop_inode_find(healer->this, healer->this, loc.gfid, &loc.inode,
+                            NULL, NULL);
+    if (ret < 0)
+        goto out;
+
+    ec_shd_selfheal(healer, healer->subvol, &loc, _gf_false);
+out:
+    if (ret == -ENOENT || ret == -ESTALE) {
+        gf_msg(healer->this->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_FAIL,
+               "Purging index for gfid %s:", uuid_utoa(loc.gfid));
+        ec_shd_index_purge(subvol, parent->inode, entry->d_name);
+    }
+    loc_wipe(&loc);
+
+    return 0;
 }
 
 int
-ec_shd_index_sweep (struct subvol_healer *healer)
+ec_shd_index_sweep(struct subvol_healer *healer)
 {
-        loc_t         loc     = {0};
-        ec_t          *ec     = NULL;
-        int           ret     = 0;
-        xlator_t      *subvol = NULL;
-        dict_t        *xdata  = NULL;
-
-        ec = healer->this->private;
-        subvol = ec->xl_list[healer->subvol];
-
-        ret = ec_shd_index_inode (healer->this, subvol, &loc.inode);
-        if (ret < 0) {
-                gf_msg (healer->this->name, GF_LOG_WARNING, errno,
-                        EC_MSG_INDEX_DIR_GET_FAIL,
-                        "unable to get index-dir on %s", subvol->name);
-                goto out;
-        }
-
-        xdata = dict_new ();
-        if (!xdata || dict_set_int32 (xdata, "get-gfid-type", 1)) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        ret = syncop_mt_dir_scan (NULL, subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
-                                  healer, ec_shd_index_heal, xdata,
-                                  ec->shd.max_threads, ec->shd.wait_qlength);
+    loc_t loc = {0};
+    ec_t *ec = NULL;
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    dict_t *xdata = NULL;
+
+    ec = healer->this->private;
+    subvol = ec->xl_list[healer->subvol];
+
+    ret = ec_shd_index_inode(healer->this, subvol, &loc.inode);
+    if (ret < 0) {
+        gf_msg(healer->this->name, GF_LOG_WARNING, errno,
+               EC_MSG_INDEX_DIR_GET_FAIL, "unable to get index-dir on %s",
+               subvol->name);
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata || dict_set_int32(xdata, "get-gfid-type", 1)) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    _mask_cancellation();
+    ret = syncop_mt_dir_scan(NULL, subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
+                             healer, ec_shd_index_heal, xdata,
+                             ec->shd.max_threads, ec->shd.wait_qlength);
+    _unmask_cancellation();
 out:
-        if (xdata)
-                dict_unref (xdata);
-        loc_wipe (&loc);
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
 
-        return ret;
+    return ret;
 }
 
 int
-ec_shd_full_heal (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                   void *data)
+ec_shd_full_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                 void *data)
 {
-        struct subvol_healer *healer = data;
-        xlator_t             *this   = healer->this;
-        ec_t                 *ec     = NULL;
-        loc_t                loc    = {0};
-        int                  ret    = 0;
-
-        ec = this->private;
-        if (ec->xl_up_count <= ec->fragments) {
-                return -ENOTCONN;
-        }
-        if (!ec->shd.enabled)
-                return -EBUSY;
-
-        if (gf_uuid_is_null(entry->d_stat.ia_gfid)) {
-                /* It's possible that an entry has been removed just after
-                 * being seen in a directory but before getting its stat info.
-                 * In this case we'll receive a NULL gfid here. Since the file
-                 * doesn't exist anymore, we can safely ignore it. */
-                return 0;
-        }
+    struct subvol_healer *healer = data;
+    xlator_t *this = healer->this;
+    ec_t *ec = NULL;
+    loc_t loc = {0};
+    int ret = 0;
+
+    ec = this->private;
+
+    if (this->cleanup_starting) {
+        return -ENOTCONN;
+    }
+
+    if (ec->xl_up_count <= ec->fragments) {
+        return -ENOTCONN;
+    }
+    if (!ec->shd.enabled)
+        return -EBUSY;
+
+    if (gf_uuid_is_null(entry->d_stat.ia_gfid)) {
+        /* It's possible that an entry has been removed just after
+         * being seen in a directory but before getting its stat info.
+         * In this case we'll receive a NULL gfid here. Since the file
+         * doesn't exist anymore, we can safely ignore it. */
+        return 0;
+    }
 
-        loc.parent = inode_ref (parent->inode);
-        loc.name   = entry->d_name;
-        gf_uuid_copy (loc.gfid, entry->d_stat.ia_gfid);
+    loc.parent = inode_ref(parent->inode);
+    loc.name = entry->d_name;
+    gf_uuid_copy(loc.gfid, entry->d_stat.ia_gfid);
 
-        /* If this fails with ENOENT/ESTALE index is stale */
-        ret = syncop_gfid_to_path (this->itable, subvol, loc.gfid,
-                                   (char **)&loc.path);
-        if (ret < 0)
-                goto out;
+    /* If this fails with ENOENT/ESTALE index is stale */
+    ret = syncop_gfid_to_path(this->itable, subvol, loc.gfid,
+                              (char **)&loc.path);
+    if (ret < 0)
+        goto out;
 
-        ret = syncop_inode_find (this, this, loc.gfid,
-                                 &loc.inode, NULL, NULL);
-        if (ret < 0)
-                goto out;
+    ret = syncop_inode_find(this, this, loc.gfid, &loc.inode, NULL, NULL);
+    if (ret < 0)
+        goto out;
 
-        ec_shd_selfheal (healer, healer->subvol, &loc);
+    ec_shd_selfheal(healer, healer->subvol, &loc, _gf_true);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        loc_wipe (&loc);
-        return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-ec_shd_full_sweep (struct subvol_healer *healer, inode_t *inode)
+ec_shd_full_sweep(struct subvol_healer *healer, inode_t *inode)
 {
-        ec_t           *ec  = NULL;
-        loc_t          loc  = {0};
-
-        ec = healer->this->private;
-        loc.inode = inode;
-        return syncop_ftw (ec->xl_list[healer->subvol], &loc,
-                           GF_CLIENT_PID_SELF_HEALD, healer,
-                           ec_shd_full_heal);
+    ec_t *ec = NULL;
+    loc_t loc = {0};
+    int ret = -1;
+
+    ec = healer->this->private;
+    loc.inode = inode;
+    _mask_cancellation();
+    ret = syncop_ftw(ec->xl_list[healer->subvol], &loc,
+                     GF_CLIENT_PID_SELF_HEALD, healer, ec_shd_full_heal);
+    _unmask_cancellation();
+    return ret;
 }
 
-
 void *
-ec_shd_index_healer (void *data)
+ec_shd_index_healer(void *data)
 {
-        struct subvol_healer *healer = NULL;
-        xlator_t *this = NULL;
-
-        healer = data;
-        THIS = this = healer->this;
-        ec_t *ec = this->private;
-
-        for (;;) {
-                ec_shd_healer_wait (healer);
-
-                ASSERT_LOCAL(this, healer);
-
-
-                if (ec->xl_up_count > ec->fragments) {
-                        gf_msg_debug (this->name, 0,
-                                "starting index sweep on subvol %s",
-                                ec_subvol_name (this, healer->subvol));
-                        ec_shd_index_sweep (healer);
-                }
-                gf_msg_debug (this->name, 0,
-                        "finished index sweep on subvol %s",
-                        ec_subvol_name (this, healer->subvol));
+    struct subvol_healer *healer = NULL;
+    xlator_t *this = NULL;
+    int run = 0;
+
+    healer = data;
+    THIS = this = healer->this;
+    ec_t *ec = this->private;
+
+    for (;;) {
+        run = ec_shd_healer_wait(healer);
+        if (run == -1)
+            break;
+
+        if (ec->xl_up_count > ec->fragments) {
+            gf_msg_debug(this->name, 0, "starting index sweep on subvol %s",
+                         ec_subvol_name(this, healer->subvol));
+            ec_shd_index_sweep(healer);
         }
+        gf_msg_debug(this->name, 0, "finished index sweep on subvol %s",
+                     ec_subvol_name(this, healer->subvol));
+    }
 
-        return NULL;
+    return NULL;
 }
 
-
 void *
-ec_shd_full_healer (void *data)
+ec_shd_full_healer(void *data)
 {
-        struct subvol_healer *healer = NULL;
-        xlator_t *this = NULL;
-        loc_t rootloc = {0};
-
-        int run = 0;
-
-        healer = data;
-        THIS = this = healer->this;
-        ec_t *ec = this->private;
-
-        rootloc.inode = this->itable->root;
-        for (;;) {
-                pthread_mutex_lock (&healer->mutex);
-                {
-                        run = __ec_shd_healer_wait (healer);
-                        if (!run)
-                                healer->running = _gf_false;
-                }
-                pthread_mutex_unlock (&healer->mutex);
-
-                if (!run)
-                        break;
-
-                ASSERT_LOCAL(this, healer);
-
-
-                if (ec->xl_up_count > ec->fragments) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                EC_MSG_FULL_SWEEP_START,
-                                "starting full sweep on subvol %s",
-                                ec_subvol_name (this, healer->subvol));
-
-                        ec_shd_selfheal (healer, healer->subvol, &rootloc);
-                        ec_shd_full_sweep (healer, this->itable->root);
-                }
-
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        EC_MSG_FULL_SWEEP_STOP,
-                        "finished full sweep on subvol %s",
-                        ec_subvol_name (this, healer->subvol));
+    struct subvol_healer *healer = NULL;
+    xlator_t *this = NULL;
+    loc_t rootloc = {0};
+
+    int run = 0;
+
+    healer = data;
+    THIS = this = healer->this;
+    ec_t *ec = this->private;
+
+    rootloc.inode = this->itable->root;
+    for (;;) {
+        run = ec_shd_healer_wait(healer);
+        if (run < 0) {
+            break;
+        } else if (run == 0) {
+            continue;
         }
 
-        return NULL;
-}
+        if (ec->xl_up_count > ec->fragments) {
+            gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_FULL_SWEEP_START,
+                   "starting full sweep on subvol %s",
+                   ec_subvol_name(this, healer->subvol));
 
+            ec_shd_selfheal(healer, healer->subvol, &rootloc, _gf_true);
+            ec_shd_full_sweep(healer, this->itable->root);
+        }
+
+        gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_FULL_SWEEP_STOP,
+               "finished full sweep on subvol %s",
+               ec_subvol_name(this, healer->subvol));
+    }
+
+    return NULL;
+}
 
 int
-ec_shd_healer_init (xlator_t *this, struct subvol_healer *healer)
+ec_shd_healer_init(xlator_t *this, struct subvol_healer *healer)
 {
-        int ret = 0;
+    int ret = 0;
 
-        ret = pthread_mutex_init (&healer->mutex, NULL);
-        if (ret)
-                goto out;
+    ret = pthread_mutex_init(&healer->mutex, NULL);
+    if (ret)
+        goto out;
 
-        ret = pthread_cond_init (&healer->cond, NULL);
-        if (ret)
-                goto out;
+    ret = pthread_cond_init(&healer->cond, NULL);
+    if (ret)
+        goto out;
 
-        healer->this = this;
-        healer->running = _gf_false;
-        healer->rerun = _gf_false;
-        healer->local = _gf_false;
+    healer->this = this;
+    healer->running = _gf_false;
+    healer->rerun = _gf_false;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-ec_shd_healer_spawn (xlator_t *this, struct subvol_healer *healer,
-                      void *(threadfn)(void *))
+ec_shd_healer_spawn(xlator_t *this, struct subvol_healer *healer,
+                    void *(threadfn)(void *))
 {
-        int ret = 0;
-
-        pthread_mutex_lock (&healer->mutex);
-        {
-                if (healer->running) {
-                        pthread_cond_signal (&healer->cond);
-                } else {
-                        ret = gf_thread_create (&healer->thread, NULL,
-                                                threadfn, healer, "ecshd");
-                        if (ret)
-                                goto unlock;
-                        healer->running = 1;
-                }
-
-                healer->rerun = 1;
+    int ret = 0;
+
+    pthread_mutex_lock(&healer->mutex);
+    {
+        if (healer->running) {
+            pthread_cond_signal(&healer->cond);
+        } else {
+            ret = gf_thread_create(&healer->thread, NULL, threadfn, healer,
+                                   "ecshd");
+            if (ret)
+                goto unlock;
+            healer->running = 1;
         }
+
+        healer->rerun = 1;
+    }
 unlock:
-        pthread_mutex_unlock (&healer->mutex);
+    pthread_mutex_unlock(&healer->mutex);
 
-        return ret;
+    return ret;
 }
 
 int
-ec_shd_full_healer_spawn (xlator_t *this, int subvol)
+ec_shd_full_healer_spawn(xlator_t *this, int subvol)
 {
-        return ec_shd_healer_spawn (this, NTH_FULL_HEALER (this, subvol),
-                                    ec_shd_full_healer);
-}
+    if (xlator_is_cleanup_starting(this))
+        return -1;
 
+    return ec_shd_healer_spawn(this, NTH_FULL_HEALER(this, subvol),
+                               ec_shd_full_healer);
+}
 
 int
-ec_shd_index_healer_spawn (xlator_t *this, int subvol)
+ec_shd_index_healer_spawn(xlator_t *this, int subvol)
 {
-        return ec_shd_healer_spawn (this, NTH_INDEX_HEALER (this, subvol),
-                                    ec_shd_index_healer);
+    if (xlator_is_cleanup_starting(this))
+        return -1;
+
+    return ec_shd_healer_spawn(this, NTH_INDEX_HEALER(this, subvol),
+                               ec_shd_index_healer);
 }
 
 void
 ec_shd_index_healer_wake(ec_t *ec)
 {
-        int32_t i;
+    int32_t i;
 
-        for (i = 0; i < ec->nodes; i++) {
-                if (((ec->xl_up >> i) & 1) != 0) {
-                        ec_shd_index_healer_spawn(ec->xl, i);
-                }
+    for (i = 0; i < ec->nodes; i++) {
+        if (((ec->xl_up >> i) & 1) != 0) {
+            ec_shd_index_healer_spawn(ec->xl, i);
         }
+    }
 }
 
 int
-ec_selfheal_daemon_init (xlator_t *this)
+ec_selfheal_daemon_init(xlator_t *this)
 {
-        ec_t    *ec = NULL;
-        ec_self_heald_t *shd = NULL;
-        int ret = -1;
-        int i = 0;
-
-        ec = this->private;
-        shd = &ec->shd;
-
-        shd->index_healers = GF_CALLOC (sizeof(*shd->index_healers),
-                                        ec->nodes,
-                                        ec_mt_subvol_healer_t);
-        if (!shd->index_healers)
-                goto out;
-
-        for (i = 0; i < ec->nodes; i++) {
-                shd->index_healers[i].subvol = i;
-                ret = ec_shd_healer_init (this, &shd->index_healers[i]);
-                if (ret)
-                        goto out;
-        }
+    ec_t *ec = NULL;
+    ec_self_heald_t *shd = NULL;
+    int ret = -1;
+    int i = 0;
+
+    ec = this->private;
+    shd = &ec->shd;
+
+    shd->index_healers = GF_CALLOC(sizeof(*shd->index_healers), ec->nodes,
+                                   ec_mt_subvol_healer_t);
+    if (!shd->index_healers)
+        goto out;
+
+    for (i = 0; i < ec->nodes; i++) {
+        shd->index_healers[i].subvol = i;
+        ret = ec_shd_healer_init(this, &shd->index_healers[i]);
+        if (ret)
+            goto out;
+    }
 
-        shd->full_healers = GF_CALLOC (sizeof(*shd->full_healers),
-                                       ec->nodes,
-                                       ec_mt_subvol_healer_t);
-        if (!shd->full_healers)
-                goto out;
-
-        for (i = 0; i < ec->nodes; i++) {
-                shd->full_healers[i].subvol = i;
-                ret = ec_shd_healer_init (this, &shd->full_healers[i]);
-                if (ret)
-                        goto out;
-        }
+    shd->full_healers = GF_CALLOC(sizeof(*shd->full_healers), ec->nodes,
+                                  ec_mt_subvol_healer_t);
+    if (!shd->full_healers)
+        goto out;
+
+    for (i = 0; i < ec->nodes; i++) {
+        shd->full_healers[i].subvol = i;
+        ret = ec_shd_healer_init(this, &shd->full_healers[i]);
+        if (ret)
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-ec_heal_op (xlator_t *this, dict_t *output, gf_xl_afr_op_t op, int xl_id)
+ec_heal_op(xlator_t *this, dict_t *output, gf_xl_afr_op_t op, int xl_id)
 {
-        char key[64] = {0};
-        int op_ret = 0;
-        ec_t *ec = NULL;
-        int     i = 0;
-        GF_UNUSED int     ret = 0;
-
-        ec = this->private;
-
-        op_ret = -1;
-        for (i = 0; i < ec->nodes; i++) {
-                snprintf (key, sizeof (key), "%d-%d-status", xl_id, i);
-
-                if (((ec->xl_up >> i) & 1) == 0) {
-                        ret = dict_set_str (output, key, "Brick is not connected");
-                } else if (!ec->up) {
-                        ret = dict_set_str (output, key,
-                                            "Disperse subvolume is not up");
-                } else if (!ec_shd_is_subvol_local (this, i)) {
-                        ret = dict_set_str (output, key, "Brick is remote");
-                } else {
-                        ret = dict_set_str (output, key, "Started self-heal");
-                        if (op == GF_SHD_OP_HEAL_FULL) {
-                                ec_shd_full_healer_spawn (this, i);
-                        } else if (op == GF_SHD_OP_HEAL_INDEX) {
-                                ec_shd_index_healer_spawn (this, i);
-                        }
-                        op_ret = 0;
-                }
+    char key[64] = {0};
+    int op_ret = 0;
+    ec_t *ec = NULL;
+    int i = 0;
+    GF_UNUSED int ret = 0;
+
+    ec = this->private;
+
+    op_ret = -1;
+    for (i = 0; i < ec->nodes; i++) {
+        snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
+
+        if (((ec->xl_up >> i) & 1) == 0) {
+            ret = dict_set_str(output, key, "Brick is not connected");
+        } else if (!ec->up) {
+            ret = dict_set_str(output, key, "Disperse subvolume is not up");
+        } else if (!ec_shd_is_subvol_local(this, i)) {
+            ret = dict_set_str(output, key, "Brick is remote");
+        } else {
+            ret = dict_set_str(output, key, "Started self-heal");
+            if (op == GF_SHD_OP_HEAL_FULL) {
+                ec_shd_full_healer_spawn(this, i);
+            } else if (op == GF_SHD_OP_HEAL_INDEX) {
+                ec_shd_index_healer_spawn(this, i);
+            }
+            op_ret = 0;
         }
-        return op_ret;
+    }
+    return op_ret;
 }
 
 int
-ec_xl_op (xlator_t *this, dict_t *input, dict_t *output)
+ec_xl_op(xlator_t *this, dict_t *input, dict_t *output)
 {
-        gf_xl_afr_op_t op     = GF_SHD_OP_INVALID;
-        int            ret    = 0;
-        int            xl_id  = 0;
+    gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
+    int ret = 0;
+    int xl_id = 0;
 
-        ret = dict_get_int32 (input, "xl-op", (int32_t *)&op);
-        if (ret)
-                goto out;
+    ret = dict_get_int32(input, "xl-op", (int32_t *)&op);
+    if (ret)
+        goto out;
 
-        ret = dict_get_int32 (input, this->name, &xl_id);
-        if (ret)
-                goto out;
+    ret = dict_get_int32(input, this->name, &xl_id);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (output, this->name, xl_id);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(output, this->name, xl_id);
+    if (ret)
+        goto out;
 
-        switch (op) {
+    switch (op) {
         case GF_SHD_OP_HEAL_FULL:
-                ret = ec_heal_op (this, output, op, xl_id);
-                break;
+            ret = ec_heal_op(this, output, op, xl_id);
+            break;
 
         case GF_SHD_OP_HEAL_INDEX:
-                ret = ec_heal_op (this, output, op, xl_id);
-                break;
+            ret = ec_heal_op(this, output, op, xl_id);
+            break;
 
         default:
-                ret = -1;
-                break;
-        }
+            ret = -1;
+            break;
+    }
 out:
-        dict_del (output, this->name);
-        return ret;
+    dict_del(output, this->name);
+    return ret;
+}
+
+void
+ec_destroy_healer_object(xlator_t *this, struct subvol_healer *healer)
+{
+    if (!healer)
+        return;
+
+    pthread_cond_destroy(&healer->cond);
+    pthread_mutex_destroy(&healer->mutex);
+}
+
+void
+ec_selfheal_daemon_fini(xlator_t *this)
+{
+    struct subvol_healer *healer = NULL;
+    ec_self_heald_t *shd = NULL;
+    ec_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+    if (!priv)
+        return;
+
+    shd = &priv->shd;
+    if (!shd->iamshd)
+        return;
+
+    for (i = 0; i < priv->nodes; i++) {
+        healer = &shd->index_healers[i];
+        ec_destroy_healer_object(this, healer);
+
+        healer = &shd->full_healers[i];
+        ec_destroy_healer_object(this, healer);
+    }
+
+    GF_FREE(shd->index_healers);
+    GF_FREE(shd->full_healers);
 }
diff --git a/xlators/cluster/ec/src/ec-heald.h b/xlators/cluster/ec/src/ec-heald.h
index 2a8488124c4..6c7da4edc10 100644
--- a/xlators/cluster/ec/src/ec-heald.h
+++ b/xlators/cluster/ec/src/ec-heald.h
@@ -11,17 +11,20 @@
 #ifndef __EC_HEALD_H__
 #define __EC_HEALD_H__
 
-#include "xlator.h"
-
-#include "ec-types.h"
+#include "ec-types.h"           // for ec_t
+#include "glusterfs/dict.h"     // for dict_t
+#include "glusterfs/globals.h"  // for xlator_t
 
 int
-ec_xl_op (xlator_t *this, dict_t *input, dict_t *output);
+ec_xl_op(xlator_t *this, dict_t *input, dict_t *output);
 
 int
-ec_selfheal_daemon_init (xlator_t *this);
+ec_selfheal_daemon_init(xlator_t *this);
 
 void
 ec_shd_index_healer_wake(ec_t *ec);
 
+void
+ec_selfheal_daemon_fini(xlator_t *this);
+
 #endif /* __EC_HEALD_H__ */
diff --git a/xlators/cluster/ec/src/ec-helpers.c b/xlators/cluster/ec/src/ec-helpers.c
index 83f96ba6cb2..48f54475e01 100644
--- a/xlators/cluster/ec/src/ec-helpers.c
+++ b/xlators/cluster/ec/src/ec-helpers.c
@@ -10,7 +10,7 @@
 
 #include <libgen.h>
 
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 
 #include "ec.h"
 #include "ec-mem-types.h"
@@ -19,25 +19,20 @@
 #include "ec-method.h"
 #include "ec-helpers.h"
 
-static const char * ec_fop_list[] =
-{
-    [-EC_FOP_HEAL] = "HEAL"
-};
+static const char *ec_fop_list[] = {[-EC_FOP_HEAL] = "HEAL"};
 
-const char * ec_bin(char * str, size_t size, uint64_t value, int32_t digits)
+const char *
+ec_bin(char *str, size_t size, uint64_t value, int32_t digits)
 {
     str += size;
 
-    if (size-- < 1)
-    {
+    if (size-- < 1) {
         goto failed;
     }
     *--str = 0;
 
-    while ((value != 0) || (digits > 0))
-    {
-        if (size-- < 1)
-        {
+    while ((value != 0) || (digits > 0)) {
+        if (size-- < 1) {
             goto failed;
         }
         *--str = '0' + (value & 1);
@@ -51,21 +46,22 @@ failed:
     return "<buffer too small>";
 }
 
-const char * ec_fop_name(int32_t id)
+const char *
+ec_fop_name(int32_t id)
 {
-    if (id >= 0)
-    {
+    if (id >= 0) {
         return gf_fop_list[id];
     }
 
     return ec_fop_list[-id];
 }
 
-void ec_trace(const char * event, ec_fop_data_t * fop, const char * fmt, ...)
+void
+ec_trace(const char *event, ec_fop_data_t *fop, const char *fmt, ...)
 {
     char str1[32], str2[32], str3[32];
-    char * msg;
-    ec_t * ec = fop->xl->private;
+    char *msg;
+    ec_t *ec = fop->xl->private;
     va_list args;
     int32_t ret;
 
@@ -73,28 +69,28 @@ void ec_trace(const char * event, ec_fop_data_t * fop, const char * fmt, ...)
     ret = vasprintf(&msg, fmt, args);
     va_end(args);
 
-    if (ret < 0)
-    {
+    if (ret < 0) {
         msg = "<memory allocation error>";
     }
 
-    gf_msg_trace ("ec", 0, "%s(%s) %p(%p) [refs=%d, winds=%d, jobs=%d] "
-                               "frame=%p/%p, min/exp=%d/%d, err=%d state=%d "
-                               "{%s:%s:%s} %s",
-           event, ec_fop_name(fop->id), fop, fop->parent, fop->refs,
-           fop->winds, fop->jobs, fop->req_frame, fop->frame, fop->minimum,
-           fop->expected, fop->error, fop->state,
-           ec_bin(str1, sizeof(str1), fop->mask, ec->nodes),
-           ec_bin(str2, sizeof(str2), fop->remaining, ec->nodes),
-           ec_bin(str3, sizeof(str3), fop->good, ec->nodes), msg);
+    gf_msg_trace("ec", 0,
+                 "%s(%s) %p(%p) [refs=%d, winds=%d, jobs=%d] "
+                 "frame=%p/%p, min/exp=%d/%d, err=%d state=%d "
+                 "{%s:%s:%s} %s",
+                 event, ec_fop_name(fop->id), fop, fop->parent, fop->refs,
+                 fop->winds, fop->jobs, fop->req_frame, fop->frame,
+                 fop->minimum, fop->expected, fop->error, fop->state,
+                 ec_bin(str1, sizeof(str1), fop->mask, ec->nodes),
+                 ec_bin(str2, sizeof(str2), fop->remaining, ec->nodes),
+                 ec_bin(str3, sizeof(str3), fop->good, ec->nodes), msg);
 
-    if (ret >= 0)
-    {
+    if (ret >= 0) {
         free(msg);
     }
 }
 
-int32_t ec_bits_consume(uint64_t * n)
+int32_t
+ec_bits_consume(uint64_t *n)
 {
     uint64_t tmp;
 
@@ -105,21 +101,18 @@ int32_t ec_bits_consume(uint64_t * n)
     return gf_bits_index(tmp);
 }
 
-size_t ec_iov_copy_to(void * dst, struct iovec * vector, int32_t count,
-                      off_t offset, size_t size)
+size_t
+ec_iov_copy_to(void *dst, struct iovec *vector, int32_t count, off_t offset,
+               size_t size)
 {
     int32_t i = 0;
     size_t total = 0, len = 0;
 
-    while (i < count)
-    {
-        if (offset < vector[i].iov_len)
-        {
-            while ((i < count) && (size > 0))
-            {
+    while (i < count) {
+        if (offset < vector[i].iov_len) {
+            while ((i < count) && (size > 0)) {
                 len = size;
-                if (len > vector[i].iov_len - offset)
-                {
+                if (len > vector[i].iov_len - offset) {
                     len = vector[i].iov_len - offset;
                 }
                 memcpy(dst, vector[i++].iov_base + offset, len);
@@ -139,15 +132,15 @@ size_t ec_iov_copy_to(void * dst, struct iovec * vector, int32_t count,
     return total;
 }
 
-int32_t ec_buffer_alloc(xlator_t *xl, size_t size, struct iobref **piobref,
-                        void **ptr)
+int32_t
+ec_buffer_alloc(xlator_t *xl, size_t size, struct iobref **piobref, void **ptr)
 {
     struct iobref *iobref = NULL;
     struct iobuf *iobuf = NULL;
     int32_t ret = -ENOMEM;
 
-    iobuf = iobuf_get_page_aligned (xl->ctx->iobuf_pool, size,
-                                    EC_METHOD_WORD_SIZE);
+    iobuf = iobuf_get_page_aligned(xl->ctx->iobuf_pool, size,
+                                   EC_METHOD_WORD_SIZE);
     if (iobuf == NULL) {
         goto out;
     }
@@ -186,12 +179,12 @@ out:
     return ret;
 }
 
-int32_t ec_dict_set_array(dict_t *dict, char *key, uint64_t value[],
-                          int32_t size)
+int32_t
+ec_dict_set_array(dict_t *dict, char *key, uint64_t value[], int32_t size)
 {
-    int         ret = -1;
-    uint64_t   *ptr = NULL;
-    int32_t     vindex;
+    int ret = -1;
+    uint64_t *ptr = NULL;
+    int32_t vindex;
 
     if (value == NULL) {
         return -EINVAL;
@@ -202,19 +195,18 @@ int32_t ec_dict_set_array(dict_t *dict, char *key, uint64_t value[],
         return -ENOMEM;
     }
     for (vindex = 0; vindex < size; vindex++) {
-         ptr[vindex] = hton64(value[vindex]);
+        ptr[vindex] = hton64(value[vindex]);
     }
     ret = dict_set_bin(dict, key, ptr, sizeof(uint64_t) * size);
     if (ret)
-         GF_FREE (ptr);
+        GF_FREE(ptr);
     return ret;
 }
 
-
 int32_t
-ec_dict_get_array (dict_t *dict, char *key, uint64_t value[], int32_t size)
+ec_dict_get_array(dict_t *dict, char *key, uint64_t value[], int32_t size)
 {
-    void    *ptr;
+    void *ptr;
     int32_t len;
     int32_t vindex;
     int32_t old_size = 0;
@@ -228,44 +220,43 @@ ec_dict_get_array (dict_t *dict, char *key, uint64_t value[], int32_t size)
         return err;
     }
 
-    if (len > (size * sizeof(uint64_t)) || (len % sizeof (uint64_t))) {
+    if (len > (size * sizeof(uint64_t)) || (len % sizeof(uint64_t))) {
         return -EINVAL;
     }
 
-    memset (value, 0, size * sizeof(uint64_t));
     /* 3.6 version ec would have stored version in 64 bit. In that case treat
      * metadata versions same as data*/
-    old_size = min (size, len/sizeof(uint64_t));
+    old_size = min(size, len / sizeof(uint64_t));
     for (vindex = 0; vindex < old_size; vindex++) {
-         value[vindex] = ntoh64(*((uint64_t *)ptr + vindex));
+        value[vindex] = ntoh64(*((uint64_t *)ptr + vindex));
     }
 
     if (old_size < size) {
-            for (vindex = old_size; vindex < size; vindex++) {
-                 value[vindex] = value[old_size-1];
-            }
+        for (vindex = old_size; vindex < size; vindex++) {
+            value[vindex] = value[old_size - 1];
+        }
     }
 
     return 0;
 }
 
 int32_t
-ec_dict_del_array (dict_t *dict, char *key, uint64_t value[], int32_t size)
+ec_dict_del_array(dict_t *dict, char *key, uint64_t value[], int32_t size)
 {
     int ret = 0;
 
-    ret = ec_dict_get_array (dict, key, value, size);
+    ret = ec_dict_get_array(dict, key, value, size);
     if (ret == 0)
-            dict_del(dict, key);
+        dict_del(dict, key);
 
     return ret;
 }
 
-
-int32_t ec_dict_set_number(dict_t * dict, char * key, uint64_t value)
+int32_t
+ec_dict_set_number(dict_t *dict, char *key, uint64_t value)
 {
-    int        ret = -1;
-    uint64_t * ptr;
+    int ret = -1;
+    uint64_t *ptr;
 
     ptr = GF_MALLOC(sizeof(value), gf_common_mt_char);
     if (ptr == NULL) {
@@ -276,14 +267,15 @@ int32_t ec_dict_set_number(dict_t * dict, char * key, uint64_t value)
 
     ret = dict_set_bin(dict, key, ptr, sizeof(value));
     if (ret)
-        GF_FREE (ptr);
+        GF_FREE(ptr);
 
     return ret;
 }
 
-int32_t ec_dict_del_number(dict_t * dict, char * key, uint64_t * value)
+int32_t
+ec_dict_del_number(dict_t *dict, char *key, uint64_t *value)
 {
-    void * ptr;
+    void *ptr;
     int32_t len, err;
 
     if (dict == NULL) {
@@ -304,24 +296,23 @@ int32_t ec_dict_del_number(dict_t * dict, char * key, uint64_t * value)
     return 0;
 }
 
-int32_t ec_dict_set_config(dict_t * dict, char * key, ec_config_t * config)
+int32_t
+ec_dict_set_config(dict_t *dict, char *key, ec_config_t *config)
 {
     int ret = -1;
-    uint64_t * ptr, data;
+    uint64_t *ptr, data;
 
-    if (config->version > EC_CONFIG_VERSION)
-    {
-        gf_msg ("ec", GF_LOG_ERROR, EINVAL,
-                EC_MSG_UNSUPPORTED_VERSION,
-                "Trying to store an unsupported config "
-                "version (%u)", config->version);
+    if (config->version > EC_CONFIG_VERSION) {
+        gf_msg("ec", GF_LOG_ERROR, EINVAL, EC_MSG_UNSUPPORTED_VERSION,
+               "Trying to store an unsupported config "
+               "version (%u)",
+               config->version);
 
         return -EINVAL;
     }
 
     ptr = GF_MALLOC(sizeof(uint64_t), gf_common_mt_char);
-    if (ptr == NULL)
-    {
+    if (ptr == NULL) {
         return -ENOMEM;
     }
 
@@ -336,14 +327,15 @@ int32_t ec_dict_set_config(dict_t * dict, char * key, ec_config_t * config)
 
     ret = dict_set_bin(dict, key, ptr, sizeof(uint64_t));
     if (ret)
-        GF_FREE (ptr);
+        GF_FREE(ptr);
 
     return ret;
 }
 
-int32_t ec_dict_del_config(dict_t * dict, char * key, ec_config_t * config)
+int32_t
+ec_dict_del_config(dict_t *dict, char *key, ec_config_t *config)
 {
-    void * ptr;
+    void *ptr;
     uint64_t data;
     int32_t len, err;
 
@@ -373,12 +365,9 @@ int32_t ec_dict_del_config(dict_t * dict, char * key, ec_config_t * config)
     }
 
     config->version = (data >> 56) & 0xff;
-    if (config->version > EC_CONFIG_VERSION)
-    {
-        gf_msg ("ec", GF_LOG_ERROR, EINVAL,
-                EC_MSG_UNSUPPORTED_VERSION,
-                "Found an unsupported config version (%u)",
-                config->version);
+    if (config->version > EC_CONFIG_VERSION) {
+        gf_msg("ec", GF_LOG_ERROR, EINVAL, EC_MSG_UNSUPPORTED_VERSION,
+               "Found an unsupported config version (%u)", config->version);
 
         return -EINVAL;
     }
@@ -394,7 +383,8 @@ int32_t ec_dict_del_config(dict_t * dict, char * key, ec_config_t * config)
     return 0;
 }
 
-gf_boolean_t ec_loc_gfid_check(xlator_t *xl, uuid_t dst, uuid_t src)
+gf_boolean_t
+ec_loc_gfid_check(xlator_t *xl, uuid_t dst, uuid_t src)
 {
     if (gf_uuid_is_null(src)) {
         return _gf_true;
@@ -407,9 +397,8 @@ gf_boolean_t ec_loc_gfid_check(xlator_t *xl, uuid_t dst, uuid_t src)
     }
 
     if (gf_uuid_compare(dst, src) != 0) {
-        gf_msg (xl->name, GF_LOG_WARNING, 0,
-                EC_MSG_GFID_MISMATCH,
-                "Mismatching GFID's in loc");
+        gf_msg(xl->name, GF_LOG_WARNING, 0, EC_MSG_GFID_MISMATCH,
+               "Mismatching GFID's in loc");
 
         return _gf_false;
     }
@@ -417,7 +406,8 @@ gf_boolean_t ec_loc_gfid_check(xlator_t *xl, uuid_t dst, uuid_t src)
     return _gf_true;
 }
 
-int32_t ec_loc_setup_inode(xlator_t *xl, inode_table_t *table, loc_t *loc)
+int32_t
+ec_loc_setup_inode(xlator_t *xl, inode_table_t *table, loc_t *loc)
 {
     int32_t ret = -EINVAL;
 
@@ -428,7 +418,7 @@ int32_t ec_loc_setup_inode(xlator_t *xl, inode_table_t *table, loc_t *loc)
     } else if (table != NULL) {
         if (!gf_uuid_is_null(loc->gfid)) {
             loc->inode = inode_find(table, loc->gfid);
-        } else if (loc->path && strchr (loc->path, '/')) {
+        } else if (loc->path && strchr(loc->path, '/')) {
             loc->inode = inode_resolve(table, (char *)loc->path);
         }
     }
@@ -439,7 +429,8 @@ out:
     return ret;
 }
 
-int32_t ec_loc_setup_parent(xlator_t *xl, inode_table_t *table, loc_t *loc)
+int32_t
+ec_loc_setup_parent(xlator_t *xl, inode_table_t *table, loc_t *loc)
 {
     char *path, *parent;
     int32_t ret = -EINVAL;
@@ -451,13 +442,11 @@ int32_t ec_loc_setup_parent(xlator_t *xl, inode_table_t *table, loc_t *loc)
     } else if (table != NULL) {
         if (!gf_uuid_is_null(loc->pargfid)) {
             loc->parent = inode_find(table, loc->pargfid);
-        } else if (loc->path && strchr (loc->path, '/')) {
+        } else if (loc->path && strchr(loc->path, '/')) {
             path = gf_strdup(loc->path);
             if (path == NULL) {
-                gf_msg (xl->name, GF_LOG_ERROR, ENOMEM,
-                        EC_MSG_NO_MEMORY,
-                        "Unable to duplicate path '%s'",
-                        loc->path);
+                gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                       "Unable to duplicate path '%s'", loc->path);
 
                 ret = -ENOMEM;
 
@@ -484,9 +473,10 @@ out:
     return ret;
 }
 
-int32_t ec_loc_setup_path(xlator_t *xl, loc_t *loc)
+int32_t
+ec_loc_setup_path(xlator_t *xl, loc_t *loc)
 {
-    uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    static uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
     char *name;
     int32_t ret = -EINVAL;
 
@@ -514,10 +504,8 @@ int32_t ec_loc_setup_path(xlator_t *xl, loc_t *loc)
 
         if (loc->name != NULL) {
             if (strcmp(loc->name, name) != 0) {
-                gf_msg (xl->name, GF_LOG_ERROR, EINVAL,
-                        EC_MSG_INVALID_LOC_NAME,
-                        "Invalid name '%s' in loc",
-                        loc->name);
+                gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_LOC_NAME,
+                       "Invalid name '%s' in loc", loc->name);
 
                 goto out;
             }
@@ -532,7 +520,8 @@ out:
     return ret;
 }
 
-int32_t ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent)
+int32_t
+ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent)
 {
     inode_table_t *table = NULL;
     char *str = NULL;
@@ -549,24 +538,20 @@ int32_t ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent)
     if (!gf_uuid_is_null(loc->pargfid)) {
         gf_uuid_copy(parent->gfid, loc->pargfid);
     }
-    if (loc->path && strchr (loc->path, '/')) {
+    if (loc->path && strchr(loc->path, '/')) {
         str = gf_strdup(loc->path);
         if (str == NULL) {
-                gf_msg (xl->name, GF_LOG_ERROR, ENOMEM,
-                        EC_MSG_NO_MEMORY,
-                        "Unable to duplicate path '%s'",
-                        loc->path);
+            gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Unable to duplicate path '%s'", loc->path);
 
-                goto out;
+            goto out;
         }
         parent->path = gf_strdup(dirname(str));
         if (parent->path == NULL) {
-                gf_msg (xl->name, GF_LOG_ERROR, ENOMEM,
-                        EC_MSG_NO_MEMORY,
-                        "Unable to duplicate path '%s'",
-                        dirname(str));
+            gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Unable to duplicate path '%s'", dirname(str));
 
-                goto out;
+            goto out;
         }
     }
 
@@ -583,9 +568,8 @@ int32_t ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent)
 
     if ((parent->inode == NULL) && (parent->path == NULL) &&
         gf_uuid_is_null(parent->gfid)) {
-        gf_msg (xl->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_LOC_PARENT_INODE_MISSING,
-                "Parent inode missing for loc_t");
+        gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_LOC_PARENT_INODE_MISSING,
+               "Parent inode missing for loc_t");
 
         ret = -EINVAL;
 
@@ -604,8 +588,8 @@ out:
     return ret;
 }
 
-int32_t ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode,
-                      struct iatt *iatt)
+int32_t
+ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode, struct iatt *iatt)
 {
     inode_table_t *table = NULL;
     int32_t ret = -EINVAL;
@@ -646,9 +630,10 @@ out:
     return ret;
 }
 
-int32_t ec_loc_from_fd(xlator_t * xl, loc_t * loc, fd_t * fd)
+int32_t
+ec_loc_from_fd(xlator_t *xl, loc_t *loc, fd_t *fd)
 {
-    ec_fd_t * ctx;
+    ec_fd_t *ctx;
     int32_t ret = -ENOMEM;
 
     memset(loc, 0, sizeof(*loc));
@@ -673,7 +658,8 @@ out:
     return ret;
 }
 
-int32_t ec_loc_from_loc(xlator_t * xl, loc_t * dst, loc_t * src)
+int32_t
+ec_loc_from_loc(xlator_t *xl, loc_t *dst, loc_t *src)
 {
     int32_t ret = -ENOMEM;
 
@@ -696,62 +682,62 @@ out:
     return ret;
 }
 
-void ec_owner_set(call_frame_t * frame, void * owner)
+void
+ec_owner_set(call_frame_t *frame, void *owner)
 {
     set_lk_owner_from_ptr(&frame->root->lk_owner, owner);
 }
 
-void ec_owner_copy(call_frame_t *frame, gf_lkowner_t *owner)
+void
+ec_owner_copy(call_frame_t *frame, gf_lkowner_t *owner)
 {
-    lk_owner_copy (&frame->root->lk_owner, owner);
+    lk_owner_copy(&frame->root->lk_owner, owner);
 }
 
 static void
-ec_stripe_cache_init (ec_t *ec, ec_inode_t *ctx)
+ec_stripe_cache_init(ec_t *ec, ec_inode_t *ctx)
 {
-        ec_stripe_list_t *stripe_cache = NULL;
+    ec_stripe_list_t *stripe_cache = NULL;
 
-        stripe_cache = &(ctx->stripe_cache);
-        if (stripe_cache->max == 0) {
-                stripe_cache->max = ec->stripe_cache;
-        }
+    stripe_cache = &(ctx->stripe_cache);
+    if (stripe_cache->max == 0) {
+        stripe_cache->max = ec->stripe_cache;
+    }
 }
 
-ec_inode_t * __ec_inode_get(inode_t * inode, xlator_t * xl)
+ec_inode_t *
+__ec_inode_get(inode_t *inode, xlator_t *xl)
 {
-    ec_inode_t * ctx = NULL;
+    ec_inode_t *ctx = NULL;
     uint64_t value = 0;
 
-    if ((__inode_ctx_get(inode, xl, &value) != 0) || (value == 0))
-    {
+    if ((__inode_ctx_get(inode, xl, &value) != 0) || (value == 0)) {
         ctx = GF_MALLOC(sizeof(*ctx), ec_mt_ec_inode_t);
-        if (ctx != NULL)
-        {
+        if (ctx != NULL) {
             memset(ctx, 0, sizeof(*ctx));
             INIT_LIST_HEAD(&ctx->heal);
             INIT_LIST_HEAD(&ctx->stripe_cache.lru);
+            ctx->heal_count = 0;
             value = (uint64_t)(uintptr_t)ctx;
-            if (__inode_ctx_set(inode, xl, &value) != 0)
-            {
+            if (__inode_ctx_set(inode, xl, &value) != 0) {
                 GF_FREE(ctx);
 
                 return NULL;
             }
         }
-    }
-    else
-    {
+    } else {
         ctx = (ec_inode_t *)(uintptr_t)value;
     }
     if (ctx)
-        ec_stripe_cache_init (xl->private, ctx);
+        ec_stripe_cache_init(xl->private, ctx);
 
     return ctx;
 }
 
-ec_inode_t * ec_inode_get(inode_t * inode, xlator_t * xl)
+ec_inode_t *
+ec_inode_get(inode_t *inode, xlator_t *xl)
 {
-    ec_inode_t * ctx = NULL;
+    ec_inode_t *ctx = NULL;
 
     LOCK(&inode->lock);
 
@@ -762,32 +748,40 @@ ec_inode_t * ec_inode_get(inode_t * inode, xlator_t * xl)
     return ctx;
 }
 
-ec_fd_t * __ec_fd_get(fd_t * fd, xlator_t * xl)
+ec_fd_t *
+__ec_fd_get(fd_t *fd, xlator_t *xl)
 {
     int i = 0;
-    ec_fd_t * ctx = NULL;
+    ec_fd_t *ctx = NULL;
+    ec_inode_t *ictx = NULL;
     uint64_t value = 0;
     ec_t *ec = xl->private;
 
     if ((__fd_ctx_get(fd, xl, &value) != 0) || (value == 0)) {
-        ctx = GF_MALLOC(sizeof(*ctx) + (sizeof (ec_fd_status_t) * ec->nodes),
+        ctx = GF_MALLOC(sizeof(*ctx) + (sizeof(ec_fd_status_t) * ec->nodes),
                         ec_mt_ec_fd_t);
         if (ctx != NULL) {
             memset(ctx, 0, sizeof(*ctx));
 
             for (i = 0; i < ec->nodes; i++) {
-                if (fd_is_anonymous (fd)) {
-                        ctx->fd_status[i] = EC_FD_OPENED;
+                if (fd_is_anonymous(fd)) {
+                    ctx->fd_status[i] = EC_FD_OPENED;
                 } else {
-                        ctx->fd_status[i] = EC_FD_NOT_OPENED;
+                    ctx->fd_status[i] = EC_FD_NOT_OPENED;
                 }
             }
 
             value = (uint64_t)(uintptr_t)ctx;
             if (__fd_ctx_set(fd, xl, value) != 0) {
-                GF_FREE (ctx);
+                GF_FREE(ctx);
                 return NULL;
             }
+            /* Only refering bad-version so no need for lock
+             * */
+            ictx = __ec_inode_get(fd->inode, xl);
+            if (ictx) {
+                ctx->bad_version = ictx->bad_version;
+            }
         }
     } else {
         ctx = (ec_fd_t *)(uintptr_t)value;
@@ -804,9 +798,10 @@ ec_fd_t * __ec_fd_get(fd_t * fd, xlator_t * xl)
     return ctx;
 }
 
-ec_fd_t * ec_fd_get(fd_t * fd, xlator_t * xl)
+ec_fd_t *
+ec_fd_get(fd_t *fd, xlator_t *xl)
 {
-    ec_fd_t * ctx = NULL;
+    ec_fd_t *ctx = NULL;
 
     LOCK(&fd->lock);
 
@@ -818,37 +813,36 @@ ec_fd_t * ec_fd_get(fd_t * fd, xlator_t * xl)
 }
 
 gf_boolean_t
-ec_is_internal_xattr (dict_t *dict, char *key, data_t *value, void *data)
+ec_is_internal_xattr(dict_t *dict, char *key, data_t *value, void *data)
 {
-        if (key &&
-            (strncmp (key, EC_XATTR_PREFIX, strlen (EC_XATTR_PREFIX)) == 0))
-                return _gf_true;
+    if (key && (strncmp(key, EC_XATTR_PREFIX, SLEN(EC_XATTR_PREFIX)) == 0))
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 void
-ec_filter_internal_xattrs (dict_t *xattr)
+ec_filter_internal_xattrs(dict_t *xattr)
 {
-        dict_foreach_match (xattr, ec_is_internal_xattr, NULL,
-                            dict_remove_foreach_fn, NULL);
+    dict_foreach_match(xattr, ec_is_internal_xattr, NULL,
+                       dict_remove_foreach_fn, NULL);
 }
 
 gf_boolean_t
-ec_is_data_fop (glusterfs_fop_t fop)
+ec_is_data_fop(glusterfs_fop_t fop)
 {
-        switch (fop) {
+    switch (fop) {
         case GF_FOP_WRITE:
         case GF_FOP_TRUNCATE:
         case GF_FOP_FTRUNCATE:
         case GF_FOP_FALLOCATE:
         case GF_FOP_DISCARD:
         case GF_FOP_ZEROFILL:
-                return _gf_true;
+            return _gf_true;
         default:
-                return _gf_false;
-        }
-        return _gf_false;
+            return _gf_false;
+    }
+    return _gf_false;
 }
 /*
 gf_boolean_t
diff --git a/xlators/cluster/ec/src/ec-helpers.h b/xlators/cluster/ec/src/ec-helpers.h
index 71b8978abf8..015db514e05 100644
--- a/xlators/cluster/ec/src/ec-helpers.h
+++ b/xlators/cluster/ec/src/ec-helpers.h
@@ -17,60 +17,77 @@
 #define EC_IS_ERR(_x) (((uintptr_t)(_x) & ~0xfffULL) == ~0xfffULL)
 #define EC_GET_ERR(_x) ((int32_t)(intptr_t)(_x))
 
-#define EC_ALIGN_CHECK(_ptr, _align) \
-    ((((uintptr_t)(_ptr)) & ((_align) - 1)) == 0)
-
-const char * ec_bin(char * str, size_t size, uint64_t value, int32_t digits);
-const char * ec_fop_name(int32_t id);
-void ec_trace(const char * event, ec_fop_data_t * fop, const char * fmt, ...);
-int32_t ec_bits_consume(uint64_t * n);
-size_t ec_iov_copy_to(void * dst, struct iovec * vector, int32_t count,
-                      off_t offset, size_t size);
-int32_t ec_buffer_alloc(xlator_t *xl, size_t size, struct iobref **piobref,
-                        void **ptr);
-int32_t ec_dict_set_array(dict_t *dict, char *key,
-                          uint64_t *value, int32_t size);
-int32_t ec_dict_get_array (dict_t *dict, char *key, uint64_t value[],
-                           int32_t size);
-
-int32_t ec_dict_del_array(dict_t *dict, char *key,
-                          uint64_t *value, int32_t size);
-int32_t ec_dict_set_number(dict_t * dict, char * key, uint64_t value);
-int32_t ec_dict_del_number(dict_t * dict, char * key, uint64_t * value);
-int32_t ec_dict_set_config(dict_t * dict, char * key, ec_config_t * config);
-int32_t ec_dict_del_config(dict_t * dict, char * key, ec_config_t * config);
-
-int32_t ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent);
-int32_t ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode,
-                      struct iatt *iatt);
-
-int32_t ec_loc_from_fd(xlator_t * xl, loc_t * loc, fd_t * fd);
-int32_t ec_loc_from_loc(xlator_t * xl, loc_t * dst, loc_t * src);
-
-void ec_owner_set(call_frame_t * frame, void * owner);
-void ec_owner_copy(call_frame_t * frame, gf_lkowner_t * owner);
-
-ec_inode_t * __ec_inode_get(inode_t * inode, xlator_t * xl);
-ec_inode_t * ec_inode_get(inode_t * inode, xlator_t * xl);
-ec_fd_t * __ec_fd_get(fd_t * fd, xlator_t * xl);
-ec_fd_t * ec_fd_get(fd_t * fd, xlator_t * xl);
+#define EC_ALIGN_CHECK(_ptr, _align) ((((uintptr_t)(_ptr)) & ((_align)-1)) == 0)
+
+const char *
+ec_bin(char *str, size_t size, uint64_t value, int32_t digits);
+const char *
+ec_fop_name(int32_t id);
+void
+ec_trace(const char *event, ec_fop_data_t *fop, const char *fmt, ...);
+int32_t
+ec_bits_consume(uint64_t *n);
+size_t
+ec_iov_copy_to(void *dst, struct iovec *vector, int32_t count, off_t offset,
+               size_t size);
+int32_t
+ec_buffer_alloc(xlator_t *xl, size_t size, struct iobref **piobref, void **ptr);
+int32_t
+ec_dict_set_array(dict_t *dict, char *key, uint64_t *value, int32_t size);
+int32_t
+ec_dict_get_array(dict_t *dict, char *key, uint64_t value[], int32_t size);
+
+int32_t
+ec_dict_del_array(dict_t *dict, char *key, uint64_t *value, int32_t size);
+int32_t
+ec_dict_set_number(dict_t *dict, char *key, uint64_t value);
+int32_t
+ec_dict_del_number(dict_t *dict, char *key, uint64_t *value);
+int32_t
+ec_dict_set_config(dict_t *dict, char *key, ec_config_t *config);
+int32_t
+ec_dict_del_config(dict_t *dict, char *key, ec_config_t *config);
+
+int32_t
+ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent);
+int32_t
+ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode, struct iatt *iatt);
+
+int32_t
+ec_loc_from_fd(xlator_t *xl, loc_t *loc, fd_t *fd);
+int32_t
+ec_loc_from_loc(xlator_t *xl, loc_t *dst, loc_t *src);
+
+void
+ec_owner_set(call_frame_t *frame, void *owner);
+void
+ec_owner_copy(call_frame_t *frame, gf_lkowner_t *owner);
+
+ec_inode_t *
+__ec_inode_get(inode_t *inode, xlator_t *xl);
+ec_inode_t *
+ec_inode_get(inode_t *inode, xlator_t *xl);
+ec_fd_t *
+__ec_fd_get(fd_t *fd, xlator_t *xl);
+ec_fd_t *
+ec_fd_get(fd_t *fd, xlator_t *xl);
 
 static inline uint32_t
 ec_adjust_size_down(ec_t *ec, uint64_t *value, gf_boolean_t scale)
 {
-        uint64_t head, tmp;
+    uint64_t head, tmp;
 
-        tmp = *value;
-        head = tmp % ec->stripe_size;
-        tmp -= head;
+    tmp = *value;
+    head = tmp % ec->stripe_size;
+    tmp -= head;
 
-        if (scale) {
-                tmp /= ec->fragments;
-        }
+    if (scale) {
+        tmp /= ec->fragments;
+    }
 
-        *value = tmp;
+    *value = tmp;
 
-        return (uint32_t)head;
+    return (uint32_t)head;
 }
 
 /* This function can cause an overflow if the passed value is too near to the
@@ -79,35 +96,35 @@ ec_adjust_size_down(ec_t *ec, uint64_t *value, gf_boolean_t scale)
 static inline int32_t
 ec_adjust_size_up(ec_t *ec, uint64_t *value, gf_boolean_t scale)
 {
-        uint64_t tmp;
-        int32_t tail;
-
-        tmp = *value;
-        /* We first adjust the value down. This never causes overflow. */
-        tail = ec_adjust_size_down(ec, &tmp, scale);
-
-        /* If the value was already aligned, tail will be 0 and nothing else
-         * needs to be done. */
-        if (tail != 0) {
-                /* Otherwise, we need to compute the real tail and adjust the
-                 * returned value to the next stripe. */
-                tail = ec->stripe_size - tail;
-                if (scale) {
-                        tmp += ec->fragment_size;
-                } else {
-                        tmp += ec->stripe_size;
-                        /* If no scaling is requested there's a posibility of
-                         * overflow. */
-                        if (tmp < ec->stripe_size) {
-                                tmp = UINT64_MAX;
-                                tail = -tail;
-                        }
-                }
+    uint64_t tmp;
+    int32_t tail;
+
+    tmp = *value;
+    /* We first adjust the value down. This never causes overflow. */
+    tail = ec_adjust_size_down(ec, &tmp, scale);
+
+    /* If the value was already aligned, tail will be 0 and nothing else
+     * needs to be done. */
+    if (tail != 0) {
+        /* Otherwise, we need to compute the real tail and adjust the
+         * returned value to the next stripe. */
+        tail = ec->stripe_size - tail;
+        if (scale) {
+            tmp += ec->fragment_size;
+        } else {
+            tmp += ec->stripe_size;
+            /* If no scaling is requested there's a possibility of
+             * overflow. */
+            if (tmp < ec->stripe_size) {
+                tmp = UINT64_MAX;
+                tail = -tail;
+            }
         }
+    }
 
-        *value = tmp;
+    *value = tmp;
 
-        return tail;
+    return tail;
 }
 
 /* This function is equivalent to ec_adjust_size_down() but with a potentially
@@ -115,19 +132,19 @@ ec_adjust_size_up(ec_t *ec, uint64_t *value, gf_boolean_t scale)
 static inline uint32_t
 ec_adjust_offset_down(ec_t *ec, off_t *value, gf_boolean_t scale)
 {
-        off_t head, tmp;
+    off_t head, tmp;
 
-        tmp = *value;
-        head = tmp % ec->stripe_size;
-        tmp -= head;
+    tmp = *value;
+    head = tmp % ec->stripe_size;
+    tmp -= head;
 
-        if (scale) {
-                tmp /= ec->fragments;
-        }
+    if (scale) {
+        tmp /= ec->fragments;
+    }
 
-        *value = tmp;
+    *value = tmp;
 
-        return (uint32_t)head;
+    return (uint32_t)head;
 }
 
 /* This function is equivalent to ec_adjust_size_up() but with a potentially
@@ -135,48 +152,49 @@ ec_adjust_offset_down(ec_t *ec, off_t *value, gf_boolean_t scale)
 static inline int32_t
 ec_adjust_offset_up(ec_t *ec, off_t *value, gf_boolean_t scale)
 {
-        uint64_t tail, tmp;
-
-        /* An offset is a signed type that can only have positive values, so
-         * we take advantage of this to avoid overflows. We simply convert it
-         * to an unsigned integer and operate normally. This won't cause an
-         * overflow. Overflow is only checked when converting back to an
-         * off_t. */
-        tmp = *value;
-        tail = ec->stripe_size;
-        tail -= (tmp + tail - 1) % tail + 1;
-        tmp += tail;
-        if (scale) {
-                /* If we are scaling, we'll never get an overflow. */
-                tmp /= ec->fragments;
-        } else {
-                /* Check if there has been an overflow. */
-                if ((off_t)tmp < 0) {
-                        tmp = GF_OFF_MAX;
-                        tail = -tail;
-                }
+    uint64_t tail, tmp;
+
+    /* An offset is a signed type that can only have positive values, so
+     * we take advantage of this to avoid overflows. We simply convert it
+     * to an unsigned integer and operate normally. This won't cause an
+     * overflow. Overflow is only checked when converting back to an
+     * off_t. */
+    tmp = *value;
+    tail = ec->stripe_size;
+    tail -= (tmp + tail - 1) % tail + 1;
+    tmp += tail;
+    if (scale) {
+        /* If we are scaling, we'll never get an overflow. */
+        tmp /= ec->fragments;
+    } else {
+        /* Check if there has been an overflow. */
+        if ((off_t)tmp < 0) {
+            tmp = GF_OFF_MAX;
+            tail = -tail;
         }
+    }
 
-        *value = (off_t)tmp;
+    *value = (off_t)tmp;
 
-        return (int32_t)tail;
+    return (int32_t)tail;
 }
 
-static inline int32_t ec_is_power_of_2(uint32_t value)
+static inline int32_t
+ec_is_power_of_2(uint32_t value)
 {
     return (value != 0) && ((value & (value - 1)) == 0);
 }
 
 gf_boolean_t
-ec_is_internal_xattr (dict_t *dict, char *key, data_t *value, void *data);
+ec_is_internal_xattr(dict_t *dict, char *key, data_t *value, void *data);
 
 void
-ec_filter_internal_xattrs (dict_t *xattr);
+ec_filter_internal_xattrs(dict_t *xattr);
 
 gf_boolean_t
-ec_is_data_fop (glusterfs_fop_t fop);
+ec_is_data_fop(glusterfs_fop_t fop);
 
 int32_t
-ec_launch_replace_heal (ec_t *ec);
+ec_launch_replace_heal(ec_t *ec);
 
 #endif /* __EC_HELPERS_H__ */
diff --git a/xlators/cluster/ec/src/ec-inode-read.c b/xlators/cluster/ec/src/ec-inode-read.c
index d58ed9e5795..dad5f4d7018 100644
--- a/xlators/cluster/ec/src/ec-inode-read.c
+++ b/xlators/cluster/ec/src/ec-inode-read.c
@@ -8,9 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
 #include "ec.h"
 #include "ec-messages.h"
 #include "ec-helpers.h"
@@ -21,8 +18,9 @@
 
 /* FOP: access */
 
-int32_t ec_access_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, dict_t *xdata)
 {
     ec_fop_data_t *fop = NULL;
     ec_cbk_data_t *cbk = NULL;
@@ -35,27 +33,27 @@ int32_t ec_access_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-    cbk = ec_cbk_data_allocate (frame, this, fop, GF_FOP_ACCESS,
-                                idx, op_ret, op_errno);
+    cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_ACCESS, idx, op_ret,
+                               op_errno);
     if (cbk) {
         if (xdata)
-               cbk->xdata = dict_ref (xdata);
-        ec_combine (cbk, NULL);
+            cbk->xdata = dict_ref(xdata);
+        ec_combine(cbk, NULL);
     }
 
 out:
-    if (fop != NULL)
-    {
-        ec_complete (fop);
+    if (fop != NULL) {
+        ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_access(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_access(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -67,19 +65,19 @@ void ec_wind_access(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 int32_t
 ec_manager_access(ec_fop_data_t *fop, int32_t state)
 {
-        ec_cbk_data_t *cbk = NULL;
+    ec_cbk_data_t *cbk = NULL;
 
-        switch (state) {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_inode (fop, &fop->loc[0], EC_QUERY_INFO, 0,
-                                   LLONG_MAX);
-            ec_lock (fop);
+            ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
+                                  EC_RANGE_FULL);
+            ec_lock(fop);
 
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
-            ec_dispatch_one (fop);
+            ec_dispatch_one(fop);
 
             return EC_STATE_PREPARE_ANSWER;
 
@@ -92,12 +90,11 @@ ec_manager_access(ec_fop_data_t *fop, int32_t state)
 
         case EC_STATE_REPORT:
             cbk = fop->answer;
-            GF_ASSERT (cbk);
+            GF_ASSERT(cbk);
             if (fop->cbks.access != NULL) {
                 if (cbk) {
-                    fop->cbks.access(fop->req_frame, fop, fop->xl,
-                                     cbk->op_ret, cbk->op_errno,
-                                     cbk->xdata);
+                    fop->cbks.access(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                     cbk->op_errno, cbk->xdata);
                 }
             }
             return EC_STATE_LOCK_REUSE;
@@ -108,8 +105,8 @@ ec_manager_access(ec_fop_data_t *fop, int32_t state)
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             if (fop->cbks.access != NULL) {
-                fop->cbks.access(fop->req_frame, fop, fop->xl, -1,
-                                 fop->error, NULL);
+                fop->cbks.access(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                 NULL);
             }
             return EC_STATE_LOCK_REUSE;
 
@@ -126,31 +123,30 @@ ec_manager_access(ec_fop_data_t *fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
-        }
+    }
 }
 
-void ec_access(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_access_cbk_t func, void * data,
-               loc_t * loc, int32_t mask, dict_t * xdata)
+void
+ec_access(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_access_cbk_t func, void *data, loc_t *loc,
+          int32_t mask, dict_t *xdata)
 {
-    ec_cbk_t callback = { .access = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.access = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(ACCESS) %p", frame);
+    gf_msg_trace("ec", 0, "EC(ACCESS) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_ACCESS, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_access,
+                               target, fop_flags, ec_wind_access,
                                ec_manager_access, callback, data);
     if (fop == NULL) {
         goto out;
@@ -160,9 +156,8 @@ void ec_access(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -170,10 +165,9 @@ void ec_access(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -191,14 +185,13 @@ out:
 
 /* FOP: getxattr */
 
-int32_t ec_combine_getxattr(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                            ec_cbk_data_t * src)
+int32_t
+ec_combine_getxattr(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_dict_compare(dst->dict, src->dict))
-    {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_DICT_MISMATCH, "Mismatching dictionary in "
-                                             "answers of 'GF_FOP_GETXATTR'");
+    if (!ec_dict_compare(dst->dict, src->dict)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_DICT_MISMATCH,
+               "Mismatching dictionary in "
+               "answers of 'GF_FOP_GETXATTR'");
 
         return 0;
     }
@@ -206,12 +199,12 @@ int32_t ec_combine_getxattr(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_getxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno, dict_t * dict,
-                        dict_t * xdata)
+int32_t
+ec_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -221,38 +214,30 @@ int32_t ec_getxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_GETXATTR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (dict != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (dict != NULL) {
                 cbk->dict = dict_ref(dict);
-                if (cbk->dict == NULL)
-                {
-                    gf_msg (this->name, GF_LOG_ERROR, 0,
-                            EC_MSG_DICT_REF_FAIL,
-                            "Failed to reference a "
-                            "dictionary.");
+                if (cbk->dict == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                           "Failed to reference a "
+                           "dictionary.");
 
                     goto out;
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                        "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -262,15 +247,15 @@ int32_t ec_getxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_getxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_getxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -280,43 +265,43 @@ void ec_wind_getxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 }
 
 void
-ec_handle_special_xattrs (ec_fop_data_t *fop)
+ec_handle_special_xattrs(ec_fop_data_t *fop)
 {
-        ec_cbk_data_t *cbk = NULL;
-        /* Stime may not be available on all the bricks, so even if some of the
-         * subvols succeed the operation, treat it as answer.*/
-        if (fop->str[0] &&
-            fnmatch (GF_XATTR_STIME_PATTERN, fop->str[0], 0) == 0) {
-                if (!fop->answer || (fop->answer->op_ret < 0)) {
-                        list_for_each_entry (cbk, &fop->cbk_list, list) {
-                                if (cbk->op_ret >= 0) {
-                                        fop->answer = cbk;
-                                        break;
-                                }
-                        }
+    ec_cbk_data_t *cbk = NULL;
+    /* Stime may not be available on all the bricks, so even if some of the
+     * subvols succeed the operation, treat it as answer.*/
+    if (fop->str[0] && fnmatch(GF_XATTR_STIME_PATTERN, fop->str[0], 0) == 0) {
+        if (!fop->answer || (fop->answer->op_ret < 0)) {
+            list_for_each_entry(cbk, &fop->cbk_list, list)
+            {
+                if (cbk->op_ret >= 0) {
+                    fop->answer = cbk;
+                    break;
                 }
+            }
         }
+    }
 }
 
-int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_getxattr(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
             /* clear-locks commands must be done without any locks acquired
                to avoid interferences. */
             if ((fop->str[0] == NULL) ||
                 (strncmp(fop->str[0], GF_XATTR_CLRLK_CMD,
-                         strlen(GF_XATTR_CLRLK_CMD)) != 0)) {
+                         SLEN(GF_XATTR_CLRLK_CMD)) != 0)) {
                 if (fop->fd == NULL) {
-                    ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO,
-                                          0, LLONG_MAX);
+                    ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
+                                          EC_RANGE_FULL);
                 } else {
                     ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0,
-                                       LLONG_MAX);
+                                       EC_RANGE_FULL);
                 }
                 ec_lock(fop);
             }
@@ -333,11 +318,11 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            ec_handle_special_xattrs (fop);
+            ec_handle_special_xattrs(fop);
             if (fop->minimum == EC_MINIMUM_ALL) {
                 cbk = ec_fop_prepare_answer(fop, _gf_true);
             } else {
-                if (ec_dispatch_one_retry (fop, &cbk)) {
+                if (ec_dispatch_one_retry(fop, &cbk)) {
                     return EC_STATE_DISPATCH;
                 }
             }
@@ -347,10 +332,10 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
                 err = ec_dict_combine(cbk, EC_COMBINE_DICT);
                 if (!ec_cbk_set_error(cbk, -err, _gf_true)) {
                     if (cbk->xdata != NULL)
-                            ec_filter_internal_xattrs (cbk->xdata);
+                        ec_filter_internal_xattrs(cbk->xdata);
 
                     if (cbk->dict != NULL)
-                            ec_filter_internal_xattrs (cbk->dict);
+                        ec_filter_internal_xattrs(cbk->dict);
                 }
             }
 
@@ -361,8 +346,7 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.getxattr != NULL)
-            {
+            if (fop->cbks.getxattr != NULL) {
                 fop->cbks.getxattr(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                    cbk->op_errno, cbk->dict, cbk->xdata);
             }
@@ -376,10 +360,9 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.getxattr != NULL)
-            {
-                fop->cbks.getxattr(fop->req_frame, fop, fop->xl, -1,
-                                   fop->error, NULL, NULL);
+            if (fop->cbks.getxattr != NULL) {
+                fop->cbks.getxattr(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                   NULL, NULL);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -397,26 +380,44 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-int32_t ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl,
-                             int32_t op_ret, int32_t op_errno, uintptr_t mask,
-                             uintptr_t good, uintptr_t bad, dict_t *xdata)
+int32_t
+ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl,
+                     int32_t op_ret, int32_t op_errno, uintptr_t mask,
+                     uintptr_t good, uintptr_t bad, uint32_t pending,
+                     dict_t *xdata)
 {
-    ec_fop_data_t *fop = cookie;
-    fop_getxattr_cbk_t func = fop->data;
+    fop_getxattr_cbk_t func = cookie;
     ec_t *ec = xl->private;
     dict_t *dict = NULL;
     char *str;
     char bin1[65], bin2[65];
 
+    /* We try to return the 'pending' information in xdata, but if this cannot
+     * be set, we will ignore it silently. We prefer to report the success or
+     * failure of the heal itself. */
+    if (xdata == NULL) {
+        xdata = dict_new();
+    } else {
+        dict_ref(xdata);
+    }
+    if (xdata != NULL) {
+        if (dict_set_uint32(xdata, EC_XATTR_HEAL_NEW, pending) != 0) {
+            /* dict_set_uint32() is marked as 'warn_unused_result' and gcc
+             * enforces to check the result in this case. However we don't
+             * really care if it succeeded or not. We'll just do the same.
+             *
+             * This empty 'if' avoids the warning, and it will be removed by
+             * the optimizer. */
+        }
+    }
+
     if (op_ret >= 0) {
         dict = dict_new();
         if (dict == NULL) {
@@ -450,25 +451,28 @@ int32_t ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl,
     }
 
 out:
-    func(frame, NULL, xl, op_ret, op_errno, dict, NULL);
+    func(frame, NULL, xl, op_ret, op_errno, dict, xdata);
 
     if (dict != NULL) {
         dict_unref(dict);
     }
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
 
     return 0;
 }
 
 void
-ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-             int32_t minimum, fop_getxattr_cbk_t func, void *data,
-             loc_t *loc, const char *name, dict_t *xdata)
+ec_getxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_getxattr_cbk_t func, void *data, loc_t *loc,
+            const char *name, dict_t *xdata)
 {
-    ec_cbk_t callback = { .getxattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.getxattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(GETXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(GETXATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
@@ -476,25 +480,23 @@ ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
     /* Special handling of an explicit self-heal request */
     if ((name != NULL) && (strcmp(name, EC_XATTR_HEAL) == 0)) {
-        ec_heal(frame, this, target, EC_MINIMUM_ONE, ec_getxattr_heal_cbk,
-                func, loc, 0, NULL);
+        ec_heal(frame, this, target, EC_MINIMUM_ONE, ec_getxattr_heal_cbk, func,
+                loc, 0, NULL);
 
         return;
     }
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_GETXATTR,
-                               EC_FLAG_LOCK_SHARED, target, minimum,
-                               ec_wind_getxattr, ec_manager_getxattr, callback,
-                               data);
+    fop = ec_fop_data_allocate(
+        frame, this, GF_FOP_GETXATTR, EC_FLAG_LOCK_SHARED, target, fop_flags,
+        ec_wind_getxattr, ec_manager_getxattr, callback, data);
     if (fop == NULL) {
         goto out;
     }
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -503,15 +505,14 @@ ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
         /* In case of list-node-uuids xattr, set flag to indicate
          * the same and use node-uuid xattr for winding fop */
         if (XATTR_IS_NODE_UUID_LIST(name)) {
-                fop->int32 = 1;
-                fop->str[0] = gf_strdup(GF_XATTR_NODE_UUID_KEY);
+            fop->int32 = 1;
+            fop->str[0] = gf_strdup(GF_XATTR_NODE_UUID_KEY);
         } else {
-                fop->str[0] = gf_strdup(name);
+            fop->str[0] = gf_strdup(name);
         }
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
@@ -519,10 +520,9 @@ ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -532,20 +532,20 @@ ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL, NULL);
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
 
 /* FOP: fgetxattr */
 
-int32_t ec_fgetxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                         int32_t op_ret, int32_t op_errno, dict_t * dict,
-                         dict_t * xdata)
+int32_t
+ec_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -555,38 +555,30 @@ int32_t ec_fgetxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FGETXATTR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (dict != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (dict != NULL) {
                 cbk->dict = dict_ref(dict);
-                if (cbk->dict == NULL)
-                {
-                    gf_msg (this->name, GF_LOG_ERROR, 0,
-                            EC_MSG_DICT_REF_FAIL,
-                            "Failed to reference a "
-                            "dictionary.");
+                if (cbk->dict == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                           "Failed to reference a "
+                           "dictionary.");
 
                     goto out;
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                        "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -596,8 +588,7 @@ int32_t ec_fgetxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
@@ -605,7 +596,7 @@ out:
 }
 
 void
-ec_wind_fgetxattr (ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+ec_wind_fgetxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -615,24 +606,23 @@ ec_wind_fgetxattr (ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 }
 
 void
-ec_fgetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-              int32_t minimum, fop_fgetxattr_cbk_t func, void *data,
-              fd_t *fd, const char *name, dict_t *xdata)
+ec_fgetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fgetxattr_cbk_t func, void *data, fd_t *fd,
+             const char *name, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fgetxattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fgetxattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FGETXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FGETXATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FGETXATTR,
-                               EC_FLAG_LOCK_SHARED, target, minimum,
-                               ec_wind_fgetxattr, ec_manager_getxattr,
-                               callback, data);
+    fop = ec_fop_data_allocate(
+        frame, this, GF_FOP_FGETXATTR, EC_FLAG_LOCK_SHARED, target, fop_flags,
+        ec_wind_fgetxattr, ec_manager_getxattr, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -642,10 +632,9 @@ ec_fgetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -653,8 +642,8 @@ ec_fgetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (name != NULL) {
         fop->str[0] = gf_strdup(name);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY, "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
@@ -662,9 +651,9 @@ ec_fgetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -674,22 +663,21 @@ ec_fgetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL, NULL);
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
 
 /* FOP: open */
 
-int32_t ec_combine_open(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                        ec_cbk_data_t * src)
+int32_t
+ec_combine_open(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (dst->fd != src->fd)
-    {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_FD_MISMATCH, "Mismatching fd in answers "
-                                             "of 'GF_FOP_OPEN': %p <-> %p",
+    if (dst->fd != src->fd) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_FD_MISMATCH,
+               "Mismatching fd in answers "
+               "of 'GF_FOP_OPEN': %p <-> %p",
                dst->fd, src->fd);
 
         return 0;
@@ -698,12 +686,12 @@ int32_t ec_combine_open(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_open_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                    int32_t op_ret, int32_t op_errno, fd_t * fd,
-                    dict_t * xdata)
+int32_t
+ec_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -713,36 +701,31 @@ int32_t ec_open_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_OPEN, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (fd != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (fd != NULL) {
                 cbk->fd = fd_ref(fd);
-                if (cbk->fd == NULL)
-                {
-                    gf_msg (this->name, GF_LOG_ERROR, 0,
-                            EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                                     "file descriptor.");
+                if (cbk->fd == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           EC_MSG_FILE_DESC_REF_FAIL,
+                           "Failed to reference a "
+                           "file descriptor.");
 
                     goto out;
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -750,20 +733,19 @@ int32_t ec_open_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
         ec_combine(cbk, ec_combine_open);
 
-        ec_update_fd_status (fd, this, idx, op_ret);
-
+        ec_update_fd_status(fd, this, idx, op_ret);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_open(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_open(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -772,10 +754,10 @@ void ec_wind_open(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->int32, fop->fd, fop->xdata);
 }
 
-int32_t ec_open_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             struct iatt *prebuf, struct iatt *postbuf,
-                             dict_t *xdata)
+int32_t
+ec_open_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
     ec_fop_data_t *fop = cookie;
     int32_t error = 0;
@@ -792,14 +774,14 @@ int32_t ec_open_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
     return 0;
 }
 
-int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_open(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
-    ec_fd_t * ctx;
+    ec_cbk_data_t *cbk;
+    ec_fd_t *ctx;
     int32_t err;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             LOCK(&fop->fd->lock);
 
@@ -811,13 +793,15 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
 
                 return EC_STATE_REPORT;
             }
-            err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]);
-            if (err != 0) {
-                UNLOCK(&fop->fd->lock);
+            if (!ctx->loc.inode) {
+                err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]);
+                if (err != 0) {
+                    UNLOCK(&fop->fd->lock);
 
-                fop->error = -err;
+                    fop->error = -err;
 
-                return EC_STATE_REPORT;
+                    return EC_STATE_REPORT;
+                }
             }
 
             ctx->flags = fop->int32;
@@ -833,7 +817,7 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
             fop->uint32 = fop->int32 & O_TRUNC;
             fop->int32 &= ~(O_APPEND | O_TRUNC);
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -864,8 +848,8 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
                     if (fop->uint32 != 0) {
                         ec_sleep(fop);
                         ec_ftruncate(fop->req_frame, fop->xl, cbk->mask,
-                                     fop->minimum, ec_open_truncate_cbk,
-                                     fop, cbk->fd, 0, NULL);
+                                     fop->minimum, ec_open_truncate_cbk, fop,
+                                     cbk->fd, 0, NULL);
                     }
                 }
             }
@@ -877,8 +861,7 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.open != NULL)
-            {
+            if (fop->cbks.open != NULL) {
                 fop->cbks.open(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                cbk->op_errno, cbk->fd, cbk->xdata);
             }
@@ -891,8 +874,7 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.open != NULL)
-            {
+            if (fop->cbks.open != NULL) {
                 fop->cbks.open(fop->req_frame, fop, fop->xl, -1, fop->error,
                                NULL, NULL);
             }
@@ -900,30 +882,30 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_open_cbk_t func, void * data, loc_t * loc,
-             int32_t flags, fd_t * fd, dict_t * xdata)
+void
+ec_open(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_open_cbk_t func, void *data, loc_t *loc,
+        int32_t flags, fd_t *fd, dict_t *xdata)
 {
-    ec_cbk_t callback = { .open = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.open = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(OPEN) %p", frame);
+    gf_msg_trace("ec", 0, "EC(OPEN) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_OPEN, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_open, ec_manager_open,
+                               target, fop_flags, ec_wind_open, ec_manager_open,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -933,8 +915,8 @@ void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -942,9 +924,9 @@ void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -952,9 +934,9 @@ void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -972,13 +954,13 @@ out:
 
 /* FOP: readlink */
 
-int32_t ec_combine_readlink(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                            ec_cbk_data_t * src)
+int32_t
+ec_combine_readlink(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
     if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_IATT_MISMATCH, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_READLINK'");
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_READLINK'");
 
         return 0;
     }
@@ -987,13 +969,13 @@ int32_t ec_combine_readlink(ec_fop_data_t * fop, ec_cbk_data_t * dst,
 }
 
 int32_t
-ec_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, const char *path,
-                 struct iatt *buf, dict_t *xdata)
+ec_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, const char *path,
+                struct iatt *buf, dict_t *xdata)
 {
-    ec_fop_data_t   *fop = NULL;
-    ec_cbk_data_t   *cbk = NULL;
-    int32_t         idx  = (int32_t)(uintptr_t)cookie;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
@@ -1002,23 +984,23 @@ ec_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-    cbk = ec_cbk_data_allocate (frame, this, fop, fop->id,
-                                idx, op_ret, op_errno);
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
+                               op_errno);
     if (cbk) {
-            if (xdata)
-                    cbk->xdata = dict_ref (xdata);
-
-            if (cbk->op_ret >= 0) {
-                    cbk->iatt[0] = *buf;
-                    cbk->str = gf_strdup (path);
-                    if (!cbk->str) {
-                            ec_cbk_set_error(cbk, ENOMEM, _gf_true);
-                    }
+        if (xdata)
+            cbk->xdata = dict_ref(xdata);
+
+        if (cbk->op_ret >= 0) {
+            cbk->iatt[0] = *buf;
+            cbk->str = gf_strdup(path);
+            if (!cbk->str) {
+                ec_cbk_set_error(cbk, ENOMEM, _gf_true);
             }
-            ec_combine (cbk, NULL);
+        }
+        ec_combine(cbk, NULL);
     }
 
 out:
@@ -1028,7 +1010,8 @@ out:
     return 0;
 }
 
-void ec_wind_readlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_readlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1037,21 +1020,21 @@ void ec_wind_readlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->size, fop->xdata);
 }
 
-int32_t ec_manager_readlink(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_readlink(ec_fop_data_t *fop, int32_t state)
 {
     ec_cbk_data_t *cbk = NULL;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_inode (fop, &fop->loc[0], EC_QUERY_INFO, 0,
-                                   LLONG_MAX);
-            ec_lock (fop);
+            ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
+                                  EC_RANGE_FULL);
+            ec_lock(fop);
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
-            ec_dispatch_one (fop);
+            ec_dispatch_one(fop);
 
             return EC_STATE_PREPARE_ANSWER;
 
@@ -1068,11 +1051,11 @@ int32_t ec_manager_readlink(ec_fop_data_t * fop, int32_t state)
 
         case EC_STATE_REPORT:
             cbk = fop->answer;
-            GF_ASSERT (cbk);
+            GF_ASSERT(cbk);
             if (fop->cbks.readlink != NULL) {
-                fop->cbks.readlink (fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                    cbk->op_errno, cbk->str, &cbk->iatt[0],
-                                    cbk->xdata);
+                fop->cbks.readlink(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                   cbk->op_errno, cbk->str, &cbk->iatt[0],
+                                   cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -1083,8 +1066,8 @@ int32_t ec_manager_readlink(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             if (fop->cbks.readlink != NULL) {
-                fop->cbks.readlink(fop->req_frame, fop, fop->xl, -1,
-                                   fop->error, NULL, NULL, NULL);
+                fop->cbks.readlink(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                   NULL, NULL, NULL);
             }
             return EC_STATE_LOCK_REUSE;
 
@@ -1100,32 +1083,31 @@ int32_t ec_manager_readlink(ec_fop_data_t * fop, int32_t state)
 
             return EC_STATE_END;
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_readlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_readlink_cbk_t func, void * data,
-                 loc_t * loc, size_t size, dict_t * xdata)
+void
+ec_readlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_readlink_cbk_t func, void *data, loc_t *loc,
+            size_t size, dict_t *xdata)
 {
-    ec_cbk_t callback = { .readlink = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.readlink = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(READLINK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(READLINK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_READLINK,
-                               EC_FLAG_LOCK_SHARED, target, minimum,
-                               ec_wind_readlink, ec_manager_readlink, callback,
-                               data);
+    fop = ec_fop_data_allocate(
+        frame, this, GF_FOP_READLINK, EC_FLAG_LOCK_SHARED, target, fop_flags,
+        ec_wind_readlink, ec_manager_readlink, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -1134,8 +1116,8 @@ void ec_readlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -1143,9 +1125,9 @@ void ec_readlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1163,13 +1145,14 @@ out:
 
 /* FOP: readv */
 
-int32_t ec_readv_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk)
+int32_t
+ec_readv_rebuild(ec_t *ec, ec_fop_data_t *fop, ec_cbk_data_t *cbk)
 {
     struct iovec vector[1];
     ec_cbk_data_t *ans = NULL;
     struct iobref *iobref = NULL;
     void *ptr;
-    size_t fsize = 0, size = 0, max = 0;
+    uint64_t fsize = 0, size = 0, max = 0;
     int32_t pos, err = -ENOMEM;
 
     if (cbk->op_ret < 0) {
@@ -1256,22 +1239,21 @@ out:
     return err;
 }
 
-int32_t ec_combine_readv(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                         ec_cbk_data_t * src)
+int32_t
+ec_combine_readv(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_vector_compare(dst->vector, dst->int32, src->vector, src->int32))
-    {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_VECTOR_MISMATCH, "Mismatching vector in "
-                                             "answers of 'GF_FOP_READ'");
+    if (!ec_vector_compare(dst->vector, dst->int32, src->vector, src->int32)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_VECTOR_MISMATCH,
+               "Mismatching vector in "
+               "answers of 'GF_FOP_READ'");
 
         return 0;
     }
 
     if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_IATT_MISMATCH, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_READ'");
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_READ'");
 
         return 0;
     }
@@ -1279,14 +1261,14 @@ int32_t ec_combine_readv(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, struct iovec * vector,
-                     int32_t count, struct iatt * stbuf,
-                     struct iobref * iobref, dict_t * xdata)
+int32_t
+ec_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iovec *vector, int32_t count,
+             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
-    ec_t * ec = this->private;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    ec_t *ec = this->private;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -1296,8 +1278,8 @@ int32_t ec_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_READ, idx, op_ret,
                                op_errno);
@@ -1308,9 +1290,9 @@ int32_t ec_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
             if (count > 0) {
                 cbk->vector = iov_dup(vector, count);
                 if (cbk->vector == NULL) {
-                    gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                            EC_MSG_NO_MEMORY, "Failed to duplicate a "
-                                                     "vector list.");
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                           "Failed to duplicate a "
+                           "vector list.");
 
                     goto out;
                 }
@@ -1322,9 +1304,9 @@ int32_t ec_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
             if (iobref != NULL) {
                 cbk->buffers = iobref_ref(iobref);
                 if (cbk->buffers == NULL) {
-                    gf_msg (this->name, GF_LOG_ERROR, 0,
-                            EC_MSG_BUF_REF_FAIL, "Failed to reference a "
-                                                     "buffer.");
+                    gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_BUF_REF_FAIL,
+                           "Failed to reference a "
+                           "buffer.");
 
                     goto out;
                 }
@@ -1333,9 +1315,9 @@ int32_t ec_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
         if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
             if (cbk->xdata == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -1356,7 +1338,8 @@ out:
     return 0;
 }
 
-void ec_wind_readv(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_readv(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1365,12 +1348,13 @@ void ec_wind_readv(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->size, fop->offset, fop->uint32, fop->xdata);
 }
 
-int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_readv(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
+    ec_t *ec = fop->xl->private;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             fop->user_size = fop->size;
             fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset,
@@ -1378,7 +1362,7 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
             fop->size += fop->head;
             ec_adjust_size_up(fop->xl->private, &fop->size, _gf_true);
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
             ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, fop->offset,
@@ -1388,6 +1372,9 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
+            if (ec->read_mask) {
+                fop->mask &= ec->read_mask;
+            }
             ec_dispatch_min(fop);
 
             return EC_STATE_PREPARE_ANSWER;
@@ -1397,8 +1384,7 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
             if (cbk != NULL) {
                 int32_t err;
 
-                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1,
-                                cbk->count);
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1, cbk->count);
 
                 err = ec_readv_rebuild(fop->xl->private, fop, cbk);
                 if (err != 0) {
@@ -1413,8 +1399,7 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.readv != NULL)
-            {
+            if (fop->cbks.readv != NULL) {
                 fop->cbks.readv(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                 cbk->op_errno, cbk->vector, cbk->int32,
                                 &cbk->iatt[0], cbk->buffers, cbk->xdata);
@@ -1429,8 +1414,7 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.readv != NULL)
-            {
+            if (fop->cbks.readv != NULL) {
                 fop->cbks.readv(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL, 0, NULL, NULL, NULL);
             }
@@ -1450,30 +1434,30 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_readv_cbk_t func, void * data, fd_t * fd,
-              size_t size, off_t offset, uint32_t flags, dict_t * xdata)
+void
+ec_readv(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_readv_cbk_t func, void *data, fd_t *fd,
+         size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .readv = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.readv = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(READ) %p", frame);
+    gf_msg_trace("ec", 0, "EC(READ) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_READ, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_readv,
+                               target, fop_flags, ec_wind_readv,
                                ec_manager_readv, callback, data);
     if (fop == NULL) {
         goto out;
@@ -1488,9 +1472,9 @@ void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -1498,9 +1482,9 @@ void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1518,9 +1502,9 @@ out:
 
 /* FOP: seek */
 
-int32_t ec_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, off_t offset,
-                    dict_t *xdata)
+int32_t
+ec_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, off_t offset, dict_t *xdata)
 {
     ec_fop_data_t *fop = NULL;
     ec_cbk_data_t *cbk = NULL;
@@ -1534,8 +1518,8 @@ int32_t ec_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_SEEK, idx, op_ret,
                                op_errno);
@@ -1563,7 +1547,8 @@ out:
     return 0;
 }
 
-void ec_wind_seek(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+void
+ec_wind_seek(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1572,126 +1557,128 @@ void ec_wind_seek(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
                       fop->offset, fop->seek, fop->xdata);
 }
 
-int32_t ec_manager_seek(ec_fop_data_t *fop, int32_t state)
+int32_t
+ec_manager_seek(ec_fop_data_t *fop, int32_t state)
 {
     ec_cbk_data_t *cbk;
-    size_t size;
+    uint64_t size;
 
     switch (state) {
-    case EC_STATE_INIT:
-        fop->user_size = fop->offset;
-        fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset,
-                                          _gf_true);
+        case EC_STATE_INIT:
+            fop->user_size = fop->offset;
+            fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset,
+                                              _gf_true);
 
-    /* Fall through */
+            /* Fall through */
 
-    case EC_STATE_LOCK:
-        ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, fop->offset, LLONG_MAX);
-        ec_lock(fop);
+        case EC_STATE_LOCK:
+            ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, fop->offset,
+                               EC_RANGE_FULL);
+            ec_lock(fop);
 
-        return EC_STATE_DISPATCH;
+            return EC_STATE_DISPATCH;
 
-    case EC_STATE_DISPATCH:
-        /* This shouldn't fail because we have the inode locked. */
-        GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode,
-                                    &size));
+        case EC_STATE_DISPATCH:
+            /* This shouldn't fail because we have the inode locked. */
+            GF_ASSERT(
+                ec_get_inode_size(fop, fop->locks[0].lock->loc.inode, &size));
 
-        if (fop->user_size >= size) {
-            ec_fop_set_error(fop, ENXIO);
+            if (fop->user_size >= size) {
+                ec_fop_set_error(fop, ENXIO);
 
-            return EC_STATE_REPORT;
-        }
+                return EC_STATE_REPORT;
+            }
 
-        ec_dispatch_one(fop);
+            ec_dispatch_one(fop);
 
-        return EC_STATE_PREPARE_ANSWER;
+            return EC_STATE_PREPARE_ANSWER;
 
-    case EC_STATE_PREPARE_ANSWER:
-        if (ec_dispatch_one_retry(fop, &cbk)) {
-            return EC_STATE_DISPATCH;
-        }
-        if ((cbk != NULL) && (cbk->op_ret >= 0)) {
-            ec_t *ec = fop->xl->private;
+        case EC_STATE_PREPARE_ANSWER:
+            if (ec_dispatch_one_retry(fop, &cbk)) {
+                return EC_STATE_DISPATCH;
+            }
+            if ((cbk != NULL) && (cbk->op_ret >= 0)) {
+                ec_t *ec = fop->xl->private;
 
-            /* This shouldn't fail because we have the inode locked. */
-            GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode,
-                                        &size));
+                /* This shouldn't fail because we have the inode locked. */
+                GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode,
+                                            &size));
 
-            cbk->offset *= ec->fragments;
-            if (cbk->offset < fop->user_size) {
-                cbk->offset = fop->user_size;
-            }
-            if (cbk->offset > size) {
-                cbk->offset = size;
+                cbk->offset *= ec->fragments;
+                if (cbk->offset < fop->user_size) {
+                    cbk->offset = fop->user_size;
+                }
+                if (cbk->offset > size) {
+                    cbk->offset = size;
+                }
             }
-        }
 
-        return EC_STATE_REPORT;
+            return EC_STATE_REPORT;
 
-    case EC_STATE_REPORT:
-        cbk = fop->answer;
+        case EC_STATE_REPORT:
+            cbk = fop->answer;
 
-        GF_ASSERT(cbk != NULL);
+            GF_ASSERT(cbk != NULL);
 
-        if (fop->cbks.seek != NULL) {
-            fop->cbks.seek(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                           cbk->op_errno, cbk->offset, cbk->xdata);
-        }
+            if (fop->cbks.seek != NULL) {
+                fop->cbks.seek(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                               cbk->op_errno, cbk->offset, cbk->xdata);
+            }
 
-        return EC_STATE_LOCK_REUSE;
+            return EC_STATE_LOCK_REUSE;
 
-    case -EC_STATE_INIT:
-    case -EC_STATE_LOCK:
-    case -EC_STATE_DISPATCH:
-    case -EC_STATE_PREPARE_ANSWER:
-    case -EC_STATE_REPORT:
-        GF_ASSERT(fop->error != 0);
+        case -EC_STATE_INIT:
+        case -EC_STATE_LOCK:
+        case -EC_STATE_DISPATCH:
+        case -EC_STATE_PREPARE_ANSWER:
+        case -EC_STATE_REPORT:
+            GF_ASSERT(fop->error != 0);
 
-        if (fop->cbks.seek != NULL) {
-            fop->cbks.seek(fop->req_frame, fop, fop->xl, -1, fop->error, 0,
-                           NULL);
-        }
+            if (fop->cbks.seek != NULL) {
+                fop->cbks.seek(fop->req_frame, fop, fop->xl, -1, fop->error, 0,
+                               NULL);
+            }
 
-        return EC_STATE_LOCK_REUSE;
+            return EC_STATE_LOCK_REUSE;
 
-    case -EC_STATE_LOCK_REUSE:
-    case EC_STATE_LOCK_REUSE:
-        ec_lock_reuse(fop);
+        case -EC_STATE_LOCK_REUSE:
+        case EC_STATE_LOCK_REUSE:
+            ec_lock_reuse(fop);
 
-        return EC_STATE_UNLOCK;
+            return EC_STATE_UNLOCK;
 
-    case -EC_STATE_UNLOCK:
-    case EC_STATE_UNLOCK:
-        ec_unlock(fop);
+        case -EC_STATE_UNLOCK:
+        case EC_STATE_UNLOCK:
+            ec_unlock(fop);
 
-        return EC_STATE_END;
+            return EC_STATE_END;
 
-    default:
-        gf_msg (fop->xl->name, GF_LOG_ERROR, 0,
-                EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s", state,
-                ec_fop_name(fop->id));
+        default:
+            gf_msg(fop->xl->name, GF_LOG_ERROR, 0, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
-        return EC_STATE_END;
+            return EC_STATE_END;
     }
 }
 
-void ec_seek(call_frame_t *frame, xlator_t *this, uintptr_t target,
-              int32_t minimum, fop_seek_cbk_t func, void *data, fd_t *fd,
-              off_t offset, gf_seek_what_t what, dict_t *xdata)
+void
+ec_seek(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_seek_cbk_t func, void *data, fd_t *fd,
+        off_t offset, gf_seek_what_t what, dict_t *xdata)
 {
-    ec_cbk_t callback = { .seek = func };
+    ec_cbk_t callback = {.seek = func};
     ec_fop_data_t *fop = NULL;
     int32_t error = EIO;
 
-    gf_msg_trace ("ec", 0, "EC(SEEK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(SEEK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_SEEK, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_seek,
-                               ec_manager_seek, callback, data);
+                               target, fop_flags, ec_wind_seek, ec_manager_seek,
+                               callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -1720,13 +1707,13 @@ out:
 
 /* FOP: stat */
 
-int32_t ec_combine_stat(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                        ec_cbk_data_t * src)
+int32_t
+ec_combine_stat(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
     if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_IATT_MISMATCH, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_STAT'");
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_STAT'");
 
         return 0;
     }
@@ -1734,12 +1721,12 @@ int32_t ec_combine_stat(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_stat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                    int32_t op_ret, int32_t op_errno, struct iatt * buf,
-                    dict_t * xdata)
+int32_t
+ec_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -1749,28 +1736,23 @@ int32_t ec_stat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_STAT, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (buf != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (buf != NULL) {
                 cbk->iatt[0] = *buf;
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -1780,15 +1762,15 @@ int32_t ec_stat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_stat(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_stat(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1797,19 +1779,20 @@ void ec_wind_stat(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->xdata);
 }
 
-int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_stat(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
             if (fop->fd == NULL) {
                 ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
-                                      LLONG_MAX);
+                                      EC_RANGE_FULL);
             } else {
-                ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0, LLONG_MAX);
+                ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0,
+                                   EC_RANGE_FULL);
             }
             ec_lock(fop);
 
@@ -1825,8 +1808,7 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
 
             if (cbk != NULL) {
                 if (cbk->iatt[0].ia_type == IA_IFREG) {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1,
-                                    cbk->count);
+                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1, cbk->count);
 
                     /* This shouldn't fail because we have the inode locked. */
                     GF_ASSERT(ec_get_inode_size(fop,
@@ -1842,18 +1824,13 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_STAT)
-            {
-                if (fop->cbks.stat != NULL)
-                {
+            if (fop->id == GF_FOP_STAT) {
+                if (fop->cbks.stat != NULL) {
                     fop->cbks.stat(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                    cbk->op_errno, &cbk->iatt[0], cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fstat != NULL)
-                {
+            } else {
+                if (fop->cbks.fstat != NULL) {
                     fop->cbks.fstat(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                     cbk->op_errno, &cbk->iatt[0], cbk->xdata);
                 }
@@ -1868,18 +1845,13 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_STAT)
-            {
-                if (fop->cbks.stat != NULL)
-                {
-                    fop->cbks.stat(fop->req_frame, fop, fop->xl, -1,
-                                   fop->error, NULL, NULL);
+            if (fop->id == GF_FOP_STAT) {
+                if (fop->cbks.stat != NULL) {
+                    fop->cbks.stat(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                   NULL, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fstat != NULL)
-                {
+            } else {
+                if (fop->cbks.fstat != NULL) {
                     fop->cbks.fstat(fop->req_frame, fop, fop->xl, -1,
                                     fop->error, NULL, NULL);
                 }
@@ -1900,30 +1872,30 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE, "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_stat(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_stat_cbk_t func, void * data, loc_t * loc,
-             dict_t * xdata)
+void
+ec_stat(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_stat_cbk_t func, void *data, loc_t *loc,
+        dict_t *xdata)
 {
-    ec_cbk_t callback = { .stat = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.stat = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(STAT) %p", frame);
+    gf_msg_trace("ec", 0, "EC(STAT) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_STAT, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_stat, ec_manager_stat,
+                               target, fop_flags, ec_wind_stat, ec_manager_stat,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -1931,8 +1903,8 @@ void ec_stat(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL, "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -1940,9 +1912,9 @@ void ec_stat(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1960,12 +1932,12 @@ out:
 
 /* FOP: fstat */
 
-int32_t ec_fstat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, struct iatt * buf,
-                     dict_t * xdata)
+int32_t
+ec_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -1975,28 +1947,23 @@ int32_t ec_fstat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSTAT, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (buf != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (buf != NULL) {
                 cbk->iatt[0] = *buf;
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -2006,15 +1973,15 @@ int32_t ec_fstat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fstat(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fstat(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -2023,23 +1990,24 @@ void ec_wind_fstat(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-void ec_fstat(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fstat_cbk_t func, void * data, fd_t * fd,
-              dict_t * xdata)
+void
+ec_fstat(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fstat_cbk_t func, void *data, fd_t *fd,
+         dict_t *xdata)
 {
-    ec_cbk_t callback = { .fstat = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fstat = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FSTAT) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSTAT) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FSTAT, EC_FLAG_LOCK_SHARED,
-                               target, minimum, ec_wind_fstat, ec_manager_stat,
-                               callback, data);
+                               target, fop_flags, ec_wind_fstat,
+                               ec_manager_stat, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -2049,9 +2017,9 @@ void ec_fstat(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL, "Failed to reference a "
-                                             "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -2059,9 +2027,9 @@ void ec_fstat(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL, "Failed to reference a "
-                                             "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
diff --git a/xlators/cluster/ec/src/ec-inode-write.c b/xlators/cluster/ec/src/ec-inode-write.c
index bf45a867971..9b5fe2a7fdc 100644
--- a/xlators/cluster/ec/src/ec-inode-write.c
+++ b/xlators/cluster/ec/src/ec-inode-write.c
@@ -8,10 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
-#include "ec.h"
 #include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
@@ -21,56 +17,55 @@
 #include "ec-mem-types.h"
 
 int32_t
-ec_update_writev_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int32_t op_ret, int32_t op_errno,
-                      struct iatt *prebuf, struct iatt *postbuf,
-                      dict_t *xdata)
+ec_update_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-    ec_fop_data_t *fop    = cookie;
-    ec_cbk_data_t *cbk    = NULL;
+    ec_fop_data_t *fop = cookie;
+    ec_cbk_data_t *cbk = NULL;
     ec_fop_data_t *parent = fop->parent;
-    int           i       = 0;
+    int i = 0;
 
     ec_trace("UPDATE_WRITEV_CBK", cookie, "ret=%d, errno=%d, parent-fop=%s",
-             op_ret, op_errno, ec_fop_name (parent->id));
+             op_ret, op_errno, ec_fop_name(parent->id));
 
     if (op_ret < 0) {
-            ec_fop_set_error (parent, op_errno);
-            goto out;
+        ec_fop_set_error(parent, op_errno);
+        goto out;
     }
-    cbk = ec_cbk_data_allocate (parent->frame, this, parent,
-                                parent->id, 0, op_ret, op_errno);
+    cbk = ec_cbk_data_allocate(parent->frame, this, parent, parent->id, 0,
+                               op_ret, op_errno);
     if (!cbk) {
-            ec_fop_set_error (parent, ENOMEM);
-            goto out;
+        ec_fop_set_error(parent, ENOMEM);
+        goto out;
     }
 
     if (xdata)
-            cbk->xdata = dict_ref (xdata);
+        cbk->xdata = dict_ref(xdata);
 
     if (prebuf)
-            cbk->iatt[i++] = *prebuf;
+        cbk->iatt[i++] = *prebuf;
 
     if (postbuf)
-            cbk->iatt[i++] = *postbuf;
+        cbk->iatt[i++] = *postbuf;
 
-    LOCK (&parent->lock);
+    LOCK(&parent->lock);
     {
-            parent->good &= fop->good;
+        parent->good &= fop->good;
 
-            if (gf_bits_count (parent->good) < parent->minimum) {
-                    __ec_fop_set_error (parent, EIO);
-            } else if (fop->error == 0 && parent->answer == NULL) {
-                    parent->answer = cbk;
-            }
+        if (gf_bits_count(parent->good) < parent->minimum) {
+            __ec_fop_set_error(parent, EIO);
+        } else if (fop->error == 0 && parent->answer == NULL) {
+            parent->answer = cbk;
+        }
     }
-    UNLOCK (&parent->lock);
+    UNLOCK(&parent->lock);
 out:
     return 0;
 }
 
-int32_t ec_update_write(ec_fop_data_t *fop, uintptr_t mask, off_t offset,
-                        size_t size)
+static int32_t
+ec_update_write(ec_fop_data_t *fop, uintptr_t mask, off_t offset, uint64_t size)
 {
     struct iobref *iobref = NULL;
     struct iobuf *iobuf = NULL;
@@ -90,13 +85,14 @@ int32_t ec_update_write(ec_fop_data_t *fop, uintptr_t mask, off_t offset,
         goto out;
     }
 
+    if (fop->locks[0].lock)
+        ec_lock_update_good(fop->locks[0].lock, fop);
     vector.iov_base = iobuf->ptr;
     vector.iov_len = size;
     memset(vector.iov_base, 0, vector.iov_len);
 
-    ec_writev(fop->frame, fop->xl, mask, fop->minimum,
-              ec_update_writev_cbk, NULL, fop->fd, &vector, 1,
-              offset, 0, iobref, NULL);
+    ec_writev(fop->frame, fop->xl, mask, fop->minimum, ec_update_writev_cbk,
+              NULL, fop->fd, &vector, 1, offset, 0, iobref, NULL);
 
     err = 0;
 
@@ -112,62 +108,63 @@ out:
 }
 
 int
-ec_inode_write_cbk (call_frame_t *frame, xlator_t *this, void *cookie,
-                    int op_ret, int op_errno, struct iatt *prestat,
-                    struct iatt *poststat, dict_t *xdata)
+ec_inode_write_cbk(call_frame_t *frame, xlator_t *this, void *cookie,
+                   int op_ret, int op_errno, struct iatt *prestat,
+                   struct iatt *poststat, dict_t *xdata)
 {
-        ec_fop_data_t *fop = NULL;
-        ec_cbk_data_t *cbk = NULL;
-        int           i    = 0;
-        int           idx  = 0;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    int i = 0;
+    int idx = 0;
 
-        VALIDATE_OR_GOTO (this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-        fop = frame->local;
-        idx = (int32_t)(uintptr_t) cookie;
+    fop = frame->local;
+    idx = (int32_t)(uintptr_t)cookie;
 
-        ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-                 frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-        cbk = ec_cbk_data_allocate (frame, this, fop, fop->id, idx, op_ret,
-                                    op_errno);
-        if (!cbk)
-                goto out;
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
+                               op_errno);
+    if (!cbk)
+        goto out;
 
-        if (op_ret < 0)
-                goto out;
+    if (op_ret < 0)
+        goto out;
 
-        if (xdata)
-                cbk->xdata = dict_ref (xdata);
+    if (xdata)
+        cbk->xdata = dict_ref(xdata);
 
-        if (prestat)
-                cbk->iatt[i++] = *prestat;
+    if (prestat)
+        cbk->iatt[i++] = *prestat;
 
-        if (poststat)
-                cbk->iatt[i++] = *poststat;
+    if (poststat)
+        cbk->iatt[i++] = *poststat;
 
 out:
-        if (cbk)
-                ec_combine (cbk, ec_combine_write);
+    if (cbk)
+        ec_combine(cbk, ec_combine_write);
 
-        if (fop)
-                ec_complete (fop);
-        return 0;
+    if (fop)
+        ec_complete(fop);
+    return 0;
 }
 /* FOP: removexattr */
 
-int32_t ec_removexattr_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *xdata)
+int32_t
+ec_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   NULL, NULL, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL,
+                              xdata);
 }
 
-void ec_wind_removexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_removexattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -177,130 +174,126 @@ void ec_wind_removexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 }
 
 void
-ec_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, dict_t *xdata)
+ec_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
 {
-        ec_fop_data_t *fop = cookie;
-        switch (fop->id) {
+    ec_fop_data_t *fop = cookie;
+    switch (fop->id) {
         case GF_FOP_SETXATTR:
-                if (fop->cbks.setxattr) {
-                        fop->cbks.setxattr (frame, cookie, this, op_ret,
-                                            op_errno, xdata);
-                }
-                break;
+            if (fop->cbks.setxattr) {
+                QUORUM_CBK(fop->cbks.setxattr, fop, frame, cookie, this, op_ret,
+                           op_errno, xdata);
+            }
+            break;
         case GF_FOP_REMOVEXATTR:
-                if (fop->cbks.removexattr) {
-                        fop->cbks.removexattr (frame, cookie, this, op_ret,
-                                               op_errno, xdata);
-                }
-                break;
+            if (fop->cbks.removexattr) {
+                QUORUM_CBK(fop->cbks.removexattr, fop, frame, cookie, this,
+                           op_ret, op_errno, xdata);
+            }
+            break;
         case GF_FOP_FSETXATTR:
-                if (fop->cbks.fsetxattr) {
-                        fop->cbks.fsetxattr (frame, cookie, this, op_ret,
-                                             op_errno, xdata);
-                }
-                break;
+            if (fop->cbks.fsetxattr) {
+                QUORUM_CBK(fop->cbks.fsetxattr, fop, frame, cookie, this,
+                           op_ret, op_errno, xdata);
+            }
+            break;
         case GF_FOP_FREMOVEXATTR:
-                if (fop->cbks.fremovexattr) {
-                        fop->cbks.fremovexattr (frame, cookie, this, op_ret,
-                                                op_errno, xdata);
-                }
-                break;
-        }
+            if (fop->cbks.fremovexattr) {
+                QUORUM_CBK(fop->cbks.fremovexattr, fop, frame, cookie, this,
+                           op_ret, op_errno, xdata);
+            }
+            break;
+    }
 }
 
 int32_t
-ec_manager_xattr (ec_fop_data_t *fop, int32_t state)
+ec_manager_xattr(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-        switch (state) {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-                if (fop->fd == NULL) {
-                        ec_lock_prepare_inode(fop, &fop->loc[0],
-                                              EC_UPDATE_META | EC_QUERY_INFO,
-                                              0, LLONG_MAX);
-                } else {
-                        ec_lock_prepare_fd(fop, fop->fd,
-                                           EC_UPDATE_META | EC_QUERY_INFO,
-                                           0, LLONG_MAX);
-                }
-                ec_lock(fop);
+            if (fop->fd == NULL) {
+                ec_lock_prepare_inode(fop, &fop->loc[0],
+                                      EC_UPDATE_META | EC_QUERY_INFO, 0,
+                                      EC_RANGE_FULL);
+            } else {
+                ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META | EC_QUERY_INFO,
+                                   0, EC_RANGE_FULL);
+            }
+            ec_lock(fop);
 
-                return EC_STATE_DISPATCH;
+            return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
-                ec_dispatch_all(fop);
+            ec_dispatch_all(fop);
 
-                return EC_STATE_PREPARE_ANSWER;
+            return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-                ec_fop_prepare_answer(fop, _gf_false);
+            ec_fop_prepare_answer(fop, _gf_false);
 
-                return EC_STATE_REPORT;
+            return EC_STATE_REPORT;
 
         case EC_STATE_REPORT:
-                cbk = fop->answer;
+            cbk = fop->answer;
 
-                GF_ASSERT(cbk != NULL);
+            GF_ASSERT(cbk != NULL);
 
-                ec_xattr_cbk (fop->req_frame, fop, fop->xl, cbk->op_ret,
-                              cbk->op_errno, cbk->xdata);
+            ec_xattr_cbk(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                         cbk->op_errno, cbk->xdata);
 
-                    return EC_STATE_LOCK_REUSE;
+            return EC_STATE_LOCK_REUSE;
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
-                GF_ASSERT(fop->error != 0);
+            GF_ASSERT(fop->error != 0);
 
-                ec_xattr_cbk (fop->req_frame, fop, fop->xl, -1, fop->error,
-                              NULL);
+            ec_xattr_cbk(fop->req_frame, fop, fop->xl, -1, fop->error, NULL);
 
-                return EC_STATE_LOCK_REUSE;
+            return EC_STATE_LOCK_REUSE;
 
         case -EC_STATE_LOCK_REUSE:
         case EC_STATE_LOCK_REUSE:
-                ec_lock_reuse(fop);
+            ec_lock_reuse(fop);
 
-                return EC_STATE_UNLOCK;
+            return EC_STATE_UNLOCK;
 
         case -EC_STATE_UNLOCK:
         case EC_STATE_UNLOCK:
-                ec_unlock(fop);
+            ec_unlock(fop);
 
-                return EC_STATE_END;
+            return EC_STATE_END;
 
         default:
-                gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                        EC_MSG_UNHANDLED_STATE,
-                        "Unhandled state %d for %s",
-                        state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
-                return EC_STATE_END;
-        }
+            return EC_STATE_END;
+    }
 }
 
 void
-ec_removexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-                int32_t minimum, fop_removexattr_cbk_t func, void *data,
-                loc_t *loc, const char *name, dict_t *xdata)
+ec_removexattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+               uint32_t fop_flags, fop_removexattr_cbk_t func, void *data,
+               loc_t *loc, const char *name, dict_t *xdata)
 {
-    ec_cbk_t callback = { .removexattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.removexattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(REMOVEXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(REMOVEXATTR) %p", frame);
 
-    VALIDATE_OR_GOTO (this, out);
-    GF_VALIDATE_OR_GOTO (this->name, frame, out);
-    GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_REMOVEXATTR, 0, target,
-                               minimum, ec_wind_removexattr, ec_manager_xattr,
+                               fop_flags, ec_wind_removexattr, ec_manager_xattr,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -308,9 +301,8 @@ ec_removexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -318,20 +310,18 @@ ec_removexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (name != NULL) {
         fop->str[0] = gf_strdup(name);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
     if (xdata != NULL) {
-        fop->xdata = dict_copy_with_ref (xdata, NULL);
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -341,23 +331,24 @@ ec_removexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL);
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: fremovexattr */
 
-int32_t ec_fremovexattr_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             dict_t *xdata)
+int32_t
+ec_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   NULL, NULL, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL,
+                              xdata);
 }
 
-void ec_wind_fremovexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fremovexattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -367,23 +358,23 @@ void ec_wind_fremovexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 }
 
 void
-ec_fremovexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-                 int32_t minimum, fop_fremovexattr_cbk_t func, void *data,
-                 fd_t *fd, const char *name, dict_t *xdata)
+ec_fremovexattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+                uint32_t fop_flags, fop_fremovexattr_cbk_t func, void *data,
+                fd_t *fd, const char *name, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fremovexattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fremovexattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FREMOVEXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FREMOVEXATTR) %p", frame);
 
-    VALIDATE_OR_GOTO (this, out);
-    GF_VALIDATE_OR_GOTO (this->name, frame, out);
-    GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FREMOVEXATTR, 0, target,
-                               minimum, ec_wind_fremovexattr, ec_manager_xattr,
-                               callback, data);
+                               fop_flags, ec_wind_fremovexattr,
+                               ec_manager_xattr, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -393,10 +384,9 @@ ec_fremovexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -404,9 +394,8 @@ ec_fremovexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (name != NULL) {
         fop->str[0] = gf_strdup(name);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
@@ -414,10 +403,9 @@ ec_fremovexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -427,24 +415,25 @@ ec_fremovexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL);
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: setattr */
 
-int32_t ec_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno,
-                        struct iatt *prestat, struct iatt *poststat,
-                        dict_t *xdata)
+int32_t
+ec_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prestat,
+               struct iatt *poststat, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
 }
 
-void ec_wind_setattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_setattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -453,21 +442,21 @@ void ec_wind_setattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], &fop->iatt, fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_setattr(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
             if (fop->fd == NULL) {
                 ec_lock_prepare_inode(fop, &fop->loc[0],
-                                      EC_UPDATE_META | EC_QUERY_INFO,
-                                      0, LLONG_MAX);
+                                      EC_UPDATE_META | EC_QUERY_INFO, 0,
+                                      EC_RANGE_FULL);
             } else {
                 ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META | EC_QUERY_INFO,
-                                   0, LLONG_MAX);
+                                   0, EC_RANGE_FULL);
             }
             ec_lock(fop);
 
@@ -482,8 +471,7 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
             cbk = ec_fop_prepare_answer(fop, _gf_false);
             if (cbk != NULL) {
                 if (cbk->iatt[0].ia_type == IA_IFREG) {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                                    cbk->count);
+                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
 
                     /* This shouldn't fail because we have the inode locked. */
                     GF_ASSERT(ec_get_inode_size(fop,
@@ -500,24 +488,17 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_SETATTR)
-            {
-                if (fop->cbks.setattr != NULL)
-                {
-                    fop->cbks.setattr(fop->req_frame, fop, fop->xl,
-                                      cbk->op_ret, cbk->op_errno,
-                                      &cbk->iatt[0], &cbk->iatt[1],
-                                      cbk->xdata);
+            if (fop->id == GF_FOP_SETATTR) {
+                if (fop->cbks.setattr != NULL) {
+                    QUORUM_CBK(fop->cbks.setattr, fop, fop->req_frame, fop,
+                               fop->xl, cbk->op_ret, cbk->op_errno,
+                               &cbk->iatt[0], &cbk->iatt[1], cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fsetattr != NULL)
-                {
-                    fop->cbks.fsetattr(fop->req_frame, fop, fop->xl,
-                                       cbk->op_ret, cbk->op_errno,
-                                       &cbk->iatt[0], &cbk->iatt[1],
-                                       cbk->xdata);
+            } else {
+                if (fop->cbks.fsetattr != NULL) {
+                    QUORUM_CBK(fop->cbks.fsetattr, fop, fop->req_frame, fop,
+                               fop->xl, cbk->op_ret, cbk->op_errno,
+                               &cbk->iatt[0], &cbk->iatt[1], cbk->xdata);
                 }
             }
 
@@ -530,18 +511,13 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_SETATTR)
-            {
-                if (fop->cbks.setattr != NULL)
-                {
+            if (fop->id == GF_FOP_SETATTR) {
+                if (fop->cbks.setattr != NULL) {
                     fop->cbks.setattr(fop->req_frame, fop, fop->xl, -1,
                                       fop->error, NULL, NULL, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fsetattr != NULL)
-                {
+            } else {
+                if (fop->cbks.fsetattr != NULL) {
                     fop->cbks.fsetattr(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL, NULL, NULL);
                 }
@@ -562,33 +538,31 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_setattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_setattr_cbk_t func, void * data,
-                loc_t * loc, struct iatt * stbuf, int32_t valid,
-                dict_t * xdata)
+void
+ec_setattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_setattr_cbk_t func, void *data, loc_t *loc,
+           struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-    ec_cbk_t callback = { .setattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.setattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(SETATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(SETATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_SETATTR, 0, target, minimum,
-                               ec_wind_setattr, ec_manager_setattr, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_SETATTR, 0, target,
+                               fop_flags, ec_wind_setattr, ec_manager_setattr,
+                               callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -597,9 +571,8 @@ void ec_setattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -610,10 +583,9 @@ void ec_setattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -631,16 +603,17 @@ out:
 
 /* FOP: fsetattr */
 
-int32_t ec_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno,
-                         struct iatt *prestat, struct iatt *poststat,
-                         dict_t *xdata)
+int32_t
+ec_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prestat,
+                struct iatt *poststat, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
 }
 
-void ec_wind_fsetattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fsetattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -649,22 +622,23 @@ void ec_wind_fsetattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, &fop->iatt, fop->int32, fop->xdata);
 }
 
-void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fsetattr_cbk_t func, void * data,
-                 fd_t * fd, struct iatt * stbuf, int32_t valid, dict_t * xdata)
+void
+ec_fsetattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fsetattr_cbk_t func, void *data, fd_t *fd,
+            struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fsetattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fsetattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FSETATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSETATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FSETATTR, 0, target,
-                               minimum, ec_wind_fsetattr, ec_manager_setattr,
+                               fop_flags, ec_wind_fsetattr, ec_manager_setattr,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -677,10 +651,9 @@ void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -691,10 +664,9 @@ void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -712,14 +684,16 @@ out:
 
 /* FOP: setxattr */
 
-int32_t ec_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int32_t
+ec_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   NULL, NULL, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL,
+                              xdata);
 }
 
-void ec_wind_setxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_setxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -729,22 +703,22 @@ void ec_wind_setxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 }
 
 void
-ec_setxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-             int32_t minimum, fop_setxattr_cbk_t func, void *data,
-             loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
+ec_setxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_setxattr_cbk_t func, void *data, loc_t *loc,
+            dict_t *dict, int32_t flags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .setxattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.setxattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(SETXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(SETXATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_SETXATTR, 0, target,
-                               minimum, ec_wind_setxattr, ec_manager_xattr,
+                               fop_flags, ec_wind_setxattr, ec_manager_xattr,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -754,9 +728,8 @@ ec_setxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -764,10 +737,9 @@ ec_setxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (dict != NULL) {
         fop->dict = dict_copy_with_ref(dict, NULL);
         if (fop->dict == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -775,10 +747,9 @@ ec_setxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -788,45 +759,41 @@ ec_setxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL);
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: fsetxattr */
 
 int32_t
-ec_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+ec_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
-    VALIDATE_OR_GOTO (this, out);
-    GF_VALIDATE_OR_GOTO (this->name, frame, out);
-    GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
-    GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSETXATTR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                        "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -836,15 +803,15 @@ ec_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fsetxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fsetxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -854,22 +821,22 @@ void ec_wind_fsetxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 }
 
 void
-ec_fsetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-              int32_t minimum, fop_fsetxattr_cbk_t func, void *data,
-              fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
+ec_fsetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fsetxattr_cbk_t func, void *data, fd_t *fd,
+             dict_t *dict, int32_t flags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fsetxattr = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fsetxattr = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FSETXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSETXATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FSETXATTR, 0, target,
-                               minimum, ec_wind_fsetxattr, ec_manager_xattr,
+                               fop_flags, ec_wind_fsetxattr, ec_manager_xattr,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -882,10 +849,9 @@ ec_fsetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -893,10 +859,9 @@ ec_fsetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (dict != NULL) {
         fop->dict = dict_copy_with_ref(dict, NULL);
         if (fop->dict == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -904,10 +869,9 @@ ec_fsetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -917,9 +881,9 @@ ec_fsetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL);
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
@@ -929,162 +893,161 @@ out:
  *
  *********************************************************************/
 
-int32_t ec_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                         struct iatt *postbuf, dict_t *xdata)
+int32_t
+ec_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
 {
-    return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prebuf, postbuf, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prebuf,
+                              postbuf, xdata);
 }
 
-void ec_wind_fallocate(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+void
+ec_wind_fallocate(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
     STACK_WIND_COOKIE(fop->frame, ec_fallocate_cbk, (void *)(uintptr_t)idx,
                       ec->xl_list[idx], ec->xl_list[idx]->fops->fallocate,
-                      fop->fd, fop->int32, fop->offset,
-                      fop->size, fop->xdata);
+                      fop->fd, fop->int32, fop->offset, fop->size, fop->xdata);
 }
 
-int32_t ec_manager_fallocate(ec_fop_data_t *fop, int32_t state)
+int32_t
+ec_manager_fallocate(ec_fop_data_t *fop, int32_t state)
 {
     ec_cbk_data_t *cbk = NULL;
 
     switch (state) {
-    case EC_STATE_INIT:
-        if (fop->size == 0) {
+        case EC_STATE_INIT:
+            if (fop->size == 0) {
                 ec_fop_set_error(fop, EINVAL);
                 return EC_STATE_REPORT;
-        }
-        if (fop->int32 & (FALLOC_FL_COLLAPSE_RANGE
-                         |FALLOC_FL_INSERT_RANGE
-                         |FALLOC_FL_ZERO_RANGE
-                         |FALLOC_FL_PUNCH_HOLE)) {
+            }
+            if (fop->int32 &
+                (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE |
+                 FALLOC_FL_ZERO_RANGE | FALLOC_FL_PUNCH_HOLE)) {
                 ec_fop_set_error(fop, ENOTSUP);
                 return EC_STATE_REPORT;
-        }
-        fop->user_size = fop->offset + fop->size;
-        fop->head = ec_adjust_offset_down (fop->xl->private, &fop->offset,
-                                           _gf_true);
-        fop->size += fop->head;
-        ec_adjust_size_up (fop->xl->private, &fop->size, _gf_true);
+            }
+            fop->user_size = fop->offset + fop->size;
+            fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset,
+                                              _gf_true);
+            fop->size += fop->head;
+            ec_adjust_size_up(fop->xl->private, &fop->size, _gf_true);
 
-        /* Fall through */
+            /* Fall through */
 
-    case EC_STATE_LOCK:
-        ec_lock_prepare_fd(fop, fop->fd,
-                           EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
-                           fop->offset, fop->size);
-        ec_lock(fop);
+        case EC_STATE_LOCK:
+            ec_lock_prepare_fd(fop, fop->fd,
+                               EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                               fop->offset, fop->size);
+            ec_lock(fop);
 
-        return EC_STATE_DISPATCH;
+            return EC_STATE_DISPATCH;
 
-    case EC_STATE_DISPATCH:
+        case EC_STATE_DISPATCH:
 
-        ec_dispatch_all(fop);
+            ec_dispatch_all(fop);
 
-        return EC_STATE_PREPARE_ANSWER;
+            return EC_STATE_PREPARE_ANSWER;
 
-    case EC_STATE_PREPARE_ANSWER:
-        cbk = ec_fop_prepare_answer(fop, _gf_false);
-        if (cbk != NULL) {
-                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                            cbk->count);
+        case EC_STATE_PREPARE_ANSWER:
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
 
                 /* This shouldn't fail because we have the inode locked. */
                 LOCK(&fop->locks[0].lock->loc.inode->lock);
                 {
-                        GF_ASSERT(__ec_get_inode_size(fop,
-                                                fop->locks[0].lock->loc.inode,
-                                                &cbk->iatt[0].ia_size));
+                    GF_ASSERT(__ec_get_inode_size(fop,
+                                                  fop->locks[0].lock->loc.inode,
+                                                  &cbk->iatt[0].ia_size));
 
-                        /*If mode has FALLOC_FL_KEEP_SIZE keep the size */
-                        if (fop->int32 & FALLOC_FL_KEEP_SIZE) {
-                                cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
-                        } else if (fop->user_size > cbk->iatt[0].ia_size) {
-                                cbk->iatt[1].ia_size = fop->user_size;
-
-                                /* This shouldn't fail because we have the inode
-                                 * locked. */
-                                GF_ASSERT(__ec_set_inode_size(fop,
-                                          fop->locks[0].lock->loc.inode,
-                                                    cbk->iatt[1].ia_size));
-                        } else {
-                                cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
-                        }
+                    /*If mode has FALLOC_FL_KEEP_SIZE keep the size */
+                    if (fop->int32 & FALLOC_FL_KEEP_SIZE) {
+                        cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
+                    } else if (fop->user_size > cbk->iatt[0].ia_size) {
+                        cbk->iatt[1].ia_size = fop->user_size;
+
+                        /* This shouldn't fail because we have the inode
+                         * locked. */
+                        GF_ASSERT(__ec_set_inode_size(
+                            fop, fop->locks[0].lock->loc.inode,
+                            cbk->iatt[1].ia_size));
+                    } else {
+                        cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
+                    }
                 }
                 UNLOCK(&fop->locks[0].lock->loc.inode->lock);
-        }
+            }
 
-        return EC_STATE_REPORT;
+            return EC_STATE_REPORT;
 
-    case EC_STATE_REPORT:
-        cbk = fop->answer;
+        case EC_STATE_REPORT:
+            cbk = fop->answer;
 
-        GF_ASSERT(cbk != NULL);
+            GF_ASSERT(cbk != NULL);
 
-        if (fop->cbks.fallocate != NULL) {
-                fop->cbks.fallocate(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                    cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
-                                    cbk->xdata);
-        }
+            if (fop->cbks.fallocate != NULL) {
+                QUORUM_CBK(fop->cbks.fallocate, fop, fop->req_frame, fop,
+                           fop->xl, cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
+            }
 
-        return EC_STATE_LOCK_REUSE;
+            return EC_STATE_LOCK_REUSE;
 
-    case -EC_STATE_INIT:
-    case -EC_STATE_LOCK:
-    case -EC_STATE_DISPATCH:
-    case -EC_STATE_PREPARE_ANSWER:
-    case -EC_STATE_REPORT:
-        GF_ASSERT(fop->error != 0);
+        case -EC_STATE_INIT:
+        case -EC_STATE_LOCK:
+        case -EC_STATE_DISPATCH:
+        case -EC_STATE_PREPARE_ANSWER:
+        case -EC_STATE_REPORT:
+            GF_ASSERT(fop->error != 0);
 
-        if (fop->cbks.fallocate != NULL) {
+            if (fop->cbks.fallocate != NULL) {
                 fop->cbks.fallocate(fop->req_frame, fop, fop->xl, -1,
                                     fop->error, NULL, NULL, NULL);
-        }
+            }
 
-        return EC_STATE_LOCK_REUSE;
+            return EC_STATE_LOCK_REUSE;
 
-    case -EC_STATE_LOCK_REUSE:
-    case EC_STATE_LOCK_REUSE:
-        ec_lock_reuse(fop);
+        case -EC_STATE_LOCK_REUSE:
+        case EC_STATE_LOCK_REUSE:
+            ec_lock_reuse(fop);
 
-        return EC_STATE_UNLOCK;
+            return EC_STATE_UNLOCK;
 
-    case -EC_STATE_UNLOCK:
-    case EC_STATE_UNLOCK:
-        ec_unlock(fop);
+        case -EC_STATE_UNLOCK:
+        case EC_STATE_UNLOCK:
+            ec_unlock(fop);
 
-        return EC_STATE_END;
+            return EC_STATE_END;
 
-    default:
-        gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_UNHANDLED_STATE,
-                "Unhandled state %d for %s",
-                state, ec_fop_name(fop->id));
+        default:
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
-        return EC_STATE_END;
+            return EC_STATE_END;
     }
 }
 
-void ec_fallocate(call_frame_t *frame, xlator_t *this, uintptr_t target,
-              int32_t minimum, fop_fallocate_cbk_t func, void *data, fd_t *fd,
-              int32_t mode, off_t offset, size_t len, dict_t *xdata)
+void
+ec_fallocate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fallocate_cbk_t func, void *data, fd_t *fd,
+             int32_t mode, off_t offset, size_t len, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fallocate = func };
+    ec_cbk_t callback = {.fallocate = func};
     ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FALLOCATE) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FALLOCATE) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FALLOCATE, 0, target,
-                               minimum, ec_wind_fallocate, ec_manager_fallocate,
-                               callback, data);
+                               fop_flags, ec_wind_fallocate,
+                               ec_manager_fallocate, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -1097,22 +1060,20 @@ void ec_fallocate(call_frame_t *frame, xlator_t *this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_FILE_DESC_REF_FAIL,
-                        "Failed to reference a "
-                        "file descriptor.");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
+            goto out;
         }
     }
 
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                        "dictionary.");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
+            goto out;
         }
     }
 
@@ -1131,66 +1092,70 @@ out:
  * File Operation : Discard
  *
  *********************************************************************/
-void ec_update_discard_write(ec_fop_data_t *fop, uintptr_t mask)
+void
+ec_update_discard_write(ec_fop_data_t *fop, uintptr_t mask)
 {
-    ec_t   *ec       = fop->xl->private;
-    off_t  off_head  = 0;
-    off_t  off_tail  = 0;
-    size_t size_head = 0;
-    size_t size_tail = 0;
-    int    error     = 0;
+    ec_t *ec = fop->xl->private;
+    off_t off_head = 0;
+    off_t off_tail = 0;
+    uint64_t size_head = 0;
+    uint64_t size_tail = 0;
+    int error = 0;
 
     off_head = fop->offset * ec->fragments - fop->int32;
     if (fop->size == 0) {
-            error = ec_update_write (fop, mask, off_head, fop->user_size);
+        error = ec_update_write(fop, mask, off_head, fop->user_size);
     } else {
-            size_head = fop->int32;
-            size_tail = (off_head + fop->user_size) % ec->stripe_size;
-            off_tail = off_head + fop->user_size - size_tail;
-            if (size_head) {
-                    error = ec_update_write (fop, mask, off_head, size_head);
-                    if (error) {
-                        goto out;
-                    }
-            }
-            if (size_tail) {
-                    error = ec_update_write (fop, mask, off_tail, size_tail);
+        size_head = fop->int32;
+        size_tail = (off_head + fop->user_size) % ec->stripe_size;
+        off_tail = off_head + fop->user_size - size_tail;
+        if (size_head) {
+            error = ec_update_write(fop, mask, off_head, size_head);
+            if (error) {
+                goto out;
             }
+        }
+        if (size_tail) {
+            error = ec_update_write(fop, mask, off_tail, size_tail);
+        }
     }
 out:
     if (error)
-            ec_fop_set_error (fop, -error);
+        ec_fop_set_error(fop, -error);
 }
 
-void ec_discard_adjust_offset_size(ec_fop_data_t *fop)
+void
+ec_discard_adjust_offset_size(ec_fop_data_t *fop)
 {
-        ec_t *ec = fop->xl->private;
+    ec_t *ec = fop->xl->private;
 
-        fop->user_size = fop->size;
-        /* If discard length covers atleast a fragment on brick, we will
-         * perform discard operation(when fop->size is non-zero) else we just
-         * write zeros.
-         */
-        fop->int32 = ec_adjust_offset_up(ec, &fop->offset, _gf_true);
-        fop->frag_range.first = fop->offset;
-        if (fop->size < fop->int32) {
-                fop->size = 0;
-        } else {
-                fop->size -= fop->int32;
-                ec_adjust_size_down(ec, &fop->size, _gf_true);
-        }
-        fop->frag_range.last = fop->offset + fop->size;
+    fop->user_size = fop->size;
+    /* If discard length covers at least a fragment on brick, we will
+     * perform discard operation(when fop->size is non-zero) else we just
+     * write zeros.
+     */
+    fop->int32 = ec_adjust_offset_up(ec, &fop->offset, _gf_true);
+    fop->frag_range.first = fop->offset;
+    if (fop->size < fop->int32) {
+        fop->size = 0;
+    } else {
+        fop->size -= fop->int32;
+        ec_adjust_size_down(ec, &fop->size, _gf_true);
+    }
+    fop->frag_range.last = fop->offset + fop->size;
 }
 
-int32_t ec_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
+int32_t
+ec_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
 {
-    return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                               prebuf, postbuf, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prebuf,
+                              postbuf, xdata);
 }
 
-void ec_wind_discard(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+void
+ec_wind_discard(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1199,145 +1164,143 @@ void ec_wind_discard(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
                       fop->fd, fop->offset, fop->size, fop->xdata);
 }
 
-int32_t ec_manager_discard(ec_fop_data_t *fop, int32_t state)
+int32_t
+ec_manager_discard(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t *cbk     = NULL;
-    off_t         fl_start = 0;
-    size_t        fl_size  = 0;
-
+    ec_cbk_data_t *cbk = NULL;
+    off_t fl_start = 0;
+    uint64_t fl_size = 0;
 
     switch (state) {
-    case EC_STATE_INIT:
-        if ((fop->size <= 0) || (fop->offset < 0)) {
+        case EC_STATE_INIT:
+            if ((fop->size <= 0) || (fop->offset < 0)) {
                 ec_fop_set_error(fop, EINVAL);
                 return EC_STATE_REPORT;
-        }
-        /* Because of the head/tail writes, "discard" happens on the remaining
-         * regions, but we need to compute region including head/tail writes
-         * so compute them separately*/
-        fl_start = fop->offset;
-        fl_size = fop->size;
-        fl_size += ec_adjust_offset_down (fop->xl->private, &fl_start,
-                                          _gf_true);
-        ec_adjust_size_up (fop->xl->private, &fl_size, _gf_true);
+            }
+            /* Because of the head/tail writes, "discard" happens on the
+             * remaining regions, but we need to compute region including
+             * head/tail writes so compute them separately*/
+            fl_start = fop->offset;
+            fl_size = fop->size;
+            fl_size += ec_adjust_offset_down(fop->xl->private, &fl_start,
+                                             _gf_true);
+            ec_adjust_size_up(fop->xl->private, &fl_size, _gf_true);
 
-        ec_discard_adjust_offset_size(fop);
+            ec_discard_adjust_offset_size(fop);
 
-    /* Fall through */
+            /* Fall through */
 
-    case EC_STATE_LOCK:
-        ec_lock_prepare_fd(fop, fop->fd,
-                           EC_UPDATE_DATA | EC_UPDATE_META |
-                           EC_QUERY_INFO, fl_start, fl_size);
-        ec_lock(fop);
+        case EC_STATE_LOCK:
+            ec_lock_prepare_fd(fop, fop->fd,
+                               EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                               fl_start, fl_size);
+            ec_lock(fop);
 
-        return EC_STATE_DISPATCH;
+            return EC_STATE_DISPATCH;
 
-    case EC_STATE_DISPATCH:
+        case EC_STATE_DISPATCH:
 
-        /* Dispatch discard fop only if we have whole fragment
-         * to deallocate */
-        if (fop->size) {
+            /* Dispatch discard fop only if we have whole fragment
+             * to deallocate */
+            if (fop->size) {
                 ec_dispatch_all(fop);
                 return EC_STATE_DELAYED_START;
-        } else {
-                /*Assume discard to have succeeded on mask*/
-                fop->good = fop->mask;
-        }
+            } else {
+                /* Assume discard to have succeeded on all bricks */
+                ec_succeed_all(fop);
+            }
 
-        /* Fall through */
+            /* Fall through */
 
-    case EC_STATE_DELAYED_START:
+        case EC_STATE_DELAYED_START:
 
-        if (fop->size) {
+            if (fop->size) {
                 if (fop->answer && fop->answer->op_ret == 0)
-                        ec_update_discard_write (fop, fop->answer->mask);
-        } else {
-                ec_update_discard_write (fop, fop->mask);
-        }
+                    ec_update_discard_write(fop, fop->answer->mask);
+            } else {
+                ec_update_discard_write(fop, fop->mask);
+            }
 
-        return EC_STATE_PREPARE_ANSWER;
+            return EC_STATE_PREPARE_ANSWER;
 
-    case EC_STATE_PREPARE_ANSWER:
-        cbk = ec_fop_prepare_answer(fop, _gf_false);
-        if (cbk != NULL) {
-                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                                cbk->count);
+        case EC_STATE_PREPARE_ANSWER:
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
 
                 /* This shouldn't fail because we have the inode locked. */
                 GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode,
                                             &cbk->iatt[0].ia_size));
 
                 cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
-        }
-        return EC_STATE_REPORT;
+            }
+            return EC_STATE_REPORT;
 
-    case EC_STATE_REPORT:
-        cbk = fop->answer;
+        case EC_STATE_REPORT:
+            cbk = fop->answer;
 
-        GF_ASSERT(cbk != NULL);
+            GF_ASSERT(cbk != NULL);
 
-        if (fop->cbks.discard != NULL) {
-                fop->cbks.discard(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                  cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
-                                  cbk->xdata);
-        }
+            if (fop->cbks.discard != NULL) {
+                QUORUM_CBK(fop->cbks.discard, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
+            }
 
-        return EC_STATE_LOCK_REUSE;
+            return EC_STATE_LOCK_REUSE;
 
-    case -EC_STATE_INIT:
-    case -EC_STATE_LOCK:
-    case -EC_STATE_DISPATCH:
-    case -EC_STATE_DELAYED_START:
-    case -EC_STATE_PREPARE_ANSWER:
-    case -EC_STATE_REPORT:
-        GF_ASSERT(fop->error != 0);
+        case -EC_STATE_INIT:
+        case -EC_STATE_LOCK:
+        case -EC_STATE_DISPATCH:
+        case -EC_STATE_DELAYED_START:
+        case -EC_STATE_PREPARE_ANSWER:
+        case -EC_STATE_REPORT:
+            GF_ASSERT(fop->error != 0);
 
-        if (fop->cbks.discard != NULL) {
-                fop->cbks.discard(fop->req_frame, fop, fop->xl, -1,
-                                  fop->error, NULL, NULL, NULL);
-        }
+            if (fop->cbks.discard != NULL) {
+                fop->cbks.discard(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                  NULL, NULL, NULL);
+            }
 
-        return EC_STATE_LOCK_REUSE;
+            return EC_STATE_LOCK_REUSE;
 
-    case -EC_STATE_LOCK_REUSE:
-    case EC_STATE_LOCK_REUSE:
-        ec_lock_reuse(fop);
+        case -EC_STATE_LOCK_REUSE:
+        case EC_STATE_LOCK_REUSE:
+            ec_lock_reuse(fop);
 
-        return EC_STATE_UNLOCK;
+            return EC_STATE_UNLOCK;
 
-    case -EC_STATE_UNLOCK:
-    case EC_STATE_UNLOCK:
-        ec_unlock(fop);
+        case -EC_STATE_UNLOCK:
+        case EC_STATE_UNLOCK:
+            ec_unlock(fop);
 
-        return EC_STATE_END;
+            return EC_STATE_END;
 
-    default:
-        gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_UNHANDLED_STATE,
-                "Unhandled state %d for %s",
-                state, ec_fop_name(fop->id));
+        default:
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
-        return EC_STATE_END;
+            return EC_STATE_END;
     }
 }
 
-void ec_discard(call_frame_t *frame, xlator_t *this, uintptr_t target,
-                int32_t minimum, fop_discard_cbk_t func, void *data, fd_t *fd,
-                off_t offset, size_t len, dict_t *xdata)
+void
+ec_discard(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_discard_cbk_t func, void *data, fd_t *fd,
+           off_t offset, size_t len, dict_t *xdata)
 {
-    ec_cbk_t callback = { .discard = func };
+    ec_cbk_t callback = {.discard = func};
     ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(DISCARD) %p", frame);
+    gf_msg_trace("ec", 0, "EC(DISCARD) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_DISCARD, 0, target,
-                               minimum, ec_wind_discard, ec_manager_discard,
+                               fop_flags, ec_wind_discard, ec_manager_discard,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -1371,33 +1334,35 @@ out:
  *
  *********************************************************************/
 
-int32_t ec_update_truncate_write (ec_fop_data_t *fop, uintptr_t mask)
+int32_t
+ec_update_truncate_write(ec_fop_data_t *fop, uintptr_t mask)
 {
-        ec_t *ec = fop->xl->private;
-        size_t size = fop->offset * ec->fragments - fop->user_size;
-        return ec_update_write (fop, mask, fop->user_size, size);
+    ec_t *ec = fop->xl->private;
+    uint64_t size = fop->offset * ec->fragments - fop->user_size;
+    return ec_update_write(fop, mask, fop->user_size, size);
 }
 
-int32_t ec_truncate_open_cbk(call_frame_t * frame, void * cookie,
-                             xlator_t * this, int32_t op_ret, int32_t op_errno,
-                             fd_t * fd, dict_t * xdata)
+int32_t
+ec_truncate_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-    ec_fop_data_t * fop = cookie;
+    ec_fop_data_t *fop = cookie;
     int32_t err;
 
     fop->parent->good &= fop->good;
     if (op_ret >= 0) {
-        fd_bind (fd);
-        err = ec_update_truncate_write (fop->parent, fop->answer->mask);
+        fd_bind(fd);
+        err = ec_update_truncate_write(fop->parent, fop->answer->mask);
         if (err != 0) {
-            ec_fop_set_error (fop->parent, -err);
+            ec_fop_set_error(fop->parent, -err);
         }
     }
 
     return 0;
 }
 
-int32_t ec_truncate_clean(ec_fop_data_t * fop)
+int32_t
+ec_truncate_clean(ec_fop_data_t *fop)
 {
     if (fop->fd == NULL) {
         fop->fd = fd_create(fop->loc[0].inode, fop->frame->root->pid);
@@ -1406,24 +1371,25 @@ int32_t ec_truncate_clean(ec_fop_data_t * fop)
         }
 
         ec_open(fop->frame, fop->xl, fop->answer->mask, fop->minimum,
-                ec_truncate_open_cbk, fop, &fop->loc[0], O_RDWR, fop->fd,
-                NULL);
+                ec_truncate_open_cbk, fop, &fop->loc[0], O_RDWR, fop->fd, NULL);
 
         return 0;
     } else {
-        return ec_update_truncate_write (fop, fop->answer->mask);
+        return ec_update_truncate_write(fop, fop->answer->mask);
     }
 }
 
-int32_t ec_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *prestat,
-                         struct iatt *poststat, dict_t *xdata)
+int32_t
+ec_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prestat,
+                struct iatt *poststat, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
 }
 
-void ec_wind_truncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_truncate(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1432,29 +1398,35 @@ void ec_wind_truncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->offset, fop->xdata);
 }
 
-int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_truncate(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
+    off_t offset_down;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             fop->user_size = fop->offset;
             ec_adjust_offset_up(fop->xl->private, &fop->offset, _gf_true);
             fop->frag_range.first = fop->offset;
             fop->frag_range.last = UINT64_MAX;
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
+            offset_down = fop->user_size;
+            ec_adjust_offset_down(fop->xl->private, &offset_down, _gf_true);
+
             if (fop->id == GF_FOP_TRUNCATE) {
-                ec_lock_prepare_inode(fop, &fop->loc[0],
-                                      EC_UPDATE_DATA | EC_UPDATE_META |
-                                      EC_QUERY_INFO, fop->offset, LLONG_MAX);
+                ec_lock_prepare_inode(
+                    fop, &fop->loc[0],
+                    EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                    offset_down, EC_RANGE_FULL);
             } else {
-                ec_lock_prepare_fd(fop, fop->fd,
-                                   EC_UPDATE_DATA | EC_UPDATE_META |
-                                   EC_QUERY_INFO, fop->offset, LLONG_MAX);
+                ec_lock_prepare_fd(
+                    fop, fop->fd,
+                    EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                    offset_down, EC_RANGE_FULL);
             }
             ec_lock(fop);
 
@@ -1470,8 +1442,7 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
             if (cbk != NULL) {
                 int32_t err;
 
-                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                                cbk->count);
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
 
                 /* This shouldn't fail because we have the inode locked. */
                 /* Inode size doesn't need to be updated under locks, because
@@ -1499,24 +1470,17 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_TRUNCATE)
-            {
-                if (fop->cbks.truncate != NULL)
-                {
-                    fop->cbks.truncate(fop->req_frame, fop, fop->xl,
-                                       cbk->op_ret, cbk->op_errno,
-                                       &cbk->iatt[0], &cbk->iatt[1],
-                                       cbk->xdata);
+            if (fop->id == GF_FOP_TRUNCATE) {
+                if (fop->cbks.truncate != NULL) {
+                    QUORUM_CBK(fop->cbks.truncate, fop, fop->req_frame, fop,
+                               fop->xl, cbk->op_ret, cbk->op_errno,
+                               &cbk->iatt[0], &cbk->iatt[1], cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.ftruncate != NULL)
-                {
-                    fop->cbks.ftruncate(fop->req_frame, fop, fop->xl,
-                                        cbk->op_ret, cbk->op_errno,
-                                        &cbk->iatt[0], &cbk->iatt[1],
-                                        cbk->xdata);
+            } else {
+                if (fop->cbks.ftruncate != NULL) {
+                    QUORUM_CBK(fop->cbks.ftruncate, fop, fop->req_frame, fop,
+                               fop->xl, cbk->op_ret, cbk->op_errno,
+                               &cbk->iatt[0], &cbk->iatt[1], cbk->xdata);
                 }
             }
 
@@ -1529,18 +1493,13 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_TRUNCATE)
-            {
-                if (fop->cbks.truncate != NULL)
-                {
+            if (fop->id == GF_FOP_TRUNCATE) {
+                if (fop->cbks.truncate != NULL) {
                     fop->cbks.truncate(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL, NULL, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.ftruncate != NULL)
-                {
+            } else {
+                if (fop->cbks.ftruncate != NULL) {
                     fop->cbks.ftruncate(fop->req_frame, fop, fop->xl, -1,
                                         fop->error, NULL, NULL, NULL);
                 }
@@ -1561,31 +1520,30 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_truncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_truncate_cbk_t func, void * data,
-                 loc_t * loc, off_t offset, dict_t * xdata)
+void
+ec_truncate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_truncate_cbk_t func, void *data, loc_t *loc,
+            off_t offset, dict_t *xdata)
 {
-    ec_cbk_t callback = { .truncate = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.truncate = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(TRUNCATE) %p", frame);
+    gf_msg_trace("ec", 0, "EC(TRUNCATE) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_TRUNCATE, 0, target,
-                               minimum, ec_wind_truncate, ec_manager_truncate,
+                               fop_flags, ec_wind_truncate, ec_manager_truncate,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -1595,9 +1553,8 @@ void ec_truncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -1605,10 +1562,9 @@ void ec_truncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1626,16 +1582,17 @@ out:
 
 /* FOP: ftruncate */
 
-int32_t ec_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iatt *prestat, struct iatt *poststat,
-                          dict_t *xdata)
+int32_t
+ec_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prestat,
+                 struct iatt *poststat, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
 }
 
-void ec_wind_ftruncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_ftruncate(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1644,23 +1601,24 @@ void ec_wind_ftruncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->offset, fop->xdata);
 }
 
-void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                  int32_t minimum, fop_ftruncate_cbk_t func, void * data,
-                  fd_t * fd, off_t offset, dict_t * xdata)
+void
+ec_ftruncate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_ftruncate_cbk_t func, void *data, fd_t *fd,
+             off_t offset, dict_t *xdata)
 {
-    ec_cbk_t callback = { .ftruncate = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.ftruncate = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FTRUNCATE) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FTRUNCATE) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FTRUNCATE, 0, target,
-                               minimum, ec_wind_ftruncate, ec_manager_truncate,
-                               callback, data);
+                               fop_flags, ec_wind_ftruncate,
+                               ec_manager_truncate, callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -1672,10 +1630,9 @@ void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -1683,10 +1640,9 @@ void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1704,95 +1660,91 @@ out:
 
 /* FOP: writev */
 static ec_stripe_t *
-ec_allocate_stripe (ec_t *ec, ec_stripe_list_t *stripe_cache)
+ec_allocate_stripe(ec_t *ec, ec_stripe_list_t *stripe_cache)
 {
-        ec_stripe_t *stripe = NULL;
+    ec_stripe_t *stripe = NULL;
 
-        if (stripe_cache->count >= stripe_cache->max) {
-                GF_ASSERT (!list_empty(&stripe_cache->lru));
-                stripe = list_first_entry(&stripe_cache->lru, ec_stripe_t, lru);
-                list_move_tail(&stripe->lru, &stripe_cache->lru);
-                GF_ATOMIC_INC(ec->stats.stripe_cache.evicts);
+    if (stripe_cache->count >= stripe_cache->max) {
+        GF_ASSERT(!list_empty(&stripe_cache->lru));
+        stripe = list_first_entry(&stripe_cache->lru, ec_stripe_t, lru);
+        list_move_tail(&stripe->lru, &stripe_cache->lru);
+        GF_ATOMIC_INC(ec->stats.stripe_cache.evicts);
+    } else {
+        stripe = GF_MALLOC(sizeof(ec_stripe_t) + ec->stripe_size,
+                           ec_mt_ec_stripe_t);
+        if (stripe != NULL) {
+            stripe_cache->count++;
+            list_add_tail(&stripe->lru, &stripe_cache->lru);
+            GF_ATOMIC_INC(ec->stats.stripe_cache.allocs);
         } else {
-                stripe = GF_MALLOC (sizeof (ec_stripe_t) + ec->stripe_size,
-                                    ec_mt_ec_stripe_t);
-                if (stripe != NULL) {
-                        stripe_cache->count++;
-                        list_add_tail (&stripe->lru, &stripe_cache->lru);
-                        GF_ATOMIC_INC(ec->stats.stripe_cache.allocs);
-                } else {
-                        GF_ATOMIC_INC(ec->stats.stripe_cache.errors);
-                }
+            GF_ATOMIC_INC(ec->stats.stripe_cache.errors);
         }
+    }
 
-        return stripe;
+    return stripe;
 }
 
 static void
-ec_write_stripe_data (ec_t *ec, ec_fop_data_t *fop,
-                      ec_stripe_t *stripe)
+ec_write_stripe_data(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe)
 {
-        off_t   base;
+    off_t base;
 
-        base = fop->size - ec->stripe_size;
-        memcpy(stripe->data, fop->vector[0].iov_base + base, ec->stripe_size);
-        stripe->frag_offset = fop->frag_range.last - ec->fragment_size;
+    base = fop->size - ec->stripe_size;
+    memcpy(stripe->data, fop->vector[0].iov_base + base, ec->stripe_size);
+    stripe->frag_offset = fop->frag_range.last - ec->fragment_size;
 }
 
 static void
-ec_add_stripe_in_cache (ec_t *ec, ec_fop_data_t *fop)
+ec_add_stripe_in_cache(ec_t *ec, ec_fop_data_t *fop)
 {
-        ec_inode_t       *ctx    = NULL;
-        ec_stripe_t      *stripe = NULL;
-        ec_stripe_list_t *stripe_cache = NULL;
-        gf_boolean_t      failed = _gf_true;
+    ec_inode_t *ctx = NULL;
+    ec_stripe_t *stripe = NULL;
+    ec_stripe_list_t *stripe_cache = NULL;
+    gf_boolean_t failed = _gf_true;
 
-        LOCK(&fop->fd->inode->lock);
+    LOCK(&fop->fd->inode->lock);
 
-        ctx = __ec_inode_get (fop->fd->inode, fop->xl);
-        if (ctx == NULL) {
-                goto out;
-        }
-
-        stripe_cache = &ctx->stripe_cache;
-        if (stripe_cache->max > 0) {
-                stripe = ec_allocate_stripe (ec, stripe_cache);
-                if (stripe == NULL) {
-                        goto out;
-                }
+    ctx = __ec_inode_get(fop->fd->inode, fop->xl);
+    if (ctx == NULL) {
+        goto out;
+    }
 
-                ec_write_stripe_data (ec, fop, stripe);
+    stripe_cache = &ctx->stripe_cache;
+    if (stripe_cache->max > 0) {
+        stripe = ec_allocate_stripe(ec, stripe_cache);
+        if (stripe == NULL) {
+            goto out;
         }
 
-        failed = _gf_false;
+        ec_write_stripe_data(ec, fop, stripe);
+    }
+
+    failed = _gf_false;
 
 out:
-        UNLOCK(&fop->fd->inode->lock);
+    UNLOCK(&fop->fd->inode->lock);
 
-        if (failed) {
-                gf_msg (ec->xl->name, GF_LOG_DEBUG, ENOMEM,
-                        EC_MSG_FILE_DESC_REF_FAIL,
-                        "Failed to create and add stripe in cache");
-        }
+    if (failed) {
+        gf_msg(ec->xl->name, GF_LOG_DEBUG, ENOMEM, EC_MSG_FILE_DESC_REF_FAIL,
+               "Failed to create and add stripe in cache");
+    }
 }
 
-int32_t ec_writev_merge_tail(call_frame_t * frame, void * cookie,
-                             xlator_t * this, int32_t op_ret, int32_t op_errno,
-                             struct iovec * vector, int32_t count,
-                             struct iatt * stbuf, struct iobref * iobref,
-                             dict_t * xdata)
+int32_t
+ec_writev_merge_tail(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                     int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                     dict_t *xdata)
 {
-    ec_t * ec = this->private;
-    ec_fop_data_t * fop = frame->local;
-    size_t size, base, tmp;
+    ec_t *ec = this->private;
+    ec_fop_data_t *fop = frame->local;
+    uint64_t size, base, tmp;
 
-    if (op_ret >= 0)
-    {
+    if (op_ret >= 0) {
         tmp = 0;
         size = fop->size - fop->user_size - fop->head;
         base = ec->stripe_size - size;
-        if (op_ret > base)
-        {
+        if (op_ret > base) {
             tmp = min(op_ret - base, size);
             ec_iov_copy_to(fop->vector[0].iov_base + fop->size - size, vector,
                            count, base, tmp);
@@ -1800,49 +1752,44 @@ int32_t ec_writev_merge_tail(call_frame_t * frame, void * cookie,
             size -= tmp;
         }
 
-        if (size > 0)
-        {
+        if (size > 0) {
             memset(fop->vector[0].iov_base + fop->size - size, 0, size);
         }
 
         if (ec->stripe_cache) {
-                ec_add_stripe_in_cache (ec, fop);
+            ec_add_stripe_in_cache(ec, fop);
         }
     }
     return 0;
 }
 
-int32_t ec_writev_merge_head(call_frame_t * frame, void * cookie,
-                             xlator_t * this, int32_t op_ret, int32_t op_errno,
-                             struct iovec * vector, int32_t count,
-                             struct iatt * stbuf, struct iobref * iobref,
-                             dict_t * xdata)
+int32_t
+ec_writev_merge_head(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                     int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                     dict_t *xdata)
 {
-    ec_t * ec = this->private;
-    ec_fop_data_t * fop = frame->local;
-    size_t size, base;
+    ec_t *ec = this->private;
+    ec_fop_data_t *fop = frame->local;
+    uint64_t size, base;
 
-    if (op_ret >= 0)
-    {
+    if (op_ret >= 0) {
         size = fop->head;
         base = 0;
 
-        if (op_ret > 0)
-        {
+        if (op_ret > 0) {
             base = min(op_ret, size);
             ec_iov_copy_to(fop->vector[0].iov_base, vector, count, 0, base);
 
             size -= base;
         }
 
-        if (size > 0)
-        {
+        if (size > 0) {
             memset(fop->vector[0].iov_base + base, 0, size);
         }
 
         size = fop->size - fop->user_size - fop->head;
-        if ((size > 0) && (fop->size == ec->stripe_size))
-        {
+        if ((size > 0) && (fop->size == ec->stripe_size)) {
             ec_writev_merge_tail(frame, cookie, this, op_ret, op_errno, vector,
                                  count, stbuf, iobref, xdata);
         }
@@ -1852,7 +1799,7 @@ int32_t ec_writev_merge_head(call_frame_t * frame, void * cookie,
 }
 
 static int
-ec_make_internal_fop_xdata (dict_t **xdata)
+ec_make_internal_fop_xdata(dict_t **xdata)
 {
     dict_t *dict = NULL;
 
@@ -1861,16 +1808,16 @@ ec_make_internal_fop_xdata (dict_t **xdata)
 
     dict = dict_new();
     if (!dict)
-       goto out;
+        goto out;
 
-    if (dict_set_str (dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes"))
-       goto out;
+    if (dict_set_str(dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes"))
+        goto out;
 
     *xdata = dict;
     return 0;
 out:
     if (dict)
-            dict_unref (dict);
+        dict_unref(dict);
     return -1;
 }
 
@@ -1889,8 +1836,7 @@ ec_writev_prepare_buffers(ec_t *ec, ec_fop_data_t *fop)
     ec_adjust_size_up(ec, &fop->size, _gf_false);
     fop->frag_range.last = fop->frag_range.first + fop->size / ec->fragments;
 
-    if ((fop->int32 != 1) || (fop->head != 0) ||
-        (fop->size > fop->user_size) ||
+    if ((fop->int32 != 1) || (fop->head != 0) || (fop->size > fop->user_size) ||
         !EC_ALIGN_CHECK(fop->vector[0].iov_base, EC_METHOD_WORD_SIZE)) {
         err = ec_buffer_alloc(ec->xl, fop->size, &iobref, &ptr);
         if (err != 0) {
@@ -1935,98 +1881,116 @@ out:
 }
 
 static void
-ec_merge_stripe_head_locked (ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe)
+ec_merge_stripe_head_locked(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe)
 {
-        size_t head, size;
+    uint32_t head, size;
 
-        head = fop->head;
-        memcpy(fop->vector[0].iov_base, stripe->data, head);
+    head = fop->head;
+    memcpy(fop->vector[0].iov_base, stripe->data, head);
 
+    size = ec->stripe_size - head;
+    if (size > fop->user_size) {
+        head += fop->user_size;
         size = ec->stripe_size - head;
-        if (size > fop->user_size) {
-                head += fop->user_size;
-                size = ec->stripe_size - head;
-                memcpy(fop->vector[0].iov_base + head, stripe->data + head,
-                       size);
-        }
+        memcpy(fop->vector[0].iov_base + head, stripe->data + head, size);
+    }
 }
 
 static void
-ec_merge_stripe_tail_locked (ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe)
+ec_merge_stripe_tail_locked(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe)
 {
-        size_t head, tail;
-        off_t  offset;
+    uint32_t head, tail;
+    off_t offset;
 
-        offset = fop->user_size + fop->head;
-        tail = fop->size - offset;
-        head = ec->stripe_size - tail;
+    offset = fop->user_size + fop->head;
+    tail = fop->size - offset;
+    head = ec->stripe_size - tail;
 
-        memcpy(fop->vector[0].iov_base + offset, stripe->data + head, tail);
+    memcpy(fop->vector[0].iov_base + offset, stripe->data + head, tail);
 }
 
 static ec_stripe_t *
-ec_get_stripe_from_cache_locked (ec_t *ec, ec_fop_data_t *fop,
-                                 uint64_t frag_offset)
+ec_get_stripe_from_cache_locked(ec_t *ec, ec_fop_data_t *fop,
+                                uint64_t frag_offset)
 {
-        ec_inode_t       *ctx    = NULL;
-        ec_stripe_t      *stripe = NULL;
-        ec_stripe_list_t *stripe_cache = NULL;
-
-        ctx = __ec_inode_get (fop->fd->inode, fop->xl);
-        if (ctx == NULL) {
-                GF_ATOMIC_INC(ec->stats.stripe_cache.errors);
-                return NULL;
-        }
+    ec_inode_t *ctx = NULL;
+    ec_stripe_t *stripe = NULL;
+    ec_stripe_list_t *stripe_cache = NULL;
 
-        stripe_cache = &ctx->stripe_cache;
-        list_for_each_entry (stripe, &stripe_cache->lru, lru) {
-                if (stripe->frag_offset == frag_offset) {
-                        list_move_tail (&stripe->lru, &stripe_cache->lru);
-                        GF_ATOMIC_INC(ec->stats.stripe_cache.hits);
-                        return stripe;
-                }
+    ctx = __ec_inode_get(fop->fd->inode, fop->xl);
+    if (ctx == NULL) {
+        GF_ATOMIC_INC(ec->stats.stripe_cache.errors);
+        return NULL;
+    }
+
+    stripe_cache = &ctx->stripe_cache;
+    list_for_each_entry(stripe, &stripe_cache->lru, lru)
+    {
+        if (stripe->frag_offset == frag_offset) {
+            list_move_tail(&stripe->lru, &stripe_cache->lru);
+            GF_ATOMIC_INC(ec->stats.stripe_cache.hits);
+            return stripe;
         }
+    }
 
-        GF_ATOMIC_INC(ec->stats.stripe_cache.misses);
+    GF_ATOMIC_INC(ec->stats.stripe_cache.misses);
 
-        return NULL;
+    return NULL;
 }
 
 static gf_boolean_t
-ec_get_and_merge_stripe (ec_t *ec, ec_fop_data_t *fop, ec_stripe_part_t which)
+ec_get_and_merge_stripe(ec_t *ec, ec_fop_data_t *fop, ec_stripe_part_t which)
 {
-        uint64_t      frag_offset;
-        ec_stripe_t  *stripe = NULL;
-        gf_boolean_t  found  = _gf_false;
+    uint64_t frag_offset;
+    ec_stripe_t *stripe = NULL;
+    gf_boolean_t found = _gf_false;
 
-        if (!ec->stripe_cache) {
-                return found;
-        }
+    if (!ec->stripe_cache) {
+        return found;
+    }
 
-        LOCK(&fop->fd->inode->lock);
-        if (which == EC_STRIPE_HEAD) {
-                frag_offset = fop->frag_range.first;
-                stripe = ec_get_stripe_from_cache_locked(ec, fop, frag_offset);
-                if (stripe) {
-                        ec_merge_stripe_head_locked (ec, fop, stripe);
-                        found = _gf_true;
-                }
+    LOCK(&fop->fd->inode->lock);
+    if (which == EC_STRIPE_HEAD) {
+        frag_offset = fop->frag_range.first;
+        stripe = ec_get_stripe_from_cache_locked(ec, fop, frag_offset);
+        if (stripe) {
+            ec_merge_stripe_head_locked(ec, fop, stripe);
+            found = _gf_true;
         }
+    }
 
-        if (which == EC_STRIPE_TAIL) {
-                frag_offset = fop->frag_range.last - ec->fragment_size;
-                stripe = ec_get_stripe_from_cache_locked(ec, fop, frag_offset);
-                if (stripe) {
-                        ec_merge_stripe_tail_locked (ec, fop, stripe);
-                        found = _gf_true;
-                }
+    if (which == EC_STRIPE_TAIL) {
+        frag_offset = fop->frag_range.last - ec->fragment_size;
+        stripe = ec_get_stripe_from_cache_locked(ec, fop, frag_offset);
+        if (stripe) {
+            ec_merge_stripe_tail_locked(ec, fop, stripe);
+            found = _gf_true;
         }
-        UNLOCK(&fop->fd->inode->lock);
+    }
+    UNLOCK(&fop->fd->inode->lock);
 
-        return found;
+    return found;
+}
+
+static uintptr_t
+ec_get_lock_good_mask(inode_t *inode, xlator_t *xl)
+{
+    ec_lock_t *lock = NULL;
+    ec_inode_t *ictx = NULL;
+    LOCK(&inode->lock);
+    {
+        ictx = __ec_inode_get(inode, xl);
+        if (ictx)
+            lock = ictx->inode_lock;
+    }
+    UNLOCK(&inode->lock);
+    if (lock)
+        return lock->good_mask;
+    return 0;
 }
 
-void ec_writev_start(ec_fop_data_t *fop)
+void
+ec_writev_start(ec_fop_data_t *fop)
 {
     ec_t *ec = fop->xl->private;
     ec_fd_t *ctx;
@@ -2061,41 +2025,51 @@ void ec_writev_start(ec_fop_data_t *fop)
     if (err != 0) {
         goto failed_fd;
     }
+    tail = fop->size - fop->user_size - fop->head;
     if (fop->head > 0) {
-        found_stripe = ec_get_and_merge_stripe (ec, fop, EC_STRIPE_HEAD);
-        if (!found_stripe) {
-                if (ec_make_internal_fop_xdata (&xdata)) {
-                        err = -ENOMEM;
-                        goto failed_xdata;
+        if (current > fop->offset) {
+            found_stripe = ec_get_and_merge_stripe(ec, fop, EC_STRIPE_HEAD);
+            if (!found_stripe) {
+                if (ec_make_internal_fop_xdata(&xdata)) {
+                    err = -ENOMEM;
+                    goto failed_xdata;
                 }
-                ec_readv(fop->frame, fop->xl, -1, EC_MINIMUM_MIN,
-                         ec_writev_merge_head,
-                         NULL, fd, ec->stripe_size, fop->offset, 0, xdata);
+                ec_readv(fop->frame, fop->xl,
+                         ec_get_lock_good_mask(fop->fd->inode, fop->xl),
+                         EC_MINIMUM_MIN, ec_writev_merge_head, NULL, fd,
+                         ec->stripe_size, fop->offset, 0, xdata);
+            }
+        } else {
+            memset(fop->vector[0].iov_base, 0, fop->head);
+            memset(fop->vector[0].iov_base + fop->size - tail, 0, tail);
+            if (ec->stripe_cache && (fop->size <= ec->stripe_size)) {
+                ec_add_stripe_in_cache(ec, fop);
+            }
         }
     }
 
-    tail = fop->size - fop->user_size - fop->head;
     if ((tail > 0) && ((fop->head == 0) || (fop->size > ec->stripe_size))) {
-            /* Current locking scheme will make sure the 'current' below will
-             * never decrease while the fop is in progress, so the checks will
-             * work as expected
-             */
+        /* Current locking scheme will make sure the 'current' below will
+         * never decrease while the fop is in progress, so the checks will
+         * work as expected
+         */
         if (current > fop->offset + fop->head + fop->user_size) {
-            found_stripe = ec_get_and_merge_stripe (ec, fop, EC_STRIPE_TAIL);
+            found_stripe = ec_get_and_merge_stripe(ec, fop, EC_STRIPE_TAIL);
             if (!found_stripe) {
-                    if (ec_make_internal_fop_xdata (&xdata)) {
-                            err = -ENOMEM;
-                            goto failed_xdata;
-                    }
-                    ec_readv(fop->frame, fop->xl, -1, EC_MINIMUM_MIN,
-                             ec_writev_merge_tail, NULL, fd, ec->stripe_size,
-                             fop->offset + fop->size - ec->stripe_size,
-                             0, xdata);
+                if (ec_make_internal_fop_xdata(&xdata)) {
+                    err = -ENOMEM;
+                    goto failed_xdata;
+                }
+                ec_readv(fop->frame, fop->xl,
+                         ec_get_lock_good_mask(fop->fd->inode, fop->xl),
+                         EC_MINIMUM_MIN, ec_writev_merge_tail, NULL, fd,
+                         ec->stripe_size,
+                         fop->offset + fop->size - ec->stripe_size, 0, xdata);
             }
         } else {
             memset(fop->vector[0].iov_base + fop->size - tail, 0, tail);
             if (ec->stripe_cache) {
-                ec_add_stripe_in_cache (ec, fop);
+                ec_add_stripe_in_cache(ec, fop);
             }
         }
     }
@@ -2112,23 +2086,25 @@ failed:
     ec_fop_set_error(fop, -err);
 }
 
-int32_t ec_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prestat,
-                       struct iatt *poststat, dict_t *xdata)
+int32_t
+ec_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *prestat, struct iatt *poststat,
+              dict_t *xdata)
 {
-        ec_t    *ec = NULL;
-        if (this && this->private) {
-                ec = this->private;
-                if ((op_ret > 0) && ((op_ret % ec->fragment_size) != 0)) {
-                        op_ret = -1;
-                        op_errno = EIO;
-                }
+    ec_t *ec = NULL;
+    if (this && this->private) {
+        ec = this->private;
+        if ((op_ret > 0) && ((op_ret % ec->fragment_size) != 0)) {
+            op_ret = -1;
+            op_errno = EIO;
         }
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    }
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
 }
 
-void ec_wind_writev(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_writev(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -2141,9 +2117,9 @@ void ec_wind_writev(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
     vector[0].iov_len = size;
 
     STACK_WIND_COOKIE(fop->frame, ec_writev_cbk, (void *)(uintptr_t)idx,
-                      ec->xl_list[idx], ec->xl_list[idx]->fops->writev,
-                      fop->fd, vector, 1, fop->offset / ec->fragments,
-                      fop->uint32, fop->buffers, fop->xdata);
+                      ec->xl_list[idx], ec->xl_list[idx]->fops->writev, fop->fd,
+                      vector, 1, fop->offset / ec->fragments, fop->uint32,
+                      fop->buffers, fop->xdata);
 }
 
 static void
@@ -2161,35 +2137,34 @@ ec_writev_encode(ec_fop_data_t *fop)
                      fop->vector[0].iov_base, blocks);
 }
 
-int32_t ec_manager_writev(ec_fop_data_t *fop, int32_t state)
+int32_t
+ec_manager_writev(ec_fop_data_t *fop, int32_t state)
 {
     ec_cbk_data_t *cbk;
     ec_fd_t *ctx = NULL;
-    ec_t    *ec = fop->xl->private;
+    ec_t *ec = fop->xl->private;
     off_t fl_start = 0;
-    size_t fl_size = LLONG_MAX;
+    uint64_t fl_size = LONG_MAX;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-                ctx = ec_fd_get(fop->fd, fop->xl);
-                if (ctx != NULL) {
-                    if ((ctx->flags & O_APPEND) == 0) {
-                            off_t user_size = 0;
-                            off_t head = 0;
-
-                            fl_start = fop->offset;
-                            user_size = iov_length(fop->vector, fop->int32);
-                            head = ec_adjust_offset_down(ec, &fl_start,
-                                                         _gf_true);
-                            fl_size = user_size + head;
-                            ec_adjust_size_up(ec, &fl_size, _gf_true);
-                    }
+            ctx = ec_fd_get(fop->fd, fop->xl);
+            if (ctx != NULL) {
+                if ((ctx->flags & O_APPEND) == 0) {
+                    off_t user_size = 0;
+                    off_t head = 0;
+
+                    fl_start = fop->offset;
+                    user_size = iov_length(fop->vector, fop->int32);
+                    head = ec_adjust_offset_down(ec, &fl_start, _gf_true);
+                    fl_size = user_size + head;
+                    ec_adjust_size_up(ec, &fl_size, _gf_true);
                 }
+            }
             ec_lock_prepare_fd(fop, fop->fd,
-                               EC_UPDATE_DATA | EC_UPDATE_META |
-                               EC_QUERY_INFO, fl_start, fl_size);
+                               EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                               fl_start, fl_size);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -2215,31 +2190,30 @@ int32_t ec_manager_writev(ec_fop_data_t *fop, int32_t state)
             cbk = ec_fop_prepare_answer(fop, _gf_false);
             if (cbk != NULL) {
                 ec_t *ec = fop->xl->private;
-                size_t size;
+                uint64_t size;
 
-                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                                cbk->count);
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
 
                 /* This shouldn't fail because we have the inode locked. */
                 LOCK(&fop->fd->inode->lock);
                 {
-                        GF_ASSERT(__ec_get_inode_size(fop, fop->fd->inode,
-                                                    &cbk->iatt[0].ia_size));
-                        cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
-                        size = fop->offset + fop->head + fop->user_size;
-                        if (size > cbk->iatt[0].ia_size) {
-                           /* Only update inode size if this is a top level fop.
-                            * Otherwise this is an internal write and the top
-                            * level fop should take care of the real inode size.
-                            */
-                            if (fop->parent == NULL) {
-                                /* This shouldn't fail because we have the inode
-                                 * locked. */
-                                GF_ASSERT(__ec_set_inode_size(fop,
-                                                        fop->fd->inode, size));
-                            }
-                            cbk->iatt[1].ia_size = size;
+                    GF_ASSERT(__ec_get_inode_size(fop, fop->fd->inode,
+                                                  &cbk->iatt[0].ia_size));
+                    cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
+                    size = fop->offset + fop->head + fop->user_size;
+                    if (size > cbk->iatt[0].ia_size) {
+                        /* Only update inode size if this is a top level fop.
+                         * Otherwise this is an internal write and the top
+                         * level fop should take care of the real inode size.
+                         */
+                        if (fop->parent == NULL) {
+                            /* This shouldn't fail because we have the inode
+                             * locked. */
+                            GF_ASSERT(
+                                __ec_set_inode_size(fop, fop->fd->inode, size));
                         }
+                        cbk->iatt[1].ia_size = size;
+                    }
                 }
                 UNLOCK(&fop->fd->inode->lock);
 
@@ -2263,11 +2237,10 @@ int32_t ec_manager_writev(ec_fop_data_t *fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.writev != NULL)
-            {
-                fop->cbks.writev(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                 cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
-                                 cbk->xdata);
+            if (fop->cbks.writev != NULL) {
+                QUORUM_CBK(fop->cbks.writev, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -2278,7 +2251,7 @@ int32_t ec_manager_writev(ec_fop_data_t *fop, int32_t state)
             fop->frame->root->uid = fop->uid;
             fop->frame->root->gid = fop->gid;
 
-        /* Fall through */
+            /* Fall through */
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
@@ -2287,8 +2260,7 @@ int32_t ec_manager_writev(ec_fop_data_t *fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.writev != NULL)
-            {
+            if (fop->cbks.writev != NULL) {
                 fop->cbks.writev(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL);
             }
@@ -2308,31 +2280,30 @@ int32_t ec_manager_writev(ec_fop_data_t *fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_writev_cbk_t func, void * data, fd_t * fd,
-               struct iovec * vector, int32_t count, off_t offset,
-               uint32_t flags, struct iobref * iobref, dict_t * xdata)
+void
+ec_writev(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_writev_cbk_t func, void *data, fd_t *fd,
+          struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+          struct iobref *iobref, dict_t *xdata)
 {
-    ec_cbk_t callback = { .writev = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.writev = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(WRITE) %p", frame);
+    gf_msg_trace("ec", 0, "EC(WRITE) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_WRITE, 0, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_WRITE, 0, target, fop_flags,
                                ec_wind_writev, ec_manager_writev, callback,
                                data);
     if (fop == NULL) {
@@ -2348,10 +2319,9 @@ void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -2359,10 +2329,9 @@ void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (count > 0) {
         fop->vector = iov_dup(vector, count);
         if (fop->vector == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a "
-                    "vector list.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a "
+                   "vector list.");
 
             goto out;
         }
@@ -2371,10 +2340,9 @@ void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (iobref != NULL) {
         fop->buffers = iobref_ref(iobref);
         if (fop->buffers == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_BUF_REF_FAIL,
-                    "Failed to reference a "
-                    "buffer.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_BUF_REF_FAIL,
+                   "Failed to reference a "
+                   "buffer.");
 
             goto out;
         }
@@ -2382,10 +2350,9 @@ void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_copy_with_ref(xdata, NULL);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
diff --git a/xlators/cluster/ec/src/ec-locks.c b/xlators/cluster/ec/src/ec-locks.c
index 996035de90b..601960d6154 100644
--- a/xlators/cluster/ec/src/ec-locks.c
+++ b/xlators/cluster/ec/src/ec-locks.c
@@ -8,52 +8,76 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
-#include "ec-method.h"
 #include "ec-fops.h"
 #include "ec-messages.h"
 
 #define EC_LOCK_MODE_NONE 0
-#define EC_LOCK_MODE_INC  1
-#define EC_LOCK_MODE_ALL  2
+#define EC_LOCK_MODE_INC 1
+#define EC_LOCK_MODE_ALL 2
 
-int32_t ec_lock_check(ec_fop_data_t *fop, uintptr_t *mask)
+int32_t
+ec_lock_check(ec_fop_data_t *fop, uintptr_t *mask)
 {
     ec_t *ec = fop->xl->private;
     ec_cbk_data_t *ans = NULL;
     ec_cbk_data_t *cbk = NULL;
-    uintptr_t locked = 0, notlocked = 0;
+    uintptr_t locked = 0;
+    int32_t good = 0;
+    int32_t eagain = 0;
+    int32_t estale = 0;
     int32_t error = -1;
 
-    list_for_each_entry(ans, &fop->cbk_list, list) {
+    /* There are some errors that we'll handle in an special way while trying
+     * to acquire a lock.
+     *
+     *   EAGAIN:  If it's found during a parallel non-blocking lock request, we
+     *            consider that there's contention on the inode, so we consider
+     *            the acquisition a failure and try again with a sequential
+     *            blocking lock request. This will ensure that we get a lock on
+     *            as many bricks as possible (ignoring EAGAIN here would cause
+     *            unnecessary triggers of self-healing).
+     *
+     *            If it's found during a sequential blocking lock request, it's
+     *            considered an error. Lock will only succeed if there are
+     *            enough other bricks locked.
+     *
+     *   ESTALE:  This can appear during parallel or sequential lock request if
+     *            the inode has just been unlinked. We consider this error is
+     *            not recoverable, but we also don't consider it as fatal. So,
+     *            if it happens during parallel lock, we won't attempt a
+     *            sequential one unless there are EAGAIN errors on other
+     *            bricks (and are enough to form a quorum), but if we reach
+     *            quorum counting the ESTALE bricks, we consider the whole
+     *            result of the operation is ESTALE instead of EIO.
+     */
+
+    list_for_each_entry(ans, &fop->cbk_list, list)
+    {
         if (ans->op_ret >= 0) {
             if (locked != 0) {
                 error = EIO;
             }
             locked |= ans->mask;
+            good = ans->count;
             cbk = ans;
-        } else {
-                if (ans->op_errno == EAGAIN) {
-                        switch (fop->uint32) {
-                        case EC_LOCK_MODE_NONE:
-                        case EC_LOCK_MODE_ALL:
-                                /* Goal is to treat non-blocking lock as failure
-                                 * even if there is a signle EAGAIN*/
-                                notlocked |= ans->mask;
-                                break;
-                        }
-                }
+        } else if (ans->op_errno == ESTALE) {
+            estale += ans->count;
+        } else if ((ans->op_errno == EAGAIN) &&
+                   (fop->uint32 != EC_LOCK_MODE_INC)) {
+            eagain += ans->count;
         }
     }
 
     if (error == -1) {
-        if (gf_bits_count(locked | notlocked) >= ec->fragments) {
-            if (notlocked == 0) {
+        /* If we have enough quorum with succeeded and EAGAIN answers, we
+         * ignore for now any ESTALE answer. If there are EAGAIN answers,
+         * we retry with a sequential blocking lock request if needed.
+         * Otherwise we succeed. */
+        if ((good + eagain) >= ec->fragments) {
+            if (eagain == 0) {
                 if (fop->answer == NULL) {
                     fop->answer = cbk;
                 }
@@ -63,24 +87,31 @@ int32_t ec_lock_check(ec_fop_data_t *fop, uintptr_t *mask)
                 error = 0;
             } else {
                 switch (fop->uint32) {
-                case EC_LOCK_MODE_NONE:
-                    error = EAGAIN;
-                    break;
-
-                case EC_LOCK_MODE_ALL:
-                    fop->uint32 = EC_LOCK_MODE_INC;
-                    break;
-
-                default:
-                    error = EIO;
-                    break;
+                    case EC_LOCK_MODE_NONE:
+                        error = EAGAIN;
+                        break;
+                    case EC_LOCK_MODE_ALL:
+                        fop->uint32 = EC_LOCK_MODE_INC;
+                        break;
+                    default:
+                        /* This shouldn't happen because eagain cannot be > 0
+                         * when fop->uint32 is EC_LOCK_MODE_INC. */
+                        error = EIO;
+                        break;
                 }
             }
         } else {
-            if (fop->answer && fop->answer->op_ret < 0)
-                    error = fop->answer->op_errno;
-            else
-                    error = EIO;
+            /* We have been unable to find enough candidates that will be able
+             * to take the lock. If we have quorum on some answer, we return
+             * it. Otherwise we check if ESTALE answers allow us to reach
+             * quorum. If so, we return ESTALE. */
+            if (fop->answer && fop->answer->op_ret < 0) {
+                error = fop->answer->op_errno;
+            } else if ((good + eagain + estale) >= ec->fragments) {
+                error = ESTALE;
+            } else {
+                error = EIO;
+            }
         }
     }
 
@@ -89,28 +120,25 @@ int32_t ec_lock_check(ec_fop_data_t *fop, uintptr_t *mask)
     return error;
 }
 
-int32_t ec_lock_unlocked(call_frame_t * frame, void * cookie,
-                         xlator_t * this, int32_t op_ret, int32_t op_errno,
-                         dict_t * xdata)
+int32_t
+ec_lock_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    if (op_ret < 0)
-    {
-        gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                EC_MSG_UNLOCK_FAILED,
-                "Failed to unlock an entry/inode");
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_UNLOCK_FAILED,
+               "Failed to unlock an entry/inode");
     }
 
     return 0;
 }
 
-int32_t ec_lock_lk_unlocked(call_frame_t * frame, void * cookie,
-                            xlator_t * this, int32_t op_ret, int32_t op_errno,
-                            struct gf_flock * flock, dict_t * xdata)
+int32_t
+ec_lock_lk_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                    dict_t *xdata)
 {
-    if (op_ret < 0)
-    {
-        gf_msg(this->name, GF_LOG_WARNING, op_errno,
-               EC_MSG_LK_UNLOCK_FAILED,
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_LK_UNLOCK_FAILED,
                "Failed to unlock an lk");
     }
 
@@ -119,11 +147,12 @@ int32_t ec_lock_lk_unlocked(call_frame_t * frame, void * cookie,
 
 /* FOP: entrylk */
 
-int32_t ec_entrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -133,20 +162,16 @@ int32_t ec_entrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_ENTRYLK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg(this->name, GF_LOG_ERROR, 0,
-                       EC_MSG_DICT_REF_FAIL,
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
                        "Failed to reference a "
                        "dictionary.");
 
@@ -158,15 +183,15 @@ int32_t ec_entrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_entrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_entrylk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -176,20 +201,19 @@ void ec_wind_entrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->entrylk_type, fop->xdata);
 }
 
-int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_entrylk(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
-            if (fop->entrylk_cmd == ENTRYLK_LOCK)
-            {
+            if (fop->entrylk_cmd == ENTRYLK_LOCK) {
                 fop->uint32 = EC_LOCK_MODE_ALL;
                 fop->entrylk_cmd = ENTRYLK_LOCK_NB;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -201,15 +225,14 @@ int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state)
             if (fop->entrylk_cmd != ENTRYLK_UNLOCK) {
                 uintptr_t mask;
 
-                ec_fop_set_error (fop, ec_lock_check(fop, &mask));
+                ec_fop_set_error(fop, ec_lock_check(fop, &mask));
                 if (fop->error != 0) {
                     if (mask != 0) {
                         if (fop->id == GF_FOP_ENTRYLK) {
-                            ec_entrylk(fop->frame, fop->xl, mask, 1,
-                                       ec_lock_unlocked, NULL, fop->str[0],
-                                       &fop->loc[0], fop->str[1],
-                                       ENTRYLK_UNLOCK, fop->entrylk_type,
-                                       fop->xdata);
+                            ec_entrylk(
+                                fop->frame, fop->xl, mask, 1, ec_lock_unlocked,
+                                NULL, fop->str[0], &fop->loc[0], fop->str[1],
+                                ENTRYLK_UNLOCK, fop->entrylk_type, fop->xdata);
                         } else {
                             ec_fentrylk(fop->frame, fop->xl, mask, 1,
                                         ec_lock_unlocked, NULL, fop->str[0],
@@ -238,18 +261,13 @@ int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_ENTRYLK)
-            {
-                if (fop->cbks.entrylk != NULL)
-                {
-                    fop->cbks.entrylk(fop->req_frame, fop, fop->xl,
-                                      cbk->op_ret, cbk->op_errno, cbk->xdata);
+            if (fop->id == GF_FOP_ENTRYLK) {
+                if (fop->cbks.entrylk != NULL) {
+                    fop->cbks.entrylk(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                      cbk->op_errno, cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fentrylk != NULL)
-                {
+            } else {
+                if (fop->cbks.fentrylk != NULL) {
                     fop->cbks.fentrylk(fop->req_frame, fop, fop->xl,
                                        cbk->op_ret, cbk->op_errno, cbk->xdata);
                 }
@@ -262,18 +280,13 @@ int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_ENTRYLK)
-            {
-                if (fop->cbks.entrylk != NULL)
-                {
+            if (fop->id == GF_FOP_ENTRYLK) {
+                if (fop->cbks.entrylk != NULL) {
                     fop->cbks.entrylk(fop->req_frame, fop, fop->xl, -1,
                                       fop->error, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fentrylk != NULL)
-                {
+            } else {
+                if (fop->cbks.fentrylk != NULL) {
                     fop->cbks.fentrylk(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL);
                 }
@@ -282,33 +295,31 @@ int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_entrylk_cbk_t func, void * data,
-                const char * volume, loc_t * loc, const char * basename,
-                entrylk_cmd cmd, entrylk_type type, dict_t * xdata)
+void
+ec_entrylk(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_entrylk_cbk_t func, void *data,
+           const char *volume, loc_t *loc, const char *basename,
+           entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-    ec_cbk_t callback = { .entrylk = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.entrylk = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(ENTRYLK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(ENTRYLK) %p", frame);
 
-    VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_ENTRYLK, 0, target, minimum,
-                               ec_wind_entrylk, ec_manager_entrylk, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_ENTRYLK, 0, target,
+                               fop_flags, ec_wind_entrylk, ec_manager_entrylk,
+                               callback, data);
     if (fop == NULL) {
         goto out;
     }
@@ -319,18 +330,16 @@ void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (volume != NULL) {
         fop->str[0] = gf_strdup(volume);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -338,9 +347,8 @@ void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (basename != NULL) {
         fop->str[1] = gf_strdup(basename);
         if (fop->str[1] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
@@ -348,10 +356,9 @@ void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -369,11 +376,12 @@ out:
 
 /* FOP: fentrylk */
 
-int32_t ec_fentrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -383,22 +391,18 @@ int32_t ec_fentrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FENTRYLK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                         "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -408,15 +412,15 @@ int32_t ec_fentrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fentrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fentrylk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -426,23 +430,23 @@ void ec_wind_fentrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->entrylk_type, fop->xdata);
 }
 
-void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fentrylk_cbk_t func, void * data,
-                 const char * volume, fd_t * fd, const char * basename,
-                 entrylk_cmd cmd, entrylk_type type, dict_t * xdata)
+void
+ec_fentrylk(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fentrylk_cbk_t func, void *data,
+            const char *volume, fd_t *fd, const char *basename, entrylk_cmd cmd,
+            entrylk_type type, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fentrylk = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.fentrylk = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FENTRYLK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FENTRYLK) %p", frame);
 
-    VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FENTRYLK, 0, target,
-                               minimum, ec_wind_fentrylk, ec_manager_entrylk,
+                               fop_flags, ec_wind_fentrylk, ec_manager_entrylk,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -456,9 +460,8 @@ void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (volume != NULL) {
         fop->str[0] = gf_strdup(volume);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
@@ -466,10 +469,9 @@ void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -477,9 +479,8 @@ void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (basename != NULL) {
         fop->str[1] = gf_strdup(basename);
         if (fop->str[1] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
@@ -487,10 +488,9 @@ void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -508,11 +508,12 @@ out:
 
 /* FOP: inodelk */
 
-int32_t ec_inodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -522,22 +523,18 @@ int32_t ec_inodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_INODELK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                        "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -547,15 +544,15 @@ int32_t ec_inodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_inodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_inodelk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -565,24 +562,22 @@ void ec_wind_inodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_inodelk(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
-            fop->flock.l_len += ec_adjust_offset_down(fop->xl->private,
-                                                      &fop->flock.l_start,
-                                                      _gf_true);
+            fop->flock.l_len += ec_adjust_offset_down(
+                fop->xl->private, &fop->flock.l_start, _gf_true);
             ec_adjust_offset_up(fop->xl->private, &fop->flock.l_len, _gf_true);
-            if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK))
-            {
+            if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK)) {
                 fop->uint32 = EC_LOCK_MODE_ALL;
                 fop->int32 = F_SETLK;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -594,7 +589,7 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
             if (fop->flock.l_type != F_UNLCK) {
                 uintptr_t mask;
 
-                ec_fop_set_error (fop, ec_lock_check(fop, &mask));
+                ec_fop_set_error(fop, ec_lock_check(fop, &mask));
                 if (fop->error != 0) {
                     if (mask != 0) {
                         ec_t *ec = fop->xl->private;
@@ -641,18 +636,13 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_INODELK)
-            {
-                if (fop->cbks.inodelk != NULL)
-                {
-                    fop->cbks.inodelk(fop->req_frame, fop, fop->xl,
-                                      cbk->op_ret, cbk->op_errno, cbk->xdata);
+            if (fop->id == GF_FOP_INODELK) {
+                if (fop->cbks.inodelk != NULL) {
+                    fop->cbks.inodelk(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                      cbk->op_errno, cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.finodelk != NULL)
-                {
+            } else {
+                if (fop->cbks.finodelk != NULL) {
                     fop->cbks.finodelk(fop->req_frame, fop, fop->xl,
                                        cbk->op_ret, cbk->op_errno, cbk->xdata);
                 }
@@ -665,18 +655,13 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_INODELK)
-            {
-                if (fop->cbks.inodelk != NULL)
-                {
+            if (fop->id == GF_FOP_INODELK) {
+                if (fop->cbks.inodelk != NULL) {
                     fop->cbks.inodelk(fop->req_frame, fop, fop->xl, -1,
                                       fop->error, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.finodelk != NULL)
-                {
+            } else {
+                if (fop->cbks.finodelk != NULL) {
                     fop->cbks.finodelk(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL);
                 }
@@ -685,55 +670,52 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_inodelk (call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
-                 uintptr_t target, int32_t minimum, fop_inodelk_cbk_t func,
-                 void *data, const char *volume, loc_t *loc, int32_t cmd,
-                 struct gf_flock *flock, dict_t *xdata)
+void
+ec_inodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
+           uintptr_t target, uint32_t fop_flags, fop_inodelk_cbk_t func,
+           void *data, const char *volume, loc_t *loc, int32_t cmd,
+           struct gf_flock *flock, dict_t *xdata)
 {
-    ec_cbk_t callback = { .inodelk = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.inodelk = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(INODELK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(INODELK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_INODELK, 0, target, minimum,
-                               ec_wind_inodelk, ec_manager_inodelk, callback,
-                               data);
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_INODELK, 0, target,
+                               fop_flags, ec_wind_inodelk, ec_manager_inodelk,
+                               callback, data);
     if (fop == NULL) {
         goto out;
     }
 
     fop->int32 = cmd;
-    ec_owner_copy (fop->frame, owner);
+    ec_owner_copy(fop->frame, owner);
 
     if (volume != NULL) {
         fop->str[0] = gf_strdup(volume);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
     if (loc != NULL) {
         if (loc_copy(&fop->loc[0], loc) != 0) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_LOC_COPY_FAIL,
-                    "Failed to copy a location.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
@@ -753,10 +735,9 @@ void ec_inodelk (call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -774,11 +755,12 @@ out:
 
 /* FOP: finodelk */
 
-int32_t ec_finodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -788,22 +770,18 @@ int32_t ec_finodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FINODELK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                        "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -813,15 +791,15 @@ int32_t ec_finodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_finodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_finodelk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -831,23 +809,24 @@ void ec_wind_finodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-void ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
-                 uintptr_t target, int32_t minimum, fop_finodelk_cbk_t func,
-                 void *data, const char *volume, fd_t *fd, int32_t cmd,
-                 struct gf_flock *flock, dict_t *xdata)
+void
+ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
+            uintptr_t target, uint32_t fop_flags, fop_finodelk_cbk_t func,
+            void *data, const char *volume, fd_t *fd, int32_t cmd,
+            struct gf_flock *flock, dict_t *xdata)
 {
-    ec_cbk_t callback = { .finodelk = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.finodelk = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(FINODELK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FINODELK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = ec_fop_data_allocate(frame, this, GF_FOP_FINODELK, 0, target,
-                               minimum, ec_wind_finodelk, ec_manager_inodelk,
+                               fop_flags, ec_wind_finodelk, ec_manager_inodelk,
                                callback, data);
     if (fop == NULL) {
         goto out;
@@ -856,14 +835,13 @@ void ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
     fop->use_fd = 1;
 
     fop->int32 = cmd;
-    ec_owner_copy (fop->frame, owner);
+    ec_owner_copy(fop->frame, owner);
 
     if (volume != NULL) {
         fop->str[0] = gf_strdup(volume);
         if (fop->str[0] == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                    EC_MSG_NO_MEMORY,
-                    "Failed to duplicate a string.");
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
@@ -871,10 +849,9 @@ void ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -894,10 +871,9 @@ void ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -915,15 +891,13 @@ out:
 
 /* FOP: lk */
 
-int32_t ec_combine_lk(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                      ec_cbk_data_t * src)
+int32_t
+ec_combine_lk(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_flock_compare(&dst->flock, &src->flock))
-    {
-        gf_msg (fop->xl->name, GF_LOG_NOTICE, 0,
-                EC_MSG_LOCK_MISMATCH,
-                "Mismatching lock in "
-                "answers of 'GF_FOP_LK'");
+    if (!ec_flock_compare(&dst->flock, &src->flock)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_LOCK_MISMATCH,
+               "Mismatching lock in "
+               "answers of 'GF_FOP_LK'");
 
         return 0;
     }
@@ -931,12 +905,12 @@ int32_t ec_combine_lk(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_lk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                  int32_t op_ret, int32_t op_errno, struct gf_flock * flock,
-                  dict_t * xdata)
+int32_t
+ec_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+          int32_t op_errno, struct gf_flock *flock, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -946,39 +920,32 @@ int32_t ec_lk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_LK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (flock != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (flock != NULL) {
                 cbk->flock.l_type = flock->l_type;
                 cbk->flock.l_whence = flock->l_whence;
                 cbk->flock.l_start = flock->l_start;
                 cbk->flock.l_len = flock->l_len;
                 cbk->flock.l_pid = flock->l_pid;
                 cbk->flock.l_owner.len = flock->l_owner.len;
-                if (flock->l_owner.len > 0)
-                {
+                if (flock->l_owner.len > 0) {
                     memcpy(cbk->flock.l_owner.data, flock->l_owner.data,
                            flock->l_owner.len);
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        EC_MSG_DICT_REF_FAIL,
-                        "Failed to reference a "
-                        "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -988,15 +955,15 @@ int32_t ec_lk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_lk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_lk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1005,20 +972,19 @@ void ec_wind_lk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->int32, &fop->flock, fop->xdata);
 }
 
-int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_lk(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
-            if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK))
-            {
+            if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK)) {
                 fop->uint32 = EC_LOCK_MODE_ALL;
                 fop->int32 = F_SETLK;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -1030,7 +996,7 @@ int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state)
             if (fop->flock.l_type != F_UNLCK) {
                 uintptr_t mask;
 
-                ec_fop_set_error (fop, ec_lock_check(fop, &mask));
+                ec_fop_set_error(fop, ec_lock_check(fop, &mask));
                 if (fop->error != 0) {
                     if (mask != 0) {
                         struct gf_flock flock = {0};
@@ -1040,11 +1006,10 @@ int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state)
                         flock.l_start = fop->flock.l_start;
                         flock.l_len = fop->flock.l_len;
                         flock.l_pid = fop->flock.l_pid;
-                        lk_owner_copy (&flock.l_owner, &fop->flock.l_owner);
+                        lk_owner_copy(&flock.l_owner, &fop->flock.l_owner);
 
-                        ec_lk(fop->frame, fop->xl, mask, 1,
-                              ec_lock_lk_unlocked, NULL, fop->fd, F_SETLK,
-                              &flock, fop->xdata);
+                        ec_lk(fop->frame, fop->xl, mask, 1, ec_lock_lk_unlocked,
+                              NULL, fop->fd, F_SETLK, &flock, fop->xdata);
                     }
 
                     if (fop->error < 0) {
@@ -1068,8 +1033,7 @@ int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.lk != NULL)
-            {
+            if (fop->cbks.lk != NULL) {
                 fop->cbks.lk(fop->req_frame, fop, fop->xl, cbk->op_ret,
                              cbk->op_errno, &cbk->flock, cbk->xdata);
             }
@@ -1081,40 +1045,36 @@ int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.lk != NULL)
-            {
-                fop->cbks.lk(fop->req_frame, fop, fop->xl, -1, fop->error,
-                             NULL, NULL);
+            if (fop->cbks.lk != NULL) {
+                fop->cbks.lk(fop->req_frame, fop, fop->xl, -1, fop->error, NULL,
+                             NULL);
             }
 
-
             return EC_STATE_END;
 
         default:
-            gf_msg (fop->xl->name, GF_LOG_ERROR, EINVAL,
-                    EC_MSG_UNHANDLED_STATE,
-                    "Unhandled state %d for %s",
-                    state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-           int32_t minimum, fop_lk_cbk_t func, void * data, fd_t * fd,
-           int32_t cmd, struct gf_flock * flock, dict_t * xdata)
+void
+ec_lk(call_frame_t *frame, xlator_t *this, uintptr_t target, uint32_t fop_flags,
+      fop_lk_cbk_t func, void *data, fd_t *fd, int32_t cmd,
+      struct gf_flock *flock, dict_t *xdata)
 {
-    ec_cbk_t callback = { .lk = func };
-    ec_fop_data_t * fop = NULL;
+    ec_cbk_t callback = {.lk = func};
+    ec_fop_data_t *fop = NULL;
     int32_t error = ENOMEM;
 
-    gf_msg_trace ("ec", 0, "EC(LK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(LK) %p", frame);
 
-    VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_LK, 0, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_LK, 0, target, fop_flags,
                                ec_wind_lk, ec_manager_lk, callback, data);
     if (fop == NULL) {
         goto out;
@@ -1127,10 +1087,9 @@ void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (fd != NULL) {
         fop->fd = fd_ref(fd);
         if (fop->fd == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_FILE_DESC_REF_FAIL,
-                    "Failed to reference a "
-                    "file descriptor.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
@@ -1150,10 +1109,9 @@ void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
         if (fop->xdata == NULL) {
-            gf_msg (this->name, GF_LOG_ERROR, 0,
-                    EC_MSG_DICT_REF_FAIL,
-                    "Failed to reference a "
-                    "dictionary.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
diff --git a/xlators/cluster/ec/src/ec-mem-types.h b/xlators/cluster/ec/src/ec-mem-types.h
index 8109a422d9d..3252c4c1c58 100644
--- a/xlators/cluster/ec/src/ec-mem-types.h
+++ b/xlators/cluster/ec/src/ec-mem-types.h
@@ -11,15 +11,13 @@
 #ifndef __EC_MEM_TYPES_H__
 #define __EC_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
-enum gf_ec_mem_types_
-{
+enum gf_ec_mem_types_ {
     ec_mt_ec_t = gf_common_mt_end + 1,
     ec_mt_xlator_t,
     ec_mt_ec_inode_t,
     ec_mt_ec_fd_t,
-    ec_mt_ec_heal_t,
     ec_mt_subvol_healer_t,
     ec_mt_ec_gf_t,
     ec_mt_ec_code_t,
diff --git a/xlators/cluster/ec/src/ec-messages.h b/xlators/cluster/ec/src/ec-messages.h
index f5138ad8e57..72e98f11286 100644
--- a/xlators/cluster/ec/src/ec-messages.h
+++ b/xlators/cluster/ec/src/ec-messages.h
@@ -11,7 +11,7 @@
 #ifndef _EC_MESSAGES_H_
 #define _EC_MESSAGES_H_
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
 
 /* To add new message IDs, append new identifiers at the end of the list.
  *
@@ -23,83 +23,39 @@
  * glfs-message-id.h.
  */
 
-GLFS_MSGID(EC,
-        EC_MSG_INVALID_CONFIG,
-        EC_MSG_HEAL_FAIL,
-        EC_MSG_DICT_COMBINE_FAIL,
-        EC_MSG_STIME_COMBINE_FAIL,
-        EC_MSG_INVALID_DICT_NUMS,
-        EC_MSG_IATT_COMBINE_FAIL,
-        EC_MSG_INVALID_FORMAT,
-        EC_MSG_DICT_GET_FAILED,
-        EC_MSG_UNHANDLED_STATE,
-        EC_MSG_FILE_DESC_REF_FAIL,
-        EC_MSG_LOC_COPY_FAIL,
-        EC_MSG_BUF_REF_FAIL,
-        EC_MSG_DICT_REF_FAIL,
-        EC_MSG_LK_UNLOCK_FAILED,
-        EC_MSG_UNLOCK_FAILED,
-        EC_MSG_LOC_PARENT_INODE_MISSING,
-        EC_MSG_INVALID_LOC_NAME,
-        EC_MSG_NO_MEMORY,
-        EC_MSG_GFID_MISMATCH,
-        EC_MSG_UNSUPPORTED_VERSION,
-        EC_MSG_FD_CREATE_FAIL,
-        EC_MSG_READDIRP_REQ_PREP_FAIL,
-        EC_MSG_LOOKUP_REQ_PREP_FAIL,
-        EC_MSG_INODE_REF_FAIL,
-        EC_MSG_LOOKUP_READAHEAD_FAIL,
-        EC_MSG_FRAME_MISMATCH,
-        EC_MSG_XLATOR_MISMATCH,
-        EC_MSG_VECTOR_MISMATCH,
-        EC_MSG_IATT_MISMATCH,
-        EC_MSG_FD_MISMATCH,
-        EC_MSG_DICT_MISMATCH,
-        EC_MSG_INDEX_DIR_GET_FAIL,
-        EC_MSG_PREOP_LOCK_FAILED,
-        EC_MSG_CHILDS_INSUFFICIENT,
-        EC_MSG_OP_EXEC_UNAVAIL,
-        EC_MSG_UNLOCK_DELAY_FAILED,
-        EC_MSG_SIZE_VERS_UPDATE_FAIL,
-        EC_MSG_INVALID_REQUEST,
-        EC_MSG_INVALID_LOCK_TYPE,
-        EC_MSG_SIZE_VERS_GET_FAIL,
-        EC_MSG_FILE_SIZE_GET_FAIL,
-        EC_MSG_FOP_MISMATCH,
-        EC_MSG_SUBVOL_ID_DICT_SET_FAIL,
-        EC_MSG_SUBVOL_BUILD_FAIL,
-        EC_MSG_XLATOR_INIT_FAIL,
-        EC_MSG_NO_PARENTS,
-        EC_MSG_TIMER_CREATE_FAIL,
-        EC_MSG_TOO_MANY_SUBVOLS,
-        EC_MSG_DATA_UNAVAILABLE,
-        EC_MSG_INODE_REMOVE_FAIL,
-        EC_MSG_INVALID_REDUNDANCY,
-        EC_MSG_XLATOR_PARSE_OPT_FAIL,
-        EC_MSG_OP_FAIL_ON_SUBVOLS,
-        EC_MSG_INVALID_INODE,
-        EC_MSG_LOCK_MISMATCH,
-        EC_MSG_XDATA_MISMATCH,
-        EC_MSG_HEALING_INFO,
-        EC_MSG_HEAL_SUCCESS,
-        EC_MSG_FULL_SWEEP_START,
-        EC_MSG_FULL_SWEEP_STOP,
-        EC_MSG_INVALID_FOP,
-        EC_MSG_EC_UP,
-        EC_MSG_EC_DOWN,
-        EC_MSG_SIZE_XATTR_GET_FAIL,
-        EC_MSG_VER_XATTR_GET_FAIL,
-        EC_MSG_CONFIG_XATTR_GET_FAIL,
-        EC_MSG_CONFIG_XATTR_INVALID,
-        EC_MSG_EXTENSION,
-        EC_MSG_EXTENSION_NONE,
-        EC_MSG_EXTENSION_UNKNOWN,
-        EC_MSG_EXTENSION_UNSUPPORTED,
-        EC_MSG_EXTENSION_FAILED,
-        EC_MSG_NO_GF,
-        EC_MSG_MATRIX_FAILED,
-        EC_MSG_DYN_CREATE_FAILED,
-        EC_MSG_DYN_CODEGEN_FAILED
-);
+GLFS_MSGID(EC, EC_MSG_INVALID_CONFIG, EC_MSG_HEAL_FAIL,
+           EC_MSG_DICT_COMBINE_FAIL, EC_MSG_STIME_COMBINE_FAIL,
+           EC_MSG_INVALID_DICT_NUMS, EC_MSG_IATT_COMBINE_FAIL,
+           EC_MSG_INVALID_FORMAT, EC_MSG_DICT_GET_FAILED,
+           EC_MSG_UNHANDLED_STATE, EC_MSG_FILE_DESC_REF_FAIL,
+           EC_MSG_LOC_COPY_FAIL, EC_MSG_BUF_REF_FAIL, EC_MSG_DICT_REF_FAIL,
+           EC_MSG_LK_UNLOCK_FAILED, EC_MSG_UNLOCK_FAILED,
+           EC_MSG_LOC_PARENT_INODE_MISSING, EC_MSG_INVALID_LOC_NAME,
+           EC_MSG_NO_MEMORY, EC_MSG_GFID_MISMATCH, EC_MSG_UNSUPPORTED_VERSION,
+           EC_MSG_FD_CREATE_FAIL, EC_MSG_READDIRP_REQ_PREP_FAIL,
+           EC_MSG_LOOKUP_REQ_PREP_FAIL, EC_MSG_INODE_REF_FAIL,
+           EC_MSG_LOOKUP_READAHEAD_FAIL, EC_MSG_FRAME_MISMATCH,
+           EC_MSG_XLATOR_MISMATCH, EC_MSG_VECTOR_MISMATCH, EC_MSG_IATT_MISMATCH,
+           EC_MSG_FD_MISMATCH, EC_MSG_DICT_MISMATCH, EC_MSG_INDEX_DIR_GET_FAIL,
+           EC_MSG_PREOP_LOCK_FAILED, EC_MSG_CHILDS_INSUFFICIENT,
+           EC_MSG_OP_EXEC_UNAVAIL, EC_MSG_UNLOCK_DELAY_FAILED,
+           EC_MSG_SIZE_VERS_UPDATE_FAIL, EC_MSG_INVALID_REQUEST,
+           EC_MSG_INVALID_LOCK_TYPE, EC_MSG_SIZE_VERS_GET_FAIL,
+           EC_MSG_FILE_SIZE_GET_FAIL, EC_MSG_FOP_MISMATCH,
+           EC_MSG_SUBVOL_ID_DICT_SET_FAIL, EC_MSG_SUBVOL_BUILD_FAIL,
+           EC_MSG_XLATOR_INIT_FAIL, EC_MSG_NO_PARENTS, EC_MSG_TIMER_CREATE_FAIL,
+           EC_MSG_TOO_MANY_SUBVOLS, EC_MSG_DATA_UNAVAILABLE,
+           EC_MSG_INODE_REMOVE_FAIL, EC_MSG_INVALID_REDUNDANCY,
+           EC_MSG_XLATOR_PARSE_OPT_FAIL, EC_MSG_OP_FAIL_ON_SUBVOLS,
+           EC_MSG_INVALID_INODE, EC_MSG_LOCK_MISMATCH, EC_MSG_XDATA_MISMATCH,
+           EC_MSG_HEALING_INFO, EC_MSG_HEAL_SUCCESS, EC_MSG_FULL_SWEEP_START,
+           EC_MSG_FULL_SWEEP_STOP, EC_MSG_INVALID_FOP, EC_MSG_EC_UP,
+           EC_MSG_EC_DOWN, EC_MSG_SIZE_XATTR_GET_FAIL,
+           EC_MSG_VER_XATTR_GET_FAIL, EC_MSG_CONFIG_XATTR_GET_FAIL,
+           EC_MSG_CONFIG_XATTR_INVALID, EC_MSG_EXTENSION, EC_MSG_EXTENSION_NONE,
+           EC_MSG_EXTENSION_UNKNOWN, EC_MSG_EXTENSION_UNSUPPORTED,
+           EC_MSG_EXTENSION_FAILED, EC_MSG_NO_GF, EC_MSG_MATRIX_FAILED,
+           EC_MSG_DYN_CREATE_FAILED, EC_MSG_DYN_CODEGEN_FAILED,
+           EC_MSG_THREAD_CLEANUP_FAILED, EC_MSG_FD_BAD);
 
 #endif /* !_EC_MESSAGES_H_ */
diff --git a/xlators/cluster/ec/src/ec-method.c b/xlators/cluster/ec/src/ec-method.c
index fdc89391bfb..55faed0b193 100644
--- a/xlators/cluster/ec/src/ec-method.c
+++ b/xlators/cluster/ec/src/ec-method.c
@@ -88,11 +88,9 @@ ec_method_matrix_init(ec_matrix_list_t *list, ec_matrix_t *matrix,
                                  matrix->rows);
         for (i = 0; i < matrix->rows; i++) {
             matrix->row_data[i].values = matrix->values + i * matrix->columns;
-            matrix->row_data[i].func.interleaved =
-                ec_code_build_interleaved(matrix->code,
-                                          EC_METHOD_WORD_SIZE,
-                                          matrix->row_data[i].values,
-                                          matrix->columns);
+            matrix->row_data[i].func.interleaved = ec_code_build_interleaved(
+                matrix->code, EC_METHOD_WORD_SIZE, matrix->row_data[i].values,
+                matrix->columns);
         }
     } else {
         matrix->rows = list->rows;
@@ -100,10 +98,9 @@ ec_method_matrix_init(ec_matrix_list_t *list, ec_matrix_t *matrix,
                                 matrix->columns, rows, matrix->rows);
         for (i = 0; i < matrix->rows; i++) {
             matrix->row_data[i].values = matrix->values + i * matrix->columns;
-            matrix->row_data[i].func.linear =
-                ec_code_build_linear(matrix->code, EC_METHOD_WORD_SIZE,
-                                     matrix->row_data[i].values,
-                                     matrix->columns);
+            matrix->row_data[i].func.linear = ec_code_build_linear(
+                matrix->code, EC_METHOD_WORD_SIZE, matrix->row_data[i].values,
+                matrix->columns);
         }
     }
 }
@@ -266,8 +263,8 @@ ec_method_setup(xlator_t *xl, ec_matrix_list_t *list, const char *gen)
     int32_t err;
 
     matrix = GF_MALLOC(sizeof(ec_matrix_t) +
-                       sizeof(ec_matrix_row_t) * list->rows +
-                       sizeof(uint32_t) * list->columns * list->rows,
+                           sizeof(ec_matrix_row_t) * list->rows +
+                           sizeof(uint32_t) * list->columns * list->rows,
                        ec_mt_ec_matrix_t);
     if (matrix == NULL) {
         err = -ENOMEM;
@@ -310,9 +307,10 @@ ec_method_init(xlator_t *xl, ec_matrix_list_t *list, uint32_t columns,
     INIT_LIST_HEAD(&list->lru);
     int32_t err;
 
-    list->pool = mem_pool_new_fn(xl->ctx, sizeof(ec_matrix_t) +
-                                 sizeof(ec_matrix_row_t) * columns +
-                                 sizeof(uint32_t) * columns * columns,
+    list->pool = mem_pool_new_fn(xl->ctx,
+                                 sizeof(ec_matrix_t) +
+                                     sizeof(ec_matrix_row_t) * columns +
+                                     sizeof(uint32_t) * columns * columns,
                                  128, "ec_matrix_t");
     if (list->pool == NULL) {
         err = -ENOMEM;
@@ -370,8 +368,8 @@ ec_method_fini(ec_matrix_list_t *list)
 
     GF_ASSERT(list->count == 0);
 
-    if (list->pool)/*Init was successful*/
-            LOCK_DESTROY(&list->lock);
+    if (list->pool) /*Init was successful*/
+        LOCK_DESTROY(&list->lock);
 
     ec_method_matrix_release(list->encode);
     GF_FREE(list->encode);
@@ -393,29 +391,28 @@ ec_method_update(xlator_t *xl, ec_matrix_list_t *list, const char *gen)
 }
 
 void
-ec_method_encode(ec_matrix_list_t *list, size_t size, void *in, void **out)
+ec_method_encode(ec_matrix_list_t *list, uint64_t size, void *in, void **out)
 {
     ec_matrix_t *matrix;
-    size_t pos;
+    uint64_t pos;
     uint32_t i;
 
     matrix = list->encode;
     for (pos = 0; pos < size; pos += list->stripe) {
         for (i = 0; i < matrix->rows; i++) {
-            matrix->row_data[i].func.linear(out[i], in, pos,
-                                            matrix->row_data[i].values,
-                                            list->columns);
+            matrix->row_data[i].func.linear(
+                out[i], in, pos, matrix->row_data[i].values, list->columns);
             out[i] += EC_METHOD_CHUNK_SIZE;
         }
     }
 }
 
 int32_t
-ec_method_decode(ec_matrix_list_t *list, size_t size, uintptr_t mask,
+ec_method_decode(ec_matrix_list_t *list, uint64_t size, uintptr_t mask,
                  uint32_t *rows, void **in, void *out)
 {
     ec_matrix_t *matrix;
-    size_t pos;
+    uint64_t pos;
     uint32_t i;
 
     matrix = ec_method_matrix_get(list, mask, rows);
@@ -424,9 +421,8 @@ ec_method_decode(ec_matrix_list_t *list, size_t size, uintptr_t mask,
     }
     for (pos = 0; pos < size; pos += EC_METHOD_CHUNK_SIZE) {
         for (i = 0; i < matrix->rows; i++) {
-            matrix->row_data[i].func.interleaved(out, in, pos,
-                                                 matrix->row_data[i].values,
-                                                 list->columns);
+            matrix->row_data[i].func.interleaved(
+                out, in, pos, matrix->row_data[i].values, list->columns);
             out += EC_METHOD_CHUNK_SIZE;
         }
     }
diff --git a/xlators/cluster/ec/src/ec-method.h b/xlators/cluster/ec/src/ec-method.h
index 9ba5069ff0e..f91233b2f88 100644
--- a/xlators/cluster/ec/src/ec-method.h
+++ b/xlators/cluster/ec/src/ec-method.h
@@ -11,8 +11,6 @@
 #ifndef __EC_METHOD_H__
 #define __EC_METHOD_H__
 
-#include "xlator.h"
-
 #include "ec-types.h"
 #include "ec-galois.h"
 
@@ -24,7 +22,7 @@
 /* Determines the maximum size of the matrix used to encode/decode data */
 #define EC_METHOD_MAX_FRAGMENTS 16
 /* Determines the maximum number of usable elements in the Galois Field */
-#define EC_METHOD_MAX_NODES     (EC_GF_SIZE - 1)
+#define EC_METHOD_MAX_NODES (EC_GF_SIZE - 1)
 
 #define EC_METHOD_WORD_SIZE 64
 
@@ -34,16 +32,17 @@ int32_t
 ec_method_init(xlator_t *xl, ec_matrix_list_t *list, uint32_t columns,
                uint32_t rows, uint32_t max, const char *gen);
 
-void ec_method_fini(ec_matrix_list_t *list);
+void
+ec_method_fini(ec_matrix_list_t *list);
 
 int32_t
 ec_method_update(xlator_t *xl, ec_matrix_list_t *list, const char *gen);
 
 void
-ec_method_encode(ec_matrix_list_t *list, size_t size, void *in, void **out);
+ec_method_encode(ec_matrix_list_t *list, uint64_t size, void *in, void **out);
 
 int32_t
-ec_method_decode(ec_matrix_list_t *list, size_t size, uintptr_t mask,
+ec_method_decode(ec_matrix_list_t *list, uint64_t size, uintptr_t mask,
                  uint32_t *rows, void **in, void *out);
 
 #endif /* __EC_METHOD_H__ */
diff --git a/xlators/cluster/ec/src/ec-types.h b/xlators/cluster/ec/src/ec-types.h
index 15b4c77abfe..de9b89bb2c9 100644
--- a/xlators/cluster/ec/src/ec-types.h
+++ b/xlators/cluster/ec/src/ec-types.h
@@ -11,10 +11,10 @@
 #ifndef __EC_TYPES_H__
 #define __EC_TYPES_H__
 
-#include "xlator.h"
-#include "timer.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/timer.h>
 #include "libxlator.h"
-#include "atomic.h"
+#include <glusterfs/atomic.h>
 
 #define EC_GF_MAX_REGS 16
 
@@ -128,182 +128,175 @@ typedef void (*ec_wind_f)(ec_t *, ec_fop_data_t *, int32_t);
 typedef int32_t (*ec_handler_f)(ec_fop_data_t *, int32_t);
 typedef void (*ec_resume_f)(ec_fop_data_t *, int32_t);
 
-enum _ec_read_policy {
-        EC_ROUND_ROBIN,
-        EC_GFID_HASH,
-        EC_READ_POLICY_MAX
-};
+enum _ec_read_policy { EC_ROUND_ROBIN, EC_GFID_HASH, EC_READ_POLICY_MAX };
 
 enum _ec_heal_need {
-        EC_HEAL_NONEED,
-        EC_HEAL_MAYBE,
-        EC_HEAL_MUST
+    EC_HEAL_NONEED,
+    EC_HEAL_MAYBE,
+    EC_HEAL_MUST,
+    EC_HEAL_PURGE_INDEX
 };
 
-enum _ec_stripe_part {
-        EC_STRIPE_HEAD,
-        EC_STRIPE_TAIL
-};
+enum _ec_stripe_part { EC_STRIPE_HEAD, EC_STRIPE_TAIL };
 
 /* Enumartions to indicate FD status. */
-typedef enum {
-    EC_FD_NOT_OPENED,
-    EC_FD_OPENED,
-    EC_FD_OPENING
-} ec_fd_status_t;
+typedef enum { EC_FD_NOT_OPENED, EC_FD_OPENED, EC_FD_OPENING } ec_fd_status_t;
 
 struct _ec_config {
     uint32_t version;
-    uint8_t  algorithm;
-    uint8_t  gf_word_size;
-    uint8_t  bricks;
-    uint8_t  redundancy;
+    uint8_t algorithm;
+    uint8_t gf_word_size;
+    uint8_t bricks;
+    uint8_t redundancy;
     uint32_t chunk_size;
 };
 
 struct _ec_fd {
-    loc_t     loc;
+    loc_t loc;
     uintptr_t open;
-    int32_t   flags;
+    int32_t flags;
+    uint64_t bad_version;
     ec_fd_status_t fd_status[0];
 };
 
 struct _ec_stripe {
-    struct list_head lru;         /* LRU list member */
-    uint64_t         frag_offset; /* Fragment offset of this stripe */
-    char             data[];      /* Contents of the stripe */
+    struct list_head lru; /* LRU list member */
+    uint64_t frag_offset; /* Fragment offset of this stripe */
+    char data[];          /* Contents of the stripe */
 };
 
 struct _ec_stripe_list {
-    struct          list_head lru;
-    uint32_t        count;
-    uint32_t        max;
+    struct list_head lru;
+    uint32_t count;
+    uint32_t max;
 };
 
 struct _ec_inode {
-    ec_lock_t        *inode_lock;
-    gf_boolean_t      have_info;
-    gf_boolean_t      have_config;
-    gf_boolean_t      have_version;
-    gf_boolean_t      have_size;
-    ec_config_t       config;
-    uint64_t          pre_version[2];
-    uint64_t          post_version[2];
-    uint64_t          pre_size;
-    uint64_t          post_size;
-    uint64_t          dirty[2];
-    struct list_head  heal;
-    ec_stripe_list_t  stripe_cache;
+    ec_lock_t *inode_lock;
+    gf_boolean_t have_info;
+    gf_boolean_t have_config;
+    gf_boolean_t have_version;
+    gf_boolean_t have_size;
+    int32_t heal_count;
+    ec_config_t config;
+    uint64_t pre_version[2];
+    uint64_t post_version[2];
+    uint64_t pre_size;
+    uint64_t post_size;
+    uint64_t dirty[2];
+    struct list_head heal;
+    ec_stripe_list_t stripe_cache;
+    uint64_t bad_version;
 };
 
-
 typedef int32_t (*fop_heal_cbk_t)(call_frame_t *, void *, xlator_t *, int32_t,
                                   int32_t, uintptr_t, uintptr_t, uintptr_t,
-                                  dict_t *);
+                                  uint32_t, dict_t *);
 typedef int32_t (*fop_fheal_cbk_t)(call_frame_t *, void *, xlator_t *, int32_t,
                                    int32_t, uintptr_t, uintptr_t, uintptr_t,
-                                   dict_t *);
+                                   uint32_t, dict_t *);
 
 union _ec_cbk {
-    fop_access_cbk_t       access;
-    fop_create_cbk_t       create;
-    fop_discard_cbk_t      discard;
-    fop_entrylk_cbk_t      entrylk;
-    fop_fentrylk_cbk_t     fentrylk;
-    fop_fallocate_cbk_t    fallocate;
-    fop_flush_cbk_t        flush;
-    fop_fsync_cbk_t        fsync;
-    fop_fsyncdir_cbk_t     fsyncdir;
-    fop_getxattr_cbk_t     getxattr;
-    fop_fgetxattr_cbk_t    fgetxattr;
-    fop_heal_cbk_t         heal;
-    fop_fheal_cbk_t        fheal;
-    fop_inodelk_cbk_t      inodelk;
-    fop_finodelk_cbk_t     finodelk;
-    fop_link_cbk_t         link;
-    fop_lk_cbk_t           lk;
-    fop_lookup_cbk_t       lookup;
-    fop_mkdir_cbk_t        mkdir;
-    fop_mknod_cbk_t        mknod;
-    fop_open_cbk_t         open;
-    fop_opendir_cbk_t      opendir;
-    fop_readdir_cbk_t      readdir;
-    fop_readdirp_cbk_t     readdirp;
-    fop_readlink_cbk_t     readlink;
-    fop_readv_cbk_t        readv;
-    fop_removexattr_cbk_t  removexattr;
+    fop_access_cbk_t access;
+    fop_create_cbk_t create;
+    fop_discard_cbk_t discard;
+    fop_entrylk_cbk_t entrylk;
+    fop_fentrylk_cbk_t fentrylk;
+    fop_fallocate_cbk_t fallocate;
+    fop_flush_cbk_t flush;
+    fop_fsync_cbk_t fsync;
+    fop_fsyncdir_cbk_t fsyncdir;
+    fop_getxattr_cbk_t getxattr;
+    fop_fgetxattr_cbk_t fgetxattr;
+    fop_heal_cbk_t heal;
+    fop_fheal_cbk_t fheal;
+    fop_inodelk_cbk_t inodelk;
+    fop_finodelk_cbk_t finodelk;
+    fop_link_cbk_t link;
+    fop_lk_cbk_t lk;
+    fop_lookup_cbk_t lookup;
+    fop_mkdir_cbk_t mkdir;
+    fop_mknod_cbk_t mknod;
+    fop_open_cbk_t open;
+    fop_opendir_cbk_t opendir;
+    fop_readdir_cbk_t readdir;
+    fop_readdirp_cbk_t readdirp;
+    fop_readlink_cbk_t readlink;
+    fop_readv_cbk_t readv;
+    fop_removexattr_cbk_t removexattr;
     fop_fremovexattr_cbk_t fremovexattr;
-    fop_rename_cbk_t       rename;
-    fop_rmdir_cbk_t        rmdir;
-    fop_setattr_cbk_t      setattr;
-    fop_fsetattr_cbk_t     fsetattr;
-    fop_setxattr_cbk_t     setxattr;
-    fop_fsetxattr_cbk_t    fsetxattr;
-    fop_stat_cbk_t         stat;
-    fop_fstat_cbk_t        fstat;
-    fop_statfs_cbk_t       statfs;
-    fop_symlink_cbk_t      symlink;
-    fop_truncate_cbk_t     truncate;
-    fop_ftruncate_cbk_t    ftruncate;
-    fop_unlink_cbk_t       unlink;
-    fop_writev_cbk_t       writev;
-    fop_xattrop_cbk_t      xattrop;
-    fop_fxattrop_cbk_t     fxattrop;
-    fop_zerofill_cbk_t     zerofill;
-    fop_seek_cbk_t         seek;
-    fop_ipc_cbk_t          ipc;
+    fop_rename_cbk_t rename;
+    fop_rmdir_cbk_t rmdir;
+    fop_setattr_cbk_t setattr;
+    fop_fsetattr_cbk_t fsetattr;
+    fop_setxattr_cbk_t setxattr;
+    fop_fsetxattr_cbk_t fsetxattr;
+    fop_stat_cbk_t stat;
+    fop_fstat_cbk_t fstat;
+    fop_statfs_cbk_t statfs;
+    fop_symlink_cbk_t symlink;
+    fop_truncate_cbk_t truncate;
+    fop_ftruncate_cbk_t ftruncate;
+    fop_unlink_cbk_t unlink;
+    fop_writev_cbk_t writev;
+    fop_xattrop_cbk_t xattrop;
+    fop_fxattrop_cbk_t fxattrop;
+    fop_zerofill_cbk_t zerofill;
+    fop_seek_cbk_t seek;
+    fop_ipc_cbk_t ipc;
 };
 
 struct _ec_lock {
-    ec_inode_t        *ctx;
-    gf_timer_t        *timer;
+    ec_inode_t *ctx;
+    gf_timer_t *timer;
 
     /* List of owners of this lock. All fops added to this list are running
      * concurrently. */
-    struct list_head   owners;
+    struct list_head owners;
 
     /* List of fops waiting to be an owner of the lock. Fops are added to this
      * list when the current owner has an incompatible access (conflicting lock)
      * or the lock is not acquired yet. */
-    struct list_head   waiting;
+    struct list_head waiting;
 
     /* List of fops that will wait until the next unlock/lock cycle. This
      * happens when the currently acquired lock is decided to be released as
      * soon as possible. In this case, all frozen fops will be continued only
      * after the lock is reacquired. */
-    struct list_head   frozen;
-
-    uintptr_t          mask;
-    uintptr_t          good_mask;
-    uintptr_t          healing;
-    uint32_t           refs_owners;  /* Refs for fops owning the lock */
-    uint32_t           refs_pending; /* Refs assigned to fops being prepared */
-    uint32_t           waiting_flags; /*Track xattrop/dirty marking*/
-    gf_boolean_t       acquired;
-    gf_boolean_t       unlock_now;
-    gf_boolean_t       release;
-    gf_boolean_t       query;
-    fd_t              *fd;
-    loc_t              loc;
+    struct list_head frozen;
+
+    uintptr_t mask;
+    uintptr_t good_mask;
+    uintptr_t healing;
+    uint32_t refs_owners;   /* Refs for fops owning the lock */
+    uint32_t refs_pending;  /* Refs assigned to fops being prepared */
+    uint32_t waiting_flags; /*Track xattrop/dirty marking*/
+    gf_boolean_t acquired;
+    gf_boolean_t contention;
+    gf_boolean_t unlock_now;
+    gf_boolean_t release;
+    gf_boolean_t query;
+    fd_t *fd;
+    loc_t loc;
     union {
-        entrylk_type     type;
-        struct gf_flock  flock;
+        entrylk_type type;
+        struct gf_flock flock;
     };
 };
 
 struct _ec_lock_link {
-    ec_lock_t        *lock;
-    ec_fop_data_t    *fop;
-    struct list_head  owner_list;
-    struct list_head  wait_list;
-    gf_boolean_t      update[2];
-    gf_boolean_t      dirty[2];
-    gf_boolean_t      optimistic_changelog;
-    loc_t            *base;
-    uint64_t          size;
-    uint32_t          waiting_flags;
-    off_t             fl_start;
-    off_t             fl_end;
+    ec_lock_t *lock;
+    ec_fop_data_t *fop;
+    struct list_head owner_list;
+    struct list_head wait_list;
+    gf_boolean_t update[2];
+    gf_boolean_t dirty[2];
+    gf_boolean_t optimistic_changelog;
+    loc_t *base;
+    uint64_t size;
+    uint32_t waiting_flags;
+    off_t fl_start;
+    off_t fl_end;
 };
 
 /* This structure keeps a range of fragment offsets affected by a fop. Since
@@ -320,111 +313,114 @@ struct _ec_fragment_range {
 /* EC xlator data structure to collect all the data required to perform
  * the file operation.*/
 struct _ec_fop_data {
-    int32_t            id;           /* ID of the file operation */
-    int32_t            refs;
-    int32_t            state;
-    int32_t            minimum;      /* Mininum number of successful
-                                        operation required to conclude a
-                                        fop as successful */
-    int32_t            expected;
-    int32_t            winds;
-    int32_t            jobs;
-    int32_t            error;
-    ec_fop_data_t     *parent;
-    xlator_t          *xl;           /* points to EC xlator */
-    call_frame_t      *req_frame;    /* frame of the calling xlator */
-    call_frame_t      *frame;        /* frame used by this fop */
-    struct list_head   cbk_list;     /* sorted list of groups of answers */
-    struct list_head   answer_list;  /* list of answers */
-    struct list_head   pending_list; /* member of ec_t.pending_fops */
-    ec_cbk_data_t     *answer;       /* accepted answer */
-    int32_t            lock_count;
-    int32_t            locked;
-    ec_lock_link_t     locks[2];
-    int32_t            first_lock;
-    gf_lock_t          lock;
-
-    uint32_t           flags;
-    uint32_t           first;
-    uintptr_t          mask;
-    uintptr_t          healing; /*Dispatch is done but call is successful only
-                                  if fop->minimum number of subvolumes succeed
-                                  which are not healing*/
-    uintptr_t          remaining;
-    uintptr_t          received; /* Mask of responses */
-    uintptr_t          good;
-
-    uid_t              uid;
-    gid_t              gid;
-
-    ec_wind_f          wind;          /* Function to wind to */
-    ec_handler_f       handler;       /* FOP manager function */
-    ec_resume_f        resume;
-    ec_cbk_t           cbks;          /* Callback function for this FOP */
-    void              *data;
-    ec_heal_t         *heal;
-    struct list_head   healer;
-
-    uint64_t           user_size;
-    uint32_t           head;
-
-    int32_t            use_fd;        /* Indicates whether this FOP uses FD or
-                                         not */
-
-    dict_t            *xdata;
-    dict_t            *dict;
-    int32_t            int32;
-    uint32_t           uint32;
-    uint64_t           size;
-    off_t              offset;
-    mode_t             mode[2];
-    entrylk_cmd        entrylk_cmd;
-    entrylk_type       entrylk_type;
+    int32_t id; /* ID of the file operation */
+    int32_t refs;
+    int32_t state;
+    uint32_t minimum; /* Minimum number of successful
+                         operation required to conclude a
+                         fop as successful */
+    int32_t expected;
+    int32_t winds;
+    int32_t jobs;
+    int32_t error;
+    ec_fop_data_t *parent;
+    xlator_t *xl;                  /* points to EC xlator */
+    call_frame_t *req_frame;       /* frame of the calling xlator */
+    call_frame_t *frame;           /* frame used by this fop */
+    struct list_head cbk_list;     /* sorted list of groups of answers */
+    struct list_head answer_list;  /* list of answers */
+    struct list_head pending_list; /* member of ec_t.pending_fops */
+    ec_cbk_data_t *answer;         /* accepted answer */
+    int32_t lock_count;
+    int32_t locked;
+    gf_lock_t lock;
+    ec_lock_link_t locks[2];
+    int32_t first_lock;
+
+    uint32_t fop_flags; /* Flags passed by the caller. */
+    uint32_t flags;     /* Internal flags. */
+    uint32_t first;
+    uintptr_t mask;
+    uintptr_t healing; /*Dispatch is done but call is successful only
+                         if fop->minimum number of subvolumes succeed
+                         which are not healing*/
+    uintptr_t remaining;
+    uintptr_t received; /* Mask of responses */
+    uintptr_t good;
+
+    uid_t uid;
+    gid_t gid;
+
+    ec_wind_f wind;       /* Function to wind to */
+    ec_handler_f handler; /* FOP manager function */
+    ec_resume_f resume;
+    ec_cbk_t cbks; /* Callback function for this FOP */
+    void *data;
+    ec_heal_t *heal;
+    struct list_head healer;
+
+    uint64_t user_size;
+    uint32_t head;
+
+    int32_t use_fd; /* Indicates whether this FOP uses FD or
+                       not */
+
+    dict_t *xdata;
+    dict_t *dict;
+    int32_t int32;
+    uint32_t uint32;
+    uint64_t size;
+    off_t offset;
+    mode_t mode[2];
+    entrylk_cmd entrylk_cmd;
+    entrylk_type entrylk_type;
     gf_xattrop_flags_t xattrop_flags;
-    dev_t              dev;
-    inode_t           *inode;
-    fd_t              *fd;              /* FD of the file on which FOP is
-                                           being carried upon */
-    struct iatt        iatt;
-    char              *str[2];
-    loc_t              loc[2];          /* Holds the location details for
-                                           the file */
-    struct gf_flock    flock;
-    struct iovec      *vector;
-    struct iobref     *buffers;
-    gf_seek_what_t     seek;
-    ec_fragment_range_t  frag_range; /* This will hold the range of stripes
-                                         affected by the fop. */
+    dev_t dev;
+    inode_t *inode;
+    fd_t *fd; /* FD of the file on which FOP is
+                 being carried upon */
+    struct iatt iatt;
+    char *str[2];
+    loc_t loc[2]; /* Holds the location details for
+                     the file */
+    struct gf_flock flock;
+    struct iovec *vector;
+    struct iobref *buffers;
+    gf_seek_what_t seek;
+    ec_fragment_range_t frag_range; /* This will hold the range of stripes
+                                        affected by the fop. */
+    char *errstr;                   /*String of fop name, path and gfid
+                                     to be used in gf_msg. */
 };
 
 struct _ec_cbk_data {
-    struct list_head  list;        /* item in the sorted list of groups */
-    struct list_head  answer_list; /* item in the list of answers */
-    ec_fop_data_t    *fop;
-    ec_cbk_data_t    *next;        /* next answer in the same group */
-    uint32_t          idx;
-    int32_t           op_ret;
-    int32_t           op_errno;
-    int32_t           count;
-    uintptr_t         mask;
-
-    dict_t           *xdata;
-    dict_t           *dict;
-    int32_t           int32;
-    uintptr_t         uintptr[3];
-    uint64_t          size;
-    uint64_t          version[2];
-    inode_t          *inode;
-    fd_t             *fd;
-    struct statvfs    statvfs;
-    struct iatt       iatt[5];
-    struct gf_flock   flock;
-    struct iovec     *vector;
-    struct iobref    *buffers;
-    char             *str;
-    gf_dirent_t       entries;
-    off_t             offset;
-    gf_seek_what_t    what;
+    struct list_head list;        /* item in the sorted list of groups */
+    struct list_head answer_list; /* item in the list of answers */
+    ec_fop_data_t *fop;
+    ec_cbk_data_t *next; /* next answer in the same group */
+    uint32_t idx;
+    int32_t op_ret;
+    int32_t op_errno;
+    int32_t count;
+    uintptr_t mask;
+
+    dict_t *xdata;
+    dict_t *dict;
+    int32_t int32;
+    uintptr_t uintptr[3];
+    uint64_t size;
+    uint64_t version[2];
+    inode_t *inode;
+    fd_t *fd;
+    struct statvfs statvfs;
+    struct iatt iatt[5];
+    struct gf_flock flock;
+    struct iovec *vector;
+    struct iobref *buffers;
+    char *str;
+    gf_dirent_t entries;
+    off_t offset;
+    gf_seek_what_t what;
 };
 
 enum _ec_gf_opcode {
@@ -439,26 +435,26 @@ enum _ec_gf_opcode {
 
 struct _ec_gf_op {
     ec_gf_opcode_t op;
-    uint32_t       arg1;
-    uint32_t       arg2;
-    uint32_t       arg3;
+    uint32_t arg1;
+    uint32_t arg2;
+    uint32_t arg3;
 };
 
 struct _ec_gf_mul {
-    uint32_t    regs;
-    uint32_t    map[EC_GF_MAX_REGS];
+    uint32_t regs;
+    uint32_t map[EC_GF_MAX_REGS];
     ec_gf_op_t *ops;
 };
 
 struct _ec_gf {
-    uint32_t      bits;
-    uint32_t      size;
-    uint32_t      mod;
-    uint32_t      min_ops;
-    uint32_t      max_ops;
-    uint32_t      avg_ops;
-    uint32_t     *log;
-    uint32_t     *pow;
+    uint32_t bits;
+    uint32_t size;
+    uint32_t mod;
+    uint32_t min_ops;
+    uint32_t max_ops;
+    uint32_t avg_ops;
+    uint32_t *log;
+    uint32_t *pow;
     ec_gf_mul_t **table;
 };
 
@@ -481,208 +477,214 @@ struct _ec_code_gen {
 };
 
 struct _ec_code {
-    gf_lock_t         lock;
-    struct list_head  spaces;
-    ec_gf_t          *gf;
-    ec_code_gen_t    *gen;
+    gf_lock_t lock;
+    struct list_head spaces;
+    ec_gf_t *gf;
+    ec_code_gen_t *gen;
 };
 
 struct _ec_code_arg {
-    uint32_t          value;
+    uint32_t value;
 };
 
 struct _ec_code_op {
-    ec_gf_opcode_t    op;
-    ec_code_arg_t     arg1;
-    ec_code_arg_t     arg2;
-    ec_code_arg_t     arg3;
+    ec_gf_opcode_t op;
+    ec_code_arg_t arg1;
+    ec_code_arg_t arg2;
+    ec_code_arg_t arg3;
 };
 
 struct _ec_code_builder {
-    ec_code_t        *code;
-    uint64_t          address;
-    uint8_t          *data;
-    uint32_t          size;
-    int32_t           error;
-    uint32_t          regs;
-    uint32_t          bits;
-    uint32_t          width;
-    uint32_t          count;
-    uint32_t          base;
-    uint32_t          map[EC_GF_MAX_REGS];
-    gf_boolean_t      linear;
-    uint64_t          loop;
-    ec_code_op_t      ops[0];
+    ec_code_t *code;
+    uint64_t address;
+    uint8_t *data;
+    uint32_t size;
+    int32_t error;
+    uint32_t regs;
+    uint32_t bits;
+    uint32_t width;
+    uint32_t count;
+    uint32_t base;
+    uint32_t map[EC_GF_MAX_REGS];
+    gf_boolean_t linear;
+    uint64_t loop;
+    ec_code_op_t ops[0];
 };
 
 struct _ec_code_chunk {
-    struct list_head  list;
-    size_t            size;
-    ec_code_space_t  *space;
+    struct list_head list;
+    size_t size;
+    ec_code_space_t *space;
 };
 
 struct _ec_code_space {
-    struct list_head  list;
-    struct list_head  chunks;
-    ec_code_t        *code;
-    void             *exec;
-    size_t            size;
+    struct list_head list;
+    struct list_head chunks;
+    ec_code_t *code;
+    void *exec;
+    size_t size;
 };
 
-
 union _ec_code_func {
-    ec_code_func_linear_t      linear;
+    ec_code_func_linear_t linear;
     ec_code_func_interleaved_t interleaved;
 };
 
 struct _ec_matrix_row {
-    ec_code_func_t  func;
-    uint32_t       *values;
+    ec_code_func_t func;
+    uint32_t *values;
 };
 
 struct _ec_matrix {
     struct list_head lru;
-    uint32_t         refs;
-    uint32_t         columns;
-    uint32_t         rows;
-    uintptr_t        mask;
-    ec_code_t       *code;
-    uint32_t        *values;
-    ec_matrix_row_t  row_data[0];
+    uint32_t refs;
+    uint32_t columns;
+    uint32_t rows;
+    uintptr_t mask;
+    ec_code_t *code;
+    uint32_t *values;
+    ec_matrix_row_t row_data[0];
 };
 
 struct _ec_matrix_list {
-    struct list_head   lru;
-    gf_lock_t          lock;
-    uint32_t           columns;
-    uint32_t           rows;
-    uint32_t           max;
-    uint32_t           count;
-    uint32_t           stripe;
-    struct mem_pool   *pool;
-    ec_gf_t           *gf;
-    ec_code_t         *code;
-    ec_matrix_t       *encode;
-    ec_matrix_t      **objects;
+    struct list_head lru;
+    gf_lock_t lock;
+    uint32_t columns;
+    uint32_t rows;
+    uint32_t max;
+    uint32_t count;
+    uint32_t stripe;
+    struct mem_pool *pool;
+    ec_gf_t *gf;
+    ec_code_t *code;
+    ec_matrix_t *encode;
+    ec_matrix_t **objects;
 };
 
 struct _ec_heal {
-    struct list_head  list;
-    gf_lock_t         lock;
-    xlator_t         *xl;
-    ec_fop_data_t    *fop;
-    void             *data;
-    ec_fop_data_t    *lookup;
-    loc_t             loc;
-    struct iatt       iatt;
-    char             *symlink;
-    fd_t             *fd;
-    int32_t           partial;
-    int32_t           done;
-    int32_t           error;
-    gf_boolean_t      nameheal;
-    uintptr_t         available;
-    uintptr_t         good;
-    uintptr_t         bad;
-    uintptr_t         open;
-    uintptr_t         fixed;
-    uint64_t          offset;
-    uint64_t          size;
-    uint64_t          total_size;
-    uint64_t          version[2];
-    uint64_t          raw_size;
+    struct list_head list;
+    gf_lock_t lock;
+    xlator_t *xl;
+    ec_fop_data_t *fop;
+    void *data;
+    ec_fop_data_t *lookup;
+    loc_t loc;
+    struct iatt iatt;
+    char *symlink;
+    fd_t *fd;
+    int32_t partial;
+    int32_t done;
+    int32_t error;
+    gf_boolean_t nameheal;
+    uintptr_t available;
+    uintptr_t good;
+    uintptr_t bad;
+    uintptr_t open;
+    uintptr_t fixed;
+    uint64_t offset;
+    uint64_t size;
+    uint64_t total_size;
+    uint64_t version[2];
+    uint64_t raw_size;
 };
 
 struct subvol_healer {
-        xlator_t        *this;
-        int              subvol;
-        gf_boolean_t     local;
-        gf_boolean_t     running;
-        gf_boolean_t     rerun;
-        pthread_mutex_t  mutex;
-        pthread_cond_t   cond;
-        pthread_t        thread;
+    xlator_t *this;
+    int subvol;
+    gf_boolean_t running;
+    gf_boolean_t rerun;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_t thread;
 };
 
 struct _ec_self_heald {
-        gf_boolean_t            iamshd;
-        gf_boolean_t            enabled;
-        int                     timeout;
-        uint32_t                max_threads;
-        uint32_t                wait_qlength;
-        struct subvol_healer   *index_healers;
-        struct subvol_healer   *full_healers;
+    gf_boolean_t iamshd;
+    gf_boolean_t enabled;
+    int timeout;
+    uint32_t max_threads;
+    uint32_t wait_qlength;
+    struct subvol_healer *index_healers;
+    struct subvol_healer *full_healers;
 };
 
 struct _ec_statistics {
-        struct {
-                gf_atomic_t hits;    /* Cache hits. */
-                gf_atomic_t misses;  /* Cache misses. */
-                gf_atomic_t updates; /* Number of times an existing stripe has
-                                        been updated with new content. */
-                gf_atomic_t invals;  /* Number of times an existing stripe has
-                                        been invalidated because of truncates
-                                        or discards. */
-                gf_atomic_t evicts;  /* Number of times that an existing entry
-                                        has been evicted to make room for newer
-                                        entries. */
-                gf_atomic_t allocs;  /* Number of memory allocations made to
-                                        store stripes. */
-                gf_atomic_t errors;  /* Number of errors that have caused extra
-                                        requests. (Basically memory allocation
-                                        errors). */
-        } stripe_cache;
+    struct {
+        gf_atomic_t hits;    /* Cache hits. */
+        gf_atomic_t misses;  /* Cache misses. */
+        gf_atomic_t updates; /* Number of times an existing stripe has
+                                been updated with new content. */
+        gf_atomic_t invals;  /* Number of times an existing stripe has
+                                been invalidated because of truncates
+                                or discards. */
+        gf_atomic_t evicts;  /* Number of times that an existing entry
+                                has been evicted to make room for newer
+                                entries. */
+        gf_atomic_t allocs;  /* Number of memory allocations made to
+                                store stripes. */
+        gf_atomic_t errors;  /* Number of errors that have caused extra
+                                requests. (Basically memory allocation
+                                errors). */
+    } stripe_cache;
+    struct {
+        gf_atomic_t attempted; /*Number of heals attempted on
+                                files/directories*/
+        gf_atomic_t completed; /*Number of heals complted on files/directories*/
+    } shd;
 };
 
 struct _ec {
-    xlator_t          *xl;
-    int32_t            healers;
-    int32_t            heal_waiters;
-    int32_t            nodes;                /* Total number of bricks(n) */
-    int32_t            bits_for_nodes;
-    int32_t            fragments;            /* Data bricks(k) */
-    int32_t            redundancy;           /* Redundant bricks(m) */
-    uint32_t           fragment_size;        /* Size of fragment/chunk on a
-                                                brick. */
-    uint32_t           stripe_size;          /* (fragment_size * fragments)
-                                                maximum size of user data
-                                                stored in one stripe. */
-    int32_t            up;                   /* Represents whether EC volume is
-                                                up or not. */
-    uint32_t           idx;
-    uint32_t           xl_up_count;          /* Number of UP bricks. */
-    uintptr_t          xl_up;                /* Bit flag representing UP
-                                                bricks */
-    uint32_t           xl_notify_count;      /* Number of notifications. */
-    uintptr_t          xl_notify;            /* Bit flag representing
-                                                notification for bricks. */
-    uintptr_t          node_mask;
-    xlator_t         **xl_list;
-    gf_lock_t          lock;
-    gf_timer_t        *timer;
-    gf_boolean_t       shutdown;
-    gf_boolean_t       eager_lock;
-    gf_boolean_t       other_eager_lock;
-    gf_boolean_t       optimistic_changelog;
-    gf_boolean_t       parallel_writes;
-    uint32_t           stripe_cache;
-    uint32_t           background_heals;
-    uint32_t           heal_wait_qlen;
-    uint32_t           self_heal_window_size; /* max size of read/writes */
-    uint32_t           eager_lock_timeout;
-    uint32_t           other_eager_lock_timeout;
-    struct list_head   pending_fops;
-    struct list_head   heal_waiting;
-    struct list_head   healing;
-    struct mem_pool   *fop_pool;
-    struct mem_pool   *cbk_pool;
-    struct mem_pool   *lock_pool;
-    ec_self_heald_t    shd;
-    char               vol_uuid[UUID_SIZE + 1];
-    dict_t            *leaf_to_subvolid;
-    ec_read_policy_t   read_policy;
-    ec_matrix_list_t   matrix;
-    ec_statistics_t    stats;
+    xlator_t *xl;
+    int32_t healers;
+    int32_t heal_waiters;
+    int32_t nodes; /* Total number of bricks(n) */
+    int32_t bits_for_nodes;
+    int32_t fragments;      /* Data bricks(k) */
+    int32_t redundancy;     /* Redundant bricks(m) */
+    uint32_t fragment_size; /* Size of fragment/chunk on a
+                               brick. */
+    uint32_t stripe_size;   /* (fragment_size * fragments)
+                               maximum size of user data
+                               stored in one stripe. */
+    int32_t up;             /* Represents whether EC volume is
+                               up or not. */
+    uint32_t idx;
+    uint32_t xl_up_count;     /* Number of UP bricks. */
+    uintptr_t xl_up;          /* Bit flag representing UP
+                                 bricks */
+    uint32_t xl_notify_count; /* Number of notifications. */
+    uintptr_t xl_notify;      /* Bit flag representing
+                                 notification for bricks. */
+    uintptr_t node_mask;
+    uintptr_t read_mask;         /*Stores user defined read-mask*/
+    gf_atomic_t async_fop_count; /* Number of on going asynchronous fops. */
+    xlator_t **xl_list;
+    gf_lock_t lock;
+    gf_timer_t *timer;
+    gf_boolean_t shutdown;
+    gf_boolean_t eager_lock;
+    gf_boolean_t other_eager_lock;
+    gf_boolean_t optimistic_changelog;
+    gf_boolean_t parallel_writes;
+    uint32_t stripe_cache;
+    uint32_t quorum_count;
+    uint32_t background_heals;
+    uint32_t heal_wait_qlen;
+    uint32_t self_heal_window_size; /* max size of read/writes */
+    uint32_t eager_lock_timeout;
+    uint32_t other_eager_lock_timeout;
+    struct list_head pending_fops;
+    struct list_head heal_waiting;
+    struct list_head healing;
+    struct mem_pool *fop_pool;
+    struct mem_pool *cbk_pool;
+    struct mem_pool *lock_pool;
+    ec_self_heald_t shd;
+    char vol_uuid[UUID_SIZE + 1];
+    dict_t *leaf_to_subvolid;
+    ec_read_policy_t read_policy;
+    ec_matrix_list_t matrix;
+    ec_statistics_t stats;
 };
 
 #endif /* __EC_TYPES_H__ */
diff --git a/xlators/cluster/ec/src/ec.c b/xlators/cluster/ec/src/ec.c
index ac7e8492133..7344be4968d 100644
--- a/xlators/cluster/ec/src/ec.c
+++ b/xlators/cluster/ec/src/ec.c
@@ -8,10 +8,10 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "defaults.h"
-#include "statedump.h"
-#include "compat-errno.h"
-#include "upcall-utils.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/upcall-utils.h>
 
 #include "ec.h"
 #include "ec-messages.h"
@@ -23,50 +23,48 @@
 #include "ec-method.h"
 #include "ec-code.h"
 #include "ec-heald.h"
-#include "events.h"
+#include <glusterfs/events.h>
 
 static char *ec_read_policies[EC_READ_POLICY_MAX + 1] = {
-        [EC_ROUND_ROBIN] = "round-robin",
-        [EC_GFID_HASH] = "gfid-hash",
-        [EC_READ_POLICY_MAX] = NULL
-};
+    [EC_ROUND_ROBIN] = "round-robin",
+    [EC_GFID_HASH] = "gfid-hash",
+    [EC_READ_POLICY_MAX] = NULL};
 
 #define EC_INTERNAL_XATTR_OR_GOTO(name, xattr, op_errno, label)                \
-        do {                                                                   \
-                if (ec_is_internal_xattr (NULL, (char *)name, NULL, NULL)) {   \
-                        op_errno = EPERM;                                      \
-                        goto label;                                            \
-                }                                                              \
-                if (name && (strlen (name) == 0) && xattr) {                   \
-                        /* Bulk [f]removexattr/[f]setxattr */                  \
-                        GF_IF_INTERNAL_XATTR_GOTO (EC_XATTR_PREFIX"*", xattr,  \
-                                                   op_errno, label);           \
-                }                                                              \
-        } while (0)
-
-int32_t ec_parse_options(xlator_t * this)
-{
-    ec_t * ec = this->private;
+    do {                                                                       \
+        if (ec_is_internal_xattr(NULL, (char *)name, NULL, NULL)) {            \
+            op_errno = EPERM;                                                  \
+            goto label;                                                        \
+        }                                                                      \
+        if (name && (strlen(name) == 0) && xattr) {                            \
+            /* Bulk [f]removexattr/[f]setxattr */                              \
+            GF_IF_INTERNAL_XATTR_GOTO(EC_XATTR_PREFIX "*", xattr, op_errno,    \
+                                      label);                                  \
+        }                                                                      \
+    } while (0)
+
+int32_t
+ec_parse_options(xlator_t *this)
+{
+    ec_t *ec = this->private;
     int32_t error = EINVAL;
     uintptr_t mask;
 
     GF_OPTION_INIT("redundancy", ec->redundancy, int32, out);
     ec->fragments = ec->nodes - ec->redundancy;
     if ((ec->redundancy < 1) || (ec->redundancy >= ec->fragments) ||
-        (ec->fragments > EC_MAX_FRAGMENTS))
-    {
-        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_INVALID_REDUNDANCY,
-                "Invalid redundancy (must be between "
-                "1 and %d)", (ec->nodes - 1) / 2);
+        (ec->fragments > EC_MAX_FRAGMENTS)) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_REDUNDANCY,
+               "Invalid redundancy (must be between "
+               "1 and %d)",
+               (ec->nodes - 1) / 2);
 
         goto out;
     }
 
     ec->bits_for_nodes = 1;
     mask = 2;
-    while (ec->nodes > mask)
-    {
+    while (ec->nodes > mask) {
         ec->bits_for_nodes++;
         mask <<= 1;
     }
@@ -74,9 +72,10 @@ int32_t ec_parse_options(xlator_t * this)
     ec->fragment_size = EC_METHOD_CHUNK_SIZE;
     ec->stripe_size = ec->fragment_size * ec->fragments;
 
-    gf_msg_debug ("ec", 0, "Initialized with: nodes=%u, fragments=%u, "
-                               "stripe_size=%u, node_mask=%lX",
-           ec->nodes, ec->fragments, ec->stripe_size, ec->node_mask);
+    gf_msg_debug("ec", 0,
+                 "Initialized with: nodes=%u, fragments=%u, "
+                 "stripe_size=%u, node_mask=%" PRIxFAST32,
+                 ec->nodes, ec->fragments, ec->stripe_size, ec->node_mask);
 
     error = 0;
 
@@ -84,30 +83,28 @@ out:
     return error;
 }
 
-int32_t ec_prepare_childs(xlator_t * this)
+int32_t
+ec_prepare_childs(xlator_t *this)
 {
-    ec_t * ec = this->private;
-    xlator_list_t * child = NULL;
+    ec_t *ec = this->private;
+    xlator_list_t *child = NULL;
     int32_t count = 0;
 
-    for (child = this->children; child != NULL; child = child->next)
-    {
+    for (child = this->children; child != NULL; child = child->next) {
         count++;
     }
-    if (count > EC_MAX_NODES)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_TOO_MANY_SUBVOLS, "Too many subvolumes");
+    if (count > EC_MAX_NODES) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_TOO_MANY_SUBVOLS,
+               "Too many subvolumes");
 
         return EINVAL;
     }
     ec->nodes = count;
 
     ec->xl_list = GF_CALLOC(count, sizeof(ec->xl_list[0]), ec_mt_xlator_t);
-    if (ec->xl_list == NULL)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                EC_MSG_NO_MEMORY, "Allocation of xlator list failed");
+    if (ec->xl_list == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Allocation of xlator list failed");
 
         return ENOMEM;
     }
@@ -115,8 +112,7 @@ int32_t ec_prepare_childs(xlator_t * this)
     ec->xl_up_count = 0;
 
     count = 0;
-    for (child = this->children; child != NULL; child = child->next)
-    {
+    for (child = this->children; child != NULL; child = child->next) {
         ec->xl_list[count++] = child->xlator;
     }
 
@@ -125,43 +121,42 @@ int32_t ec_prepare_childs(xlator_t * this)
 
 /* This function transforms the subvol to subvol-id*/
 static int
-_subvol_to_subvolid (dict_t *this, char *key, data_t *value, void *data)
-{
-        ec_t *ec = data;
-        xlator_t *subvol = NULL;
-        int      i = 0;
-        int     ret = -1;
-
-        subvol = data_to_ptr (value);
-        for (i = 0; i < ec->nodes; i++) {
-                if (ec->xl_list[i] == subvol) {
-                        ret = dict_set_int32 (this, key, i);
-                        /* -1 stops dict_foreach and returns -1*/
-                        if (ret < 0)
-                                ret = -1;
-                        goto out;
-                }
+_subvol_to_subvolid(dict_t *this, char *key, data_t *value, void *data)
+{
+    ec_t *ec = data;
+    xlator_t *subvol = NULL;
+    int i = 0;
+    int ret = -1;
+
+    subvol = data_to_ptr(value);
+    for (i = 0; i < ec->nodes; i++) {
+        if (ec->xl_list[i] == subvol) {
+            ret = dict_set_int32(this, key, i);
+            /* -1 stops dict_foreach and returns -1*/
+            if (ret < 0)
+                ret = -1;
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-ec_subvol_to_subvol_id_transform (ec_t *ec, dict_t *leaf_to_subvolid)
+ec_subvol_to_subvol_id_transform(ec_t *ec, dict_t *leaf_to_subvolid)
 {
-        return dict_foreach (leaf_to_subvolid, _subvol_to_subvolid, ec);
+    return dict_foreach(leaf_to_subvolid, _subvol_to_subvolid, ec);
 }
 
-void __ec_destroy_private(xlator_t * this)
+void
+__ec_destroy_private(xlator_t *this)
 {
-    ec_t * ec = this->private;
+    ec_t *ec = this->private;
 
-    if (ec != NULL)
-    {
+    if (ec != NULL) {
         LOCK(&ec->lock);
 
-        if (ec->timer != NULL)
-        {
+        if (ec->timer != NULL) {
             gf_timer_call_cancel(this->ctx, ec->timer);
             ec->timer = NULL;
         }
@@ -179,31 +174,27 @@ void __ec_destroy_private(xlator_t * this)
         sleep(2);
 
         this->private = NULL;
-        if (ec->xl_list != NULL)
-        {
+        if (ec->xl_list != NULL) {
             GF_FREE(ec->xl_list);
             ec->xl_list = NULL;
         }
 
-        if (ec->fop_pool != NULL)
-        {
+        if (ec->fop_pool != NULL) {
             mem_pool_destroy(ec->fop_pool);
         }
 
-        if (ec->cbk_pool != NULL)
-        {
+        if (ec->cbk_pool != NULL) {
             mem_pool_destroy(ec->cbk_pool);
         }
 
-        if (ec->lock_pool != NULL)
-        {
+        if (ec->lock_pool != NULL) {
             mem_pool_destroy(ec->lock_pool);
         }
 
         LOCK_DESTROY(&ec->lock);
 
         if (ec->leaf_to_subvolid)
-                dict_unref (ec->leaf_to_subvolid);
+            dict_unref(ec->leaf_to_subvolid);
 
         ec_method_fini(&ec->matrix);
 
@@ -211,13 +202,13 @@ void __ec_destroy_private(xlator_t * this)
     }
 }
 
-int32_t mem_acct_init(xlator_t * this)
+int32_t
+mem_acct_init(xlator_t *this)
 {
-    if (xlator_mem_acct_init(this, ec_mt_end + 1) != 0)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                EC_MSG_NO_MEMORY, "Memory accounting initialization "
-                                         "failed.");
+    if (xlator_mem_acct_init(this, ec_mt_end + 1) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Memory accounting initialization "
+               "failed.");
 
         return -1;
     }
@@ -226,442 +217,582 @@ int32_t mem_acct_init(xlator_t * this)
 }
 
 void
-ec_configure_background_heal_opts (ec_t *ec, int background_heals,
-                                   int heal_wait_qlen)
+ec_configure_background_heal_opts(ec_t *ec, int background_heals,
+                                  int heal_wait_qlen)
 {
-        if (background_heals == 0) {
-                ec->heal_wait_qlen = 0;
-        } else {
-                ec->heal_wait_qlen = heal_wait_qlen;
-        }
-        ec->background_heals = background_heals;
+    if (background_heals == 0) {
+        ec->heal_wait_qlen = 0;
+    } else {
+        ec->heal_wait_qlen = heal_wait_qlen;
+    }
+    ec->background_heals = background_heals;
 }
 
 int
-ec_assign_read_policy (ec_t *ec, char *read_policy)
+ec_assign_read_policy(ec_t *ec, char *read_policy)
 {
-        int read_policy_idx = -1;
+    int read_policy_idx = -1;
 
-        read_policy_idx = gf_get_index_by_elem (ec_read_policies, read_policy);
-        if (read_policy_idx < 0 || read_policy_idx >= EC_READ_POLICY_MAX)
-                return -1;
+    read_policy_idx = gf_get_index_by_elem(ec_read_policies, read_policy);
+    if (read_policy_idx < 0 || read_policy_idx >= EC_READ_POLICY_MAX)
+        return -1;
 
-        ec->read_policy = read_policy_idx;
-        return 0;
+    ec->read_policy = read_policy_idx;
+    return 0;
 }
 
 int32_t
-reconfigure (xlator_t *this, dict_t *options)
-{
-        ec_t     *ec              = this->private;
-        char     *read_policy     = NULL;
-        char     *extensions      = NULL;
-        uint32_t heal_wait_qlen   = 0;
-        uint32_t background_heals = 0;
-        int32_t  ret              = -1;
-        int32_t  err;
-
-        GF_OPTION_RECONF ("cpu-extensions", extensions, options, str, failed);
-
-        GF_OPTION_RECONF ("self-heal-daemon", ec->shd.enabled, options, bool,
-                          failed);
-        GF_OPTION_RECONF ("iam-self-heal-daemon", ec->shd.iamshd, options,
-                          bool, failed);
-        GF_OPTION_RECONF ("eager-lock", ec->eager_lock, options,
-                          bool, failed);
-        GF_OPTION_RECONF ("other-eager-lock", ec->other_eager_lock, options,
-                          bool, failed);
-        GF_OPTION_RECONF ("eager-lock-timeout", ec->eager_lock_timeout,
-                          options, uint32, failed);
-        GF_OPTION_RECONF ("other-eager-lock-timeout",
-                          ec->other_eager_lock_timeout, options, uint32,
-                          failed);
-        GF_OPTION_RECONF ("background-heals", background_heals, options,
-                          uint32, failed);
-        GF_OPTION_RECONF ("heal-wait-qlength", heal_wait_qlen, options,
-                          uint32, failed);
-        GF_OPTION_RECONF ("self-heal-window-size", ec->self_heal_window_size,
-                          options, uint32, failed);
-        GF_OPTION_RECONF ("heal-timeout", ec->shd.timeout, options,
-                          int32, failed);
-        ec_configure_background_heal_opts (ec, background_heals,
-                                           heal_wait_qlen);
-        GF_OPTION_RECONF ("shd-max-threads", ec->shd.max_threads,
-                          options, uint32, failed);
-        GF_OPTION_RECONF ("shd-wait-qlength", ec->shd.wait_qlength,
-                          options, uint32, failed);
-
-        GF_OPTION_RECONF ("read-policy", read_policy, options, str, failed);
-
-        GF_OPTION_RECONF ("optimistic-change-log", ec->optimistic_changelog,
-                          options, bool, failed);
-        GF_OPTION_RECONF ("parallel-writes", ec->parallel_writes,
-                          options, bool, failed);
-        GF_OPTION_RECONF ("stripe-cache", ec->stripe_cache, options, uint32,
-                          failed);
-        ret = 0;
-        if (ec_assign_read_policy (ec, read_policy)) {
-                ret = -1;
-        }
+reconfigure(xlator_t *this, dict_t *options)
+{
+    ec_t *ec = this->private;
+    char *read_policy = NULL;
+    char *extensions = NULL;
+    uint32_t heal_wait_qlen = 0;
+    uint32_t background_heals = 0;
+    int32_t ret = -1;
+    int32_t err;
 
-        err = ec_method_update(this, &ec->matrix, extensions);
-        if (err != 0) {
-                ret = -1;
-        }
+    GF_OPTION_RECONF("cpu-extensions", extensions, options, str, failed);
+
+    GF_OPTION_RECONF("self-heal-daemon", ec->shd.enabled, options, bool,
+                     failed);
+    GF_OPTION_RECONF("iam-self-heal-daemon", ec->shd.iamshd, options, bool,
+                     failed);
+    GF_OPTION_RECONF("eager-lock", ec->eager_lock, options, bool, failed);
+    GF_OPTION_RECONF("other-eager-lock", ec->other_eager_lock, options, bool,
+                     failed);
+    GF_OPTION_RECONF("eager-lock-timeout", ec->eager_lock_timeout, options,
+                     uint32, failed);
+    GF_OPTION_RECONF("other-eager-lock-timeout", ec->other_eager_lock_timeout,
+                     options, uint32, failed);
+    GF_OPTION_RECONF("background-heals", background_heals, options, uint32,
+                     failed);
+    GF_OPTION_RECONF("heal-wait-qlength", heal_wait_qlen, options, uint32,
+                     failed);
+    GF_OPTION_RECONF("self-heal-window-size", ec->self_heal_window_size,
+                     options, uint32, failed);
+    GF_OPTION_RECONF("heal-timeout", ec->shd.timeout, options, int32, failed);
+    ec_configure_background_heal_opts(ec, background_heals, heal_wait_qlen);
+    GF_OPTION_RECONF("shd-max-threads", ec->shd.max_threads, options, uint32,
+                     failed);
+    GF_OPTION_RECONF("shd-wait-qlength", ec->shd.wait_qlength, options, uint32,
+                     failed);
+
+    GF_OPTION_RECONF("read-policy", read_policy, options, str, failed);
+
+    GF_OPTION_RECONF("optimistic-change-log", ec->optimistic_changelog, options,
+                     bool, failed);
+    GF_OPTION_RECONF("parallel-writes", ec->parallel_writes, options, bool,
+                     failed);
+    GF_OPTION_RECONF("stripe-cache", ec->stripe_cache, options, uint32, failed);
+    GF_OPTION_RECONF("quorum-count", ec->quorum_count, options, uint32, failed);
+    ret = 0;
+    if (ec_assign_read_policy(ec, read_policy)) {
+        ret = -1;
+    }
+
+    err = ec_method_update(this, &ec->matrix, extensions);
+    if (err != 0) {
+        ret = -1;
+    }
 
 failed:
-        return ret;
+    return ret;
 }
 
 glusterfs_event_t
-ec_get_event_from_state (ec_t *ec)
-{
-        int               down_count = 0;
-
-        if (ec->xl_up_count >= ec->fragments) {
-                /* If ec is up but some subvolumes are yet to notify, give
-                 * grace time for other subvols to notify to prevent start of
-                 * I/O which may result in self-heals */
-                if (ec->xl_notify_count < ec->nodes)
-                        return GF_EVENT_MAXVAL;
-
-                return GF_EVENT_CHILD_UP;
-        } else {
-                down_count = ec->xl_notify_count - ec->xl_up_count;
-                if (down_count > ec->redundancy)
-                        return GF_EVENT_CHILD_DOWN;
-        }
+ec_get_event_from_state(ec_t *ec)
+{
+    int down_count = 0;
+
+    if (ec->xl_up_count >= ec->fragments) {
+        /* If ec is up but some subvolumes are yet to notify, give
+         * grace time for other subvols to notify to prevent start of
+         * I/O which may result in self-heals */
+        if (ec->xl_notify_count < ec->nodes)
+            return GF_EVENT_MAXVAL;
+
+        return GF_EVENT_CHILD_UP;
+    } else {
+        down_count = ec->xl_notify_count - ec->xl_up_count;
+        if (down_count > ec->redundancy)
+            return GF_EVENT_CHILD_DOWN;
+    }
 
-        return GF_EVENT_MAXVAL;
+    return GF_EVENT_MAXVAL;
 }
 
 void
-ec_up (xlator_t *this, ec_t *ec)
+ec_up(xlator_t *this, ec_t *ec)
 {
-        if (ec->timer != NULL) {
-                gf_timer_call_cancel (this->ctx, ec->timer);
-                ec->timer = NULL;
-        }
+    char str1[32], str2[32];
 
-        ec->up = 1;
-        gf_msg (this->name, GF_LOG_INFO, 0, EC_MSG_EC_UP, "Going UP");
+    if (ec->timer != NULL) {
+        gf_timer_call_cancel(this->ctx, ec->timer);
+        ec->timer = NULL;
+    }
+
+    ec->up = 1;
+    gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_EC_UP,
+           "Going UP : Child UP = %s Child Notify = %s",
+           ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes),
+           ec_bin(str2, sizeof(str2), ec->xl_notify, ec->nodes));
 
-        gf_event (EVENT_EC_MIN_BRICKS_UP, "subvol=%s", this->name);
+    gf_event(EVENT_EC_MIN_BRICKS_UP, "subvol=%s", this->name);
 }
 
 void
-ec_down (xlator_t *this, ec_t *ec)
+ec_down(xlator_t *this, ec_t *ec)
 {
-        if (ec->timer != NULL) {
-                gf_timer_call_cancel(this->ctx, ec->timer);
-                ec->timer = NULL;
-        }
+    char str1[32], str2[32];
 
-        ec->up = 0;
-        gf_msg (this->name, GF_LOG_INFO, 0, EC_MSG_EC_DOWN, "Going DOWN");
+    if (ec->timer != NULL) {
+        gf_timer_call_cancel(this->ctx, ec->timer);
+        ec->timer = NULL;
+    }
+
+    ec->up = 0;
+    gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_EC_DOWN,
+           "Going DOWN : Child UP = %s Child Notify = %s",
+           ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes),
+           ec_bin(str2, sizeof(str2), ec->xl_notify, ec->nodes));
 
-        gf_event (EVENT_EC_MIN_BRICKS_NOT_UP, "subvol=%s", this->name);
+    gf_event(EVENT_EC_MIN_BRICKS_NOT_UP, "subvol=%s", this->name);
 }
 
 void
-ec_notify_cbk (void *data)
+ec_notify_cbk(void *data)
 {
-        ec_t *ec = data;
-        glusterfs_event_t event = GF_EVENT_MAXVAL;
-        gf_boolean_t propagate = _gf_false;
+    ec_t *ec = data;
+    glusterfs_event_t event = GF_EVENT_MAXVAL;
+    gf_boolean_t propagate = _gf_false;
+    gf_boolean_t launch_heal = _gf_false;
 
-        LOCK(&ec->lock);
-        {
-                if (!ec->timer) {
-                        /*
-                         * Either child_up/child_down is already sent to parent
-                         * This is a spurious wake up.
-                         */
-                        goto unlock;
-                }
-
-                gf_timer_call_cancel (ec->xl->ctx, ec->timer);
-                ec->timer = NULL;
-
-                /* The timeout has expired, so any subvolume that has not
-                 * already reported its state, will be considered to be down.
-                 * We mark as if all bricks had reported. */
-                ec->xl_notify = (1ULL << ec->nodes) - 1ULL;
-                ec->xl_notify_count = ec->nodes;
-
-                /* Since we have marked all subvolumes as notified, it's
-                 * guaranteed that ec_get_event_from_state() will return
-                 * CHILD_UP or CHILD_DOWN, but not MAXVAL. */
-                event = ec_get_event_from_state (ec);
-                if (event == GF_EVENT_CHILD_UP) {
-                        /* We are ready to bring the volume up. If there are
-                         * still bricks DOWN, they will be healed when they
-                         * come up. */
-                        ec_up (ec->xl, ec);
-                }
-
-                propagate = _gf_true;
+    LOCK(&ec->lock);
+    {
+        if (!ec->timer) {
+            /*
+             * Either child_up/child_down is already sent to parent
+             * This is a spurious wake up.
+             */
+            goto unlock;
         }
-unlock:
-        UNLOCK(&ec->lock);
 
-        if (propagate) {
-                if ((event == GF_EVENT_CHILD_UP) && ec->shd.iamshd) {
-                        /* We have just brought the volume UP, so we trigger
-                         * a self-heal check on the root directory. */
-                        ec_launch_replace_heal (ec);
-                }
-
-                default_notify (ec->xl, event, NULL);
+        gf_timer_call_cancel(ec->xl->ctx, ec->timer);
+        ec->timer = NULL;
+
+        /* The timeout has expired, so any subvolume that has not
+         * already reported its state, will be considered to be down.
+         * We mark as if all bricks had reported. */
+        ec->xl_notify = (1ULL << ec->nodes) - 1ULL;
+        ec->xl_notify_count = ec->nodes;
+
+        /* Since we have marked all subvolumes as notified, it's
+         * guaranteed that ec_get_event_from_state() will return
+         * CHILD_UP or CHILD_DOWN, but not MAXVAL. */
+        event = ec_get_event_from_state(ec);
+        if (event == GF_EVENT_CHILD_UP) {
+            /* We are ready to bring the volume up. If there are
+             * still bricks DOWN, they will be healed when they
+             * come up. */
+            ec_up(ec->xl, ec);
+
+            if (ec->shd.iamshd && !ec->shutdown) {
+                launch_heal = _gf_true;
+                GF_ATOMIC_INC(ec->async_fop_count);
+            }
         }
 
+        propagate = _gf_true;
+    }
+unlock:
+    UNLOCK(&ec->lock);
+
+    if (launch_heal) {
+        /* We have just brought the volume UP, so we trigger
+         * a self-heal check on the root directory. */
+        ec_launch_replace_heal(ec);
+    }
+    if (propagate) {
+        default_notify(ec->xl, event, NULL);
+    }
 }
 
 void
-ec_launch_notify_timer (xlator_t *this, ec_t *ec)
-{
-        struct timespec delay = {0, };
-
-        gf_msg_debug (this->name, 0, "Initiating child-down timer");
-        delay.tv_sec = 10;
-        delay.tv_nsec = 0;
-        ec->timer = gf_timer_call_after (this->ctx, delay, ec_notify_cbk, ec);
-        if (ec->timer == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        EC_MSG_TIMER_CREATE_FAIL, "Cannot create timer "
-                        "for delayed initialization");
-        }
+ec_launch_notify_timer(xlator_t *this, ec_t *ec)
+{
+    struct timespec delay = {
+        0,
+    };
+
+    gf_msg_debug(this->name, 0, "Initiating child-down timer");
+    delay.tv_sec = 10;
+    delay.tv_nsec = 0;
+    ec->timer = gf_timer_call_after(this->ctx, delay, ec_notify_cbk, ec);
+    if (ec->timer == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_TIMER_CREATE_FAIL,
+               "Cannot create timer "
+               "for delayed initialization");
+    }
 }
 
 gf_boolean_t
 ec_disable_delays(ec_t *ec)
 {
-        ec->shutdown = _gf_true;
+    ec->shutdown = _gf_true;
 
-        return list_empty (&ec->pending_fops);
+    return __ec_is_last_fop(ec);
 }
 
 void
-ec_pending_fops_completed(ec_t *ec)
+ec_cleanup_healer_object(ec_t *ec)
 {
-        if (ec->shutdown) {
-                default_notify (ec->xl, GF_EVENT_PARENT_DOWN, NULL);
+    struct subvol_healer *healer = NULL;
+    ec_self_heald_t *shd = NULL;
+    void *res = NULL;
+    int i = 0;
+    gf_boolean_t is_join = _gf_false;
+
+    shd = &ec->shd;
+    if (!shd->iamshd)
+        return;
+
+    for (i = 0; i < ec->nodes; i++) {
+        healer = &shd->index_healers[i];
+        pthread_mutex_lock(&healer->mutex);
+        {
+            healer->rerun = 1;
+            if (healer->running) {
+                pthread_cond_signal(&healer->cond);
+                is_join = _gf_true;
+            }
+        }
+        pthread_mutex_unlock(&healer->mutex);
+        if (is_join) {
+            pthread_join(healer->thread, &res);
+            is_join = _gf_false;
         }
+
+        healer = &shd->full_healers[i];
+        pthread_mutex_lock(&healer->mutex);
+        {
+            healer->rerun = 1;
+            if (healer->running) {
+                pthread_cond_signal(&healer->cond);
+                is_join = _gf_true;
+            }
+        }
+        pthread_mutex_unlock(&healer->mutex);
+        if (is_join) {
+            pthread_join(healer->thread, &res);
+            is_join = _gf_false;
+        }
+    }
+}
+void
+ec_pending_fops_completed(ec_t *ec)
+{
+    if (ec->shutdown) {
+        default_notify(ec->xl, GF_EVENT_PARENT_DOWN, NULL);
+    }
 }
 
 static gf_boolean_t
 ec_set_up_state(ec_t *ec, uintptr_t index_mask, uintptr_t new_state)
 {
-        uintptr_t current_state = 0;
+    uintptr_t current_state = 0;
 
-        if ((ec->xl_notify & index_mask) == 0) {
-                ec->xl_notify |= index_mask;
-                ec->xl_notify_count++;
-        }
-        current_state = ec->xl_up & index_mask;
-        if (current_state != new_state) {
-                ec->xl_up ^= index_mask;
-                ec->xl_up_count += (current_state ? -1 : 1);
+    if (xlator_is_cleanup_starting(ec->xl))
+        return _gf_false;
 
-                return _gf_true;
-        }
+    if ((ec->xl_notify & index_mask) == 0) {
+        ec->xl_notify |= index_mask;
+        ec->xl_notify_count++;
+    }
+    current_state = ec->xl_up & index_mask;
+    if (current_state != new_state) {
+        ec->xl_up ^= index_mask;
+        ec->xl_up_count += (current_state ? -1 : 1);
 
-        return _gf_false;
+        return _gf_true;
+    }
+
+    return _gf_false;
 }
 
 static gf_boolean_t
 ec_upcall(ec_t *ec, struct gf_upcall *upcall)
 {
-        struct gf_upcall_cache_invalidation *ci = NULL;
-        struct gf_upcall_inodelk_contention *lc = NULL;
-        inode_t *inode;
+    struct gf_upcall_cache_invalidation *ci = NULL;
+    struct gf_upcall_inodelk_contention *lc = NULL;
+    inode_t *inode;
+    inode_table_t *table;
 
-        switch (upcall->event_type) {
+    switch (upcall->event_type) {
         case GF_UPCALL_CACHE_INVALIDATION:
-                ci = upcall->data;
-                ci->flags |= UP_INVAL_ATTR;
-                return _gf_true;
+            ci = upcall->data;
+            ci->flags |= UP_INVAL_ATTR;
+            return _gf_true;
 
         case GF_UPCALL_INODELK_CONTENTION:
-                lc = upcall->data;
-                if (strcmp(lc->domain, ec->xl->name) != 0) {
-                        /* The lock is not owned by EC, ignore it. */
-                        return _gf_true;
-                }
-                inode = inode_find(((xlator_t *)ec->xl->graph->top)->itable,
-                                   upcall->gfid);
-                /* If inode is not found, it means that it's already released,
-                 * so we can ignore it. Probably it has been released and
-                 * destroyed while the contention notification was being sent.
-                 */
-                if (inode != NULL) {
-                        ec_lock_release(ec, inode);
-                        inode_unref(inode);
-                }
-
+            lc = upcall->data;
+            if (strcmp(lc->domain, ec->xl->name) != 0) {
+                /* The lock is not owned by EC, ignore it. */
+                return _gf_true;
+            }
+            table = ((xlator_t *)ec->xl->graph->top)->itable;
+            if (table == NULL) {
+                /* Self-heal daemon doesn't have an inode table on the top
+                 * xlator because it doesn't need it. In this case we should
+                 * use the inode table managed by EC itself where all inodes
+                 * being healed should be present. However self-heal doesn't
+                 * use eager-locking and inodelk's are already released as
+                 * soon as possible. In this case we can safely ignore these
+                 * notifications. */
                 return _gf_false;
+            }
+            inode = inode_find(table, upcall->gfid);
+            /* If inode is not found, it means that it's already released,
+             * so we can ignore it. Probably it has been released and
+             * destroyed while the contention notification was being sent.
+             */
+            if (inode != NULL) {
+                ec_lock_release(ec, inode);
+                inode_unref(inode);
+            }
+
+            return _gf_false;
 
         default:
-                return _gf_true;
-        }
+            return _gf_true;
+    }
 }
 
 int32_t
-ec_notify (xlator_t *this, int32_t event, void *data, void *data2)
-{
-        ec_t             *ec              = this->private;
-        int32_t           idx             = 0;
-        int32_t           error           = 0;
-        glusterfs_event_t old_event       = GF_EVENT_MAXVAL;
-        dict_t            *input          = NULL;
-        dict_t            *output         = NULL;
-        gf_boolean_t      propagate       = _gf_true;
-        gf_boolean_t      needs_shd_check = _gf_false;
-        int32_t           orig_event      = event;
-        uintptr_t         mask            = 0;
-
-        gf_msg_trace (this->name, 0, "NOTIFY(%d): %p, %p",
-                event, data, data2);
-
-        if (event == GF_EVENT_UPCALL) {
-                propagate = ec_upcall(ec, data);
-                goto done;
+ec_notify(xlator_t *this, int32_t event, void *data, void *data2)
+{
+    ec_t *ec = this->private;
+    int32_t idx = 0;
+    int32_t error = 0;
+    glusterfs_event_t old_event = GF_EVENT_MAXVAL;
+    dict_t *input = NULL;
+    dict_t *output = NULL;
+    gf_boolean_t propagate = _gf_true;
+    gf_boolean_t needs_shd_check = _gf_false;
+    int32_t orig_event = event;
+    uintptr_t mask = 0;
+
+    gf_msg_trace(this->name, 0, "NOTIFY(%d): %p, %p", event, data, data2);
+
+    if (event == GF_EVENT_UPCALL) {
+        propagate = ec_upcall(ec, data);
+        goto done;
+    }
+
+    if (event == GF_EVENT_TRANSLATOR_OP) {
+        if (!ec->up) {
+            error = -1;
+        } else {
+            input = data;
+            output = data2;
+            error = ec_xl_op(this, input, output);
+        }
+        goto out;
+    }
+
+    for (idx = 0; idx < ec->nodes; idx++) {
+        if (ec->xl_list[idx] == data) {
+            break;
         }
+    }
 
-        if (event == GF_EVENT_TRANSLATOR_OP) {
-                if (!ec->up) {
-                        error = -1;
-                } else {
-                        input = data;
-                        output = data2;
-                        error = ec_xl_op (this, input, output);
-                }
-                goto out;
+    LOCK(&ec->lock);
+
+    if (event == GF_EVENT_PARENT_UP) {
+        /*
+         * Start a timer which sends appropriate event to parent
+         * xlator to prevent the 'mount' syscall from hanging.
+         */
+        ec_launch_notify_timer(this, ec);
+        goto unlock;
+    } else if (event == GF_EVENT_PARENT_DOWN) {
+        /* If there aren't pending fops running after we have waken up
+         * them, we immediately propagate the notification. */
+        propagate = ec_disable_delays(ec);
+        ec_cleanup_healer_object(ec);
+        goto unlock;
+    }
+
+    if (idx < ec->nodes) { /* CHILD_* events */
+        old_event = ec_get_event_from_state(ec);
+
+        mask = 1ULL << idx;
+        if (event == GF_EVENT_CHILD_UP) {
+            /* We need to trigger a selfheal if a brick changes
+             * to UP state. */
+            if (ec_set_up_state(ec, mask, mask) && ec->shd.iamshd &&
+                !ec->shutdown) {
+                needs_shd_check = _gf_true;
+            }
+        } else if (event == GF_EVENT_CHILD_DOWN) {
+            ec_set_up_state(ec, mask, 0);
         }
 
-        for (idx = 0; idx < ec->nodes; idx++) {
-                if (ec->xl_list[idx] == data) {
-                        break;
-                }
+        event = ec_get_event_from_state(ec);
+
+        if (event == GF_EVENT_CHILD_UP) {
+            if (!ec->up) {
+                ec_up(this, ec);
+            }
+        } else {
+            /* If the volume is not UP, it's irrelevant if one
+             * brick has come up. We cannot heal anything. */
+            needs_shd_check = _gf_false;
+
+            if ((event == GF_EVENT_CHILD_DOWN) && ec->up) {
+                ec_down(this, ec);
+            }
         }
 
-        LOCK (&ec->lock);
-
-        if (event == GF_EVENT_PARENT_UP) {
-                /*
-                 * Start a timer which sends appropriate event to parent
-                 * xlator to prevent the 'mount' syscall from hanging.
-                 */
-                ec_launch_notify_timer (this, ec);
-                goto unlock;
-        } else if (event == GF_EVENT_PARENT_DOWN) {
-                /* If there aren't pending fops running after we have waken up
-                 * them, we immediately propagate the notification. */
-                propagate = ec_disable_delays(ec);
-                goto unlock;
+        if (event != GF_EVENT_MAXVAL) {
+            if (event == old_event) {
+                if (orig_event == GF_EVENT_CHILD_UP)
+                    event = GF_EVENT_SOME_DESCENDENT_UP;
+                else /* orig_event has to be GF_EVENT_CHILD_DOWN */
+                    event = GF_EVENT_SOME_DESCENDENT_DOWN;
+            }
+        } else {
+            propagate = _gf_false;
+            needs_shd_check = _gf_false;
         }
 
-        if (idx < ec->nodes) { /* CHILD_* events */
-                old_event = ec_get_event_from_state (ec);
-
-                mask = 1ULL << idx;
-                if (event == GF_EVENT_CHILD_UP) {
-                        /* We need to trigger a selfheal if a brick changes
-                         * to UP state. */
-                        needs_shd_check = ec_set_up_state(ec, mask, mask);
-                } else if (event == GF_EVENT_CHILD_DOWN) {
-                        ec_set_up_state(ec, mask, 0);
-                }
-
-                event = ec_get_event_from_state (ec);
-
-                if (event == GF_EVENT_CHILD_UP) {
-                        if (!ec->up) {
-                                ec_up (this, ec);
-                        }
-                } else {
-                        /* If the volume is not UP, it's irrelevant if one
-                         * brick has come up. We cannot heal anything. */
-                        needs_shd_check = _gf_false;
-
-                        if ((event == GF_EVENT_CHILD_DOWN) && ec->up) {
-                                ec_down (this, ec);
-                        }
-                }
-
-                if (event != GF_EVENT_MAXVAL) {
-                        if (event == old_event) {
-                                if (orig_event == GF_EVENT_CHILD_UP)
-                                        event = GF_EVENT_SOME_DESCENDENT_UP;
-                                else /* orig_event has to be GF_EVENT_CHILD_DOWN */
-                                        event = GF_EVENT_SOME_DESCENDENT_DOWN;
-                        }
-                } else {
-                        propagate = _gf_false;
-                }
+        if (needs_shd_check) {
+            GF_ATOMIC_INC(ec->async_fop_count);
         }
+    }
 unlock:
-        UNLOCK (&ec->lock);
+    UNLOCK(&ec->lock);
 
 done:
-        if (propagate) {
-                if (needs_shd_check && ec->shd.iamshd) {
-                        ec_launch_replace_heal (ec);
-                }
-
-                error = default_notify (this, event, data);
-        }
+    if (needs_shd_check) {
+        ec_launch_replace_heal(ec);
+    }
+    if (propagate) {
+        error = default_notify(this, event, data);
+    }
 
 out:
-        return error;
+    return error;
 }
 
 int32_t
-notify (xlator_t *this, int32_t event, void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int ret = -1;
-        va_list         ap;
-        void *data2 = NULL;
+    int ret = -1;
+    va_list ap;
+    void *data2 = NULL;
 
-        va_start (ap, data);
-        data2 = va_arg (ap, dict_t*);
-        va_end (ap);
-        ret = ec_notify (this, event, data, data2);
+    va_start(ap, data);
+    data2 = va_arg(ap, dict_t *);
+    va_end(ap);
+    ret = ec_notify(this, event, data, data2);
 
-        return ret;
+    return ret;
 }
 
 static void
 ec_statistics_init(ec_t *ec)
 {
-        GF_ATOMIC_INIT(ec->stats.stripe_cache.hits, 0);
-        GF_ATOMIC_INIT(ec->stats.stripe_cache.misses, 0);
-        GF_ATOMIC_INIT(ec->stats.stripe_cache.updates, 0);
-        GF_ATOMIC_INIT(ec->stats.stripe_cache.invals, 0);
-        GF_ATOMIC_INIT(ec->stats.stripe_cache.evicts, 0);
-        GF_ATOMIC_INIT(ec->stats.stripe_cache.allocs, 0);
-        GF_ATOMIC_INIT(ec->stats.stripe_cache.errors, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.hits, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.misses, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.updates, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.invals, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.evicts, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.allocs, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.errors, 0);
+    GF_ATOMIC_INIT(ec->stats.shd.attempted, 0);
+    GF_ATOMIC_INIT(ec->stats.shd.completed, 0);
+}
+
+static int
+ec_assign_read_mask(ec_t *ec, char *read_mask_str)
+{
+    char *mask = NULL;
+    char *maskptr = NULL;
+    char *saveptr = NULL;
+    char *id_str = NULL;
+    int id = 0;
+    int ret = 0;
+    uintptr_t read_mask = 0;
+
+    if (!read_mask_str) {
+        ec->read_mask = 0;
+        ret = 0;
+        goto out;
+    }
+
+    mask = gf_strdup(read_mask_str);
+    if (!mask) {
+        ret = -1;
+        goto out;
+    }
+    maskptr = mask;
+
+    for (;;) {
+        id_str = strtok_r(maskptr, ":", &saveptr);
+        if (id_str == NULL)
+            break;
+        if (gf_string2int(id_str, &id)) {
+            gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL,
+                   "In read-mask \"%s\" id %s is not a valid integer",
+                   read_mask_str, id_str);
+            ret = -1;
+            goto out;
+        }
+
+        if ((id < 0) || (id >= ec->nodes)) {
+            gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL,
+                   "In read-mask \"%s\" id %d is not in range [0 - %d]",
+                   read_mask_str, id, ec->nodes - 1);
+            ret = -1;
+            goto out;
+        }
+        read_mask |= (1UL << id);
+        maskptr = NULL;
+    }
+
+    if (gf_bits_count(read_mask) < ec->fragments) {
+        gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL,
+               "read-mask \"%s\" should contain at least %d ids", read_mask_str,
+               ec->fragments);
+        ret = -1;
+        goto out;
+    }
+    ec->read_mask = read_mask;
+    ret = 0;
+out:
+    GF_FREE(mask);
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-    ec_t *ec          = NULL;
+    ec_t *ec = NULL;
     char *read_policy = NULL;
-    char *extensions  = NULL;
+    char *extensions = NULL;
     int32_t err;
+    char *read_mask_str = NULL;
 
-    if (this->parents == NULL)
-    {
-        gf_msg (this->name, GF_LOG_WARNING, 0,
-                EC_MSG_NO_PARENTS, "Volume does not have parents.");
+    if (this->parents == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, EC_MSG_NO_PARENTS,
+               "Volume does not have parents.");
     }
 
     ec = GF_MALLOC(sizeof(*ec), ec_mt_ec_t);
-    if (ec == NULL)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                EC_MSG_NO_MEMORY, "Failed to allocate private memory.");
+    if (ec == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to allocate private memory.");
 
         return -1;
     }
@@ -672,6 +803,7 @@ init (xlator_t *this)
     ec->xl = this;
     LOCK_INIT(&ec->lock);
 
+    GF_ATOMIC_INIT(ec->async_fop_count, 0);
     INIT_LIST_HEAD(&ec->pending_fops);
     INIT_LIST_HEAD(&ec->heal_waiting);
     INIT_LIST_HEAD(&ec->healing);
@@ -680,26 +812,23 @@ init (xlator_t *this)
     ec->cbk_pool = mem_pool_new(ec_cbk_data_t, 4096);
     ec->lock_pool = mem_pool_new(ec_lock_t, 1024);
     if ((ec->fop_pool == NULL) || (ec->cbk_pool == NULL) ||
-        (ec->lock_pool == NULL))
-    {
-        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                EC_MSG_NO_MEMORY, "Failed to create memory pools.");
+        (ec->lock_pool == NULL)) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to create memory pools.");
 
         goto failed;
     }
 
-    if (ec_prepare_childs(this) != 0)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, 0,
-                EC_MSG_XLATOR_INIT_FAIL, "Failed to initialize xlator");
+    if (ec_prepare_childs(this) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL,
+               "Failed to initialize xlator");
 
         goto failed;
     }
 
-    if (ec_parse_options(this) != 0)
-    {
-        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                EC_MSG_XLATOR_PARSE_OPT_FAIL, "Failed to parse xlator options");
+    if (ec_parse_options(this) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_XLATOR_PARSE_OPT_FAIL,
+               "Failed to parse xlator options");
 
         goto failed;
     }
@@ -709,58 +838,65 @@ init (xlator_t *this)
     err = ec_method_init(this, &ec->matrix, ec->fragments, ec->nodes,
                          ec->nodes * 2, extensions);
     if (err != 0) {
-        gf_msg (this->name, GF_LOG_ERROR, -err, EC_MSG_MATRIX_FAILED,
-                "Failed to initialize matrix management");
+        gf_msg(this->name, GF_LOG_ERROR, -err, EC_MSG_MATRIX_FAILED,
+               "Failed to initialize matrix management");
 
         goto failed;
     }
 
-    GF_OPTION_INIT ("self-heal-daemon", ec->shd.enabled, bool, failed);
-    GF_OPTION_INIT ("iam-self-heal-daemon", ec->shd.iamshd, bool, failed);
-    GF_OPTION_INIT ("eager-lock", ec->eager_lock, bool, failed);
-    GF_OPTION_INIT ("other-eager-lock", ec->other_eager_lock, bool, failed);
-    GF_OPTION_INIT ("eager-lock-timeout", ec->eager_lock_timeout, uint32,
-                    failed);
-    GF_OPTION_INIT ("other-eager-lock-timeout", ec->other_eager_lock_timeout,
-                    uint32, failed);
-    GF_OPTION_INIT ("background-heals", ec->background_heals, uint32, failed);
-    GF_OPTION_INIT ("heal-wait-qlength", ec->heal_wait_qlen, uint32, failed);
-    GF_OPTION_INIT ("self-heal-window-size", ec->self_heal_window_size, uint32,
-                    failed);
-    ec_configure_background_heal_opts (ec, ec->background_heals,
-                                       ec->heal_wait_qlen);
-    GF_OPTION_INIT ("read-policy", read_policy, str, failed);
-    if (ec_assign_read_policy (ec, read_policy))
-            goto failed;
-
-    GF_OPTION_INIT ("shd-max-threads", ec->shd.max_threads, uint32, failed);
-    GF_OPTION_INIT ("shd-wait-qlength", ec->shd.wait_qlength, uint32, failed);
-    GF_OPTION_INIT ("optimistic-change-log", ec->optimistic_changelog, bool, failed);
-    GF_OPTION_INIT ("parallel-writes", ec->parallel_writes, bool, failed);
-    GF_OPTION_INIT ("stripe-cache", ec->stripe_cache, uint32, failed);
-
-    this->itable = inode_table_new (EC_SHD_INODE_LRU_LIMIT, this);
+    GF_OPTION_INIT("self-heal-daemon", ec->shd.enabled, bool, failed);
+    GF_OPTION_INIT("iam-self-heal-daemon", ec->shd.iamshd, bool, failed);
+    GF_OPTION_INIT("eager-lock", ec->eager_lock, bool, failed);
+    GF_OPTION_INIT("other-eager-lock", ec->other_eager_lock, bool, failed);
+    GF_OPTION_INIT("eager-lock-timeout", ec->eager_lock_timeout, uint32,
+                   failed);
+    GF_OPTION_INIT("other-eager-lock-timeout", ec->other_eager_lock_timeout,
+                   uint32, failed);
+    GF_OPTION_INIT("background-heals", ec->background_heals, uint32, failed);
+    GF_OPTION_INIT("heal-wait-qlength", ec->heal_wait_qlen, uint32, failed);
+    GF_OPTION_INIT("self-heal-window-size", ec->self_heal_window_size, uint32,
+                   failed);
+    ec_configure_background_heal_opts(ec, ec->background_heals,
+                                      ec->heal_wait_qlen);
+    GF_OPTION_INIT("read-policy", read_policy, str, failed);
+    if (ec_assign_read_policy(ec, read_policy))
+        goto failed;
+
+    GF_OPTION_INIT("heal-timeout", ec->shd.timeout, int32, failed);
+    GF_OPTION_INIT("shd-max-threads", ec->shd.max_threads, uint32, failed);
+    GF_OPTION_INIT("shd-wait-qlength", ec->shd.wait_qlength, uint32, failed);
+    GF_OPTION_INIT("optimistic-change-log", ec->optimistic_changelog, bool,
+                   failed);
+    GF_OPTION_INIT("parallel-writes", ec->parallel_writes, bool, failed);
+    GF_OPTION_INIT("stripe-cache", ec->stripe_cache, uint32, failed);
+    GF_OPTION_INIT("quorum-count", ec->quorum_count, uint32, failed);
+    GF_OPTION_INIT("ec-read-mask", read_mask_str, str, failed);
+
+    if (ec_assign_read_mask(ec, read_mask_str))
+        goto failed;
+
+    this->itable = inode_table_new(EC_SHD_INODE_LRU_LIMIT, this);
     if (!this->itable)
-            goto failed;
+        goto failed;
 
     if (ec->shd.iamshd)
-            ec_selfheal_daemon_init (this);
-    gf_msg_debug (this->name, 0, "Disperse translator initialized.");
+        ec_selfheal_daemon_init(this);
+    gf_msg_debug(this->name, 0, "Disperse translator initialized.");
 
-    ec->leaf_to_subvolid = dict_new ();
+    ec->leaf_to_subvolid = dict_new();
     if (!ec->leaf_to_subvolid)
-            goto failed;
-    if (glusterfs_reachable_leaves (this, ec->leaf_to_subvolid)) {
-        gf_msg (this->name, GF_LOG_ERROR, 0,
-                EC_MSG_SUBVOL_BUILD_FAIL, "Failed to build subvol "
-                "dictionary");
+        goto failed;
+    if (glusterfs_reachable_leaves(this, ec->leaf_to_subvolid)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_SUBVOL_BUILD_FAIL,
+               "Failed to build subvol "
+               "dictionary");
         goto failed;
     }
 
-    if (ec_subvol_to_subvol_id_transform (ec, ec->leaf_to_subvolid) < 0) {
-        gf_msg (this->name, GF_LOG_ERROR, 0,
-                EC_MSG_SUBVOL_ID_DICT_SET_FAIL, "Failed to build subvol-id "
-                "dictionary");
+    if (ec_subvol_to_subvol_id_transform(ec, ec->leaf_to_subvolid) < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_SUBVOL_ID_DICT_SET_FAIL,
+               "Failed to build subvol-id "
+               "dictionary");
         goto failed;
     }
 
@@ -774,13 +910,16 @@ failed:
     return -1;
 }
 
-void fini(xlator_t * this)
+void
+fini(xlator_t *this)
 {
+    ec_selfheal_daemon_fini(this);
     __ec_destroy_private(this);
 }
 
-int32_t ec_gf_access(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     int32_t mask, dict_t * xdata)
+int32_t
+ec_gf_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+             dict_t *xdata)
 {
     ec_access(frame, this, -1, EC_MINIMUM_ONE, default_access_cbk, NULL, loc,
               mask, xdata);
@@ -788,9 +927,9 @@ int32_t ec_gf_access(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_create(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     int32_t flags, mode_t mode, mode_t umask, fd_t * fd,
-                     dict_t * xdata)
+int32_t
+ec_gf_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
     ec_create(frame, this, -1, EC_MINIMUM_MIN, default_create_cbk, NULL, loc,
               flags, mode, umask, fd, xdata);
@@ -798,53 +937,58 @@ int32_t ec_gf_create(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_discard(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                      off_t offset, size_t len, dict_t * xdata)
+int32_t
+ec_gf_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              size_t len, dict_t *xdata)
 {
-    ec_discard(frame, this, -1, EC_MINIMUM_MIN, default_discard_cbk,
-               NULL, fd, offset, len, xdata);
+    ec_discard(frame, this, -1, EC_MINIMUM_MIN, default_discard_cbk, NULL, fd,
+               offset, len, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_entrylk(call_frame_t * frame, xlator_t * this,
-                      const char * volume, loc_t * loc, const char * basename,
-                      entrylk_cmd cmd, entrylk_type type, dict_t * xdata)
+int32_t
+ec_gf_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+              loc_t *loc, const char *basename, entrylk_cmd cmd,
+              entrylk_type type, dict_t *xdata)
 {
-    int32_t minimum = EC_MINIMUM_ALL;
+    uint32_t fop_flags = EC_MINIMUM_ALL;
+
     if (cmd == ENTRYLK_UNLOCK)
-            minimum = EC_MINIMUM_ONE;
-    ec_entrylk(frame, this, -1, minimum, default_entrylk_cbk, NULL,
-               volume, loc, basename, cmd, type, xdata);
+        fop_flags = EC_MINIMUM_ONE;
+    ec_entrylk(frame, this, -1, fop_flags, default_entrylk_cbk, NULL, volume,
+               loc, basename, cmd, type, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_fentrylk(call_frame_t * frame, xlator_t * this,
-                       const char * volume, fd_t * fd, const char * basename,
-                       entrylk_cmd cmd, entrylk_type type, dict_t * xdata)
+int32_t
+ec_gf_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+               fd_t *fd, const char *basename, entrylk_cmd cmd,
+               entrylk_type type, dict_t *xdata)
 {
-    int32_t minimum = EC_MINIMUM_ALL;
+    uint32_t fop_flags = EC_MINIMUM_ALL;
+
     if (cmd == ENTRYLK_UNLOCK)
-            minimum = EC_MINIMUM_ONE;
-    ec_fentrylk(frame, this, -1, minimum, default_fentrylk_cbk, NULL,
-                volume, fd, basename, cmd, type, xdata);
+        fop_flags = EC_MINIMUM_ONE;
+    ec_fentrylk(frame, this, -1, fop_flags, default_fentrylk_cbk, NULL, volume,
+                fd, basename, cmd, type, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_fallocate(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                        int32_t mode, off_t offset, size_t len,
-                        dict_t * xdata)
+int32_t
+ec_gf_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+                off_t offset, size_t len, dict_t *xdata)
 {
-    ec_fallocate(frame, this, -1, EC_MINIMUM_MIN, default_fallocate_cbk,
-                 NULL, fd, mode, offset, len, xdata);
+    ec_fallocate(frame, this, -1, EC_MINIMUM_MIN, default_fallocate_cbk, NULL,
+                 fd, mode, offset, len, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_flush(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                    dict_t * xdata)
+int32_t
+ec_gf_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
     ec_flush(frame, this, -1, EC_MINIMUM_MIN, default_flush_cbk, NULL, fd,
              xdata);
@@ -852,8 +996,9 @@ int32_t ec_gf_flush(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_fsync(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                    int32_t datasync, dict_t * xdata)
+int32_t
+ec_gf_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+            dict_t *xdata)
 {
     ec_fsync(frame, this, -1, EC_MINIMUM_MIN, default_fsync_cbk, NULL, fd,
              datasync, xdata);
@@ -861,136 +1006,139 @@ int32_t ec_gf_fsync(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_fsyncdir(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       int32_t datasync, dict_t * xdata)
+int32_t
+ec_gf_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+               dict_t *xdata)
 {
-    ec_fsyncdir(frame, this, -1, EC_MINIMUM_MIN, default_fsyncdir_cbk, NULL,
-                fd, datasync, xdata);
+    ec_fsyncdir(frame, this, -1, EC_MINIMUM_MIN, default_fsyncdir_cbk, NULL, fd,
+                datasync, xdata);
 
     return 0;
 }
 
 int
-ec_marker_populate_args (call_frame_t *frame, int type, int *gauge,
-                         xlator_t **subvols)
+ec_marker_populate_args(call_frame_t *frame, int type, int *gauge,
+                        xlator_t **subvols)
 {
-        xlator_t *this = frame->this;
-        ec_t *ec = this->private;
+    xlator_t *this = frame->this;
+    ec_t *ec = this->private;
 
-        memcpy (subvols, ec->xl_list, sizeof (*subvols) * ec->nodes);
+    memcpy(subvols, ec->xl_list, sizeof(*subvols) * ec->nodes);
 
-        if (type == MARKER_XTIME_TYPE) {
-                /*Don't error out on ENOENT/ENOTCONN */
-                gauge[MCNT_NOTFOUND] = 0;
-                gauge[MCNT_ENOTCONN] = 0;
-        }
+    if (type == MARKER_XTIME_TYPE) {
+        /*Don't error out on ENOENT/ENOTCONN */
+        gauge[MCNT_NOTFOUND] = 0;
+        gauge[MCNT_ENOTCONN] = 0;
+    }
 
-        return ec->nodes;
+    return ec->nodes;
 }
 
 int32_t
-ec_handle_heal_commands (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                         const char *name, dict_t *xdata)
+ec_handle_heal_commands(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        const char *name, dict_t *xdata)
 {
-        dict_t  *dict_rsp = NULL;
-        int     op_ret = -1;
-        int     op_errno = ENOMEM;
+    dict_t *dict_rsp = NULL;
+    int op_ret = -1;
+    int op_errno = ENOMEM;
 
-        if (!name || strcmp (name, GF_HEAL_INFO))
-                return -1;
+    if (!name || strcmp(name, GF_HEAL_INFO))
+        return -1;
 
-        op_errno = -ec_get_heal_info (this, loc, &dict_rsp);
-        if (op_errno <= 0) {
-                op_errno = op_ret = 0;
-        }
+    op_errno = -ec_get_heal_info(this, loc, &dict_rsp);
+    if (op_errno <= 0) {
+        op_errno = op_ret = 0;
+    }
 
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict_rsp, NULL);
-        if (dict_rsp)
-                dict_unref (dict_rsp);
-        return 0;
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict_rsp, NULL);
+    if (dict_rsp)
+        dict_unref(dict_rsp);
+    return 0;
 }
 
 int32_t
-ec_gf_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata)
+ec_gf_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        int     error = 0;
-        ec_t    *ec = this->private;
-        int32_t minimum = EC_MINIMUM_ONE;
+    int error = 0;
+    ec_t *ec = this->private;
+    int32_t fop_flags = EC_MINIMUM_ONE;
 
-        if (name && strcmp (name, EC_XATTR_HEAL) != 0) {
-                EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out);
-        }
+    if (name && strcmp(name, EC_XATTR_HEAL) != 0) {
+        EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out);
+    }
 
-        if (ec_handle_heal_commands (frame, this, loc, name, xdata) == 0)
-                return 0;
+    if (ec_handle_heal_commands(frame, this, loc, name, xdata) == 0)
+        return 0;
 
-        if (cluster_handle_marker_getxattr (frame, loc, name, ec->vol_uuid,
-                                            NULL, ec_marker_populate_args) == 0)
-                return 0;
+    if (cluster_handle_marker_getxattr(frame, loc, name, ec->vol_uuid, NULL,
+                                       ec_marker_populate_args) == 0)
+        return 0;
 
-        if (name &&
-            ((fnmatch (GF_XATTR_STIME_PATTERN, name, 0) == 0) ||
-             XATTR_IS_NODE_UUID(name) || XATTR_IS_NODE_UUID_LIST(name))) {
-                minimum = EC_MINIMUM_ALL;
-        }
+    if (name && ((fnmatch(GF_XATTR_STIME_PATTERN, name, 0) == 0) ||
+                 XATTR_IS_NODE_UUID(name) || XATTR_IS_NODE_UUID_LIST(name))) {
+        fop_flags = EC_MINIMUM_ALL;
+    }
 
-        ec_getxattr (frame, this, -1, minimum, default_getxattr_cbk,
-                     NULL, loc, name, xdata);
+    ec_getxattr(frame, this, -1, fop_flags, default_getxattr_cbk, NULL, loc,
+                name, xdata);
 
-        return 0;
+    return 0;
 out:
-        error = ENODATA;
-        STACK_UNWIND_STRICT (getxattr, frame, -1, error, NULL, NULL);
-        return 0;
+    error = ENODATA;
+    STACK_UNWIND_STRICT(getxattr, frame, -1, error, NULL, NULL);
+    return 0;
 }
 
 int32_t
-ec_gf_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name, dict_t *xdata)
+ec_gf_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
 {
-        int     error = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out);
 
-        ec_fgetxattr (frame, this, -1, EC_MINIMUM_ONE, default_fgetxattr_cbk,
-                      NULL, fd, name, xdata);
-        return 0;
+    ec_fgetxattr(frame, this, -1, EC_MINIMUM_ONE, default_fgetxattr_cbk, NULL,
+                 fd, name, xdata);
+    return 0;
 out:
-        error = ENODATA;
-        STACK_UNWIND_STRICT (fgetxattr, frame, -1, error, NULL, NULL);
-        return 0;
+    error = ENODATA;
+    STACK_UNWIND_STRICT(fgetxattr, frame, -1, error, NULL, NULL);
+    return 0;
 }
 
-int32_t ec_gf_inodelk(call_frame_t * frame, xlator_t * this,
-                      const char * volume, loc_t * loc, int32_t cmd,
-                      struct gf_flock * flock, dict_t * xdata)
+int32_t
+ec_gf_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+              loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-    int32_t minimum = EC_MINIMUM_ALL;
+    int32_t fop_flags = EC_MINIMUM_ALL;
+
     if (flock->l_type == F_UNLCK)
-            minimum = EC_MINIMUM_ONE;
+        fop_flags = EC_MINIMUM_ONE;
 
-    ec_inodelk(frame, this, &frame->root->lk_owner, -1, minimum,
+    ec_inodelk(frame, this, &frame->root->lk_owner, -1, fop_flags,
                default_inodelk_cbk, NULL, volume, loc, cmd, flock, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_finodelk(call_frame_t * frame, xlator_t * this,
-                       const char * volume, fd_t * fd, int32_t cmd,
-                       struct gf_flock * flock, dict_t * xdata)
+int32_t
+ec_gf_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+               fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-    int32_t minimum = EC_MINIMUM_ALL;
+    int32_t fop_flags = EC_MINIMUM_ALL;
+
     if (flock->l_type == F_UNLCK)
-            minimum = EC_MINIMUM_ONE;
-    ec_finodelk(frame, this, &frame->root->lk_owner, -1, minimum,
+        fop_flags = EC_MINIMUM_ONE;
+    ec_finodelk(frame, this, &frame->root->lk_owner, -1, fop_flags,
                 default_finodelk_cbk, NULL, volume, fd, cmd, flock, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_link(call_frame_t * frame, xlator_t * this, loc_t * oldloc,
-                   loc_t * newloc, dict_t * xdata)
+int32_t
+ec_gf_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
     ec_link(frame, this, -1, EC_MINIMUM_MIN, default_link_cbk, NULL, oldloc,
             newloc, xdata);
@@ -998,20 +1146,22 @@ int32_t ec_gf_link(call_frame_t * frame, xlator_t * this, loc_t * oldloc,
     return 0;
 }
 
-int32_t ec_gf_lk(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                 int32_t cmd, struct gf_flock * flock, dict_t * xdata)
+int32_t
+ec_gf_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+         struct gf_flock *flock, dict_t *xdata)
 {
-    int32_t minimum = EC_MINIMUM_ALL;
+    int32_t fop_flags = EC_MINIMUM_ALL;
+
     if (flock->l_type == F_UNLCK)
-            minimum = EC_MINIMUM_ONE;
-    ec_lk(frame, this, -1, minimum, default_lk_cbk, NULL, fd, cmd,
-          flock, xdata);
+        fop_flags = EC_MINIMUM_ONE;
+    ec_lk(frame, this, -1, fop_flags, default_lk_cbk, NULL, fd, cmd, flock,
+          xdata);
 
     return 0;
 }
 
-int32_t ec_gf_lookup(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     dict_t * xdata)
+int32_t
+ec_gf_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
     ec_lookup(frame, this, -1, EC_MINIMUM_MIN, default_lookup_cbk, NULL, loc,
               xdata);
@@ -1019,8 +1169,9 @@ int32_t ec_gf_lookup(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_mkdir(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                    mode_t mode, mode_t umask, dict_t * xdata)
+int32_t
+ec_gf_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            mode_t umask, dict_t *xdata)
 {
     ec_mkdir(frame, this, -1, EC_MINIMUM_MIN, default_mkdir_cbk, NULL, loc,
              mode, umask, xdata);
@@ -1028,8 +1179,9 @@ int32_t ec_gf_mkdir(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_mknod(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                    mode_t mode, dev_t rdev, mode_t umask, dict_t * xdata)
+int32_t
+ec_gf_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            dev_t rdev, mode_t umask, dict_t *xdata)
 {
     ec_mknod(frame, this, -1, EC_MINIMUM_MIN, default_mknod_cbk, NULL, loc,
              mode, rdev, umask, xdata);
@@ -1037,17 +1189,19 @@ int32_t ec_gf_mknod(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_open(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                   int32_t flags, fd_t * fd, dict_t * xdata)
+int32_t
+ec_gf_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           fd_t *fd, dict_t *xdata)
 {
-    ec_open(frame, this, -1, EC_MINIMUM_MIN, default_open_cbk, NULL, loc,
-            flags, fd, xdata);
+    ec_open(frame, this, -1, EC_MINIMUM_MIN, default_open_cbk, NULL, loc, flags,
+            fd, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_opendir(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                      fd_t * fd, dict_t * xdata)
+int32_t
+ec_gf_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+              dict_t *xdata)
 {
     ec_opendir(frame, this, -1, EC_MINIMUM_MIN, default_opendir_cbk, NULL, loc,
                fd, xdata);
@@ -1055,8 +1209,9 @@ int32_t ec_gf_opendir(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_readdir(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                      size_t size, off_t offset, dict_t * xdata)
+int32_t
+ec_gf_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, dict_t *xdata)
 {
     ec_readdir(frame, this, -1, EC_MINIMUM_ONE, default_readdir_cbk, NULL, fd,
                size, offset, xdata);
@@ -1064,17 +1219,19 @@ int32_t ec_gf_readdir(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_readdirp(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       size_t size, off_t offset, dict_t * xdata)
+int32_t
+ec_gf_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, dict_t *xdata)
 {
-    ec_readdirp(frame, this, -1, EC_MINIMUM_ONE, default_readdirp_cbk, NULL,
-                fd, size, offset, xdata);
+    ec_readdirp(frame, this, -1, EC_MINIMUM_ONE, default_readdirp_cbk, NULL, fd,
+                size, offset, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_readlink(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                       size_t size, dict_t * xdata)
+int32_t
+ec_gf_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+               dict_t *xdata)
 {
     ec_readlink(frame, this, -1, EC_MINIMUM_ONE, default_readlink_cbk, NULL,
                 loc, size, xdata);
@@ -1082,60 +1239,63 @@ int32_t ec_gf_readlink(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_readv(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                    size_t size, off_t offset, uint32_t flags, dict_t * xdata)
+int32_t
+ec_gf_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
 {
-    ec_readv(frame, this, -1, EC_MINIMUM_MIN, default_readv_cbk, NULL, fd,
-             size, offset, flags, xdata);
+    ec_readv(frame, this, -1, EC_MINIMUM_MIN, default_readv_cbk, NULL, fd, size,
+             offset, flags, xdata);
 
     return 0;
 }
 
 int32_t
-ec_gf_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   const char *name, dict_t *xdata)
+ec_gf_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
 {
-        int     error  = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO (name, xdata, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO(name, xdata, error, out);
 
-        ec_removexattr (frame, this, -1, EC_MINIMUM_MIN,
-                        default_removexattr_cbk, NULL, loc, name, xdata);
+    ec_removexattr(frame, this, -1, EC_MINIMUM_MIN, default_removexattr_cbk,
+                   NULL, loc, name, xdata);
 
-        return 0;
+    return 0;
 out:
-        STACK_UNWIND_STRICT (removexattr, frame, -1, error, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(removexattr, frame, -1, error, NULL);
+    return 0;
 }
 
 int32_t
-ec_gf_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    const char *name, dict_t *xdata)
+ec_gf_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
 {
-        int     error  = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO (name, xdata, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO(name, xdata, error, out);
 
-        ec_fremovexattr (frame, this, -1, EC_MINIMUM_MIN,
-                         default_fremovexattr_cbk, NULL, fd, name, xdata);
+    ec_fremovexattr(frame, this, -1, EC_MINIMUM_MIN, default_fremovexattr_cbk,
+                    NULL, fd, name, xdata);
 
-        return 0;
+    return 0;
 out:
-        STACK_UNWIND_STRICT (fremovexattr, frame, -1, error, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fremovexattr, frame, -1, error, NULL);
+    return 0;
 }
 
-int32_t ec_gf_rename(call_frame_t * frame, xlator_t * this, loc_t * oldloc,
-                     loc_t * newloc, dict_t * xdata)
+int32_t
+ec_gf_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
-    ec_rename(frame, this, -1, EC_MINIMUM_MIN, default_rename_cbk, NULL,
-              oldloc, newloc, xdata);
+    ec_rename(frame, this, -1, EC_MINIMUM_MIN, default_rename_cbk, NULL, oldloc,
+              newloc, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_rmdir(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                    int xflags, dict_t * xdata)
+int32_t
+ec_gf_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+            dict_t *xdata)
 {
     ec_rmdir(frame, this, -1, EC_MINIMUM_MIN, default_rmdir_cbk, NULL, loc,
              xflags, xdata);
@@ -1143,8 +1303,9 @@ int32_t ec_gf_rmdir(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_setattr(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                      struct iatt * stbuf, int32_t valid, dict_t * xdata)
+int32_t
+ec_gf_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
     ec_setattr(frame, this, -1, EC_MINIMUM_MIN, default_setattr_cbk, NULL, loc,
                stbuf, valid, xdata);
@@ -1152,51 +1313,52 @@ int32_t ec_gf_setattr(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_fsetattr(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       struct iatt * stbuf, int32_t valid, dict_t * xdata)
+int32_t
+ec_gf_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-    ec_fsetattr(frame, this, -1, EC_MINIMUM_MIN, default_fsetattr_cbk, NULL,
-                fd, stbuf, valid, xdata);
+    ec_fsetattr(frame, this, -1, EC_MINIMUM_MIN, default_fsetattr_cbk, NULL, fd,
+                stbuf, valid, xdata);
 
     return 0;
 }
 
 int32_t
-ec_gf_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                dict_t *dict, int32_t flags, dict_t *xdata)
+ec_gf_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
 {
-        int     error = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO ("", dict, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO("", dict, error, out);
 
-        ec_setxattr (frame, this, -1, EC_MINIMUM_MIN, default_setxattr_cbk,
-                     NULL, loc, dict, flags, xdata);
+    ec_setxattr(frame, this, -1, EC_MINIMUM_MIN, default_setxattr_cbk, NULL,
+                loc, dict, flags, xdata);
 
-        return 0;
+    return 0;
 out:
-        STACK_UNWIND_STRICT (setxattr, frame, -1, error, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(setxattr, frame, -1, error, NULL);
+    return 0;
 }
 
 int32_t
-ec_gf_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 dict_t *dict, int32_t flags, dict_t *xdata)
+ec_gf_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                int32_t flags, dict_t *xdata)
 {
-        int     error = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO ("", dict, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO("", dict, error, out);
 
-        ec_fsetxattr (frame, this, -1, EC_MINIMUM_MIN, default_fsetxattr_cbk,
-                      NULL, fd, dict, flags, xdata);
+    ec_fsetxattr(frame, this, -1, EC_MINIMUM_MIN, default_fsetxattr_cbk, NULL,
+                 fd, dict, flags, xdata);
 
-        return 0;
+    return 0;
 out:
-        STACK_UNWIND_STRICT (fsetxattr, frame, -1, error, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fsetxattr, frame, -1, error, NULL);
+    return 0;
 }
 
-int32_t ec_gf_stat(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                   dict_t * xdata)
+int32_t
+ec_gf_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
     ec_stat(frame, this, -1, EC_MINIMUM_MIN, default_stat_cbk, NULL, loc,
             xdata);
@@ -1204,8 +1366,8 @@ int32_t ec_gf_stat(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_fstat(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                    dict_t * xdata)
+int32_t
+ec_gf_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
     ec_fstat(frame, this, -1, EC_MINIMUM_MIN, default_fstat_cbk, NULL, fd,
              xdata);
@@ -1213,8 +1375,8 @@ int32_t ec_gf_fstat(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_statfs(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     dict_t * xdata)
+int32_t
+ec_gf_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
     ec_statfs(frame, this, -1, EC_MINIMUM_MIN, default_statfs_cbk, NULL, loc,
               xdata);
@@ -1222,9 +1384,9 @@ int32_t ec_gf_statfs(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_symlink(call_frame_t * frame, xlator_t * this,
-                      const char * linkname, loc_t * loc, mode_t umask,
-                      dict_t * xdata)
+int32_t
+ec_gf_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+              loc_t *loc, mode_t umask, dict_t *xdata)
 {
     ec_symlink(frame, this, -1, EC_MINIMUM_MIN, default_symlink_cbk, NULL,
                linkname, loc, umask, xdata);
@@ -1232,8 +1394,9 @@ int32_t ec_gf_symlink(call_frame_t * frame, xlator_t * this,
     return 0;
 }
 
-int32_t ec_gf_truncate(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                       off_t offset, dict_t * xdata)
+int32_t
+ec_gf_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+               dict_t *xdata)
 {
     ec_truncate(frame, this, -1, EC_MINIMUM_MIN, default_truncate_cbk, NULL,
                 loc, offset, xdata);
@@ -1241,8 +1404,9 @@ int32_t ec_gf_truncate(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_ftruncate(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                        off_t offset, dict_t * xdata)
+int32_t
+ec_gf_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                dict_t *xdata)
 {
     ec_ftruncate(frame, this, -1, EC_MINIMUM_MIN, default_ftruncate_cbk, NULL,
                  fd, offset, xdata);
@@ -1250,8 +1414,9 @@ int32_t ec_gf_ftruncate(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_unlink(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     int xflags, dict_t * xdata)
+int32_t
+ec_gf_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+             dict_t *xdata)
 {
     ec_unlink(frame, this, -1, EC_MINIMUM_MIN, default_unlink_cbk, NULL, loc,
               xflags, xdata);
@@ -1259,9 +1424,10 @@ int32_t ec_gf_unlink(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_writev(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                     struct iovec * vector, int32_t count, off_t offset,
-                     uint32_t flags, struct iobref * iobref, dict_t * xdata)
+int32_t
+ec_gf_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+             struct iobref *iobref, dict_t *xdata)
 {
     ec_writev(frame, this, -1, EC_MINIMUM_MIN, default_writev_cbk, NULL, fd,
               vector, count, offset, flags, iobref, xdata);
@@ -1269,9 +1435,9 @@ int32_t ec_gf_writev(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_xattrop(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                      gf_xattrop_flags_t optype, dict_t * xattr,
-                      dict_t * xdata)
+int32_t
+ec_gf_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
     ec_xattrop(frame, this, -1, EC_MINIMUM_MIN, default_xattrop_cbk, NULL, loc,
                optype, xattr, xdata);
@@ -1279,47 +1445,49 @@ int32_t ec_gf_xattrop(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_fxattrop(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       gf_xattrop_flags_t optype, dict_t * xattr,
-                       dict_t * xdata)
+int32_t
+ec_gf_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-    ec_fxattrop(frame, this, -1, EC_MINIMUM_MIN, default_fxattrop_cbk, NULL,
-                fd, optype, xattr, xdata);
+    ec_fxattrop(frame, this, -1, EC_MINIMUM_MIN, default_fxattrop_cbk, NULL, fd,
+                optype, xattr, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_zerofill(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       off_t offset, off_t len, dict_t * xdata)
+int32_t
+ec_gf_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               off_t len, dict_t *xdata)
 {
     default_zerofill_failure_cbk(frame, ENOTSUP);
 
     return 0;
 }
 
-int32_t ec_gf_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                   gf_seek_what_t what, dict_t *xdata)
+int32_t
+ec_gf_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           gf_seek_what_t what, dict_t *xdata)
 {
-    ec_seek(frame, this, -1, EC_MINIMUM_ONE, default_seek_cbk, NULL, fd,
-            offset, what, xdata);
+    ec_seek(frame, this, -1, EC_MINIMUM_ONE, default_seek_cbk, NULL, fd, offset,
+            what, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_ipc(call_frame_t *frame, xlator_t *this, int32_t op,
-                  dict_t *xdata)
+int32_t
+ec_gf_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
     ec_ipc(frame, this, -1, EC_MINIMUM_MIN, default_ipc_cbk, NULL, op, xdata);
     return 0;
 }
 
-int32_t ec_gf_forget(xlator_t * this, inode_t * inode)
+int32_t
+ec_gf_forget(xlator_t *this, inode_t *inode)
 {
     uint64_t value = 0;
-    ec_inode_t * ctx = NULL;
+    ec_inode_t *ctx = NULL;
 
-    if ((inode_ctx_del(inode, this, &value) == 0) && (value != 0))
-    {
+    if ((inode_ctx_del(inode, this, &value) == 0) && (value != 0)) {
         ctx = (ec_inode_t *)(uintptr_t)value;
         /* We can only forget an inode if it has been unlocked, so the stripe
          * cache should also be empty. */
@@ -1330,38 +1498,41 @@ int32_t ec_gf_forget(xlator_t * this, inode_t * inode)
     return 0;
 }
 
-void ec_gf_release_fd(xlator_t * this, fd_t * fd)
+void
+ec_gf_release_fd(xlator_t *this, fd_t *fd)
 {
     uint64_t value = 0;
-    ec_fd_t * ctx = NULL;
+    ec_fd_t *ctx = NULL;
 
-    if ((fd_ctx_del(fd, this, &value) == 0) && (value != 0))
-    {
+    if ((fd_ctx_del(fd, this, &value) == 0) && (value != 0)) {
         ctx = (ec_fd_t *)(uintptr_t)value;
         loc_wipe(&ctx->loc);
         GF_FREE(ctx);
     }
 }
 
-int32_t ec_gf_release(xlator_t * this, fd_t * fd)
+int32_t
+ec_gf_release(xlator_t *this, fd_t *fd)
 {
     ec_gf_release_fd(this, fd);
 
     return 0;
 }
 
-int32_t ec_gf_releasedir(xlator_t * this, fd_t * fd)
+int32_t
+ec_gf_releasedir(xlator_t *this, fd_t *fd)
 {
     ec_gf_release_fd(this, fd);
 
     return 0;
 }
 
-int32_t ec_dump_private(xlator_t *this)
+int32_t
+ec_dump_private(xlator_t *this)
 {
     ec_t *ec = NULL;
-    char  key_prefix[GF_DUMP_MAX_BUF_LEN];
-    char  tmp[65];
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    char tmp[65];
 
     GF_ASSERT(this);
 
@@ -1369,7 +1540,8 @@ int32_t ec_dump_private(xlator_t *this)
     GF_ASSERT(ec);
 
     snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
-    gf_proc_dump_add_section(key_prefix);
+    gf_proc_dump_add_section("%s", key_prefix);
+    gf_proc_dump_write("up", "%u", ec->up);
     gf_proc_dump_write("nodes", "%u", ec->nodes);
     gf_proc_dump_write("redundancy", "%u", ec->redundancy);
     gf_proc_dump_write("fragment_size", "%u", ec->fragment_size);
@@ -1377,107 +1549,106 @@ int32_t ec_dump_private(xlator_t *this)
     gf_proc_dump_write("childs_up", "%u", ec->xl_up_count);
     gf_proc_dump_write("childs_up_mask", "%s",
                        ec_bin(tmp, sizeof(tmp), ec->xl_up, ec->nodes));
+    if (ec->read_mask) {
+        gf_proc_dump_write("read-mask", "%s",
+                           ec_bin(tmp, sizeof(tmp), ec->read_mask, ec->nodes));
+    }
     gf_proc_dump_write("background-heals", "%d", ec->background_heals);
     gf_proc_dump_write("heal-wait-qlength", "%d", ec->heal_wait_qlen);
-    gf_proc_dump_write("self-heal-window-size", "%"PRIu32,
+    gf_proc_dump_write("self-heal-window-size", "%" PRIu32,
                        ec->self_heal_window_size);
     gf_proc_dump_write("healers", "%d", ec->healers);
     gf_proc_dump_write("heal-waiters", "%d", ec->heal_waiters);
     gf_proc_dump_write("read-policy", "%s", ec_read_policies[ec->read_policy]);
+    gf_proc_dump_write("parallel-writes", "%d", ec->parallel_writes);
+    gf_proc_dump_write("quorum-count", "%u", ec->quorum_count);
 
     snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s.stats.stripe_cache",
              this->type, this->name);
-    gf_proc_dump_add_section(key_prefix);
+    gf_proc_dump_add_section("%s", key_prefix);
 
-    gf_proc_dump_write("hits", "%llu",
+    gf_proc_dump_write("hits", "%" GF_PRI_ATOMIC,
                        GF_ATOMIC_GET(ec->stats.stripe_cache.hits));
-    gf_proc_dump_write("misses", "%llu",
+    gf_proc_dump_write("misses", "%" GF_PRI_ATOMIC,
                        GF_ATOMIC_GET(ec->stats.stripe_cache.misses));
-    gf_proc_dump_write("updates", "%llu",
+    gf_proc_dump_write("updates", "%" GF_PRI_ATOMIC,
                        GF_ATOMIC_GET(ec->stats.stripe_cache.updates));
-    gf_proc_dump_write("invalidations", "%llu",
+    gf_proc_dump_write("invalidations", "%" GF_PRI_ATOMIC,
                        GF_ATOMIC_GET(ec->stats.stripe_cache.invals));
-    gf_proc_dump_write("evicts", "%llu",
+    gf_proc_dump_write("evicts", "%" GF_PRI_ATOMIC,
                        GF_ATOMIC_GET(ec->stats.stripe_cache.evicts));
-    gf_proc_dump_write("allocations", "%llu",
+    gf_proc_dump_write("allocations", "%" GF_PRI_ATOMIC,
                        GF_ATOMIC_GET(ec->stats.stripe_cache.allocs));
-    gf_proc_dump_write("errors", "%llu",
+    gf_proc_dump_write("errors", "%" GF_PRI_ATOMIC,
                        GF_ATOMIC_GET(ec->stats.stripe_cache.errors));
+    gf_proc_dump_write("heals-attempted", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.shd.attempted));
+    gf_proc_dump_write("heals-completed", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.shd.completed));
 
     return 0;
 }
 
-struct xlator_fops fops =
-{
-    .lookup       = ec_gf_lookup,
-    .stat         = ec_gf_stat,
-    .fstat        = ec_gf_fstat,
-    .truncate     = ec_gf_truncate,
-    .ftruncate    = ec_gf_ftruncate,
-    .access       = ec_gf_access,
-    .readlink     = ec_gf_readlink,
-    .mknod        = ec_gf_mknod,
-    .mkdir        = ec_gf_mkdir,
-    .unlink       = ec_gf_unlink,
-    .rmdir        = ec_gf_rmdir,
-    .symlink      = ec_gf_symlink,
-    .rename       = ec_gf_rename,
-    .link         = ec_gf_link,
-    .create       = ec_gf_create,
-    .open         = ec_gf_open,
-    .readv        = ec_gf_readv,
-    .writev       = ec_gf_writev,
-    .flush        = ec_gf_flush,
-    .fsync        = ec_gf_fsync,
-    .opendir      = ec_gf_opendir,
-    .readdir      = ec_gf_readdir,
-    .readdirp     = ec_gf_readdirp,
-    .fsyncdir     = ec_gf_fsyncdir,
-    .statfs       = ec_gf_statfs,
-    .setxattr     = ec_gf_setxattr,
-    .getxattr     = ec_gf_getxattr,
-    .fsetxattr    = ec_gf_fsetxattr,
-    .fgetxattr    = ec_gf_fgetxattr,
-    .removexattr  = ec_gf_removexattr,
-    .fremovexattr = ec_gf_fremovexattr,
-    .lk           = ec_gf_lk,
-    .inodelk      = ec_gf_inodelk,
-    .finodelk     = ec_gf_finodelk,
-    .entrylk      = ec_gf_entrylk,
-    .fentrylk     = ec_gf_fentrylk,
-    .xattrop      = ec_gf_xattrop,
-    .fxattrop     = ec_gf_fxattrop,
-    .setattr      = ec_gf_setattr,
-    .fsetattr     = ec_gf_fsetattr,
-    .fallocate    = ec_gf_fallocate,
-    .discard      = ec_gf_discard,
-    .zerofill     = ec_gf_zerofill,
-    .seek         = ec_gf_seek,
-    .ipc          = ec_gf_ipc
-};
-
-struct xlator_cbks cbks =
-{
-    .forget            = ec_gf_forget,
-    .release           = ec_gf_release,
-    .releasedir        = ec_gf_releasedir
-};
-
-struct xlator_dumpops dumpops = {
-    .priv = ec_dump_private
-};
-
-struct volume_options options[] =
-{
+struct xlator_fops fops = {.lookup = ec_gf_lookup,
+                           .stat = ec_gf_stat,
+                           .fstat = ec_gf_fstat,
+                           .truncate = ec_gf_truncate,
+                           .ftruncate = ec_gf_ftruncate,
+                           .access = ec_gf_access,
+                           .readlink = ec_gf_readlink,
+                           .mknod = ec_gf_mknod,
+                           .mkdir = ec_gf_mkdir,
+                           .unlink = ec_gf_unlink,
+                           .rmdir = ec_gf_rmdir,
+                           .symlink = ec_gf_symlink,
+                           .rename = ec_gf_rename,
+                           .link = ec_gf_link,
+                           .create = ec_gf_create,
+                           .open = ec_gf_open,
+                           .readv = ec_gf_readv,
+                           .writev = ec_gf_writev,
+                           .flush = ec_gf_flush,
+                           .fsync = ec_gf_fsync,
+                           .opendir = ec_gf_opendir,
+                           .readdir = ec_gf_readdir,
+                           .readdirp = ec_gf_readdirp,
+                           .fsyncdir = ec_gf_fsyncdir,
+                           .statfs = ec_gf_statfs,
+                           .setxattr = ec_gf_setxattr,
+                           .getxattr = ec_gf_getxattr,
+                           .fsetxattr = ec_gf_fsetxattr,
+                           .fgetxattr = ec_gf_fgetxattr,
+                           .removexattr = ec_gf_removexattr,
+                           .fremovexattr = ec_gf_fremovexattr,
+                           .lk = ec_gf_lk,
+                           .inodelk = ec_gf_inodelk,
+                           .finodelk = ec_gf_finodelk,
+                           .entrylk = ec_gf_entrylk,
+                           .fentrylk = ec_gf_fentrylk,
+                           .xattrop = ec_gf_xattrop,
+                           .fxattrop = ec_gf_fxattrop,
+                           .setattr = ec_gf_setattr,
+                           .fsetattr = ec_gf_fsetattr,
+                           .fallocate = ec_gf_fallocate,
+                           .discard = ec_gf_discard,
+                           .zerofill = ec_gf_zerofill,
+                           .seek = ec_gf_seek,
+                           .ipc = ec_gf_ipc};
+
+struct xlator_cbks cbks = {.forget = ec_gf_forget,
+                           .release = ec_gf_release,
+                           .releasedir = ec_gf_releasedir};
+
+struct xlator_dumpops dumpops = {.priv = ec_dump_private};
+
+struct volume_options options[] = {
+    {.key = {"redundancy"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "{{ volume.redundancy }}",
+     .description = "Maximum number of bricks that can fail "
+                    "simultaneously without losing data."},
     {
-        .key = { "redundancy" },
-        .type = GF_OPTION_TYPE_INT,
-        .default_value = "{{ volume.redundancy }}",
-        .description = "Maximum number of bricks that can fail "
-                       "simultaneously without losing data."
-    },
-    {
-        .key = { "self-heal-daemon" },
+        .key = {"self-heal-daemon"},
         .type = GF_OPTION_TYPE_BOOL,
         .description = "self-heal daemon enable/disable",
         .default_value = "enable",
@@ -1485,193 +1656,218 @@ struct volume_options options[] =
         .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
         .tags = {"disperse"},
     },
-    { .key = {"iam-self-heal-daemon"},
-      .type = GF_OPTION_TYPE_BOOL,
-      .default_value = "off",
-      .description = "This option differentiates if the disperse "
-                     "translator is running as part of self-heal-daemon "
-                     "or not."
-    },
-    { .key = {"eager-lock"},
-      .type = GF_OPTION_TYPE_BOOL,
-      .default_value = "on",
-      .op_version = {GD_OP_VERSION_3_7_10},
-      .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
-      .tags = {"disperse"},
-      .description = "Enable/Disable eager lock for regular files on a "
-                     "disperse volume. If a fop takes a lock and completes "
-                     "its operation, it waits for next 1 second before "
-                     "releasing the lock, to see if the lock can be reused "
-                     "for next fop from the same client. If ec finds any lock "
-                     "contention within 1 second it releases the lock "
-                     "immediately before time expires. This improves the "
-                     "performance of file operations. However, as it takes "
-                     "lock on first brick, for few operations like read, "
-                     "discovery of lock contention might take long time and "
-                     "can actually degrade the performance. If eager lock is "
-                     "disabled, lock will be released as soon as fop "
-                     "completes."
-    },
-    { .key = {"other-eager-lock"},
-      .type = GF_OPTION_TYPE_BOOL,
-      .default_value = "on",
-      .op_version = { GD_OP_VERSION_3_13_0 },
-      .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
-      .tags = { "disperse" },
-      .description = "It's equivalent to the eager-lock option but for non "
-                     "regular files."
-    },
-    { .key = {"eager-lock-timeout"},
-      .type = GF_OPTION_TYPE_INT,
-      .min = 1,
-      .max = 60,
-      .default_value = "1",
-      .op_version = { GD_OP_VERSION_4_0_0 },
-      .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
-      .tags = { "disperse", "locks", "timeout" },
-      .description = "Maximum time (in seconds) that a lock on an inode is "
-                     "kept held if no new operations on the inode are "
-                     "received."
-    },
-    { .key = {"other-eager-lock-timeout"},
-      .type = GF_OPTION_TYPE_INT,
-      .min = 1,
-      .max = 60,
-      .default_value = "1",
-      .op_version = { GD_OP_VERSION_4_0_0 },
-      .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
-      .tags = { "disperse", "locks", "timeout" },
-      .description = "It's equivalent ot eager-lock-timeout option but for "
-                     "non regular files."
-    },
-    { .key = {"background-heals"},
-      .type = GF_OPTION_TYPE_INT,
-      .min = 0,/*Disabling background heals*/
-      .max = 256,
-      .default_value = "8",
-      .op_version = {GD_OP_VERSION_3_7_3},
-      .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
-      .tags = {"disperse"},
-      .description = "This option can be used to control number of parallel"
-                     " heals",
-    },
-    { .key = {"heal-wait-qlength"},
-      .type = GF_OPTION_TYPE_INT,
-      .min = 0,
-      .max = 65536, /*Around 100MB as of now with sizeof(ec_fop_data_t) at 1800*/
-      .default_value = "128",
-      .op_version = {GD_OP_VERSION_3_7_3},
-      .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
-      .tags = {"disperse"},
-      .description = "This option can be used to control number of heals"
-                     " that can wait",
-    },
-    { .key  = {"heal-timeout"},
-      .type = GF_OPTION_TYPE_INT,
-      .min  = 60,
-      .max  = INT_MAX,
-      .default_value = "600",
-      .op_version = {GD_OP_VERSION_3_7_3},
-      .flags = OPT_FLAG_SETTABLE,
-      .tags = {"disperse"},
-      .description = "time interval for checking the need to self-heal "
-                     "in self-heal-daemon"
-    },
-    { .key = {"read-policy" },
-      .type = GF_OPTION_TYPE_STR,
-      .value = {"round-robin", "gfid-hash"},
-      .default_value = "gfid-hash",
-      .op_version = {GD_OP_VERSION_3_7_6},
-      .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
-      .tags = {"disperse"},
-      .description = "inode-read fops happen only on 'k' number of bricks in"
-              " n=k+m disperse subvolume. 'round-robin' selects the read"
-              " subvolume using round-robin algo. 'gfid-hash' selects read"
-              " subvolume based on hash of the gfid of that file/directory.",
-    },
-    { .key   = {"shd-max-threads"},
-      .type  = GF_OPTION_TYPE_INT,
-      .min   = 1,
-      .max   = 64,
-      .default_value = "1",
-      .op_version = {GD_OP_VERSION_3_9_0},
-      .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-      .tags = {"disperse"},
-      .description = "Maximum number of parallel heals SHD can do per local "
-                      "brick.  This can substantially lower heal times, "
-                      "but can also crush your bricks if you don't have "
-                      "the storage hardware to support this."
+    {.key = {"iam-self-heal-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option differentiates if the disperse "
+                    "translator is running as part of self-heal-daemon "
+                    "or not."},
+    {.key = {"eager-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {GD_OP_VERSION_3_7_10},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "Enable/Disable eager lock for regular files on a "
+                    "disperse volume. If a fop takes a lock and completes "
+                    "its operation, it waits for next 1 second before "
+                    "releasing the lock, to see if the lock can be reused "
+                    "for next fop from the same client. If ec finds any lock "
+                    "contention within 1 second it releases the lock "
+                    "immediately before time expires. This improves the "
+                    "performance of file operations. However, as it takes "
+                    "lock on first brick, for few operations like read, "
+                    "discovery of lock contention might take long time and "
+                    "can actually degrade the performance. If eager lock is "
+                    "disabled, lock will be released as soon as fop "
+                    "completes."},
+    {.key = {"other-eager-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {GD_OP_VERSION_3_13_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "It's equivalent to the eager-lock option but for non "
+                    "regular files."},
+    {.key = {"eager-lock-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 60,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse", "locks", "timeout"},
+     .description = "Maximum time (in seconds) that a lock on an inode is "
+                    "kept held if no new operations on the inode are "
+                    "received."},
+    {.key = {"other-eager-lock-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 60,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse", "locks", "timeout"},
+     .description = "It's equivalent to eager-lock-timeout option but for "
+                    "non regular files."},
+    {
+        .key = {"background-heals"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 0, /*Disabling background heals*/
+        .max = 256,
+        .default_value = "8",
+        .op_version = {GD_OP_VERSION_3_7_3},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .tags = {"disperse"},
+        .description = "This option can be used to control number of parallel"
+                       " heals",
     },
-    { .key   = {"shd-wait-qlength"},
-      .type  = GF_OPTION_TYPE_INT,
-      .min   = 1,
-      .max   = 65536,
-      .default_value = "1024",
-      .op_version = {GD_OP_VERSION_3_9_0},
-      .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-      .tags = {"disperse"},
-      .description = "This option can be used to control number of heals"
-                     " that can wait in SHD per subvolume"
+    {
+        .key = {"heal-wait-qlength"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 0,
+        .max =
+            65536, /*Around 100MB as of now with sizeof(ec_fop_data_t) at 1800*/
+        .default_value = "128",
+        .op_version = {GD_OP_VERSION_3_7_3},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .tags = {"disperse"},
+        .description = "This option can be used to control number of heals"
+                       " that can wait",
     },
+    {.key = {"heal-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 60,
+     .max = INT_MAX,
+     .default_value = "600",
+     .op_version = {GD_OP_VERSION_3_7_3},
+     .flags = OPT_FLAG_SETTABLE,
+     .tags = {"disperse"},
+     .description = "time interval for checking the need to self-heal "
+                    "in self-heal-daemon"},
     {
-        .key = { "cpu-extensions" },
+        .key = {"read-policy"},
         .type = GF_OPTION_TYPE_STR,
-        .value = { "none", "auto", "x64", "sse", "avx" },
-        .default_value = "auto",
-        .op_version = {GD_OP_VERSION_3_9_0},
+        .value = {"round-robin", "gfid-hash"},
+        .default_value = "gfid-hash",
+        .op_version = {GD_OP_VERSION_3_7_6},
         .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
         .tags = {"disperse"},
-        .description = "force the cpu extensions to be used to accelerate the "
-                       "galois field computations."
-    },
-    { .key  = {"self-heal-window-size"},
-      .type = GF_OPTION_TYPE_INT,
-      .min  = 1,
-      .max  = 1024,
-      .default_value = "1",
-      .op_version = {GD_OP_VERSION_3_11_0},
-      .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
-      .tags = {"disperse"},
-      .description = "Maximum number blocks(128KB) per file for which "
-                     "self-heal process would be applied simultaneously."
+        .description =
+            "inode-read fops happen only on 'k' number of bricks in"
+            " n=k+m disperse subvolume. 'round-robin' selects the read"
+            " subvolume using round-robin algo. 'gfid-hash' selects read"
+            " subvolume based on hash of the gfid of that file/directory.",
     },
-    { .key = {"optimistic-change-log"},
-      .type = GF_OPTION_TYPE_BOOL,
-      .default_value = "on",
-      .op_version = {GD_OP_VERSION_3_10_1},
-      .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT,
-      .tags = {"disperse"},
-      .description =  "Set/Unset dirty flag for every update fop at the start"
-                      "of the fop. If OFF, this option impacts performance of"
-                      "entry  operations or metadata operations as it will"
-                      "set dirty flag at the start and unset it at the end of"
-                      "ALL update fop. If ON and all the bricks are good,"
-                      "dirty flag will be set at the start only for file fops"
-                      "For metadata and entry fops dirty flag will not be set"
-                      "at the start, if all the bricks are good. This does"
-                      "not impact performance for metadata operations and"
-                      "entry operation but has a very small window to miss"
-                      "marking entry as dirty in case it is required to be"
-                      "healed"
+    {.key = {"shd-max-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 64,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_3_9_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "Maximum number of parallel heals SHD can do per local "
+                    "brick.  This can substantially lower heal times, "
+                    "but can also crush your bricks if you don't have "
+                    "the storage hardware to support this."},
+    {.key = {"shd-wait-qlength"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 65536,
+     .default_value = "1024",
+     .op_version = {GD_OP_VERSION_3_9_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "This option can be used to control number of heals"
+                    " that can wait in SHD per subvolume"},
+    {.key = {"cpu-extensions"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {"none", "auto", "x64", "sse", "avx"},
+     .default_value = "auto",
+     .op_version = {GD_OP_VERSION_3_9_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "force the cpu extensions to be used to accelerate the "
+                    "galois field computations."},
+    {.key = {"self-heal-window-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 1024,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "Maximum number blocks(128KB) per file for which "
+                    "self-heal process would be applied simultaneously."},
+    {.key = {"optimistic-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {GD_OP_VERSION_3_10_1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT,
+     .tags = {"disperse"},
+     .description = "Set/Unset dirty flag for every update fop at the start"
+                    "of the fop. If OFF, this option impacts performance of"
+                    "entry  operations or metadata operations as it will"
+                    "set dirty flag at the start and unset it at the end of"
+                    "ALL update fop. If ON and all the bricks are good,"
+                    "dirty flag will be set at the start only for file fops"
+                    "For metadata and entry fops dirty flag will not be set"
+                    "at the start, if all the bricks are good. This does"
+                    "not impact performance for metadata operations and"
+                    "entry operation but has a very small window to miss"
+                    "marking entry as dirty in case it is required to be"
+                    "healed"},
+    {.key = {"parallel-writes"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "This controls if writes can be wound in parallel as long"
+                    "as it doesn't modify same stripes"},
+    {.key = {"stripe-cache"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0, /*Disabling stripe_cache*/
+     .max = EC_STRIPE_CACHE_MAX_SIZE,
+     .default_value = "4",
+     .description = "This option will keep the last stripe of write fop"
+                    "in memory. If next write falls in this stripe, we need"
+                    "not to read it again from backend and we can save READ"
+                    "fop going over the network. This will improve performance,"
+                    "specially for sequential writes. However, this will also"
+                    "lead to extra memory consumption, maximum "
+                    "(cache size * stripe size) Bytes per open file."},
+    {
+        .key = {"quorum-count"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = "0",
+        .description =
+            "This option can be used to define how many successes on"
+            "the bricks constitute a success to the application. This"
+            " count should be in the range"
+            "[disperse-data-count,  disperse-count] (inclusive)",
     },
-    { .key = {"parallel-writes"},
-      .type = GF_OPTION_TYPE_BOOL,
-      .default_value = "on",
-      .description = "This controls if writes can be wound in parallel as long"
-                     "as it doesn't modify same stripes"
+    {
+        .key = {"ec-read-mask"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = NULL,
+        .description = "This option can be used to choose which bricks can be"
+                       " used for reading data/metadata of a file/directory",
     },
-    {   .key = {"stripe-cache"},
-        .type = GF_OPTION_TYPE_INT,
-        .min = 0,/*Disabling stripe_cache*/
-        .max = EC_STRIPE_CACHE_MAX_SIZE,
-        .default_value = "4",
-        .description =  "This option will keep the last stripe of write fop"
-                        "in memory. If next write falls in this stripe, we need"
-                        "not to read it again from backend and we can save READ"
-                        "fop going over the network. This will improve performance,"
-                        "specially for sequential writes. However, this will also"
-                        "lead to extra memory consumption, maximum "
-                        "(cache size * stripe size) Bytes per open file."
+    {
+        .key = {NULL},
     },
-    { .key = {NULL} }
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "disperse",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/cluster/ec/src/ec.h b/xlators/cluster/ec/src/ec.h
index e2ec9cf04f5..6f6de6d5981 100644
--- a/xlators/cluster/ec/src/ec.h
+++ b/xlators/cluster/ec/src/ec.h
@@ -13,15 +13,16 @@
 
 #include "ec-method.h"
 
-#define EC_XATTR_PREFIX  "trusted.ec."
-#define EC_XATTR_CONFIG  EC_XATTR_PREFIX"config"
-#define EC_XATTR_SIZE    EC_XATTR_PREFIX"size"
-#define EC_XATTR_VERSION EC_XATTR_PREFIX"version"
-#define EC_XATTR_HEAL    EC_XATTR_PREFIX"heal"
-#define EC_XATTR_DIRTY   EC_XATTR_PREFIX"dirty"
-#define EC_STRIPE_CACHE_MAX_SIZE    10
+#define EC_XATTR_PREFIX "trusted.ec."
+#define EC_XATTR_CONFIG EC_XATTR_PREFIX "config"
+#define EC_XATTR_SIZE EC_XATTR_PREFIX "size"
+#define EC_XATTR_VERSION EC_XATTR_PREFIX "version"
+#define EC_XATTR_HEAL EC_XATTR_PREFIX "heal"
+#define EC_XATTR_HEAL_NEW EC_XATTR_PREFIX "heal-new"
+#define EC_XATTR_DIRTY EC_XATTR_PREFIX "dirty"
+#define EC_STRIPE_CACHE_MAX_SIZE 10
 #define EC_VERSION_SIZE 2
-#define EC_SHD_INODE_LRU_LIMIT          10
+#define EC_SHD_INODE_LRU_LIMIT 10
 
 #define EC_MAX_FRAGMENTS EC_METHOD_MAX_FRAGMENTS
 /* The maximum number of nodes is derived from the maximum allowed fragments
diff --git a/xlators/cluster/stripe/src/Makefile.am b/xlators/cluster/stripe/src/Makefile.am
deleted file mode 100644
index 2b594567db1..00000000000
--- a/xlators/cluster/stripe/src/Makefile.am
+++ /dev/null
@@ -1,22 +0,0 @@
-xlator_LTLIBRARIES = stripe.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
-
-stripe_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-
-
-stripe_la_SOURCES = stripe.c stripe-helpers.c \
-	$(top_builddir)/xlators/lib/src/libxlator.c
-
-stripe_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = stripe.h stripe-mem-types.h \
-	$(top_builddir)/xlators/lib/src/libxlator.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/xlators/lib/src \
-	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/cluster/stripe/src/stripe-helpers.c b/xlators/cluster/stripe/src/stripe-helpers.c
deleted file mode 100644
index 217f4d2b6e8..00000000000
--- a/xlators/cluster/stripe/src/stripe-helpers.c
+++ /dev/null
@@ -1,677 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include <fnmatch.h>
-
-#include "stripe.h"
-#include "byte-order.h"
-#include "mem-types.h"
-#include "logging.h"
-
-void
-stripe_local_wipe (stripe_local_t *local)
-{
-        if (!local)
-                goto out;
-
-        loc_wipe (&local->loc);
-        loc_wipe (&local->loc2);
-
-        if (local->fd)
-                fd_unref (local->fd);
-
-        if (local->inode)
-                inode_unref (local->inode);
-
-        if (local->xattr)
-                dict_unref (local->xattr);
-
-        if (local->xdata)
-                dict_unref (local->xdata);
-
-out:
-        return;
-}
-
-
-
-int
-stripe_aggregate (dict_t *this, char *key, data_t *value, void *data)
-{
-        dict_t  *dst  = NULL;
-        int64_t *ptr  = 0, *size = NULL;
-        int32_t  ret  = -1;
-
-        dst = data;
-
-        if (strcmp (key, QUOTA_SIZE_KEY) == 0) {
-                ret = dict_get_bin (dst, key, (void **)&size);
-                if (ret < 0) {
-                        size = GF_CALLOC (1, sizeof (int64_t),
-                                          gf_common_mt_char);
-                        if (size == NULL) {
-                                gf_log ("stripe", GF_LOG_WARNING,
-                                        "memory allocation failed");
-                                goto out;
-                        }
-                        ret = dict_set_bin (dst, key, size, sizeof (int64_t));
-                        if (ret < 0) {
-                                gf_log ("stripe", GF_LOG_WARNING,
-                                        "stripe aggregate dict set failed");
-                                GF_FREE (size);
-                                goto out;
-                        }
-                }
-
-                ptr = data_to_bin (value);
-                if (ptr == NULL) {
-                        gf_log ("stripe", GF_LOG_WARNING, "data to bin failed");
-                        goto out;
-                }
-
-                *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
-        } else if (strcmp (key, GF_CONTENT_KEY)) {
-                /* No need to aggregate 'CONTENT' data */
-                ret = dict_set (dst, key, value);
-                if (ret)
-                        gf_log ("stripe", GF_LOG_WARNING, "xattr dict set failed");
-        }
-
-out:
-        return 0;
-}
-
-
-void
-stripe_aggregate_xattr (dict_t *dst, dict_t *src)
-{
-        if ((dst == NULL) || (src == NULL)) {
-                goto out;
-        }
-
-        dict_foreach (src, stripe_aggregate, dst);
-out:
-        return;
-}
-
-
-int32_t
-stripe_xattr_aggregate (char *buffer, stripe_local_t *local, int32_t *total)
-{
-        int32_t              i     = 0;
-        int32_t              ret   = -1;
-        int32_t              len   = 0;
-        char                *sbuf  = NULL;
-        stripe_xattr_sort_t *xattr = NULL;
-
-        if (!buffer || !local || !local->xattr_list)
-                goto out;
-
-        sbuf = buffer;
-
-        for (i = 0; i < local->nallocs; i++) {
-                xattr = local->xattr_list + i;
-                len = xattr->xattr_len - 1; /* length includes \0 */
-
-                if (len && xattr && xattr->xattr_value) {
-                        memcpy (buffer, xattr->xattr_value, len);
-                        buffer += len;
-                        *buffer++ = ' ';
-                }
-        }
-
-        *--buffer = '\0';
-        if (total)
-                *total = buffer - sbuf;
-        ret = 0;
-
- out:
-        return ret;
-}
-
-int32_t
-stripe_free_xattr_str (stripe_local_t *local)
-{
-        int32_t              i     = 0;
-        int32_t              ret   = -1;
-        stripe_xattr_sort_t *xattr = NULL;
-
-        if (!local || !local->xattr_list)
-                goto out;
-
-        for (i = 0; i < local->nallocs; i++) {
-                xattr = local->xattr_list + i;
-
-                if (xattr && xattr->xattr_value)
-                        GF_FREE (xattr->xattr_value);
-        }
-
-        ret = 0;
- out:
-        return ret;
-}
-
-
-int32_t
-stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
-                            void **xattr_serz)
-{
-        int32_t              ret   = -1, i = 0, len = 0;
-        dict_t              *tmp1  = NULL, *tmp2 = NULL;
-        char                *buf   = NULL;
-        stripe_xattr_sort_t *xattr = NULL;
-
-        if (xattr_serz == NULL) {
-                goto out;
-        }
-
-        tmp2 = dict_new ();
-
-        if (tmp2 == NULL) {
-                goto out;
-        }
-
-        for (i = 0; i < local->nallocs; i++) {
-                xattr = local->xattr_list + i;
-                len = xattr->xattr_len;
-
-                if (len && xattr && xattr->xattr_value) {
-                        ret = dict_reset (tmp2);
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "dict_reset failed (%s)",
-                                        strerror (-ret));
-                        }
-
-                        ret = dict_unserialize (xattr->xattr_value,
-                                                xattr->xattr_len,
-                                                &tmp2);
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "dict_unserialize failed (%s)",
-                                        strerror (-ret));
-                                ret = -1;
-                                goto out;
-                        }
-
-                        tmp1 = dict_copy (tmp2, tmp1);
-                        if (tmp1 == NULL) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "dict_copy failed (%s)",
-                                        strerror (-ret));
-                                ret = -1;
-                                goto out;
-                        }
-                }
-        }
-
-        len = dict_serialized_length (tmp1);
-        if (len > 0) {
-                buf = GF_CALLOC (1, len, gf_common_mt_dict_t);
-                if (buf == NULL) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_serialize (tmp1, buf);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "dict_serialize failed (%s)", strerror (-ret));
-                        GF_FREE(buf);
-                        ret = -1;
-                        goto out;
-                }
-
-                *xattr_serz = buf;
-        }
-
-        ret = 0;
-out:
-        if (tmp1 != NULL) {
-                dict_unref (tmp1);
-        }
-
-        if (tmp2 != NULL) {
-                dict_unref (tmp2);
-        }
-
-        return ret;
-}
-
-
-int32_t
-stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
-                            char **xattr_serz)
-{
-        int      ret             = -1;
-        int32_t  padding         = 0;
-        int32_t  tlen            = 0;
-        char stripe_size_str[20] = {0,};
-        char    *pathinfo_serz   = NULL;
-
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "Possible NULL deref");
-                goto out;
-        }
-
-        (void) snprintf (stripe_size_str, 20, "%"PRId64,
-                         (long long) (local->fctx) ? local->fctx->stripe_size : 0);
-
-        /* extra bytes for decorations (brackets and <>'s) */
-        padding = strlen (this->name) + strlen (STRIPE_PATHINFO_HEADER)
-                + strlen (stripe_size_str) + 7;
-        local->xattr_total_len += (padding + 2);
-
-        pathinfo_serz = GF_CALLOC (local->xattr_total_len, sizeof (char),
-                                   gf_common_mt_char);
-        if (!pathinfo_serz)
-                goto out;
-
-        /* xlator info */
-        (void) sprintf (pathinfo_serz, "(<"STRIPE_PATHINFO_HEADER"%s:[%s]> ",
-                        this->name, stripe_size_str);
-
-        ret = stripe_xattr_aggregate (pathinfo_serz + padding, local, &tlen);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Cannot aggregate pathinfo list");
-                GF_FREE(pathinfo_serz);
-                goto out;
-        }
-
-        *(pathinfo_serz + padding + tlen) = ')';
-        *(pathinfo_serz + padding + tlen + 1) = '\0';
-
-        *xattr_serz = pathinfo_serz;
-
-        ret = 0;
- out:
-        return ret;
-}
-
-/**
- * stripe_get_matching_bs - Get the matching block size for the given path.
- */
-int32_t
-stripe_get_matching_bs (const char *path, stripe_private_t *priv)
-{
-        struct stripe_options *trav       = NULL;
-        uint64_t               block_size = 0;
-
-        GF_VALIDATE_OR_GOTO ("stripe", priv, out);
-        GF_VALIDATE_OR_GOTO ("stripe", path, out);
-
-        LOCK (&priv->lock);
-        {
-                block_size = priv->block_size;
-                trav = priv->pattern;
-                while (trav) {
-                        if (!fnmatch (trav->path_pattern, path, FNM_NOESCAPE)) {
-                                block_size = trav->block_size;
-                                break;
-                        }
-                        trav = trav->next;
-                }
-        }
-        UNLOCK (&priv->lock);
-
-out:
-        return block_size;
-}
-
-int32_t
-stripe_ctx_handle (xlator_t *this, call_frame_t *prev, stripe_local_t *local,
-                   dict_t *dict)
-{
-        char            key[256]       = {0,};
-        data_t         *data            = NULL;
-        int32_t         index           = 0;
-        stripe_private_t *priv          = NULL;
-
-        priv = this->private;
-
-
-        if (!local->fctx) {
-                local->fctx =  GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
-                                         gf_stripe_mt_stripe_fd_ctx_t);
-                if (!local->fctx) {
-                        local->op_errno = ENOMEM;
-                        local->op_ret = -1;
-                        goto out;
-                }
-
-                local->fctx->static_array = 0;
-        }
-        /* Stripe block size */
-        sprintf (key, "trusted.%s.stripe-size", this->name);
-        data = dict_get (dict, key);
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-size");
-                goto out;
-        } else {
-                if (!local->fctx->stripe_size) {
-                        local->fctx->stripe_size =
-                                     data_to_int64 (data);
-                }
-
-                if (local->fctx->stripe_size != data_to_int64 (data)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "stripe-size mismatch in blocks");
-                        local->xattr_self_heal_needed = 1;
-                }
-        }
-
-        /* Stripe count */
-        sprintf (key, "trusted.%s.stripe-count", this->name);
-        data = dict_get (dict, key);
-
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-count");
-                goto out;
-        }
-        if (!local->fctx->xl_array) {
-                local->fctx->stripe_count = data_to_int32 (data);
-                if (!local->fctx->stripe_count) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "error with stripe-count xattr");
-                        local->op_ret   = -1;
-                        local->op_errno = EIO;
-                        goto out;
-                }
-
-                local->fctx->xl_array = GF_CALLOC (local->fctx->stripe_count,
-                                                   sizeof (xlator_t *),
-                                                   gf_stripe_mt_xlator_t);
-
-                if (!local->fctx->xl_array) {
-                        local->op_errno = ENOMEM;
-                        local->op_ret   = -1;
-                        goto out;
-                }
-        }
-        if (local->fctx->stripe_count != data_to_int32 (data)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error with stripe-count xattr (%d != %d)",
-                        local->fctx->stripe_count, data_to_int32 (data));
-                local->op_ret   = -1;
-                local->op_errno = EIO;
-                goto out;
-        }
-
-        /* index */
-        sprintf (key, "trusted.%s.stripe-index", this->name);
-        data = dict_get (dict, key);
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-index");
-                goto out;
-        }
-        index = data_to_int32 (data);
-        if (index > priv->child_count) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error with stripe-index xattr (%d)", index);
-                local->op_ret   = -1;
-                local->op_errno = EIO;
-                goto out;
-        }
-        if (local->fctx->xl_array) {
-                if (!local->fctx->xl_array[index])
-                        local->fctx->xl_array[index] = prev->this;
-        }
-
-	sprintf(key, "trusted.%s.stripe-coalesce", this->name);
-	data = dict_get(dict, key);
-	if (!data) {
-		/*
-		 * The file was probably created prior to coalesce support.
-		 * Assume non-coalesce mode for this file to maintain backwards
-		 * compatibility.
-		 */
-		gf_log(this->name, GF_LOG_DEBUG, "missing stripe-coalesce "
-			"attr, assume non-coalesce mode");
-		local->fctx->stripe_coalesce = 0;
-	} else {
-		local->fctx->stripe_coalesce = data_to_int32(data);
-	}
-
-
-out:
-        return 0;
-}
-
-int32_t
-stripe_xattr_request_build (xlator_t *this, dict_t *dict, uint64_t stripe_size,
-                            uint32_t stripe_count, uint32_t stripe_index,
-			    uint32_t stripe_coalesce)
-{
-        char            key[256]       = {0,};
-        int32_t         ret             = -1;
-
-        sprintf (key, "trusted.%s.stripe-size", this->name);
-        ret = dict_set_int64 (dict, key, stripe_size);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-
-        sprintf (key, "trusted.%s.stripe-count", this->name);
-        ret = dict_set_int32 (dict, key, stripe_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-
-        sprintf (key, "trusted.%s.stripe-index", this->name);
-        ret = dict_set_int32 (dict, key, stripe_index);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-
-	sprintf(key, "trusted.%s.stripe-coalesce", this->name);
-	ret = dict_set_int32(dict, key, stripe_coalesce);
-	if (ret) {
-		gf_log(this->name, GF_LOG_WARNING,
-			"failed to set %s in xattr_req_dict", key);
-		goto out;
-	}
-out:
-        return ret;
-}
-
-
-static int
-set_default_block_size (stripe_private_t *priv, char *num)
-{
-
-        int             ret = -1;
-        GF_VALIDATE_OR_GOTO ("stripe", THIS, out);
-        GF_VALIDATE_OR_GOTO (THIS->name, priv, out);
-        GF_VALIDATE_OR_GOTO (THIS->name, num, out);
-
-
-        if (gf_string2bytesize_uint64 (num, &priv->block_size) != 0) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "invalid number format \"%s\"", num);
-                goto out;
-        }
-
-        ret = 0;
-
- out:
-        return ret;
-
-}
-
-
-int
-set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data)
-{
-        int                    ret = -1;
-        char                  *tmp_str = NULL;
-        char                  *tmp_str1 = NULL;
-        char                  *dup_str = NULL;
-        char                  *stripe_str = NULL;
-        char                  *pattern = NULL;
-        char                  *num = NULL;
-        struct stripe_options *temp_stripeopt = NULL;
-        struct stripe_options *stripe_opt = NULL;
-
-        if (!this || !priv || !data)
-                goto out;
-
-        /* Get the pattern for striping.
-           "option block-size *avi:10MB" etc */
-        stripe_str = strtok_r (data, ",", &tmp_str);
-        while (stripe_str) {
-                dup_str = gf_strdup (stripe_str);
-                stripe_opt = GF_CALLOC (1, sizeof (struct stripe_options),
-                                        gf_stripe_mt_stripe_options);
-                if (!stripe_opt) {
-                        goto out;
-                }
-
-                pattern = strtok_r (dup_str, ":", &tmp_str1);
-                num = strtok_r (NULL, ":", &tmp_str1);
-                if (!num) {
-                        num = pattern;
-                        pattern = "*";
-                        ret = set_default_block_size (priv, num);
-                        if (ret)
-                                goto out;
-                }
-                if (gf_string2bytesize_uint64 (num, &stripe_opt->block_size) != 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid number format \"%s\"", num);
-                        goto out;
-                }
-
-                if (stripe_opt->block_size < STRIPE_MIN_BLOCK_SIZE) {
-                        gf_log (this->name, GF_LOG_ERROR, "Invalid Block-size: "
-                                "%s. Should be atleast %llu bytes", num,
-                                STRIPE_MIN_BLOCK_SIZE);
-                        goto out;
-                }
-                if (stripe_opt->block_size % 512) {
-                        gf_log (this->name, GF_LOG_ERROR, "Block-size: %s should"
-                                " be a multiple of 512 bytes", num);
-                        goto out;
-                }
-
-                memcpy (stripe_opt->path_pattern, pattern, strlen (pattern));
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "block-size : pattern %s : size %"PRId64,
-                        stripe_opt->path_pattern, stripe_opt->block_size);
-
-                if (priv->pattern)
-                        temp_stripeopt = NULL;
-                else
-                        temp_stripeopt = priv->pattern;
-
-                stripe_opt->next = temp_stripeopt;
-
-                priv->pattern = stripe_opt;
-                stripe_opt = NULL;
-
-                GF_FREE (dup_str);
-                dup_str = NULL;
-
-                stripe_str = strtok_r (NULL, ",", &tmp_str);
-        }
-
-        ret = 0;
-out:
-
-        GF_FREE (dup_str);
-
-        GF_FREE (stripe_opt);
-
-        return ret;
-}
-
-int32_t
-stripe_iatt_merge (struct iatt *from, struct iatt *to)
-{
-        if (to->ia_size < from->ia_size)
-                to->ia_size = from->ia_size;
-        if (to->ia_mtime < from->ia_mtime)
-                to->ia_mtime = from->ia_mtime;
-        if (to->ia_ctime < from->ia_ctime)
-                to->ia_ctime = from->ia_ctime;
-        if (to->ia_atime < from->ia_atime)
-                to->ia_atime = from->ia_atime;
-        return 0;
-}
-
-off_t
-coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count)
-{
-	size_t line_size = 0;
-	uint64_t stripe_num = 0;
-	off_t coalesced_offset = 0;
-
-	line_size = stripe_size * stripe_count;
-	stripe_num = offset / line_size;
-
-	coalesced_offset = (stripe_num * stripe_size) +
-		(offset % stripe_size);
-
-	return coalesced_offset;
-}
-
-off_t
-uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
-		int stripe_index)
-{
-        uint64_t nr_full_stripe_chunks = 0, mod = 0;
-
-        if (!size)
-                return size;
-
-	/*
-	 * Estimate the number of fully written stripes from the
-	 * local file size. Each stripe_size chunk corresponds to
-	 * a stripe.
-	 */
-        nr_full_stripe_chunks = (size / stripe_size) * stripe_count;
-        mod = size % stripe_size;
-
-        if (!mod) {
-                /*
-		 * There is no remainder, thus we could have overestimated
-		 * the size of the file in terms of chunks. Trim the number
-		 * of chunks by the following stripe members and leave it
-		 * up to those nodes to respond with a larger size (if
-		 * necessary).
-		 */
-                nr_full_stripe_chunks -= stripe_count -
-                        (stripe_index + 1);
-                size = nr_full_stripe_chunks * stripe_size;
-        } else {
-		/*
-		 * There is a remainder and thus we own the last chunk of the
-		 * file. Add the preceding stripe members of the final stripe
-		 * along with the remainder to calculate the exact size.
-		 */
-                nr_full_stripe_chunks += stripe_index;
-                size = nr_full_stripe_chunks * stripe_size + mod;
-        }
-
-        return size;
-}
diff --git a/xlators/cluster/stripe/src/stripe-mem-types.h b/xlators/cluster/stripe/src/stripe-mem-types.h
deleted file mode 100644
index e9ac9cf4648..00000000000
--- a/xlators/cluster/stripe/src/stripe-mem-types.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef __STRIPE_MEM_TYPES_H__
-#define __STRIPE_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_stripe_mem_types_ {
-        gf_stripe_mt_iovec = gf_common_mt_end + 1,
-        gf_stripe_mt_stripe_replies,
-        gf_stripe_mt_stripe_fd_ctx_t,
-        gf_stripe_mt_char,
-        gf_stripe_mt_int8_t,
-        gf_stripe_mt_int32_t,
-        gf_stripe_mt_xlator_t,
-        gf_stripe_mt_stripe_private_t,
-        gf_stripe_mt_stripe_options,
-        gf_stripe_mt_xattr_sort_t,
-        gf_stripe_mt_end
-};
-#endif
-
diff --git a/xlators/cluster/stripe/src/stripe.c b/xlators/cluster/stripe/src/stripe.c
deleted file mode 100644
index fae80c3e200..00000000000
--- a/xlators/cluster/stripe/src/stripe.c
+++ /dev/null
@@ -1,5782 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/**
- * xlators/cluster/stripe:
- *    Stripe translator, stripes the data across its child nodes,
- *    as per the options given in the volfile. The striping works
- *    fairly simple. It writes files at different offset as per
- *    calculation. So, 'ls -l' output at the real posix level will
- *    show file size bigger than the actual size. But when one does
- *    'df' or 'du <file>', real size of the file on the server is shown.
- *
- * WARNING:
- *  Stripe translator can't regenerate data if a child node gets disconnected.
- *  So, no 'self-heal' for stripe. Hence the advice, use stripe only when its
- *  very much necessary, or else, use it in combination with AFR, to have a
- *  backup copy.
- */
-#include <fnmatch.h>
-
-#include "stripe.h"
-#include "libxlator.h"
-#include "byte-order.h"
-#include "statedump.h"
-
-struct volume_options options[];
-
-int32_t
-stripe_sh_chown_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *preop, struct iatt *postop, dict_t *xdata)
-{
-        int             callcnt = -1;
-        stripe_local_t *local   = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_DESTROY (frame);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_sh_make_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, inode_t *inode,
-                          struct iatt *buf, struct iatt *preparent,
-                          struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-        call_frame_t    *prev = NULL;
-
-        if (!frame || !frame->local || !cookie || !this) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        STACK_WIND (frame, stripe_sh_chown_cbk, prev->this,
-                    prev->this->fops->setattr, &local->loc,
-                    &local->stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL);
-
-out:
-        return 0;
-}
-
-int32_t
-stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
-                        stripe_local_t *local)
-{
-        xlator_list_t    *trav   = NULL;
-        call_frame_t     *rframe = NULL;
-        stripe_local_t   *rlocal = NULL;
-        stripe_private_t *priv   = NULL;
-        dict_t           *xdata  = NULL;
-        int               ret    = 0;
-
-        if (!local || !this || !frame) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        if (!(IA_ISREG (local->stbuf.ia_type) ||
-              IA_ISDIR (local->stbuf.ia_type)))
-                return 0;
-
-        priv = this->private;
-        trav = this->children;
-        rframe = copy_frame (frame);
-        if (!rframe) {
-                goto out;
-        }
-        rlocal = mem_get0 (this->local_pool);
-        if (!rlocal) {
-                goto out;
-        }
-        rframe->local = rlocal;
-        rlocal->call_count = priv->child_count;
-        loc_copy (&rlocal->loc, &local->loc);
-        memcpy (&rlocal->stbuf, &local->stbuf, sizeof (struct iatt));
-
-        xdata = dict_new ();
-        if (!xdata)
-                goto out;
-
-        ret = dict_set_gfuuid (xdata, "gfid-req", local->stbuf.ia_gfid, true);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set gfid-req", local->loc.path);
-
-        while (trav) {
-                if (IA_ISREG (local->stbuf.ia_type)) {
-                        STACK_WIND (rframe, stripe_sh_make_entry_cbk,
-                                    trav->xlator, trav->xlator->fops->mknod,
-                                    &local->loc,
-                                    st_mode_from_ia (local->stbuf.ia_prot,
-                                                     local->stbuf.ia_type),
-                                    0, 0, xdata);
-                }
-                if (IA_ISDIR (local->stbuf.ia_type)) {
-                        STACK_WIND (rframe, stripe_sh_make_entry_cbk,
-                                    trav->xlator, trav->xlator->fops->mkdir,
-                                    &local->loc,
-                                    st_mode_from_ia (local->stbuf.ia_prot,
-                                                     local->stbuf.ia_type),
-                                    0, xdata);
-                }
-                trav = trav->next;
-        }
-
-        if (xdata)
-                dict_unref (xdata);
-        return 0;
-
-out:
-        if (rframe)
-                STRIPE_STACK_DESTROY (rframe);
-
-        return 0;
-}
-
-
-int32_t
-stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
-{
-        int32_t         callcnt     = 0;
-        stripe_local_t *local       = NULL;
-        call_frame_t   *prev        = NULL;
-        int             ret         = 0;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        if ((op_errno != ENOENT) && (op_errno != ESTALE))
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "%s returned error %s",
-                                        prev->this->name,
-                                        strerror (op_errno));
-                        if (local->op_errno != ESTALE)
-                                local->op_errno = op_errno;
-                        if (((op_errno != ENOENT) && (op_errno != ENOTCONN)
-                              && (op_errno != ESTALE)) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                        if (op_errno == ENOENT)
-                                local->entry_self_heal_needed = 1;
-                }
-
-                if (op_ret >= 0) {
-                        local->op_ret = 0;
-                        if (IA_ISREG (buf->ia_type)) {
-                                ret = stripe_ctx_handle (this, prev, local,
-                                                         xdata);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                 "Error getting fctx info from"
-                                                 " dict");
-                        }
-
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->stbuf      = *buf;
-                                local->postparent = *postparent;
-                                local->inode = inode_ref (inode);
-                                if (xdata)
-                                        local->xdata = dict_ref (xdata);
-                                if (local->xattr) {
-                                        stripe_aggregate_xattr (local->xdata,
-                                                                local->xattr);
-                                        dict_unref (local->xattr);
-                                        local->xattr = NULL;
-                                }
-                        }
-
-                        if (!local->xdata && !local->xattr) {
-                                local->xattr = dict_ref (xdata);
-                        } else if (local->xdata) {
-                                stripe_aggregate_xattr (local->xdata, xdata);
-                        } else if (local->xattr) {
-                                stripe_aggregate_xattr (local->xattr, xdata);
-                        }
-
-                        local->stbuf_blocks      += buf->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-
-                        if (gf_uuid_is_null (local->ia_gfid))
-                                gf_uuid_copy (local->ia_gfid, buf->ia_gfid);
-
-                        /* Make sure the gfid on all the nodes are same */
-                        if (gf_uuid_compare (local->ia_gfid, buf->ia_gfid)) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "%s: gfid different on subvolume %s",
-                                        local->loc.path, prev->this->name);
-                        }
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->op_ret == 0 && local->entry_self_heal_needed &&
-                    !gf_uuid_is_null (local->loc.inode->gfid))
-                        stripe_entry_self_heal (frame, this, local);
-
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        inode_ctx_put (local->inode, this,
-                                       (uint64_t) (long)local->fctx);
-                }
-
-                STRIPE_STACK_UNWIND (lookup, frame, local->op_ret,
-                                     local->op_errno, local->inode,
-                                     &local->stbuf, local->xdata,
-                                     &local->postparent);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               dict_t *xdata)
-{
-        stripe_local_t   *local    = NULL;
-        xlator_list_t    *trav     = NULL;
-        stripe_private_t *priv     = NULL;
-        int32_t           op_errno = EINVAL;
-        int64_t           filesize = 0;
-        int               ret      = 0;
-        uint64_t          tmpctx   = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-
-        inode_ctx_get (local->inode, this, &tmpctx);
-        if (tmpctx)
-                local->fctx = (stripe_fd_ctx_t*) (long)tmpctx;
-
-        /* quick-read friendly changes */
-        if (xdata && dict_get (xdata, GF_CONTENT_KEY)) {
-                ret = dict_get_int64 (xdata, GF_CONTENT_KEY, &filesize);
-                if (!ret && (filesize > priv->block_size))
-                        dict_del (xdata, GF_CONTENT_KEY);
-        }
-
-        /* get stripe-size xattr on lookup. This would be required for
-         * open/read/write/pathinfo calls. Hence we send down the request
-         * even when type == IA_INVAL */
-
-	/*
-	 * We aren't guaranteed to have xdata here. We need the format info for
-	 * the file, so allocate xdata if necessary.
-	 */
-	if (!xdata)
-		xdata = dict_new();
-	else
-		xdata = dict_ref(xdata);
-
-        if (xdata && (IA_ISREG (loc->inode->ia_type) ||
-            (loc->inode->ia_type == IA_INVAL))) {
-                ret = stripe_xattr_request_build (this, xdata, 8, 4, 4, 0);
-                if (ret)
-                        gf_log (this->name , GF_LOG_ERROR, "Failed to build"
-                                " xattr request for %s", loc->path);
-
-        }
-
-        /* Every time in stripe lookup, all child nodes
-           should be looked up */
-        local->call_count = priv->child_count;
-        while (trav) {
-                STACK_WIND (frame, stripe_lookup_cbk, trav->xlator,
-                            trav->xlator->fops->lookup, loc, xdata);
-                trav = trav->next;
-        }
-
-	dict_unref(xdata);
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->stbuf = *buf;
-                        }
-
-                        local->stbuf_blocks += buf->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->stbuf.ia_size   = local->stbuf_size;
-                        local->stbuf.ia_blocks = local->stbuf_blocks;
-                }
-
-                STRIPE_STACK_UNWIND (stat, frame, local->op_ret,
-                                     local->op_errno, &local->stbuf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-	if (IA_ISREG(loc->inode->ia_type)) {
-		inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
-		if (!fctx)
-			goto err;
-		local->fctx = fctx;
-	}
-
-        while (trav) {
-                STACK_WIND (frame, stripe_stat_cbk, trav->xlator,
-                            trav->xlator->fops->stat, loc, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-
-err:
-        STRIPE_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct statvfs *stbuf, dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-        int32_t         callcnt = 0;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-        local = frame->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret && (op_errno != ENOTCONN)) {
-                        local->op_errno = op_errno;
-                }
-                if (op_ret == 0) {
-                        struct statvfs *dict_buf = &local->statvfs_buf;
-                        dict_buf->f_bsize   = stbuf->f_bsize;
-                        dict_buf->f_frsize  = stbuf->f_frsize;
-                        dict_buf->f_blocks += stbuf->f_blocks;
-                        dict_buf->f_bfree  += stbuf->f_bfree;
-                        dict_buf->f_bavail += stbuf->f_bavail;
-                        dict_buf->f_files  += stbuf->f_files;
-                        dict_buf->f_ffree  += stbuf->f_ffree;
-                        dict_buf->f_favail += stbuf->f_favail;
-                        dict_buf->f_fsid    = stbuf->f_fsid;
-                        dict_buf->f_flag    = stbuf->f_flag;
-                        dict_buf->f_namemax = stbuf->f_namemax;
-                        local->op_ret = 0;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_UNWIND (statfs, frame, local->op_ret,
-                                     local->op_errno, &local->statvfs_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        trav = this->children;
-        priv = this->private;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        local->op_errno = ENOTCONN;
-        frame->local = local;
-
-        local->call_count = priv->child_count;
-        while (trav) {
-                STACK_WIND (frame, stripe_statfs_cbk, trav->xlator,
-                            trav->xlator->fops->statfs, loc, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-
-int32_t
-stripe_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->pre_buf  = *prebuf;
-                                local->post_buf = *postbuf;
-                        }
-
-                        local->prebuf_blocks  += prebuf->ia_blocks;
-                        local->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, local->fctx, prev);
-			correct_file_size(postbuf, local->fctx, prev);
-
-                        if (local->prebuf_size < prebuf->ia_size)
-                                local->prebuf_size = prebuf->ia_size;
-
-                        if (local->postbuf_size < postbuf->ia_size)
-                                local->postbuf_size = postbuf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->pre_buf.ia_blocks  = local->prebuf_blocks;
-                        local->pre_buf.ia_size    = local->prebuf_size;
-                        local->post_buf.ia_blocks = local->postbuf_blocks;
-                        local->post_buf.ia_size   = local->postbuf_size;
-                }
-
-                STRIPE_STACK_UNWIND (truncate, frame, local->op_ret,
-                                     local->op_errno, &local->pre_buf,
-                                     &local->post_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = EINVAL;
-	int		  i, eof_idx;
-	off_t		  dest_offset, tmp_offset;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-	inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
-	if (!fctx) {
-		gf_log(this->name, GF_LOG_ERROR, "no stripe context");
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local->fctx = fctx;
-	eof_idx = (offset / fctx->stripe_size) % fctx->stripe_count;
-
-	for (i = 0; i < fctx->stripe_count; i++) {
-		if (!fctx->xl_array[i]) {
-			gf_log(this->name, GF_LOG_ERROR,
-				"no xlator at index %d", i);
-			op_errno = EINVAL;
-			goto err;
-		}
-
-		if (fctx->stripe_coalesce) {
-			/*
-	 		 * The node that owns EOF is truncated to the exact
-	 		 * coalesced offset. Nodes prior to this index should
-	 		 * be rounded up to the size of the complete stripe,
-	 		 * while nodes after this index should be rounded down
-			 * to the size of the previous stripe.
-			 */
-			if (i < eof_idx)
-				tmp_offset = roof(offset, fctx->stripe_size *
-						fctx->stripe_count);
-			else if (i > eof_idx)
-				tmp_offset = floor(offset, fctx->stripe_size *
-						fctx->stripe_count);
-			else
-				tmp_offset = offset;
-
-			dest_offset = coalesced_offset(tmp_offset,
-					fctx->stripe_size, fctx->stripe_count);
-		} else {
-			dest_offset = offset;
-		}
-
-		STACK_WIND(frame, stripe_truncate_cbk, fctx->xl_array[i],
-			fctx->xl_array[i]->fops->truncate, loc, dest_offset,
-			NULL);
-	}
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *preop, struct iatt *postop, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->pre_buf  = *preop;
-                                local->post_buf = *postop;
-                        }
-
-                        local->prebuf_blocks  += preop->ia_blocks;
-                        local->postbuf_blocks += postop->ia_blocks;
-
-			correct_file_size(preop, local->fctx, prev);
-			correct_file_size(postop, local->fctx, prev);
-
-                        if (local->prebuf_size < preop->ia_size)
-                                local->prebuf_size = preop->ia_size;
-                        if (local->postbuf_size < postop->ia_size)
-                                local->postbuf_size = postop->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->pre_buf.ia_blocks  = local->prebuf_blocks;
-                        local->pre_buf.ia_size    = local->prebuf_size;
-                        local->post_buf.ia_blocks = local->postbuf_blocks;
-                        local->post_buf.ia_size   = local->postbuf_size;
-                }
-
-                STRIPE_STACK_UNWIND (setattr, frame, local->op_ret,
-                                     local->op_errno, &local->pre_buf,
-                                     &local->post_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-	stripe_fd_ctx_t	 *fctx = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        if (!IA_ISDIR (loc->inode->ia_type) &&
-            !IA_ISREG (loc->inode->ia_type)) {
-                local->call_count = 1;
-                STACK_WIND (frame, stripe_setattr_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid, NULL);
-                return 0;
-        }
-
-	if (IA_ISREG(loc->inode->ia_type)) {
-		inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
-		if (!fctx)
-			goto err;
-		local->fctx = fctx;
-	}
-
-        local->call_count = priv->child_count;
-        while (trav) {
-                STACK_WIND (frame, stripe_setattr_cbk,
-                            trav->xlator, trav->xlator->fops->setattr,
-                            loc, stbuf, valid, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_setattr_cbk, trav->xlator,
-                            trav->xlator->fops->fsetattr, fd, stbuf, valid, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_stack_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent,
-                         dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-
-                        local->stbuf.ia_blocks      += buf->ia_blocks;
-                        local->preparent.ia_blocks  += preoldparent->ia_blocks;
-                        local->postparent.ia_blocks += postoldparent->ia_blocks;
-                        local->pre_buf.ia_blocks    += prenewparent->ia_blocks;
-                        local->post_buf.ia_blocks   += postnewparent->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf.ia_size < buf->ia_size)
-                                local->stbuf.ia_size =  buf->ia_size;
-
-                        if (local->preparent.ia_size < preoldparent->ia_size)
-                                local->preparent.ia_size = preoldparent->ia_size;
-
-                        if (local->postparent.ia_size < postoldparent->ia_size)
-                                local->postparent.ia_size = postoldparent->ia_size;
-
-                        if (local->pre_buf.ia_size < prenewparent->ia_size)
-                                local->pre_buf.ia_size = prenewparent->ia_size;
-
-                        if (local->post_buf.ia_size < postnewparent->ia_size)
-                                local->post_buf.ia_size = postnewparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                STRIPE_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                                     &local->stbuf, &local->preparent,
-                                     &local->postparent,  &local->pre_buf,
-                                     &local->post_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_first_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent,
-                         dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-        xlator_list_t  *trav = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        if (op_ret == -1) {
-                goto unwind;
-        }
-
-        local = frame->local;
-        trav = this->children;
-
-        local->stbuf      = *buf;
-        local->preparent  = *preoldparent;
-        local->postparent = *postoldparent;
-        local->pre_buf    = *prenewparent;
-        local->post_buf   = *postnewparent;
-
-        local->op_ret = 0;
-        local->call_count--;
-
-        trav = trav->next; /* Skip first child */
-        while (trav) {
-                STACK_WIND (frame, stripe_stack_rename_cbk,
-                            trav->xlator, trav->xlator->fops->rename,
-                            &local->loc, &local->loc2, NULL);
-                trav = trav->next;
-        }
-        return 0;
-
-unwind:
-        STRIPE_STACK_UNWIND (rename, frame, -1, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent, NULL);
-        return 0;
-}
-
-int32_t
-stripe_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc, dict_t *xdata)
-{
-        stripe_private_t *priv = NULL;
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-	stripe_fd_ctx_t	 *fctx = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (oldloc->path, err);
-        VALIDATE_OR_GOTO (oldloc->inode, err);
-        VALIDATE_OR_GOTO (newloc, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* If any one node is down, don't allow rename */
-        if (priv->nodes_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-
-        local->op_ret = -1;
-        loc_copy (&local->loc, oldloc);
-        loc_copy (&local->loc2, newloc);
-
-        local->call_count = priv->child_count;
-
-	if (IA_ISREG(oldloc->inode->ia_type)) {
-		inode_ctx_get(oldloc->inode, this, (uint64_t *) &fctx);
-		if (!fctx)
-			goto err;
-		local->fctx = fctx;
-	}
-
-        STACK_WIND (frame, stripe_first_rename_cbk, trav->xlator,
-                    trav->xlator->fops->rename, oldloc, newloc, NULL);
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL, NULL);
-        return 0;
-}
-int32_t
-stripe_first_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
-                        prev->this->name, strerror (op_errno));
-                goto out;
-        }
-        local->op_ret = 0;
-        local->preparent  = *preparent;
-        local->postparent = *postparent;
-        local->preparent_blocks  += preparent->ia_blocks;
-        local->postparent_blocks += postparent->ia_blocks;
-
-        STRIPE_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
-                            &local->preparent, &local->postparent, xdata);
-        return 0;
-out:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-
-        return 0;
-}
-
-
-
-
-int32_t
-stripe_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if (op_errno != ENOENT) {
-                                local->failed = 1;
-                                local->op_ret = op_ret;
-                        }
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (callcnt == 1) {
-                if (local->failed) {
-                        op_errno = local->op_errno;
-                        goto out;
-                }
-                STACK_WIND(frame, stripe_first_unlink_cbk, FIRST_CHILD (this),
-                           FIRST_CHILD (this)->fops->unlink, &local->loc,
-                           local->xflag, local->xdata);
-        }
-        return 0;
-out:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-
-        return 0;
-}
-
-int32_t
-stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int xflag, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Don't unlink a file if a node is down */
-        if (priv->nodes_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        loc_copy (&local->loc, loc);
-        local->xflag = xflag;
-
-        if (xdata)
-            local->xdata = dict_ref (xdata);
-
-        frame->local = local;
-        local->call_count = priv->child_count;
-        trav = trav->next; /* Skip the first child */
-
-        while (trav) {
-                STACK_WIND (frame, stripe_unlink_cbk,
-                            trav->xlator, trav->xlator->fops->unlink,
-                            loc, xflag, xdata);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_first_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno,struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (op_ret == -1) {
-                goto err;
-        }
-
-        local = frame->local;
-        local->op_ret = 0;
-
-        local->call_count--; /* First child successful */
-
-        local->preparent  = *preparent;
-        local->postparent = *postparent;
-        local->preparent_size  = preparent->ia_size;
-        local->postparent_size = postparent->ia_size;
-        local->preparent_blocks  += preparent->ia_blocks;
-        local->postparent_blocks += postparent->ia_blocks;
-
-        STRIPE_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                             &local->preparent, &local->postparent, xdata);
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (rmdir, frame, op_ret, op_errno, NULL, NULL, NULL);
-        return 0;
-
-}
-
-int32_t
-stripe_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        if (op_errno != ENOENT)
-                                local->failed = 1;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (callcnt == 1) {
-                if (local->failed)
-                        goto out;
-                STACK_WIND (frame, stripe_first_rmdir_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->rmdir, &local->loc,
-                            local->flags, NULL);
-        }
-        return 0;
-out:
-        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* don't delete a directory if any of the subvolume is down */
-        if (priv->nodes_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-        local->flags = flags;
-        local->call_count = priv->child_count;
-        trav = trav->next; /* skip the first child */
-
-        while (trav) {
-                STACK_WIND (frame, stripe_rmdir_cbk,  trav->xlator,
-                            trav->xlator->fops->rmdir, loc, flags, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_mknod_ifreg_fail_unlink_cbk (call_frame_t *frame, void *cookie,
-                                    xlator_t *this, int32_t op_ret,
-                                    int32_t op_errno, struct iatt *preparent,
-                                    struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
-                                     local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-
-/**
- */
-int32_t
-stripe_mknod_ifreg_setxattr_cbk (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *xdata)
-{
-        int32_t           callcnt = 0;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        call_frame_t     *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-
-	LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->op_ret == -1) {
-                        local->call_count = priv->child_count;
-                        while (trav) {
-                                STACK_WIND (frame,
-                                            stripe_mknod_ifreg_fail_unlink_cbk,
-                                            trav->xlator,
-                                            trav->xlator->fops->unlink,
-                                            &local->loc, 0, NULL);
-                                trav = trav->next;
-                        }
-                        return 0;
-                }
-
-                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
-                                     local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
-{
-        int32_t           callcnt = 0;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        call_frame_t     *prev = NULL;
-        xlator_list_t    *trav = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                        local->op_errno = op_errno;
-                }
-                if (op_ret >= 0) {
-                        local->op_ret = op_ret;
-
-                        /* Can be used as a mechanism to understand if mknod
-                           was successful in at least one place */
-                        if (gf_uuid_is_null (local->ia_gfid))
-                                gf_uuid_copy (local->ia_gfid, buf->ia_gfid);
-
-			if (stripe_ctx_handle(this, prev, local, xdata))
-				gf_log(this->name, GF_LOG_ERROR,
-					"Error getting fctx info from dict");
-
-                        local->stbuf_blocks += buf->ia_blocks;
-                        local->preparent_blocks  += preparent->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->preparent_size < preparent->ia_size)
-                                local->preparent_size = preparent->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if ((local->op_ret == -1) && !gf_uuid_is_null (local->ia_gfid)) {
-                        /* ia_gfid set means, at least on one node 'mknod'
-                           is successful */
-                        local->call_count = priv->child_count;
-                        trav = this->children;
-                        while (trav) {
-                                STACK_WIND (frame,
-                                            stripe_mknod_ifreg_fail_unlink_cbk,
-                                            trav->xlator,
-                                            trav->xlator->fops->unlink,
-                                            &local->loc, 0, NULL);
-                                trav = trav->next;
-                        }
-                        return 0;
-                }
-
-
-                if (local->op_ret != -1) {
-                        local->preparent.ia_blocks  = local->preparent_blocks;
-                        local->preparent.ia_size    = local->preparent_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-                        inode_ctx_put (local->inode, this,
-                                       (uint64_t)(long) local->fctx);
-
-                }
-                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
-                                     local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_mknod_first_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        call_frame_t     *prev = NULL;
-        xlator_list_t    *trav = NULL;
-        int               i    = 1;
-        dict_t           *dict           = NULL;
-        int               ret            = 0;
-        int               need_unref     = 0;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-        trav = this->children;
-
-        local->call_count--;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
-                        prev->this->name, strerror (op_errno));
-                local->failed = 1;
-                local->op_errno = op_errno;
-                goto out;
-        }
-
-        local->op_ret = op_ret;
-
-        local->stbuf      = *buf;
-        local->preparent  = *preparent;
-        local->postparent = *postparent;
-
-        if (gf_uuid_is_null (local->ia_gfid))
-                gf_uuid_copy (local->ia_gfid, buf->ia_gfid);
-        local->preparent.ia_blocks  = local->preparent_blocks;
-        local->preparent.ia_size    = local->preparent_size;
-        local->postparent.ia_blocks = local->postparent_blocks;
-        local->postparent.ia_size   = local->postparent_size;
-        local->stbuf.ia_size        = local->stbuf_size;
-        local->stbuf.ia_blocks      = local->stbuf_blocks;
-
-        trav = trav->next;
-        while (trav) {
-                if (priv->xattr_supported) {
-                        dict = dict_new ();
-                        if (!dict) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to allocate dict %s", local->loc.path);
-                        }
-                        need_unref = 1;
-
-                        dict_copy (local->xattr, dict);
-
-                        ret = stripe_xattr_request_build (this, dict,
-                                                          local->stripe_size,
-                                                          priv->child_count, i,
-							  priv->coalesce);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to build xattr request");
-
-                } else {
-                        dict = local->xattr;
-                }
-
-                STACK_WIND (frame, stripe_mknod_ifreg_cbk,
-                            trav->xlator, trav->xlator->fops->mknod,
-                            &local->loc, local->mode, local->rdev, 0, dict);
-                trav = trav->next;
-                i++;
-
-                if (dict && need_unref)
-                        dict_unref (dict);
-        }
-
-        return 0;
-
-out:
-
-       STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, NULL, NULL, NULL, NULL, NULL);
-       return 0;
-}
-
-
-int32_t
-stripe_single_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, inode_t *inode,
-                         struct iatt *buf, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
-{
-        STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
-
-
-int
-stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, mode_t umask, dict_t *xdata)
-{
-        stripe_private_t *priv           = NULL;
-        stripe_local_t   *local          = NULL;
-        int32_t           op_errno       = EINVAL;
-        int32_t           i              = 0;
-        dict_t           *dict           = NULL;
-        int               ret            = 0;
-        int               need_unref     = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        if (S_ISREG(mode)) {
-                /* NOTE: on older kernels (older than 2.6.9),
-                   creat() fops is sent as mknod() + open(). Hence handling
-                   S_IFREG files is necessary */
-                if (priv->nodes_down) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Some node down, returning EIO");
-                        op_errno = EIO;
-                        goto err;
-                }
-
-                /* Initialization */
-                local = mem_get0 (this->local_pool);
-                if (!local) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                local->op_ret = -1;
-                local->op_errno = ENOTCONN;
-                local->stripe_size = stripe_get_matching_bs (loc->path, priv);
-                frame->local = local;
-                local->inode = inode_ref (loc->inode);
-                loc_copy (&local->loc, loc);
-                local->xattr = dict_copy_with_ref (xdata, NULL);
-                local->mode = mode;
-                local->umask = umask;
-                local->rdev = rdev;
-
-                /* Every time in stripe lookup, all child nodes should
-                   be looked up */
-                local->call_count = priv->child_count;
-
-                if (priv->xattr_supported) {
-                        dict = dict_new ();
-                        if (!dict) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to allocate dict %s", loc->path);
-                        }
-                        need_unref = 1;
-
-                        dict_copy (xdata, dict);
-
-                        ret = stripe_xattr_request_build (this, dict,
-                                                          local->stripe_size,
-                                                          priv->child_count,
-                                                          i, priv->coalesce);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to build xattr request");
-                } else {
-                        dict = xdata;
-                }
-
-                STACK_WIND (frame, stripe_mknod_first_ifreg_cbk,
-                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
-                            loc, mode, rdev, umask, dict);
-
-                        if (dict && need_unref)
-                                dict_unref (dict);
-                return 0;
-        }
-
-        STACK_WIND (frame, stripe_single_mknod_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, rdev, umask, xdata);
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t  *local   = NULL;
-        call_frame_t    *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret >= 0) {
-                        local->op_ret = 0;
-
-                        local->stbuf_blocks      += buf->ia_blocks;
-                        local->preparent_blocks  += preparent->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->preparent_size < preparent->ia_size)
-                                local->preparent_size = preparent->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed != -1) {
-                        local->preparent.ia_blocks  = local->preparent_blocks;
-                        local->preparent.ia_size    = local->preparent_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-                }
-                STRIPE_STACK_UNWIND (mkdir, frame, local->op_ret,
-                                     local->op_errno, local->inode,
-                                     &local->stbuf, &local->preparent,
-                                     &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_first_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t  *local   = NULL;
-        call_frame_t    *prev = NULL;
-        xlator_list_t        *trav = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-        trav = this->children;
-
-        local->call_count--; /* first child is successful */
-        trav = trav->next;   /* skip first child */
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
-                        prev->this->name, strerror (op_errno));
-                local->op_errno = op_errno;
-                goto out;
-        }
-
-        local->op_ret = 0;
-
-        local->inode      = inode_ref (inode);
-        local->stbuf      = *buf;
-        local->postparent = *postparent;
-        local->preparent  = *preparent;
-
-        local->stbuf_blocks      += buf->ia_blocks;
-        local->preparent_blocks  += preparent->ia_blocks;
-        local->postparent_blocks += postparent->ia_blocks;
-
-        local->stbuf_size = buf->ia_size;
-        local->preparent_size = preparent->ia_size;
-        local->postparent_size = postparent->ia_size;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_mkdir_cbk, trav->xlator,
-                            trav->xlator->fops->mkdir, &local->loc, local->mode,
-                            local->umask, local->xdata);
-                trav = trav->next;
-        }
-        return 0;
-out:
-        STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
-
-        return 0;
-
-}
-
-
-int
-stripe_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              mode_t umask, dict_t *xdata)
-{
-        stripe_private_t *priv = NULL;
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        local->call_count = priv->child_count;
-        if (xdata)
-                local->xdata = dict_ref (xdata);
-        local->mode  = mode;
-        local->umask = umask;
-        loc_copy (&local->loc, loc);
-        frame->local = local;
-
-        /* Every time in stripe lookup, all child nodes should be looked up */
-        STACK_WIND (frame, stripe_first_mkdir_cbk, trav->xlator,
-                    trav->xlator->fops->mkdir, loc, mode, umask, xdata);
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t  *local   = NULL;
-        call_frame_t    *prev = NULL;
-	stripe_fd_ctx_t *fctx = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret >= 0) {
-                        local->op_ret = 0;
-
-			if (IA_ISREG(inode->ia_type)) {
-				inode_ctx_get(inode, this, (uint64_t *) &fctx);
-				if (!fctx) {
-					gf_log(this->name, GF_LOG_ERROR,
-						"failed to get stripe context");
-					op_ret = -1;
-					op_errno = EINVAL;
-				}
-			}
-
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->inode      = inode_ref (inode);
-                                local->stbuf      = *buf;
-                                local->postparent = *postparent;
-                                local->preparent  = *preparent;
-                        }
-                        local->stbuf_blocks      += buf->ia_blocks;
-                        local->preparent_blocks  += preparent->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-			correct_file_size(buf, fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->preparent_size < preparent->ia_size)
-                                local->preparent_size = preparent->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->preparent.ia_blocks  = local->preparent_blocks;
-                        local->preparent.ia_size    = local->preparent_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-                }
-                STRIPE_STACK_UNWIND (link, frame, local->op_ret,
-                                     local->op_errno, local->inode,
-                                     &local->stbuf, &local->preparent,
-                                     &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (oldloc->path, err);
-        VALIDATE_OR_GOTO (oldloc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* If any one node is down, don't allow link operation */
-        if (priv->nodes_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        /* Every time in stripe lookup, all child
-           nodes should be looked up */
-        while (trav) {
-                STACK_WIND (frame, stripe_link_cbk,
-                            trav->xlator, trav->xlator->fops->link,
-                            oldloc, newloc, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_create_fail_unlink_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret,
-                               int32_t op_errno, struct iatt *preparent,
-                               struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_UNWIND (create, frame, local->op_ret, local->op_errno,
-                                     local->fd, local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd,
-                   inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
-{
-        int32_t           callcnt = 0;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        call_frame_t     *prev = NULL;
-        xlator_list_t    *trav = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->failed = 1;
-                        local->op_errno = op_errno;
-                }
-
-                if (op_ret >= 0) {
-			if (IA_ISREG(buf->ia_type)) {
-				if (stripe_ctx_handle(this, prev, local, xdata))
-					gf_log(this->name, GF_LOG_ERROR,
-						"Error getting fctx info from "
-						"dict");
-			}
-
-                        local->op_ret = op_ret;
-
-                        local->stbuf_blocks += buf->ia_blocks;
-                        local->preparent_blocks  += preparent->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->preparent_size < preparent->ia_size)
-                                local->preparent_size = preparent->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret == -1) {
-                        local->call_count = priv->child_count;
-                        trav = this->children;
-                        while (trav) {
-                                STACK_WIND (frame,
-                                            stripe_create_fail_unlink_cbk,
-                                            trav->xlator,
-                                            trav->xlator->fops->unlink,
-                                            &local->loc, 0, NULL);
-                                trav = trav->next;
-                        }
-
-                        return 0;
-                }
-
-                if (local->op_ret >= 0) {
-                        local->preparent.ia_blocks  = local->preparent_blocks;
-                        local->preparent.ia_size    = local->preparent_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-
-			stripe_copy_xl_array(local->fctx->xl_array,
-				             priv->xl_array,
-					     local->fctx->stripe_count);
-			inode_ctx_put(local->inode, this,
-					(uint64_t) local->fctx);
-                }
-
-                /* Create itself has failed.. so return
-                   without setxattring */
-                STRIPE_STACK_UNWIND (create, frame, local->op_ret,
-                                     local->op_errno, local->fd,
-                                     local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-
-out:
-        return 0;
-}
-
-
-
-int32_t
-stripe_first_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd,
-                   inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        call_frame_t     *prev = NULL;
-        xlator_list_t    *trav = NULL;
-        int               i    = 1;
-        dict_t           *dict = NULL;
-        loc_t            *loc  = NULL;
-        int32_t           need_unref = 0;
-        int32_t           ret  = -1;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-        trav = this->children;
-        loc = &local->loc;
-
-        --local->call_count;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
-                        prev->this->name, strerror (op_errno));
-                 local->failed = 1;
-                 local->op_errno = op_errno;
-        }
-
-        local->op_ret = 0;
-        /* Get the mapping in inode private */
-        /* Get the stat buf right */
-        local->stbuf      = *buf;
-        local->preparent  = *preparent;
-        local->postparent = *postparent;
-
-        local->stbuf_blocks += buf->ia_blocks;
-        local->preparent_blocks  += preparent->ia_blocks;
-        local->postparent_blocks += postparent->ia_blocks;
-
-        if (local->stbuf_size < buf->ia_size)
-              local->stbuf_size = buf->ia_size;
-        if (local->preparent_size < preparent->ia_size)
-              local->preparent_size = preparent->ia_size;
-        if (local->postparent_size < postparent->ia_size)
-              local->postparent_size = postparent->ia_size;
-
-        if (local->failed)
-                local->op_ret = -1;
-
-        if (local->op_ret == -1) {
-                local->call_count = 1;
-                STACK_WIND (frame, stripe_create_fail_unlink_cbk,
-                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
-                            &local->loc, 0, NULL);
-                return 0;
-        }
-
-        if (local->op_ret >= 0) {
-                local->preparent.ia_blocks  = local->preparent_blocks;
-                local->preparent.ia_size    = local->preparent_size;
-                local->postparent.ia_blocks = local->postparent_blocks;
-                local->postparent.ia_size   = local->postparent_size;
-                local->stbuf.ia_size        = local->stbuf_size;
-                local->stbuf.ia_blocks      = local->stbuf_blocks;
-        }
-
-        /* Send a setxattr request to nodes where the
-           files are created */
-        trav = trav->next;
-        while (trav) {
-                if (priv->xattr_supported) {
-                        dict = dict_new ();
-                        if (!dict) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to allocate dict %s", loc->path);
-                        }
-                        need_unref = 1;
-
-                        dict_copy (local->xattr, dict);
-
-                        ret = stripe_xattr_request_build (this, dict,
-                                                          local->stripe_size,
-                                                          priv->child_count,
-                                                          i, priv->coalesce);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to build xattr request");
-                } else {
-                        dict = local->xattr;
-                }
-
-                STACK_WIND (frame, stripe_create_cbk, trav->xlator,
-                            trav->xlator->fops->create, &local->loc,
-                            local->flags, local->mode, local->umask, local->fd,
-                            dict);
-                trav = trav->next;
-                if (need_unref && dict)
-                        dict_unref (dict);
-                i++;
-        }
-
-out:
-        return 0;
-}
-
-
-
-/**
- * stripe_create - If a block-size is specified for the 'name', create the
- *    file in all the child nodes. If not, create it in only first child.
- *
- * @name- complete path of the file to be created.
- */
-int32_t
-stripe_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        stripe_private_t *priv = NULL;
-        stripe_local_t   *local = NULL;
-        int32_t           op_errno = EINVAL;
-        int               ret            = 0;
-        int               need_unref     = 0;
-        int               i              = 0;
-        dict_t           *dict           = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-
-        /* files created in O_APPEND mode does not allow lseek() on fd */
-        flags &= ~O_APPEND;
-
-        if (priv->first_child_down || priv->nodes_down) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "First node down, returning EIO");
-                op_errno = EIO;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        local->op_errno = ENOTCONN;
-        local->stripe_size = stripe_get_matching_bs (loc->path, priv);
-        frame->local = local;
-        local->inode = inode_ref (loc->inode);
-        loc_copy (&local->loc, loc);
-        local->fd = fd_ref (fd);
-        local->flags = flags;
-        local->mode = mode;
-        local->umask = umask;
-        if (xdata)
-                local->xattr = dict_ref (xdata);
-
-        local->call_count = priv->child_count;
-        /* Send a setxattr request to nodes where the
-           files are created */
-
-        if (priv->xattr_supported) {
-                dict = dict_new ();
-                if (!dict) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to allocate dict %s", loc->path);
-                }
-                need_unref = 1;
-
-                dict_copy (xdata, dict);
-
-                ret = stripe_xattr_request_build (this, dict,
-                                                  local->stripe_size,
-                                                  priv->child_count,
-                                                  i, priv->coalesce);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to build xattr request");
-        } else {
-                        dict = xdata;
-        }
-
-
-        STACK_WIND (frame, stripe_first_create_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->create, loc, flags, mode,
-                    umask, fd, dict);
-
-        if (need_unref && dict)
-                dict_unref (dict);
-
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL, xdata);
-        return 0;
-}
-
-int32_t
-stripe_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                        local->op_errno = op_errno;
-                }
-
-                if (op_ret >= 0)
-                        local->op_ret = op_ret;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                STRIPE_STACK_UNWIND (open, frame, local->op_ret,
-                                     local->op_errno, local->fd, xdata);
-        }
-out:
-        return 0;
-}
-
-
-/**
- * stripe_open -
- */
-int32_t
-stripe_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             int32_t flags, fd_t *fd, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        /* files opened in O_APPEND mode does not allow lseek() on fd */
-        flags &= ~O_APPEND;
-
-        local->fd = fd_ref (fd);
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-
-        /* Striped files */
-        local->flags = flags;
-        local->call_count = priv->child_count;
-        local->stripe_size = stripe_get_matching_bs (loc->path, priv);
-
-        while (trav) {
-                STACK_WIND (frame, stripe_open_cbk, trav->xlator,
-                            trav->xlator->fops->open,
-                            &local->loc, local->flags, local->fd,
-                            xdata);
-                trav = trav->next;
-        }
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                }
-
-                if (op_ret >= 0)
-                        local->op_ret = op_ret;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_UNWIND (opendir, frame, local->op_ret,
-                                     local->op_errno, local->fd, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->call_count = priv->child_count;
-        local->fd = fd_ref (fd);
-
-        while (trav) {
-                STACK_WIND (frame, stripe_opendir_cbk, trav->xlator,
-                            trav->xlator->fops->opendir, loc, fd, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-                if (op_ret >= 0) {
-                        if (FIRST_CHILD(this) == prev->this) {
-                                /* First successful call, copy the *lock */
-                                local->op_ret = op_ret;
-                                local->lock = *lock;
-                        }
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-                STRIPE_STACK_UNWIND (lk, frame, local->op_ret,
-                                     local->op_errno, &local->lock, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-           struct gf_flock *lock, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        trav = this->children;
-        priv = this->private;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_lk_cbk, trav->xlator,
-                            trav->xlator->fops->lk, fd, cmd, lock, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-                if (op_ret >= 0)
-                        local->op_ret = op_ret;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                STRIPE_STACK_UNWIND (flush, frame, local->op_ret,
-                                     local->op_errno, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_flush_cbk,  trav->xlator,
-                            trav->xlator->fops->flush, fd, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-
-
-int32_t
-stripe_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-                if (op_ret >= 0) {
-                        local->op_ret = op_ret;
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->pre_buf  = *prebuf;
-                                local->post_buf = *postbuf;
-                        }
-                        local->prebuf_blocks  += prebuf->ia_blocks;
-                        local->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, local->fctx, prev);
-			correct_file_size(postbuf, local->fctx, prev);
-
-                        if (local->prebuf_size < prebuf->ia_size)
-                                local->prebuf_size = prebuf->ia_size;
-
-                        if (local->postbuf_size < postbuf->ia_size)
-                                local->postbuf_size = postbuf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->pre_buf.ia_blocks  = local->prebuf_blocks;
-                        local->pre_buf.ia_size    = local->prebuf_size;
-                        local->post_buf.ia_blocks = local->postbuf_blocks;
-                        local->post_buf.ia_size   = local->postbuf_size;
-                }
-
-                STRIPE_STACK_UNWIND (fsync, frame, local->op_ret,
-                                     local->op_errno, &local->pre_buf,
-                                     &local->post_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-
-	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
-	if (!fctx) {
-		op_errno = EINVAL;
-		goto err;
-	}
-	local->fctx = fctx;
-        local->op_ret = -1;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_fsync_cbk, trav->xlator,
-                            trav->xlator->fops->fsync, fd, flags, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-
-                        if (FIRST_CHILD(this) == prev->this)
-                                local->stbuf = *buf;
-
-                        local->stbuf_blocks += buf->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->stbuf.ia_size   = local->stbuf_size;
-                        local->stbuf.ia_blocks = local->stbuf_blocks;
-                }
-
-                STRIPE_STACK_UNWIND (fstat, frame, local->op_ret,
-                                     local->op_errno, &local->stbuf, NULL);
-        }
-
-out:
-        return 0;
-}
-
-int32_t
-stripe_fstat (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-	if (IA_ISREG(fd->inode->ia_type)) {
-		inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
-		if (!fctx)
-			goto err;
-		local->fctx = fctx;
-	}
-
-        while (trav) {
-                STACK_WIND (frame, stripe_fstat_cbk, trav->xlator,
-                            trav->xlator->fops->fstat, fd, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-	int		  i, eof_idx;
-	off_t		  dest_offset, tmp_offset;
-        int32_t		  op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
-	if (!fctx) {
-		gf_log(this->name, GF_LOG_ERROR, "no stripe context");
-		op_errno = EINVAL;
-		goto err;
-	}
-	if (!fctx->stripe_count) {
-		gf_log(this->name, GF_LOG_ERROR, "no stripe count");
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local->fctx = fctx;
-	eof_idx = (offset / fctx->stripe_size) % fctx->stripe_count;
-
-	for (i = 0; i < fctx->stripe_count; i++) {
-		if (!fctx->xl_array[i]) {
-			gf_log(this->name, GF_LOG_ERROR, "no xlator at index "
-				"%d", i);
-			op_errno = EINVAL;
-			goto err;
-		}
-
-		if (fctx->stripe_coalesce) {
-			if (i < eof_idx)
-				tmp_offset = roof(offset, fctx->stripe_size *
-						fctx->stripe_count);
-			else if (i > eof_idx)
-				tmp_offset = floor(offset, fctx->stripe_size *
-						fctx->stripe_count);
-			else
-				tmp_offset = offset;
-
-			dest_offset = coalesced_offset(tmp_offset,
-				fctx->stripe_size, fctx->stripe_count);
-		} else {
-			dest_offset = offset;
-		}
-
-		STACK_WIND(frame, stripe_truncate_cbk, fctx->xl_array[i],
-			fctx->xl_array[i]->fops->ftruncate, fd, dest_offset,
-			NULL);
-	}
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-                if (op_ret >= 0)
-                        local->op_ret = op_ret;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                STRIPE_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                     local->op_errno, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_fsyncdir_cbk, trav->xlator,
-                            trav->xlator->fops->fsyncdir, fd, flags, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        int32_t         i = 0;
-        int32_t         callcnt = 0;
-        int32_t         count = 0;
-        stripe_local_t *local = NULL;
-        struct iovec   *vec = NULL;
-        struct iatt     tmp_stbuf = {0,};
-        struct iobref  *tmp_iobref = NULL;
-        struct iobuf   *iobuf = NULL;
-	call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        local = frame->local;
-	prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                if (op_ret != -1) {
-			correct_file_size(buf, local->fctx, prev);
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-		}
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                op_ret = 0;
-
-                /* Keep extra space for filling in '\0's */
-                vec = GF_CALLOC ((local->count * 2), sizeof (struct iovec),
-                                 gf_stripe_mt_iovec);
-                if (!vec) {
-                        op_ret = -1;
-                        goto done;
-                }
-
-                for (i = 0; i < local->wind_count; i++) {
-                        if (local->replies[i].op_ret) {
-                                memcpy ((vec + count), local->replies[i].vector,
-                                        (local->replies[i].count * sizeof (struct iovec)));
-                                count +=  local->replies[i].count;
-                                op_ret += local->replies[i].op_ret;
-                        }
-                        if ((local->replies[i].op_ret <
-                             local->replies[i].requested_size) &&
-                            (local->stbuf_size > (local->offset + op_ret))) {
-                                /* Fill in 0s here */
-                                vec[count].iov_len  =
-                                        (local->replies[i].requested_size -
-                                         local->replies[i].op_ret);
-                                iobuf = iobuf_get2 (this->ctx->iobuf_pool,
-                                                    vec[count].iov_len);
-                                if (!iobuf) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Out of memory.");
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        goto done;
-                                }
-                                memset (iobuf->ptr, 0, vec[count].iov_len);
-                                vec[count].iov_base = iobuf->ptr;
-
-                                iobref_add (local->iobref, iobuf);
-                                iobuf_unref(iobuf);
-
-                                op_ret += vec[count].iov_len;
-                                count++;
-                        }
-                        GF_FREE (local->replies[i].vector);
-                }
-
-                /* ENOENT signals EOF to the NFS-server */
-                if (op_ret != -1 && op_ret < local->readv_size &&
-                    (local->offset + op_ret == buf->ia_size))
-                        op_errno = ENOENT;
-
-                /* FIXME: notice that st_ino, and st_dev (gen) will be
-                 * different than what inode will have. Make sure this doesn't
-                 * cause any bugs at higher levels */
-                memcpy (&tmp_stbuf, &local->replies[0].stbuf,
-                        sizeof (struct iatt));
-                tmp_stbuf.ia_size = local->stbuf_size;
-
-        done:
-                GF_FREE (local->replies);
-                tmp_iobref = local->iobref;
-                STRIPE_STACK_UNWIND (readv, frame, op_ret, op_errno, vec,
-                                     count, &tmp_stbuf, tmp_iobref, NULL);
-
-                iobref_unref (tmp_iobref);
-                GF_FREE (vec);
-        }
-out:
-        return 0;
-}
-
-/**
- * stripe_readv_cbk - get all the striped reads, and order it properly, send it
- *        to above layer after putting it in a single vector.
- */
-int32_t
-stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                  int32_t count, struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
-{
-        int32_t         index = 0;
-        int32_t         callcnt = 0;
-        int32_t         final_count = 0;
-        int32_t         need_to_check_proper_size = 0;
-        call_frame_t   *mframe = NULL;
-        stripe_local_t *mlocal = NULL;
-        stripe_local_t *local = NULL;
-        struct iovec   *final_vec = NULL;
-        struct iatt     tmp_stbuf = {0,};
-        struct iatt    *tmp_stbuf_p = NULL; //need it for a warning
-        struct iobref  *tmp_iobref = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-	call_frame_t	*prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto end;
-        }
-
-        local  = frame->local;
-        index  = local->node_index;
-	prev = cookie;
-        mframe = local->orig_frame;
-        if (!mframe)
-                goto out;
-
-        mlocal = mframe->local;
-        if (!mlocal)
-                goto out;
-
-        fctx = mlocal->fctx;
-
-        LOCK (&mframe->lock);
-        {
-                mlocal->replies[index].op_ret = op_ret;
-                mlocal->replies[index].op_errno = op_errno;
-                mlocal->replies[index].requested_size = local->readv_size;
-                if (op_ret >= 0) {
-                        mlocal->replies[index].stbuf  = *stbuf;
-                        mlocal->replies[index].count  = count;
-                        mlocal->replies[index].vector = iov_dup (vector, count);
-
-			correct_file_size(stbuf, fctx, prev);
-
-                        if (local->stbuf_size < stbuf->ia_size)
-                                local->stbuf_size = stbuf->ia_size;
-                        local->stbuf_blocks += stbuf->ia_blocks;
-
-                        if (!mlocal->iobref)
-                                mlocal->iobref = iobref_new ();
-                        iobref_merge (mlocal->iobref, iobref);
-                }
-                callcnt = ++mlocal->call_count;
-        }
-        UNLOCK(&mframe->lock);
-
-        if (callcnt == mlocal->wind_count) {
-                op_ret = 0;
-
-                for (index=0; index < mlocal->wind_count; index++) {
-                        /* check whether each stripe returned
-                         * 'expected' number of bytes */
-                        if (mlocal->replies[index].op_ret == -1) {
-                                op_ret = -1;
-                                op_errno = mlocal->replies[index].op_errno;
-                                break;
-                        }
-                        /* TODO: handle the 'holes' within the read range
-                           properly */
-                        if (mlocal->replies[index].op_ret <
-                            mlocal->replies[index].requested_size) {
-                                need_to_check_proper_size = 1;
-                        }
-
-                        op_ret       += mlocal->replies[index].op_ret;
-                        mlocal->count += mlocal->replies[index].count;
-                }
-                if (op_ret == -1)
-                        goto done;
-                if (need_to_check_proper_size)
-                        goto check_size;
-
-                final_vec = GF_CALLOC (mlocal->count, sizeof (struct iovec),
-                                       gf_stripe_mt_iovec);
-
-                if (!final_vec) {
-                        op_ret = -1;
-                        goto done;
-                }
-
-                for (index = 0; index < mlocal->wind_count; index++) {
-                        memcpy ((final_vec + final_count),
-                                mlocal->replies[index].vector,
-                                (mlocal->replies[index].count *
-                                 sizeof (struct iovec)));
-                        final_count +=  mlocal->replies[index].count;
-                        GF_FREE (mlocal->replies[index].vector);
-                }
-
-                /* FIXME: notice that st_ino, and st_dev (gen) will be
-                 * different than what inode will have. Make sure this doesn't
-                 * cause any bugs at higher levels */
-                memcpy (&tmp_stbuf, &mlocal->replies[0].stbuf,
-                        sizeof (struct iatt));
-                tmp_stbuf.ia_size = local->stbuf_size;
-                tmp_stbuf.ia_blocks = local->stbuf_blocks;
-
-        done:
-                /* */
-                GF_FREE (mlocal->replies);
-                tmp_iobref = mlocal->iobref;
-                /* work around for nfs truncated read. Bug 3774 */
-                tmp_stbuf_p = &tmp_stbuf;
-                WIPE (tmp_stbuf_p);
-                STRIPE_STACK_UNWIND (readv, mframe, op_ret, op_errno, final_vec,
-                                     final_count, &tmp_stbuf, tmp_iobref, NULL);
-
-                iobref_unref (tmp_iobref);
-                GF_FREE (final_vec);
-        }
-
-        goto out;
-
-check_size:
-        mlocal->call_count = fctx->stripe_count;
-
-        for (index = 0; index < fctx->stripe_count; index++) {
-                STACK_WIND (mframe, stripe_readv_fstat_cbk,
-                            (fctx->xl_array[index]),
-                            (fctx->xl_array[index])->fops->fstat,
-                            mlocal->fd, NULL);
-        }
-
-out:
-        STRIPE_STACK_DESTROY (frame);
-end:
-        return 0;
-}
-
-
-int32_t
-stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              size_t size, off_t offset, uint32_t flags, dict_t *xdata)
-{
-        int32_t           op_errno = EINVAL;
-        int32_t           idx = 0;
-        int32_t           index = 0;
-        int32_t           num_stripe = 0;
-        int32_t           off_index = 0;
-        size_t            frame_size = 0;
-        off_t             rounded_end = 0;
-        uint64_t          tmp_fctx = 0;
-        uint64_t          stripe_size = 0;
-        off_t             rounded_start = 0;
-        off_t             frame_offset = offset;
-	off_t		  dest_offset = 0;
-        stripe_local_t   *local = NULL;
-        call_frame_t     *rframe = NULL;
-        stripe_local_t   *rlocal = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EBADFD;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                goto err;
-        }
-        /* The file is stripe across the child nodes. Send the read request
-         * to the child nodes appropriately after checking which region of
-         * the file is in which child node. Always '0-<stripe_size>' part of
-         * the file resides in the first child.
-         */
-        rounded_start = floor (offset, stripe_size);
-        rounded_end = roof (offset+size, stripe_size);
-        num_stripe = (rounded_end- rounded_start)/stripe_size;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-
-        /* This is where all the vectors should be copied. */
-        local->replies = GF_CALLOC (num_stripe, sizeof (struct stripe_replies),
-                                    gf_stripe_mt_stripe_replies);
-        if (!local->replies) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        off_index = (offset / stripe_size) % fctx->stripe_count;
-        local->wind_count = num_stripe;
-        local->readv_size = size;
-        local->offset     = offset;
-        local->fd         = fd_ref (fd);
-        local->fctx       = fctx;
-
-        for (index = off_index; index < (num_stripe + off_index); index++) {
-                rframe = copy_frame (frame);
-                rlocal = mem_get0 (this->local_pool);
-                if (!rlocal) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-
-                frame_size = min (roof (frame_offset+1, stripe_size),
-                                  (offset + size)) - frame_offset;
-
-                rlocal->node_index = index - off_index;
-                rlocal->orig_frame = frame;
-                rlocal->readv_size = frame_size;
-                rframe->local = rlocal;
-                idx = (index % fctx->stripe_count);
-
-		if (fctx->stripe_coalesce)
-			dest_offset = coalesced_offset(frame_offset,
-				stripe_size, fctx->stripe_count);
-		else
-			dest_offset = frame_offset;
-
-                STACK_WIND (rframe, stripe_readv_cbk, fctx->xl_array[idx],
-                            fctx->xl_array[idx]->fops->readv,
-                            fd, frame_size, dest_offset, flags, xdata);
-
-                frame_offset += frame_size;
-        }
-
-        return 0;
-err:
-        if (rframe)
-                STRIPE_STACK_DESTROY (rframe);
-
-        STRIPE_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-	stripe_local_t *mlocal = NULL;
-        call_frame_t   *prev = NULL;
-	call_frame_t   *mframe = NULL;
-	struct stripe_replies *reply = NULL;
-	int32_t		i = 0;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-	mframe = local->orig_frame;
-	mlocal = mframe->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = ++mlocal->call_count;
-
-		mlocal->replies[local->node_index].op_ret = op_ret;
-		mlocal->replies[local->node_index].op_errno = op_errno;
-
-                if (op_ret >= 0) {
-                        mlocal->post_buf = *postbuf;
-                        mlocal->pre_buf = *prebuf;
-
-			mlocal->prebuf_blocks  += prebuf->ia_blocks;
-			mlocal->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, mlocal->fctx, prev);
-			correct_file_size(postbuf, mlocal->fctx, prev);
-
-			if (mlocal->prebuf_size < prebuf->ia_size)
-				mlocal->prebuf_size = prebuf->ia_size;
-			if (mlocal->postbuf_size < postbuf->ia_size)
-				mlocal->postbuf_size = postbuf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
-		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
-		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
-		mlocal->post_buf.ia_size = mlocal->postbuf_size;
-		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
-
-		/*
-		 * Only return the number of consecutively written bytes up until
-		 * the first error. Only return an error if it occurs first.
-		 *
-		 * When a short write occurs, the application should retry at the
-		 * appropriate offset, at which point we'll potentially pass back
-		 * the error.
-		 */
-		for (i = 0, reply = mlocal->replies; i < mlocal->wind_count;
-			i++, reply++) {
-			if (reply->op_ret == -1) {
-				gf_log(this->name, GF_LOG_DEBUG, "reply %d "
-					"returned error %s", i,
-					strerror(reply->op_errno));
-				if (!mlocal->op_ret) {
-					mlocal->op_ret = -1;
-					mlocal->op_errno = reply->op_errno;
-				}
-				break;
-			}
-
-			mlocal->op_ret += reply->op_ret;
-
-			if (reply->op_ret < reply->requested_size)
-				break;
-		}
-
-		GF_FREE(mlocal->replies);
-
-                STRIPE_STACK_UNWIND (writev, mframe, mlocal->op_ret,
-                                     mlocal->op_errno, &mlocal->pre_buf,
-                                     &mlocal->post_buf, NULL);
-        }
-out:
-	STRIPE_STACK_DESTROY(frame);
-        return 0;
-}
-
-int32_t
-stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               struct iovec *vector, int32_t count, off_t offset,
-               uint32_t flags, struct iobref *iobref, dict_t *xdata)
-{
-        struct iovec     *tmp_vec = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-        int32_t           idx = 0;
-        int32_t           total_size = 0;
-        int32_t           offset_offset = 0;
-        int32_t           remaining_size = 0;
-        int32_t           tmp_count = count;
-        off_t             fill_size = 0;
-        uint64_t          stripe_size = 0;
-        uint64_t          tmp_fctx = 0;
-	off_t		  dest_offset = 0;
-	off_t		  rounded_start = 0;
-	off_t		  rounded_end = 0;
-	int32_t		  total_chunks = 0;
-	call_frame_t	  *wframe = NULL;
-	stripe_local_t	  *wlocal = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EINVAL;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        /* File has to be stripped across the child nodes */
-        total_size = iov_length (vector, count);
-        remaining_size = total_size;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->stripe_size = stripe_size;
-	local->fctx = fctx;
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-	rounded_start = floor(offset, stripe_size);
-	rounded_end = roof(offset + total_size, stripe_size);
-	total_chunks = (rounded_end - rounded_start) / stripe_size;
-	local->replies = GF_CALLOC(total_chunks, sizeof(struct stripe_replies),
-				gf_stripe_mt_stripe_replies);
-	if (!local->replies) {
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-	total_chunks = 0;
-        while (1) {
-		wframe = copy_frame(frame);
-		wlocal = mem_get0(this->local_pool);
-		if (!wlocal) {
-			op_errno = ENOMEM;
-			goto err;
-		}
-		wlocal->orig_frame = frame;
-		wframe->local = wlocal;
-
-                /* Send striped chunk of the vector to child
-                   nodes appropriately. */
-                idx = (((offset + offset_offset) /
-                        local->stripe_size) % fctx->stripe_count);
-
-                fill_size = (local->stripe_size -
-                             ((offset + offset_offset) % local->stripe_size));
-                if (fill_size > remaining_size)
-                        fill_size = remaining_size;
-
-                remaining_size -= fill_size;
-
-                tmp_count = iov_subset (vector, count, offset_offset,
-                                        offset_offset + fill_size, NULL);
-                tmp_vec = GF_CALLOC (tmp_count, sizeof (struct iovec),
-                                     gf_stripe_mt_iovec);
-                if (!tmp_vec) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                tmp_count = iov_subset (vector, count, offset_offset,
-                                        offset_offset + fill_size, tmp_vec);
-
-                local->wind_count++;
-                if (remaining_size == 0)
-                        local->unwind = 1;
-
-		/*
-		 * Store off the request index (with respect to the chunk of the
-		 * initial offset) and the size of the request. This is required
-		 * in the callback to calculate an appropriate return value in
-		 * the event of a write failure in one or more requests.
-		 */
-		wlocal->node_index = total_chunks;
-		local->replies[total_chunks].requested_size = fill_size;
-
-		dest_offset = offset + offset_offset;
-		if (fctx->stripe_coalesce)
-			dest_offset = coalesced_offset(dest_offset,
-					local->stripe_size, fctx->stripe_count);
-
-                STACK_WIND (wframe, stripe_writev_cbk, fctx->xl_array[idx],
-                            fctx->xl_array[idx]->fops->writev, fd, tmp_vec,
-                            tmp_count, dest_offset, flags, iobref,
-                            xdata);
-
-                GF_FREE (tmp_vec);
-                offset_offset += fill_size;
-		total_chunks++;
-                if (remaining_size == 0)
-                        break;
-        }
-
-        return 0;
-err:
-	if (wframe)
-		STRIPE_STACK_DESTROY(wframe);
-
-        STRIPE_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-	stripe_local_t *mlocal = NULL;
-        call_frame_t   *prev = NULL;
-	call_frame_t   *mframe = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-	mframe = local->orig_frame;
-	mlocal = mframe->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = ++mlocal->call_count;
-
-                if (op_ret == 0) {
-                        mlocal->post_buf = *postbuf;
-                        mlocal->pre_buf = *prebuf;
-
-			mlocal->prebuf_blocks  += prebuf->ia_blocks;
-			mlocal->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, mlocal->fctx, prev);
-			correct_file_size(postbuf, mlocal->fctx, prev);
-
-			if (mlocal->prebuf_size < prebuf->ia_size)
-				mlocal->prebuf_size = prebuf->ia_size;
-			if (mlocal->postbuf_size < postbuf->ia_size)
-				mlocal->postbuf_size = postbuf->ia_size;
-                }
-
-		/* return the first failure */
-		if (mlocal->op_ret == 0) {
-			mlocal->op_ret = op_ret;
-			mlocal->op_errno = op_errno;
-		}
-        }
-        UNLOCK (&frame->lock);
-
-        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
-		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
-		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
-		mlocal->post_buf.ia_size = mlocal->postbuf_size;
-		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
-
-                STRIPE_STACK_UNWIND (fallocate, mframe, mlocal->op_ret,
-                                     mlocal->op_errno, &mlocal->pre_buf,
-                                     &mlocal->post_buf, NULL);
-        }
-out:
-	STRIPE_STACK_DESTROY(frame);
-        return 0;
-}
-
-int32_t
-stripe_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-		 off_t offset, size_t len, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-        int32_t           idx = 0;
-        int32_t           offset_offset = 0;
-        int32_t           remaining_size = 0;
-        off_t             fill_size = 0;
-        uint64_t          stripe_size = 0;
-        uint64_t          tmp_fctx = 0;
-	off_t		  dest_offset = 0;
-	call_frame_t	  *fframe = NULL;
-	stripe_local_t	  *flocal = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EINVAL;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        remaining_size = len;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->stripe_size = stripe_size;
-	local->fctx = fctx;
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        while (1) {
-		fframe = copy_frame(frame);
-		flocal = mem_get0(this->local_pool);
-		if (!flocal) {
-			op_errno = ENOMEM;
-			goto err;
-		}
-		flocal->orig_frame = frame;
-		fframe->local = flocal;
-
-		/* send fallocate request to the associated child node */
-                idx = (((offset + offset_offset) /
-                        local->stripe_size) % fctx->stripe_count);
-
-                fill_size = (local->stripe_size -
-                             ((offset + offset_offset) % local->stripe_size));
-                if (fill_size > remaining_size)
-                        fill_size = remaining_size;
-
-                remaining_size -= fill_size;
-
-                local->wind_count++;
-                if (remaining_size == 0)
-                        local->unwind = 1;
-
-		dest_offset = offset + offset_offset;
-		if (fctx->stripe_coalesce)
-			dest_offset = coalesced_offset(dest_offset,
-					local->stripe_size, fctx->stripe_count);
-
-		/*
-		 * TODO: Create a separate handler for coalesce mode that sends a
-		 * single fallocate per-child (since the ranges are linear).
-		 */
-		STACK_WIND(fframe, stripe_fallocate_cbk, fctx->xl_array[idx],
-			   fctx->xl_array[idx]->fops->fallocate, fd, mode,
-			   dest_offset, fill_size, xdata);
-
-                offset_offset += fill_size;
-                if (remaining_size == 0)
-                        break;
-        }
-
-        return 0;
-err:
-	if (fframe)
-		STRIPE_STACK_DESTROY(fframe);
-
-        STRIPE_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-	stripe_local_t *mlocal = NULL;
-        call_frame_t   *prev = NULL;
-	call_frame_t   *mframe = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-	mframe = local->orig_frame;
-	mlocal = mframe->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = ++mlocal->call_count;
-
-                if (op_ret == 0) {
-                        mlocal->post_buf = *postbuf;
-                        mlocal->pre_buf = *prebuf;
-
-			mlocal->prebuf_blocks  += prebuf->ia_blocks;
-			mlocal->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, mlocal->fctx, prev);
-			correct_file_size(postbuf, mlocal->fctx, prev);
-
-			if (mlocal->prebuf_size < prebuf->ia_size)
-				mlocal->prebuf_size = prebuf->ia_size;
-			if (mlocal->postbuf_size < postbuf->ia_size)
-				mlocal->postbuf_size = postbuf->ia_size;
-                }
-
-		/* return the first failure */
-		if (mlocal->op_ret == 0) {
-			mlocal->op_ret = op_ret;
-			mlocal->op_errno = op_errno;
-		}
-        }
-        UNLOCK (&frame->lock);
-
-        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
-		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
-		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
-		mlocal->post_buf.ia_size = mlocal->postbuf_size;
-		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
-
-                STRIPE_STACK_UNWIND (discard, mframe, mlocal->op_ret,
-                                     mlocal->op_errno, &mlocal->pre_buf,
-                                     &mlocal->post_buf, NULL);
-        }
-out:
-        if (frame)
-                STRIPE_STACK_DESTROY(frame);
-
-        return 0;
-}
-
-int32_t
-stripe_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	       size_t len, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-        int32_t           idx = 0;
-        int32_t           offset_offset = 0;
-        int32_t           remaining_size = 0;
-        off_t             fill_size = 0;
-        uint64_t          stripe_size = 0;
-        uint64_t          tmp_fctx = 0;
-	off_t		  dest_offset = 0;
-	call_frame_t	  *fframe = NULL;
-	stripe_local_t	  *flocal = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EINVAL;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        remaining_size = len;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->stripe_size = stripe_size;
-	local->fctx = fctx;
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        while (1) {
-		fframe = copy_frame(frame);
-		flocal = mem_get0(this->local_pool);
-		if (!flocal) {
-			op_errno = ENOMEM;
-			goto err;
-		}
-		flocal->orig_frame = frame;
-		fframe->local = flocal;
-
-		/* send discard request to the associated child node */
-                idx = (((offset + offset_offset) /
-                        local->stripe_size) % fctx->stripe_count);
-
-                fill_size = (local->stripe_size -
-                             ((offset + offset_offset) % local->stripe_size));
-                if (fill_size > remaining_size)
-                        fill_size = remaining_size;
-
-                remaining_size -= fill_size;
-
-                local->wind_count++;
-                if (remaining_size == 0)
-                        local->unwind = 1;
-
-		dest_offset = offset + offset_offset;
-		if (fctx->stripe_coalesce)
-			dest_offset = coalesced_offset(dest_offset,
-					local->stripe_size, fctx->stripe_count);
-
-		/*
-		 * TODO: Create a separate handler for coalesce mode that sends a
-		 * single discard per-child (since the ranges are linear).
-		 */
-		STACK_WIND(fframe, stripe_discard_cbk, fctx->xl_array[idx],
-			   fctx->xl_array[idx]->fops->discard, fd, dest_offset,
-			   fill_size, xdata);
-
-                offset_offset += fill_size;
-                if (remaining_size == 0)
-                        break;
-        }
-
-        return 0;
-err:
-	if (fframe)
-		STRIPE_STACK_DESTROY(fframe);
-
-        STRIPE_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt    = 0;
-        stripe_local_t *local      = NULL;
-        stripe_local_t *mlocal     = NULL;
-        call_frame_t   *prev       = NULL;
-        call_frame_t   *mframe     = NULL;
-
-        GF_ASSERT (frame);
-
-        if (!this || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-        mframe = local->orig_frame;
-        mlocal = mframe->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = ++mlocal->call_count;
-
-                if (op_ret == 0) {
-                        mlocal->post_buf = *postbuf;
-                        mlocal->pre_buf = *prebuf;
-
-                        mlocal->prebuf_blocks  += prebuf->ia_blocks;
-                        mlocal->postbuf_blocks += postbuf->ia_blocks;
-
-                        correct_file_size(prebuf, mlocal->fctx, prev);
-                        correct_file_size(postbuf, mlocal->fctx, prev);
-
-                        if (mlocal->prebuf_size < prebuf->ia_size)
-                                mlocal->prebuf_size = prebuf->ia_size;
-                        if (mlocal->postbuf_size < postbuf->ia_size)
-                                mlocal->postbuf_size = postbuf->ia_size;
-                }
-
-                /* return the first failure */
-                if (mlocal->op_ret == 0) {
-                        mlocal->op_ret = op_ret;
-                        mlocal->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
-                mlocal->pre_buf.ia_size = mlocal->prebuf_size;
-                mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
-                mlocal->post_buf.ia_size = mlocal->postbuf_size;
-                mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
-
-                STRIPE_STACK_UNWIND (zerofill, mframe, mlocal->op_ret,
-                                     mlocal->op_errno, &mlocal->pre_buf,
-                                     &mlocal->post_buf, NULL);
-        }
-out:
-        STRIPE_STACK_DESTROY(frame);
-        return 0;
-}
-
-int32_t
-stripe_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               off_t len, dict_t *xdata)
-{
-        stripe_local_t   *local            = NULL;
-        stripe_fd_ctx_t  *fctx             = NULL;
-        int32_t           op_errno         = 1;
-        int32_t           idx              = 0;
-        int32_t           offset_offset    = 0;
-        int32_t           remaining_size   = 0;
-        off_t             fill_size        = 0;
-        uint64_t          stripe_size      = 0;
-        uint64_t          tmp_fctx         = 0;
-        off_t             dest_offset      = 0;
-        call_frame_t     *fframe           = NULL;
-        stripe_local_t   *flocal           = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EINVAL;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        remaining_size = len;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->stripe_size = stripe_size;
-        local->fctx = fctx;
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        while (1) {
-                fframe = copy_frame(frame);
-                flocal = mem_get0(this->local_pool);
-                if (!flocal) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                flocal->orig_frame = frame;
-                fframe->local = flocal;
-
-                idx = (((offset + offset_offset) /
-                        local->stripe_size) % fctx->stripe_count);
-
-                fill_size = (local->stripe_size -
-                             ((offset + offset_offset) % local->stripe_size));
-                if (fill_size > remaining_size)
-                        fill_size = remaining_size;
-
-                remaining_size -= fill_size;
-
-                local->wind_count++;
-                if (remaining_size == 0)
-                        local->unwind = 1;
-
-                dest_offset = offset + offset_offset;
-                if (fctx->stripe_coalesce)
-                        dest_offset = coalesced_offset(dest_offset,
-                                                       local->stripe_size,
-                                                       fctx->stripe_count);
-
-                STACK_WIND(fframe, stripe_zerofill_cbk, fctx->xl_array[idx],
-                           fctx->xl_array[idx]->fops->zerofill, fd,
-                           dest_offset, fill_size, xdata);
-                offset_offset += fill_size;
-                if (remaining_size == 0)
-                        break;
-        }
-
-        return 0;
-err:
-        if (fframe)
-                STRIPE_STACK_DESTROY(fframe);
-
-        STRIPE_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_seek (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-             gf_seek_what_t what, dict_t *xdata)
-{
-        /* TBD */
-        gf_log (this->name, GF_LOG_INFO, "seek called on %s.",
-                uuid_utoa (fd->inode->gfid));
-        STRIPE_STACK_UNWIND (seek, frame, -1, ENOTSUP, 0, NULL);
-        return 0;
-}
-
-int32_t
-stripe_release (xlator_t *this, fd_t *fd)
-{
-	return 0;
-}
-
-int
-stripe_forget (xlator_t *this, inode_t *inode)
-{
-        uint64_t          tmp_fctx = 0;
-        stripe_fd_ctx_t  *fctx = NULL;
-
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (inode, err);
-
-        (void) inode_ctx_del (inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                goto err;
-        }
-
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-
-        if (!fctx->static_array)
-                GF_FREE (fctx->xl_array);
-
-        GF_FREE (fctx);
-err:
-        return 0;
-}
-
-int32_t
-notify (xlator_t *this, int32_t event, void *data, ...)
-{
-        stripe_private_t *priv = NULL;
-        int               down_client = 0;
-        int               i = 0;
-        gf_boolean_t      heard_from_all_children = _gf_false;
-
-        if (!this)
-                return 0;
-
-        priv = this->private;
-        if (!priv)
-                return 0;
-
-        switch (event)
-        {
-        case GF_EVENT_CHILD_UP:
-        {
-                /* get an index number to set */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (data == priv->xl_array[i])
-                                break;
-                }
-
-                if (priv->child_count == i) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "got GF_EVENT_CHILD_UP bad subvolume %s",
-                                data? ((xlator_t *)data)->name: NULL);
-                        break;
-                }
-
-                LOCK (&priv->lock);
-                {
-                        if (data == FIRST_CHILD (this))
-                                priv->first_child_down = 0;
-                        priv->last_event[i] = event;
-                }
-                UNLOCK (&priv->lock);
-        }
-        break;
-        case GF_EVENT_CHILD_CONNECTING:
-        {
-                // 'CONNECTING' doesn't ensure its CHILD_UP, so do nothing
-                goto out;
-        }
-        case GF_EVENT_CHILD_DOWN:
-        {
-                /* get an index number to set */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (data == priv->xl_array[i])
-                                break;
-                }
-
-                if (priv->child_count == i) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "got GF_EVENT_CHILD_DOWN bad subvolume %s",
-                                data? ((xlator_t *)data)->name: NULL);
-                        break;
-                }
-
-                LOCK (&priv->lock);
-                {
-                        if (data == FIRST_CHILD (this))
-                                priv->first_child_down = 1;
-                        priv->last_event[i] = event;
-                }
-                UNLOCK (&priv->lock);
-        }
-        break;
-
-        default:
-        {
-                /* */
-                default_notify (this, event, data);
-                goto out;
-        }
-        break;
-        }
-
-        // Consider child as down if it's last_event is not CHILD_UP
-        for (i = 0, down_client = 0; i < priv->child_count; i++)
-                if (priv->last_event[i] != GF_EVENT_CHILD_UP)
-                        down_client++;
-
-        LOCK (&priv->lock);
-        {
-                priv->nodes_down = down_client;
-        }
-        UNLOCK (&priv->lock);
-
-        heard_from_all_children = _gf_true;
-        for (i = 0; i < priv->child_count; i++)
-                if (!priv->last_event[i])
-                        heard_from_all_children = _gf_false;
-
-        if (heard_from_all_children)
-                default_notify (this, event, data);
-out:
-        return 0;
-}
-
-int
-stripe_setxattr_cbk (call_frame_t *frame, void *cookie,
-                     xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
-{
-        int ret = -1;
-        int call_cnt = 0;
-        stripe_local_t *local = NULL;
-
-        if (!frame || !frame->local || !this) {
-                gf_log ("", GF_LOG_ERROR, "Possible NULL deref");
-                return ret;
-        }
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                call_cnt = --local->wind_count;
-
-                /**
-                 * We overwrite ->op_* values here for subsequent faliure
-                 * conditions, hence we propagate the last errno down the
-                 * stack.
-                 */
-                if (op_ret < 0) {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                        goto unlock;
-                }
-        }
-
- unlock:
-        UNLOCK (&frame->lock);
-
-        if (!call_cnt) {
-                STRIPE_STACK_UNWIND (setxattr, frame, local->op_ret,
-                                     local->op_errno, xdata);
-        }
-
-        return 0;
-}
-
-#ifdef HAVE_BD_XLATOR
-int
-stripe_is_bd (dict_t *this, char *key, data_t *value, void *data)
-{
-        gf_boolean_t *is_bd = data;
-
-        if (data == NULL)
-                return 0;
-
-        if (XATTR_IS_BD (key))
-            *is_bd = _gf_true;
-
-        return 0;
-}
-
-static gf_boolean_t
-stripe_setxattr_is_bd (dict_t *dict)
-{
-        gf_boolean_t is_bd = _gf_false;
-
-        if (dict == NULL)
-                goto out;
-
-        dict_foreach (dict, stripe_is_bd, &is_bd);
-out:
-        return is_bd;
-}
-#else
-#define stripe_setxattr_is_bd(dict) _gf_false
-#endif
-
-int
-stripe_setxattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
-{
-        int32_t           op_errno = EINVAL;
-        xlator_list_t    *trav     = NULL;
-        stripe_private_t *priv     = NULL;
-        stripe_local_t   *local    = NULL;
-        int               i        = 0;
-        gf_boolean_t      is_bd    = _gf_false;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
-                                   op_errno, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-        local->wind_count = priv->child_count;
-        local->op_ret = local->op_errno = 0;
-
-        is_bd = stripe_setxattr_is_bd (dict);
-
-        /**
-         * Set xattrs for directories on all subvolumes. Additionally
-         * this power is only given to a special client. Bd xlator
-         * also needs xattrs for regular files (ie LVs)
-         */
-        if (((frame->root->pid == GF_CLIENT_PID_GSYNCD) &&
-             IA_ISDIR (loc->inode->ia_type)) || is_bd) {
-                for (i = 0; i < priv->child_count; i++, trav = trav->next) {
-                        STACK_WIND (frame, stripe_setxattr_cbk,
-                                    trav->xlator, trav->xlator->fops->setxattr,
-                                    loc, dict, flags, xdata);
-                }
-        } else {
-                local->wind_count = 1;
-                STACK_WIND (frame, stripe_setxattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->setxattr,
-                            loc, dict, flags, xdata);
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (setxattr, frame, -1,  op_errno, NULL);
-        return 0;
-}
-
-
-int
-stripe_fsetxattr_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
-{
-        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-
-int
-stripe_is_special_key (dict_t *this,
-                       char *key,
-                       data_t *value,
-                       void *data)
-{
-        gf_boolean_t *is_special = NULL;
-
-        if (data == NULL) {
-                goto out;
-        }
-
-        is_special = data;
-
-        if (XATTR_IS_LOCKINFO (key) || XATTR_IS_BD (key))
-                *is_special = _gf_true;
-
-out:
-        return 0;
-}
-
-int32_t
-stripe_fsetxattr_everyone_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               dict_t *xdata)
-{
-        int             call_count = 0;
-        stripe_local_t *local      = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                call_count = --local->wind_count;
-
-                if (op_ret < 0) {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (call_count == 0) {
-                STRIPE_STACK_UNWIND (fsetxattr, frame, local->op_ret,
-                                     local->op_errno, NULL);
-        }
-        return 0;
-}
-
-int
-stripe_fsetxattr_to_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                              dict_t *dict, int flags, dict_t *xdata)
-{
-        xlator_list_t    *trav  = NULL;
-        stripe_private_t *priv  = NULL;
-        int               ret   = -1;
-        stripe_local_t   *local = NULL;
-
-        priv = this->private;
-
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                goto out;
-        }
-
-        frame->local = local;
-
-        local->wind_count = priv->child_count;
-
-        trav = this->children;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_fsetxattr_everyone_cbk,
-                            trav->xlator, trav->xlator->fops->fsetxattr,
-                            fd, dict, flags, xdata);
-                trav = trav->next;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-static gf_boolean_t
-stripe_fsetxattr_is_special (dict_t *dict)
-{
-        gf_boolean_t is_spl = _gf_false;
-
-        if (dict == NULL) {
-                goto out;
-        }
-
-        dict_foreach (dict, stripe_is_special_key, &is_spl);
-
-out:
-        return is_spl;
-}
-
-int
-stripe_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  dict_t *dict, int flags, dict_t *xdata)
-{
-        int32_t      op_ret = -1, ret = -1, op_errno = EINVAL;
-        gf_boolean_t is_spl = _gf_false;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
-                                   op_errno, err);
-
-        is_spl = stripe_fsetxattr_is_special (dict);
-        if (is_spl) {
-                ret = stripe_fsetxattr_to_everyone (frame, this, fd, dict,
-                                                    flags, xdata);
-                if (ret < 0) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-
-                goto out;
-        }
-
-        STACK_WIND (frame, stripe_fsetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-out:
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
-        return 0;
-}
-
-int
-stripe_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        STRIPE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-stripe_removexattr (call_frame_t *frame, xlator_t *this,
-                    loc_t *loc, const char *name, dict_t *xdata)
-{
-        int32_t         op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (this, err);
-
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.*stripe*",
-                                 name, op_errno, err);
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        STACK_WIND (frame, stripe_removexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (removexattr, frame, -1,  op_errno, NULL);
-        return 0;
-}
-
-
-int
-stripe_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-stripe_fremovexattr (call_frame_t *frame, xlator_t *this,
-                     fd_t *fd, const char *name, dict_t *xdata)
-{
-        int32_t         op_ret   = -1;
-        int32_t         op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.*stripe*",
-                                 name, op_errno, err);
-
-        STACK_WIND (frame, stripe_fremovexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
- err:
-        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int32_t
-stripe_readdirp_lookup_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int op_ret, int op_errno,
-                            inode_t *inode, struct iatt *stbuf,
-                            dict_t *xattr, struct iatt *parent)
-{
-        stripe_local_t          *local          = NULL;
-        call_frame_t            *main_frame     = NULL;
-        stripe_local_t          *main_local     = NULL;
-        gf_dirent_t             *entry          = NULL;
-        call_frame_t            *prev           = NULL;
-        int                      done           = 0;
-
-        local = frame->local;
-        prev = cookie;
-
-        entry = local->dirent;
-
-        main_frame = local->orig_frame;
-        main_local = main_frame->local;
-        LOCK (&frame->lock);
-        {
-
-                local->call_count--;
-                if (!local->call_count)
-                        done = 1;
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        local->op_ret = op_ret;
-                        goto unlock;
-                }
-
-		if (stripe_ctx_handle(this, prev, local, xattr))
-			gf_log(this->name, GF_LOG_ERROR,
-				"Error getting fctx info from dict.");
-
-		correct_file_size(stbuf, local->fctx, prev);
-
-                stripe_iatt_merge (stbuf, &entry->d_stat);
-                local->stbuf_blocks += stbuf->ia_blocks;
-        }
-unlock:
-        UNLOCK(&frame->lock);
-
-        if (done) {
-                inode_ctx_put (entry->inode, this,
-                               (uint64_t) (long)local->fctx);
-
-                done = 0;
-                LOCK (&main_frame->lock);
-                {
-                        main_local->wind_count--;
-                        if (!main_local->wind_count)
-                                done = 1;
-                        if (local->op_ret == -1) {
-                                main_local->op_errno = local->op_errno;
-                                main_local->op_ret = local->op_ret;
-                        }
-                        entry->d_stat.ia_blocks = local->stbuf_blocks;
-                }
-                UNLOCK (&main_frame->lock);
-                if (done) {
-                        main_frame->local = NULL;
-                        STRIPE_STACK_UNWIND (readdir, main_frame,
-                                             main_local->op_ret,
-                                             main_local->op_errno,
-                                             &main_local->entries, NULL);
-                        gf_dirent_free (&main_local->entries);
-                        stripe_local_wipe (main_local);
-                        mem_put (main_local);
-                }
-                frame->local = NULL;
-                stripe_local_wipe (local);
-                mem_put (local);
-                STRIPE_STACK_DESTROY (frame);
-        }
-
-        return 0;
-}
-
-int32_t
-stripe_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     gf_dirent_t *orig_entries, dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-        gf_dirent_t    *local_entry = NULL;
-        gf_dirent_t    *tmp_entry = NULL;
-        xlator_list_t  *trav = NULL;
-        loc_t          loc = {0, };
-        int32_t        count = 0;
-        stripe_private_t *priv = NULL;
-        int32_t        subvols = 0;
-        dict_t         *xattrs = NULL;
-        call_frame_t   *local_frame = NULL;
-        stripe_local_t *local_ent = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-        prev  = cookie;
-        local = frame->local;
-        trav = this->children;
-        priv = this->private;
-
-        subvols = priv->child_count;
-
-        LOCK (&frame->lock);
-        {
-                local->op_errno = op_errno;
-                local->op_ret = op_ret;
-
-                if (op_ret != -1) {
-                        list_splice_init (&orig_entries->list,
-                                          &local->entries.list);
-                        local->wind_count = op_ret;
-                }
-
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "%s returned error %s",
-                        prev->this->name, strerror (op_errno));
-                goto out;
-        }
-
-        xattrs = dict_new ();
-        if (xattrs)
-                (void) stripe_xattr_request_build (this, xattrs, 0, 0, 0, 0);
-        count = op_ret;
-        list_for_each_entry_safe (local_entry, tmp_entry,
-                                  (&local->entries.list), list) {
-
-                if (!local_entry)
-                        break;
-                if (!IA_ISREG (local_entry->d_stat.ia_type) || !local_entry->inode) {
-                        LOCK (&frame->lock);
-                        {
-                                local->wind_count--;
-                                count = local->wind_count;
-                        }
-                        UNLOCK (&frame->lock);
-                        continue;
-                }
-
-                local_frame = copy_frame (frame);
-
-                if (!local_frame) {
-                        op_errno = ENOMEM;
-                        op_ret = -1;
-                        goto out;
-                }
-
-                local_ent = mem_get0 (this->local_pool);
-                if (!local_ent) {
-                        op_errno = ENOMEM;
-                        op_ret = -1;
-                        goto out;
-                }
-
-                loc.inode = inode_ref (local_entry->inode);
-
-                gf_uuid_copy (loc.gfid, local_entry->d_stat.ia_gfid);
-
-                local_ent->orig_frame = frame;
-
-                local_ent->call_count = subvols;
-
-                local_ent->dirent = local_entry;
-
-                local_frame->local = local_ent;
-
-                trav = this->children;
-                while (trav) {
-                        STACK_WIND (local_frame, stripe_readdirp_lookup_cbk,
-                                    trav->xlator, trav->xlator->fops->lookup,
-                                    &loc, xattrs);
-                        trav = trav->next;
-                }
-                loc_wipe (&loc);
-        }
-out:
-        if (!count) {
-                /* all entries are directories */
-                frame->local = NULL;
-                STRIPE_STACK_UNWIND (readdir, frame,
-                                     (local ? local->op_ret : -1),
-                                     (local ? local->op_errno : EINVAL),
-                                     (local ? &local->entries : NULL),
-                                     NULL);
-                gf_dirent_free (&local->entries);
-                stripe_local_wipe (local);
-                mem_put (local);
-        }
-        if (xattrs)
-                dict_unref (xattrs);
-        return 0;
-
-}
-int32_t
-stripe_readdirp (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, size_t size, off_t off, dict_t *xdata)
-{
-        stripe_local_t  *local  = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t   *trav = NULL;
-        int             op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-
-        local->fd = fd_ref (fd);
-
-        local->wind_count = 0;
-
-        local->count = 0;
-        local->op_ret = -1;
-        INIT_LIST_HEAD(&local->entries);
-
-        if (!trav)
-                goto err;
-
-        STACK_WIND (frame, stripe_readdirp_cbk, trav->xlator,
-                    trav->xlator->fops->readdirp, fd, size, off, xdata);
-        return 0;
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STRIPE_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
-
-        return 0;
-
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                goto out;
-
-        ret = xlator_mem_acct_init (this, gf_stripe_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-static int
-clear_pattern_list (stripe_private_t *priv)
-{
-        struct stripe_options *prev = NULL;
-        struct stripe_options *trav = NULL;
-        int                    ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("stripe", priv, out);
-
-        trav = priv->pattern;
-        priv->pattern = NULL;
-        while (trav) {
-                prev = trav;
-                trav = trav->next;
-                GF_FREE (prev);
-        }
-
-        ret = 0;
- out:
-        return ret;
-
-
-}
-
-
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-
-        stripe_private_t *priv = NULL;
-        data_t           *data = NULL;
-        int               ret = -1;
-        volume_option_t  *opt = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (this->private);
-
-        priv = this->private;
-
-
-        ret = 0;
-        LOCK (&priv->lock);
-        {
-                ret = clear_pattern_list (priv);
-                if (ret)
-                         goto unlock;
-
-                data = dict_get (options, "block-size");
-                if (data) {
-                        ret = set_stripe_block_size (this, priv, data->data);
-                        if (ret)
-                                goto unlock;
-                } else {
-                        opt = xlator_volume_option_get (this, "block-size");
-                        if (!opt) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "option 'block-size' not found");
-                                ret = -1;
-                                goto unlock;
-                        }
-
-                        if (gf_string2bytesize_uint64 (opt->default_value, &priv->block_size)){
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to set default block-size ");
-                                ret = -1;
-                                goto unlock;
-                        }
-                }
-
-		GF_OPTION_RECONF("coalesce", priv->coalesce, options, bool,
-				unlock);
-        }
- unlock:
-        UNLOCK (&priv->lock);
-        if (ret)
-                goto out;
-
-        ret = 0;
- out:
-        return ret;
-
-}
-
-/**
- * init - This function is called when xlator-graph gets initialized.
- *     The option given in volfiles are parsed here.
- * @this -
- */
-int32_t
-init (xlator_t *this)
-{
-        stripe_private_t *priv = NULL;
-        volume_option_t  *opt = NULL;
-        xlator_list_t    *trav = NULL;
-        data_t           *data = NULL;
-        int32_t           count = 0;
-        int               ret = -1;
-
-        if (!this)
-                goto out;
-
-        trav = this->children;
-        while (trav) {
-                count++;
-                trav = trav->next;
-        }
-
-        if (!count) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "stripe configured without \"subvolumes\" option. "
-                        "exiting");
-                goto out;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
-
-        if (count == 1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "stripe configured with only one \"subvolumes\" option."
-                        " please check the volume. exiting");
-                goto out;
-        }
-
-        priv = GF_CALLOC (1, sizeof (stripe_private_t),
-                          gf_stripe_mt_stripe_private_t);
-
-        if (!priv)
-                goto out;
-        priv->xl_array = GF_CALLOC (count, sizeof (xlator_t *),
-                                    gf_stripe_mt_xlator_t);
-        if (!priv->xl_array)
-                goto out;
-
-        priv->last_event = GF_CALLOC (count, sizeof (int),
-                                      gf_stripe_mt_int32_t);
-        if (!priv->last_event)
-                goto out;
-
-        priv->child_count = count;
-        LOCK_INIT (&priv->lock);
-
-        trav = this->children;
-        count = 0;
-        while (trav) {
-                priv->xl_array[count++] = trav->xlator;
-                trav = trav->next;
-        }
-
-        if (count > 256) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "maximum number of stripe subvolumes supported "
-                        "is 256");
-                goto out;
-        }
-
-        ret = 0;
-        LOCK (&priv->lock);
-        {
-                opt = xlator_volume_option_get (this, "block-size");
-                if (!opt) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "option 'block-size' not found");
-                        ret = -1;
-                        goto unlock;
-                }
-                if (gf_string2bytesize_uint64 (opt->default_value, &priv->block_size)){
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to set default block-size ");
-                        ret = -1;
-                        goto unlock;
-                }
-                /* option stripe-pattern *avi:1GB,*pdf:16K */
-                data = dict_get (this->options, "block-size");
-                if (data) {
-                        ret = set_stripe_block_size (this, priv, data->data);
-                        if (ret)
-                                goto unlock;
-                }
-        }
- unlock:
-        UNLOCK (&priv->lock);
-        if (ret)
-                goto out;
-
-        GF_OPTION_INIT ("use-xattr", priv->xattr_supported, bool, out);
-        /* notify related */
-        priv->nodes_down = priv->child_count;
-
-	GF_OPTION_INIT("coalesce", priv->coalesce, bool, out);
-
-        this->local_pool = mem_pool_new (stripe_local_t, 128);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
-
-        this->private = priv;
-
-        ret = 0;
-out:
-        if (ret) {
-                if (priv) {
-                        GF_FREE (priv->xl_array);
-                        GF_FREE (priv);
-                }
-        }
-        return ret;
-}
-
-/**
- * fini -   Free all the private variables
- * @this -
- */
-void
-fini (xlator_t *this)
-{
-        stripe_private_t      *priv = NULL;
-        struct stripe_options *prev = NULL;
-        struct stripe_options *trav = NULL;
-
-        if (!this)
-                goto out;
-
-        priv = this->private;
-        if (priv) {
-                this->private = NULL;
-                GF_FREE (priv->xl_array);
-
-                trav = priv->pattern;
-                while (trav) {
-                        prev = trav;
-                        trav = trav->next;
-                        GF_FREE (prev);
-                }
-                GF_FREE (priv->last_event);
-                LOCK_DESTROY (&priv->lock);
-                GF_FREE (priv);
-        }
-
-out:
-        return;
-}
-
-int32_t
-stripe_getxattr_unwind (call_frame_t *frame,
-                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
-
-{
-        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
-}
-
-int
-stripe_internal_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int op_ret, int op_errno, dict_t *xattr,
-                              dict_t *xdata)
-{
-
-        char        size_key[256]  = {0,};
-        char        index_key[256] = {0,};
-        char        count_key[256] = {0,};
-        char        coalesce_key[256] = {0,};
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
-
-        if (!xattr || (op_ret == -1))
-            goto out;
-
-        sprintf (size_key, "trusted.%s.stripe-size", this->name);
-        sprintf (count_key, "trusted.%s.stripe-count", this->name);
-        sprintf (index_key, "trusted.%s.stripe-index", this->name);
-	sprintf (coalesce_key, "trusted.%s.stripe-coalesce", this->name);
-
-        dict_del (xattr, size_key);
-        dict_del (xattr, count_key);
-        dict_del (xattr, index_key);
-        dict_del (xattr, coalesce_key);
-
-out:
-        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
-
-        return 0;
-
-}
-
-int
-stripe_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
-{
-        int                     call_cnt = 0;
-        stripe_local_t         *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                call_cnt = --local->wind_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!xattr || (op_ret < 0))
-                goto out;
-
-        local->op_ret = 0;
-
-        if (!local->xattr) {
-                local->xattr = dict_ref (xattr);
-        } else {
-                stripe_aggregate_xattr (local->xattr, xattr);
-        }
-
-out:
-        if (!call_cnt) {
-                STRIPE_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
-                                     local->xattr, xdata);
-        }
-
-        return 0;
-}
-
-int32_t
-stripe_vgetxattr_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int32_t op_ret, int32_t op_errno,
-                      dict_t *dict, dict_t *xdata)
-{
-        stripe_local_t      *local         = NULL;
-        int32_t              callcnt       = 0;
-        int32_t              ret           = -1;
-        long                 cky           = 0;
-        void                *xattr_val     = NULL;
-        void                *xattr_serz    = NULL;
-        stripe_xattr_sort_t *xattr         = NULL;
-        dict_t              *stripe_xattr  = NULL;
-
-        if (!frame || !frame->local || !this) {
-                gf_log ("", GF_LOG_ERROR, "Possible NULL deref");
-                return ret;
-        }
-
-        local = frame->local;
-        cky = (long) cookie;
-
-        if (local->xsel[0] == '\0') {
-                gf_log (this->name, GF_LOG_ERROR, "Empty xattr in cbk");
-                return ret;
-        }
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->wind_count;
-
-                if (!dict || (op_ret < 0))
-                        goto out;
-
-                if (!local->xattr_list)
-                        local->xattr_list = (stripe_xattr_sort_t *)
-                                GF_CALLOC (local->nallocs,
-                                           sizeof (stripe_xattr_sort_t),
-                                           gf_stripe_mt_xattr_sort_t);
-
-                if (local->xattr_list) {
-                        xattr = local->xattr_list + (int32_t) cky;
-
-                        ret = dict_get_ptr_and_len (dict, local->xsel,
-                                                    &xattr_val,
-                                                    &xattr->xattr_len);
-                        if (xattr->xattr_len == 0)
-                                goto out;
-
-                        xattr->pos = cky;
-                        xattr->xattr_value = gf_memdup (xattr_val,
-                                                        xattr->xattr_len);
-
-                        if (xattr->xattr_value != NULL)
-                                local->xattr_total_len += xattr->xattr_len + 1;
-                }
-        }
- out:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (!local->xattr_total_len)
-                        goto unwind;
-
-                stripe_xattr = dict_new ();
-                if (!stripe_xattr)
-                        goto unwind;
-
-                /* select filler based on ->xsel */
-                if (XATTR_IS_PATHINFO (local->xsel))
-                        ret = stripe_fill_pathinfo_xattr (this, local,
-                                                          (char **)&xattr_serz);
-                else if (XATTR_IS_LOCKINFO (local->xsel)) {
-                        ret = stripe_fill_lockinfo_xattr (this, local,
-                                                          &xattr_serz);
-                } else {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unknown xattr in xattr request");
-                        goto unwind;
-                }
-
-                if (!ret) {
-                        ret = dict_set_dynptr (stripe_xattr, local->xsel,
-                                               xattr_serz,
-                                               local->xattr_total_len);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Can't set %s key in dict",
-                                        local->xsel);
-                }
-
-        unwind:
-                /*
-                 * Among other things, STRIPE_STACK_UNWIND will free "local"
-                 * for us.  That means we can't dereference it afterward.
-                 * Fortunately, the actual result is in stripe_xattr now, so we
-                 * can simply clean up before unwinding.
-                 */
-                ret = stripe_free_xattr_str (local);
-                GF_FREE (local->xattr_list);
-                local->xattr_list = NULL;
-
-                STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno,
-                                     stripe_xattr, NULL);
-
-                if (stripe_xattr)
-                        dict_unref (stripe_xattr);
-        }
-
-        return ret;
-}
-
-int
-stripe_marker_populate_args (call_frame_t *frame, int type, int *gauge,
-                            xlator_t **subvols)
-{
-        xlator_t         *this  = frame->this;
-        stripe_private_t *priv  = this->private;
-        stripe_local_t   *local = frame->local;
-        int              count  = 0;
-
-        count = priv->child_count;
-        if (MARKER_XTIME_TYPE == type) {
-                if (!IA_FILE_OR_DIR (local->loc.inode->ia_type))
-                        count = 1;
-        }
-        memcpy (subvols, priv->xl_array, sizeof (*subvols) * count);
-
-        return count;
-}
-
-int32_t
-stripe_getxattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name, dict_t *xdata)
-{
-        stripe_local_t    *local    = NULL;
-        xlator_list_t     *trav     = NULL;
-        stripe_private_t  *priv     = NULL;
-        int32_t            op_errno = EINVAL;
-        int                i        = 0;
-        int                ret      = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-
-
-        if (name && strncmp (name, QUOTA_SIZE_KEY,
-                             strlen (QUOTA_SIZE_KEY)) == 0) {
-                local->wind_count = priv->child_count;
-
-                for (i = 0, trav=this->children; i < priv->child_count; i++,
-                             trav = trav->next) {
-                        STACK_WIND (frame, stripe_getxattr_cbk,
-                                    trav->xlator, trav->xlator->fops->getxattr,
-                                    loc, name, xdata);
-                }
-
-                return 0;
-        }
-
-        if (name && (XATTR_IS_PATHINFO (name))) {
-                if (IA_ISREG (loc->inode->ia_type)) {
-                        ret = inode_ctx_get (loc->inode, this,
-                                             (uint64_t *) &local->fctx);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "stripe size unavailable from fctx"
-                                        " relying on pathinfo could lead to"
-                                        " wrong results");
-                }
-
-                local->nallocs = local->wind_count = priv->child_count;
-                (void) strncpy (local->xsel, name, strlen (name));
-
-                /**
-                 * for xattrs that need info from all childs, fill ->xsel
-                 * as above and call the filler function in cbk based on
-                 * it
-                 */
-                for (i = 0, trav = this->children; i < priv->child_count; i++,
-                     trav = trav->next) {
-                        STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
-                                           (void *) (long) i, trav->xlator,
-                                           trav->xlator->fops->getxattr,
-                                           loc, name, xdata);
-                }
-
-                return 0;
-        }
-
-        if (cluster_handle_marker_getxattr (frame, loc, name, priv->vol_uuid,
-                                            stripe_getxattr_unwind,
-                                            stripe_marker_populate_args) == 0)
-                return 0;
-
-        STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
-
-        return 0;
-
-err:
-        STRIPE_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-static gf_boolean_t
-stripe_is_special_xattr (const char *name)
-{
-        gf_boolean_t    is_spl = _gf_false;
-
-        if (!name) {
-                goto out;
-        }
-
-        if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
-                      strlen (GF_XATTR_LOCKINFO_KEY))
-            || XATTR_IS_PATHINFO (name))
-                is_spl = _gf_true;
-out:
-        return is_spl;
-}
-
-int32_t
-stripe_fgetxattr_from_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                                const char *name, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv  = NULL;
-        int32_t           ret   = -1, op_errno = 0;
-        int               i     = 0;
-        xlator_list_t    *trav  = NULL;
-
-        priv = this->private;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->op_ret = -1;
-        frame->local = local;
-
-        strncpy (local->xsel, name, strlen (name));
-        local->nallocs = local->wind_count = priv->child_count;
-
-        for (i = 0, trav = this->children; i < priv->child_count; i++,
-                     trav = trav->next) {
-                STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
-                                   (void *) (long) i, trav->xlator,
-                                   trav->xlator->fops->fgetxattr,
-                                   fd, name, xdata);
-        }
-
-        return 0;
-
-err:
-        STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, NULL);
-        return ret;
-}
-
-int32_t
-stripe_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  const char *name, dict_t *xdata)
-{
-        if (stripe_is_special_xattr (name)) {
-                stripe_fgetxattr_from_everyone (frame, this, fd, name, xdata);
-                goto out;
-        }
-
-        STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
-
-out:
-        return 0;
-}
-
-
-
-int32_t
-stripe_priv_dump (xlator_t *this)
-{
-        char                    key[GF_DUMP_MAX_BUF_LEN];
-        int                     i = 0;
-        stripe_private_t       *priv = NULL;
-        int                     ret = -1;
-        struct stripe_options  *options = NULL;
-
-        GF_VALIDATE_OR_GOTO ("stripe", this, out);
-
-        priv = this->private;
-        if (!priv)
-                goto out;
-
-        ret = TRY_LOCK (&priv->lock);
-        if (ret != 0)
-                goto out;
-
-        gf_proc_dump_add_section("xlator.cluster.stripe.%s.priv", this->name);
-        gf_proc_dump_write("child_count","%d", priv->child_count);
-
-        for (i = 0; i < priv->child_count; i++) {
-                sprintf (key, "subvolumes[%d]", i);
-                gf_proc_dump_write (key, "%s.%s", priv->xl_array[i]->type,
-                                    priv->xl_array[i]->name);
-        }
-
-        options = priv->pattern;
-        while (options != NULL) {
-                gf_proc_dump_write ("path_pattern", "%s", priv->pattern->path_pattern);
-                gf_proc_dump_write ("options_block_size", "%ul", options->block_size);
-
-                options = options->next;
-        }
-
-        gf_proc_dump_write ("block_size", "%ul", priv->block_size);
-        gf_proc_dump_write ("nodes-down", "%d", priv->nodes_down);
-        gf_proc_dump_write ("first-child_down", "%d", priv->first_child_down);
-        gf_proc_dump_write ("xattr_supported", "%d", priv->xattr_supported);
-
-        UNLOCK (&priv->lock);
-
-out:
-        return ret;
-}
-
-struct xlator_fops fops = {
-        .stat           = stripe_stat,
-        .unlink         = stripe_unlink,
-        .rename         = stripe_rename,
-        .link           = stripe_link,
-        .truncate       = stripe_truncate,
-        .create         = stripe_create,
-        .open           = stripe_open,
-        .readv          = stripe_readv,
-        .writev         = stripe_writev,
-        .statfs         = stripe_statfs,
-        .flush          = stripe_flush,
-        .fsync          = stripe_fsync,
-        .ftruncate      = stripe_ftruncate,
-        .fstat          = stripe_fstat,
-        .mkdir          = stripe_mkdir,
-        .rmdir          = stripe_rmdir,
-        .lk             = stripe_lk,
-        .opendir        = stripe_opendir,
-        .fsyncdir       = stripe_fsyncdir,
-        .setattr        = stripe_setattr,
-        .fsetattr       = stripe_fsetattr,
-        .lookup         = stripe_lookup,
-        .mknod          = stripe_mknod,
-        .setxattr       = stripe_setxattr,
-        .fsetxattr      = stripe_fsetxattr,
-        .getxattr       = stripe_getxattr,
-        .fgetxattr      = stripe_fgetxattr,
-        .removexattr    = stripe_removexattr,
-        .fremovexattr   = stripe_fremovexattr,
-        .readdirp       = stripe_readdirp,
-	.fallocate	= stripe_fallocate,
-	.discard	= stripe_discard,
-        .zerofill       = stripe_zerofill,
-        .seek           = stripe_seek,
-};
-
-struct xlator_cbks cbks = {
-        .release = stripe_release,
-        .forget  = stripe_forget,
-};
-
-struct xlator_dumpops dumpops = {
-        .priv = stripe_priv_dump,
-};
-
-struct volume_options options[] = {
-        { .key  = {"block-size"},
-          .type = GF_OPTION_TYPE_SIZE_LIST,
-          .default_value = "128KB",
-          .min = STRIPE_MIN_BLOCK_SIZE,
-          .description = "Size of the stripe unit that would be read "
-          "from or written to the striped servers.",
-          .op_version = { 1 },
-          .tags = { "stripe" },
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
-        },
-        { .key  = {"use-xattr"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "true",
-          .description = "handle the stripe without the xattr",
-          .tags = { "stripe", "dev-only" },
-          .flags = OPT_FLAG_CLIENT_OPT,
-        },
-	{ .key = {"coalesce"},
-	  .type = GF_OPTION_TYPE_BOOL,
-	  .default_value = "true",
-	  .description = "Enable/Disable coalesce mode to flatten striped "
-          "files as stored on the server (i.e., eliminate holes "
-          "caused by the traditional format).",
-          .op_version = { 1 },
-          .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
-          .tags = { "stripe" },
-	},
-        { .key  = {NULL} },
-};
diff --git a/xlators/cluster/stripe/src/stripe.h b/xlators/cluster/stripe/src/stripe.h
deleted file mode 100644
index 1e2fcb4e659..00000000000
--- a/xlators/cluster/stripe/src/stripe.h
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef _STRIPE_H_
-#define _STRIPE_H_
-
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "stripe-mem-types.h"
-#include "libxlator.h"
-#include <fnmatch.h>
-#include <signal.h>
-
-#define STRIPE_PATHINFO_HEADER "STRIPE:"
-#define STRIPE_MIN_BLOCK_SIZE  (16*GF_UNIT_KB)
-
-#define STRIPE_STACK_UNWIND(fop, frame, params ...) do {           \
-                stripe_local_t *__local = NULL;                    \
-                if (frame) {                                       \
-                        __local = frame->local;                    \
-                        frame->local = NULL;                       \
-                }                                                  \
-                STACK_UNWIND_STRICT (fop, frame, params);          \
-                if (__local) {                                     \
-                        stripe_local_wipe(__local);                \
-                        mem_put (__local);       \
-                }                                                  \
-        } while (0)
-
-#define STRIPE_STACK_DESTROY(frame) do {                        \
-                stripe_local_t *__local = NULL;                 \
-                __local = frame->local;                         \
-                frame->local = NULL;                            \
-                STACK_DESTROY (frame->root);                    \
-                if (__local) {                                  \
-                        stripe_local_wipe (__local);            \
-                        mem_put (__local);    \
-                }                                               \
-        } while (0)
-
-#define STRIPE_VALIDATE_FCTX(fctx, label) do {                  \
-        int     idx = 0;                                        \
-        if (!fctx) {                                            \
-                op_errno = EINVAL;                              \
-                goto label;                                     \
-        }                                                       \
-        for (idx = 0; idx < fctx->stripe_count; idx++) {        \
-                if (!fctx->xl_array[idx]) {                     \
-                        gf_log (this->name, GF_LOG_ERROR,       \
-                                "fctx->xl_array[%d] is NULL",   \
-                                idx);                           \
-                        op_errno = ESTALE;                      \
-                        goto label;                             \
-                }                                               \
-        }                                                       \
-       } while (0)
-
-typedef struct stripe_xattr_sort {
-        int   pos;
-        int   xattr_len;
-        char *xattr_value;
-} stripe_xattr_sort_t;
-
-/**
- * struct stripe_options : This keeps the pattern and the block-size
- *     information, which is used for striping on a file.
- */
-struct stripe_options {
-        struct stripe_options *next;
-        char                   path_pattern[256];
-        uint64_t               block_size;
-};
-
-/**
- * Private structure for stripe translator
- */
-struct stripe_private {
-        struct stripe_options  *pattern;
-        xlator_t              **xl_array;
-        uint64_t                block_size;
-        gf_lock_t               lock;
-        uint8_t                 nodes_down;
-        int8_t                  first_child_down;
-        int                    *last_event;
-        int8_t                  child_count;
-        gf_boolean_t            xattr_supported;  /* default yes */
-	gf_boolean_t		coalesce;
-        char                    vol_uuid[UUID_SIZE + 1];
-};
-
-/**
- * Used to keep info about the replies received from readv/writev calls
- */
-struct stripe_replies {
-        struct iovec *vector;
-        int32_t       count;    //count of vector
-        int32_t       op_ret;   //op_ret of readv
-        int32_t       op_errno;
-        int32_t       requested_size;
-        struct iatt   stbuf;    /* 'stbuf' is also a part of reply */
-};
-
-typedef struct _stripe_fd_ctx {
-        off_t      stripe_size;
-        int        stripe_count;
-	int	   stripe_coalesce;
-        int        static_array;
-        xlator_t **xl_array;
-} stripe_fd_ctx_t;
-
-
-/**
- * Local structure to be passed with all the frames in case of STACK_WIND
- */
-struct stripe_local; /* this itself is used inside the structure; */
-
-struct stripe_local {
-        struct stripe_local *next;
-        call_frame_t        *orig_frame;
-
-        stripe_fd_ctx_t     *fctx;
-
-        /* Used by _cbk functions */
-        struct iatt          stbuf;
-        struct iatt          pre_buf;
-        struct iatt          post_buf;
-        struct iatt          preparent;
-        struct iatt          postparent;
-
-        off_t                stbuf_size;
-        off_t                prebuf_size;
-        off_t                postbuf_size;
-        off_t                preparent_size;
-        off_t                postparent_size;
-
-        blkcnt_t             stbuf_blocks;
-        blkcnt_t             prebuf_blocks;
-        blkcnt_t             postbuf_blocks;
-        blkcnt_t             preparent_blocks;
-        blkcnt_t             postparent_blocks;
-
-        struct stripe_replies *replies;
-        struct statvfs        statvfs_buf;
-        dir_entry_t          *entry;
-
-        int8_t               revalidate;
-        int8_t               failed;
-        int8_t               unwind;
-
-        size_t               readv_size;
-        int32_t              entry_count;
-        int32_t              node_index;
-        int32_t              call_count;
-        int32_t              wind_count; /* used instead of child_cound
-                                            in case of read and write */
-        int32_t              op_ret;
-        int32_t              op_errno;
-        int32_t              count;
-        int32_t              flags;
-        char                *name;
-        inode_t             *inode;
-
-        loc_t                loc;
-        loc_t                loc2;
-
-        mode_t               mode;
-        dev_t                rdev;
-        /* For File I/O fops */
-        dict_t              *xdata;
-
-        stripe_xattr_sort_t *xattr_list;
-        int32_t              xattr_total_len;
-        int32_t              nallocs;
-        char xsel[256];
-
-        /* General usage */
-        off_t                offset;
-        off_t                stripe_size;
-
-        int xattr_self_heal_needed;
-        int entry_self_heal_needed;
-
-        int8_t              *list;
-        struct gf_flock         lock;
-        fd_t                *fd;
-        void                *value;
-        struct iobref       *iobref;
-        gf_dirent_t          entries;
-        gf_dirent_t         *dirent;
-        dict_t              *xattr;
-        uuid_t               ia_gfid;
-
-        int                  xflag;
-        mode_t               umask;
-};
-
-typedef struct stripe_local   stripe_local_t;
-typedef struct stripe_private stripe_private_t;
-
-/*
- * Determine the stripe index of a particular frame based on the translator.
- */
-static inline int32_t stripe_get_frame_index(stripe_fd_ctx_t *fctx,
-					     call_frame_t *prev)
-{
-	int32_t i, idx = -1;
-
-	for (i = 0; i < fctx->stripe_count; i++) {
-		if (fctx->xl_array[i] == prev->this) {
-			idx = i;
-			break;
-		}
-	}
-
-	return idx;
-}
-
-static inline void stripe_copy_xl_array(xlator_t **dst, xlator_t **src,
-					int count)
-{
-	int i;
-
-	for (i = 0; i < count; i++)
-		dst[i] = src[i];
-}
-
-void stripe_local_wipe (stripe_local_t *local);
-int32_t stripe_ctx_handle (xlator_t *this, call_frame_t *prev,
-                           stripe_local_t *local, dict_t *dict);
-void stripe_aggregate_xattr (dict_t *dst, dict_t *src);
-int32_t stripe_xattr_request_build (xlator_t *this, dict_t *dict,
-                                    uint64_t stripe_size, uint32_t stripe_count,
-                                    uint32_t stripe_index,
-				    uint32_t stripe_coalesce);
-int32_t stripe_get_matching_bs (const char *path, stripe_private_t *priv);
-int set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data);
-int32_t stripe_iatt_merge (struct iatt *from, struct iatt *to);
-int32_t stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
-                                    char **xattr_serz);
-int32_t stripe_free_xattr_str (stripe_local_t *local);
-int32_t stripe_xattr_aggregate (char *buffer, stripe_local_t *local,
-                                int32_t *total);
-off_t coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count);
-off_t uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
-			int stripe_index);
-int32_t
-stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
-                            void **xattr_serz);
-
-/*
- * Adjust the size attribute for files if coalesce is enabled.
- */
-static inline void correct_file_size(struct iatt *buf, stripe_fd_ctx_t *fctx,
-	call_frame_t *prev)
-{
-	int index;
-
-	if (!IA_ISREG(buf->ia_type))
-		return;
-
-	if (!fctx || !fctx->stripe_coalesce)
-		return;
-
-	index = stripe_get_frame_index(fctx, prev);
-	buf->ia_size = uncoalesced_size(buf->ia_size, fctx->stripe_size,
-		fctx->stripe_count, index);
-}
-
-#endif /* _STRIPE_H_ */
diff --git a/xlators/debug/delay-gen/src/delay-gen-mem-types.h b/xlators/debug/delay-gen/src/delay-gen-mem-types.h
index 8c54f5f922e..c89a9217193 100644
--- a/xlators/debug/delay-gen/src/delay-gen-mem-types.h
+++ b/xlators/debug/delay-gen/src/delay-gen-mem-types.h
@@ -8,15 +8,14 @@
  *   cases as published by the Free Software Foundation.
  */
 
-
 #ifndef __DELAY_GEN_MEM_TYPES_H__
 #define __DELAY_GEN_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_delay_gen_mem_types_ {
-        gf_delay_gen_mt_dg_t = gf_common_mt_end + 1,
-        gf_delay_gen_mt_end
+    gf_delay_gen_mt_dg_t = gf_common_mt_end + 1,
+    gf_delay_gen_mt_end
 };
 
 #endif /* __DELAY_GEN_MEM_TYPES_H__ */
diff --git a/xlators/debug/delay-gen/src/delay-gen-messages.h b/xlators/debug/delay-gen/src/delay-gen-messages.h
index a9046ca14bf..bc98cec2885 100644
--- a/xlators/debug/delay-gen/src/delay-gen-messages.h
+++ b/xlators/debug/delay-gen/src/delay-gen-messages.h
@@ -11,7 +11,7 @@
 #ifndef __DELAY_GEN_MESSAGES_H__
 #define __DELAY_GEN_MESSAGES_H__
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
 
 /* To add new message IDs, append new identifiers at the end of the list.
  *
diff --git a/xlators/debug/delay-gen/src/delay-gen.c b/xlators/debug/delay-gen/src/delay-gen.c
index 849ebeaabcd..4698f1fd785 100644
--- a/xlators/debug/delay-gen/src/delay-gen.c
+++ b/xlators/debug/delay-gen/src/delay-gen.c
@@ -8,741 +8,690 @@
  *  cases as published by the Free Software Foundation.
  */
 
-
 #include "delay-gen.h"
 
-#define DELAY_GRANULARITY     (1 << 20)
+#define DELAY_GRANULARITY (1 << 20)
 
 #define DG_FOP(fop, name, frame, this, args...)                                \
-        do {                                                                   \
-                delay_gen (this, fop);                                         \
-                default_##name (frame, this, args);                            \
-        } while (0)
+    do {                                                                       \
+        delay_gen(this, fop);                                                  \
+        default_##name(frame, this, args);                                     \
+    } while (0)
 
 int
-delay_gen (xlator_t *this, int fop)
+delay_gen(xlator_t *this, int fop)
 {
-        dg_t             *dg = this->private;
+    dg_t *dg = this->private;
 
-        if (!dg->enable[fop] || !dg->delay_ppm)
-                return 0;
+    if (!dg->enable[fop] || !dg->delay_ppm)
+        return 0;
 
-        if ((rand () % DELAY_GRANULARITY) < dg->delay_ppm)
-                usleep (dg->delay_duration);
+    if ((rand() % DELAY_GRANULARITY) < dg->delay_ppm)
+        gf_nanosleep(dg->delay_duration * GF_US_IN_NS);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-dg_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-           dict_t *xdata)
+dg_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
 {
-        DG_FOP (GF_FOP_RENAME, rename, frame, this,  oldloc,  newloc, xdata);
-        return 0;
+    DG_FOP(GF_FOP_RENAME, rename, frame, this, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int32_t
-dg_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+dg_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_IPC, ipc, frame, this, op, xdata);
-        return 0;
+    DG_FOP(GF_FOP_IPC, ipc, frame, this, op, xdata);
+    return 0;
 }
 
 int32_t
-dg_setactivelk (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                lock_migration_info_t *locklist, dict_t *xdata)
+dg_setactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               lock_migration_info_t *locklist, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_SETACTIVELK, setactivelk, frame, this, loc,
-                locklist, xdata);
-        return 0;
+    DG_FOP(GF_FOP_SETACTIVELK, setactivelk, frame, this, loc, locklist, xdata);
+    return 0;
 }
 
 int32_t
-dg_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+dg_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_FLUSH, flush, frame, this, fd, xdata);
-        return 0;
+    DG_FOP(GF_FOP_FLUSH, flush, frame, this, fd, xdata);
+    return 0;
 }
 
 int32_t
-dg_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-            off_t off, dict_t *xdata)
+dg_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+           off_t off, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_READDIR, readdir, frame, this,  fd, size, off, xdata);
-        return 0;
+    DG_FOP(GF_FOP_READDIR, readdir, frame, this, fd, size, off, xdata);
+    return 0;
 }
 
 int32_t
-dg_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags, dict_t *xdata)
+dg_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_SETXATTR, setxattr, frame, this, loc, dict, flags,
-                xdata);
-        return 0;
+    DG_FOP(GF_FOP_SETXATTR, setxattr, frame, this, loc, dict, flags, xdata);
+    return 0;
 }
 
 int32_t
-dg_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dev_t rdev, mode_t umask, dict_t *xdata)
+dg_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_MKNOD, mknod, frame, this, loc, mode, rdev, umask,
-                xdata);
-        return 0;
+    DG_FOP(GF_FOP_MKNOD, mknod, frame, this, loc, mode, rdev, umask, xdata);
+    return 0;
 }
 
 int32_t
-dg_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags, dict_t *xdata)
+dg_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_FSETXATTR, fsetxattr, frame, this, fd, dict, flags,
-                xdata);
-        return 0;
+    DG_FOP(GF_FOP_FSETXATTR, fsetxattr, frame, this, fd, dict, flags, xdata);
+    return 0;
 }
 
 int32_t
-dg_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-          off_t offset, uint32_t flags, dict_t *xdata)
+dg_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_READ, readv, frame, this, fd, size, offset, flags,
-                xdata);
-        return 0;
+    DG_FOP(GF_FOP_READ, readv, frame, this, fd, size, offset, flags, xdata);
+    return 0;
 }
 
 int32_t
-dg_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-            loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+dg_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_INODELK, inodelk, frame, this, volume, loc, cmd, lock,
-                xdata);
-        return 0;
+    DG_FOP(GF_FOP_INODELK, inodelk, frame, this, volume, loc, cmd, lock, xdata);
+    return 0;
 }
 
 int32_t
-dg_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name, dict_t *xdata)
+dg_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
 {
-        DG_FOP (GF_FOP_FREMOVEXATTR, fremovexattr, frame, this, fd, name,
-                xdata);
-        return 0;
+    DG_FOP(GF_FOP_FREMOVEXATTR, fremovexattr, frame, this, fd, name, xdata);
+    return 0;
 }
 
 int32_t
-dg_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, dict_t *xdata)
+dg_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_OPEN, open, frame, this, loc, flags, fd, xdata);
-        return 0;
+    DG_FOP(GF_FOP_OPEN, open, frame, this, loc, flags, fd, xdata);
+    return 0;
 }
 
 int32_t
-dg_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+dg_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_XATTROP, xattrop, frame, this, loc, flags, dict, xdata);
-        return 0;
+    DG_FOP(GF_FOP_XATTROP, xattrop, frame, this, loc, flags, dict, xdata);
+    return 0;
 }
 
 int32_t
-dg_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-            loc_t *loc, const char *basename, entrylk_cmd cmd,
-            entrylk_type type, dict_t *xdata)
+dg_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata)
 {
-        DG_FOP (GF_FOP_ENTRYLK, entrylk, frame, this, volume,
-                loc, basename, cmd, type, xdata);
-        return 0;
+    DG_FOP(GF_FOP_ENTRYLK, entrylk, frame, this, volume, loc, basename, cmd,
+           type, xdata);
+    return 0;
 }
 
 int32_t
-dg_getactivelk (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+dg_getactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_GETACTIVELK, getactivelk, frame, this, loc, xdata);
-        return 0;
+    DG_FOP(GF_FOP_GETACTIVELK, getactivelk, frame, this, loc, xdata);
+    return 0;
 }
 
 int32_t
-dg_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+dg_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_FINODELK, finodelk, frame, this, volume, fd, cmd, lock,
-                xdata);
-        return 0;
+    DG_FOP(GF_FOP_FINODELK, finodelk, frame, this, volume, fd, cmd, lock,
+           xdata);
+    return 0;
 }
 
 int32_t
-dg_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+dg_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_CREATE, create, frame, this, loc, flags, mode, umask, fd,
-                xdata);
-        return 0;
+    DG_FOP(GF_FOP_CREATE, create, frame, this, loc, flags, mode, umask, fd,
+           xdata);
+    return 0;
 }
 
 int32_t
-dg_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-            size_t len, dict_t *xdata)
+dg_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           size_t len, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_DISCARD, discard, frame, this, fd, offset, len, xdata);
-        return 0;
+    DG_FOP(GF_FOP_DISCARD, discard, frame, this, fd, offset, len, xdata);
+    return 0;
 }
 
 int32_t
-dg_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          mode_t umask, dict_t *xdata)
+dg_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_MKDIR, mkdir, frame, this, loc, mode, umask, xdata);
-        return 0;
+    DG_FOP(GF_FOP_MKDIR, mkdir, frame, this, loc, mode, umask, xdata);
+    return 0;
 }
 
 int32_t
-dg_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-       struct gf_flock *lock, dict_t *xdata)
+dg_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+      struct gf_flock *lock, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_LK, lk, frame, this, fd, cmd, lock, xdata);
-        return 0;
+    DG_FOP(GF_FOP_LK, lk, frame, this, fd, cmd, lock, xdata);
+    return 0;
 }
 
 int32_t
-dg_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-           struct iovec *vector, int32_t count, off_t off, uint32_t flags,
-           struct iobref *iobref, dict_t *xdata)
+dg_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t off, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
 {
-        DG_FOP (GF_FOP_WRITE, writev, frame, this, fd,
-                vector, count, off, flags, iobref, xdata);
-        return 0;
+    DG_FOP(GF_FOP_WRITE, writev, frame, this, fd, vector, count, off, flags,
+           iobref, xdata);
+    return 0;
 }
 
 int32_t
-dg_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
-           dict_t *xdata)
+dg_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+          dict_t *xdata)
 {
-        DG_FOP (GF_FOP_ACCESS, access, frame, this, loc, mask, xdata);
-        return 0;
+    DG_FOP(GF_FOP_ACCESS, access, frame, this, loc, mask, xdata);
+    return 0;
 }
 
 int32_t
-dg_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+dg_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_LOOKUP, lookup, frame, this, loc, xdata);
-        return 0;
+    DG_FOP(GF_FOP_LOOKUP, lookup, frame, this, loc, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-          int32_t flags, dict_t *xdata)
+dg_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         dict_t *xdata)
 {
-        DG_FOP (GF_FOP_RMDIR, rmdir, frame, this, loc, flags, xdata);
-        return 0;
+    DG_FOP(GF_FOP_RMDIR, rmdir, frame, this, loc, flags, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t keep_size,
-              off_t offset, size_t len, dict_t *xdata)
+dg_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t keep_size,
+             off_t offset, size_t len, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_FALLOCATE, fallocate, frame, this, fd, keep_size, offset,
-                len, xdata);
-        return 0;
+    DG_FOP(GF_FOP_FALLOCATE, fallocate, frame, this, fd, keep_size, offset, len,
+           xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+dg_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_FSTAT, fstat, frame, this, fd, xdata);
-        return 0;
+    DG_FOP(GF_FOP_FSTAT, fstat, frame, this, fd, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_lease (call_frame_t *frame, xlator_t *this, loc_t *loc,
-          struct gf_lease *lease, dict_t *xdata)
+dg_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+         struct gf_lease *lease, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_LEASE, lease, frame, this, loc, lease, xdata);
-        return 0;
+    DG_FOP(GF_FOP_LEASE, lease, frame, this, loc, lease, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+dg_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_STAT, stat, frame, this, loc, xdata);
-        return 0;
+    DG_FOP(GF_FOP_STAT, stat, frame, this, loc, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-             dict_t *xdata)
+dg_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
 {
-        DG_FOP (GF_FOP_TRUNCATE, truncate, frame, this, loc, offset, xdata);
-        return 0;
+    DG_FOP(GF_FOP_TRUNCATE, truncate, frame, this, loc, offset, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             const char *name, dict_t *xdata)
+dg_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xdata)
 {
-        DG_FOP (GF_FOP_GETXATTR, getxattr, frame, this, loc, name, xdata);
-        return 0;
+    DG_FOP(GF_FOP_GETXATTR, getxattr, frame, this, loc, name, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-            loc_t *loc, mode_t umask, dict_t *xdata)
+dg_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+           loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_SYMLINK, symlink, frame, this, linkpath, loc, umask,
-                xdata);
-        return 0;
+    DG_FOP(GF_FOP_SYMLINK, symlink, frame, this, linkpath, loc, umask, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-             off_t len, dict_t *xdata)
+dg_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_ZEROFILL, zerofill, frame, this, fd, offset, len, xdata);
-        return 0;
+    DG_FOP(GF_FOP_ZEROFILL, zerofill, frame, this, fd, offset, len, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-             dict_t *xdata)
+dg_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+            dict_t *xdata)
 {
-        DG_FOP (GF_FOP_FSYNCDIR, fsyncdir, frame, this, fd, flags, xdata);
-        return 0;
+    DG_FOP(GF_FOP_FSYNCDIR, fsyncdir, frame, this, fd, flags, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
-              dict_t *xdata)
+dg_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
 {
-        DG_FOP (GF_FOP_FGETXATTR, fgetxattr, frame, this, fd, name, xdata);
-        return 0;
+    DG_FOP(GF_FOP_FGETXATTR, fgetxattr, frame, this, fd, name, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t off, dict_t *xdata)
+dg_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_READDIRP, readdirp, frame, this, fd, size, off,  xdata);
-        return 0;
+    DG_FOP(GF_FOP_READDIRP, readdirp, frame, this, fd, size, off, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-         dict_t *xdata)
+dg_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
 {
-        DG_FOP (GF_FOP_LINK, link, frame, this, oldloc, newloc, xdata);
-        return 0;
+    DG_FOP(GF_FOP_LINK, link, frame, this, oldloc, newloc, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+dg_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_FXATTROP, fxattrop, frame, this, fd, flags, dict, xdata);
-        return 0;
+    DG_FOP(GF_FOP_FXATTROP, fxattrop, frame, this, fd, flags, dict, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-              dict_t *xdata)
+dg_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
 {
-        DG_FOP (GF_FOP_FTRUNCATE, ftruncate, frame, this, fd, offset,  xdata);
-        return 0;
+    DG_FOP(GF_FOP_FTRUNCATE, ftruncate, frame, this, fd, offset, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-              int32_t len, dict_t *xdata)
+dg_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             int32_t len, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_RCHECKSUM, rchecksum, frame, this, fd, offset, len,
-                xdata);
-        return 0;
+    DG_FOP(GF_FOP_RCHECKSUM, rchecksum, frame, this, fd, offset, len, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           dict_t *xdata)
+dg_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          dict_t *xdata)
 {
-        DG_FOP (GF_FOP_UNLINK, unlink, frame, this, loc, flags, xdata);
-        return 0;
+    DG_FOP(GF_FOP_UNLINK, unlink, frame, this, loc, flags, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, const char *basename, entrylk_cmd cmd,
-             entrylk_type type, dict_t *xdata)
+dg_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
 {
-        DG_FOP (GF_FOP_FENTRYLK, fentrylk, frame, this, volume,  fd, basename,
-                cmd, type, xdata);
-        return 0;
+    DG_FOP(GF_FOP_FENTRYLK, fentrylk, frame, this, volume, fd, basename, cmd,
+           type, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_getspec (call_frame_t *frame, xlator_t *this, const char *key,
-            int32_t flags)
+dg_getspec(call_frame_t *frame, xlator_t *this, const char *key, int32_t flags)
 {
-        DG_FOP (GF_FOP_GETSPEC, getspec, frame, this, key, flags);
-        return 0;
+    DG_FOP(GF_FOP_GETSPEC, getspec, frame, this, key, flags);
+    return 0;
 }
 
-
-
 int32_t
-dg_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid, dict_t *xdata)
+dg_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_SETATTR, setattr, frame, this, loc, stbuf, valid, xdata);
-        return 0;
+    DG_FOP(GF_FOP_SETATTR, setattr, frame, this, loc, stbuf, valid, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-          dict_t *xdata)
+dg_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+         dict_t *xdata)
 {
-        DG_FOP (GF_FOP_FSYNC, fsync, frame, this,  fd, flags, xdata);
-        return 0;
+    DG_FOP(GF_FOP_FSYNC, fsync, frame, this, fd, flags, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+dg_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_STATFS, statfs, frame, this, loc, xdata);
-        return 0;
+    DG_FOP(GF_FOP_STATFS, statfs, frame, this, loc, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_seek (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-         gf_seek_what_t what, dict_t *xdata)
+dg_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+        gf_seek_what_t what, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_SEEK, seek, frame, this, fd, offset, what, xdata);
-        return 0;
+    DG_FOP(GF_FOP_SEEK, seek, frame, this, fd, offset, what, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+dg_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_FSETATTR, fsetattr, frame, this, fd,
-                stbuf, valid, xdata);
-        return 0;
+    DG_FOP(GF_FOP_FSETATTR, fsetattr, frame, this, fd, stbuf, valid, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-            dict_t *xdata)
+dg_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+           dict_t *xdata)
 {
-        DG_FOP (GF_FOP_OPENDIR, opendir, frame, this, loc, fd, xdata);
-        return 0;
+    DG_FOP(GF_FOP_OPENDIR, opendir, frame, this, loc, fd, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-             dict_t *xdata)
+dg_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+            dict_t *xdata)
 {
-        DG_FOP (GF_FOP_READLINK, readlink, frame, this, loc, size, xdata);
-        return 0;
+    DG_FOP(GF_FOP_READLINK, readlink, frame, this, loc, size, xdata);
+    return 0;
 }
 
-
-
 int32_t
-dg_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata)
+dg_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        DG_FOP (GF_FOP_REMOVEXATTR, removexattr, frame, this, loc, name, xdata);
-        return 0;
+    DG_FOP(GF_FOP_REMOVEXATTR, removexattr, frame, this, loc, name, xdata);
+    return 0;
 }
 
 int32_t
-dg_forget (xlator_t *this, inode_t *inode)
+dg_forget(xlator_t *this, inode_t *inode)
 {
-        return 0;
+    return 0;
 }
 
 int32_t
-dg_release (xlator_t *this, fd_t *fd)
+dg_release(xlator_t *this, fd_t *fd)
 {
-        return 0;
+    return 0;
 }
 
 int32_t
-dg_releasedir (xlator_t *this, fd_t *fd)
+dg_releasedir(xlator_t *this, fd_t *fd)
 {
-        return 0;
+    return 0;
 }
 
 static int
-delay_gen_parse_fill_fops (dg_t *dg, char *enable_fops)
-{
-        char            *op_no_str = NULL;
-        int              op_no = -1;
-        int              i = 0;
-        int              ret = 0;
-        xlator_t        *this = THIS;
-        char            *saveptr = NULL;
-        char            *dup_enable_fops = NULL;
-
-        if (strlen (enable_fops) == 0) {
-                for (i = GF_FOP_NULL + 1; i < GF_FOP_MAXVALUE; i++)
-                        dg->enable[i] = 1;
-        } else {
-                dup_enable_fops = gf_strdup (enable_fops);
-                if (!dup_enable_fops) {
-                        ret = -1;
-                        goto out;
-                }
-                op_no_str = strtok_r (dup_enable_fops, ",", &saveptr);
-                while (op_no_str) {
-                        op_no = gf_fop_int (op_no_str);
-                        if (op_no == -1) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Wrong option value %s", op_no_str);
-                                ret = -1;
-                                goto out;
-                        } else {
-                                dg->enable[op_no] = 1;
-                        }
-
-                        op_no_str = strtok_r (NULL, ",", &saveptr);
-                }
+delay_gen_parse_fill_fops(dg_t *dg, char *enable_fops)
+{
+    char *op_no_str = NULL;
+    int op_no = -1;
+    int i = 0;
+    int ret = 0;
+    xlator_t *this = THIS;
+    char *saveptr = NULL;
+    char *dup_enable_fops = NULL;
+
+    if (strlen(enable_fops) == 0) {
+        for (i = GF_FOP_NULL + 1; i < GF_FOP_MAXVALUE; i++)
+            dg->enable[i] = 1;
+    } else {
+        dup_enable_fops = gf_strdup(enable_fops);
+        if (!dup_enable_fops) {
+            ret = -1;
+            goto out;
         }
+        op_no_str = strtok_r(dup_enable_fops, ",", &saveptr);
+        while (op_no_str) {
+            op_no = gf_fop_int(op_no_str);
+            if (op_no == -1) {
+                gf_log(this->name, GF_LOG_WARNING, "Wrong option value %s",
+                       op_no_str);
+                ret = -1;
+                goto out;
+            } else {
+                dg->enable[op_no] = 1;
+            }
+
+            op_no_str = strtok_r(NULL, ",", &saveptr);
+        }
+    }
 out:
-        GF_FREE (dup_enable_fops);
-        return ret;
+    GF_FREE(dup_enable_fops);
+    return ret;
 }
 
 void
-delay_gen_set_delay_ppm (dg_t *dg, double percent)
+delay_gen_set_delay_ppm(dg_t *dg, double percent)
 {
-        double ppm;
+    double ppm;
 
-        ppm = (percent / 100.0) * (double) DELAY_GRANULARITY;
-        dg->delay_ppm = ppm;
+    ppm = (percent / 100.0) * (double)DELAY_GRANULARITY;
+    dg->delay_ppm = ppm;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        dg_t            *dg = NULL;
-        int32_t          ret = 0;
-        double          delay_percent = 0;
-        char            *delay_enable_fops = NULL;
+    dg_t *dg = NULL;
+    int32_t ret = 0;
+    double delay_percent = 0;
+    char *delay_enable_fops = NULL;
 
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "delay-gen not configured with one subvolume");
-                ret = -1;
-                goto out;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "delay-gen not configured with one subvolume");
+        ret = -1;
+        goto out;
+    }
 
-        dg = GF_CALLOC (1, sizeof (*dg), gf_delay_gen_mt_dg_t);
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
 
-        if (!dg) {
-                ret = -1;
-                goto out;
-        }
+    dg = GF_CALLOC(1, sizeof(*dg), gf_delay_gen_mt_dg_t);
 
+    if (!dg) {
         ret = -1;
+        goto out;
+    }
 
-        GF_OPTION_INIT ("delay-percentage", delay_percent, percent, out);
-        GF_OPTION_INIT ("enable", delay_enable_fops, str, out);
-        GF_OPTION_INIT ("delay-duration", dg->delay_duration, int32, out);
+    ret = -1;
 
-        delay_gen_set_delay_ppm (dg, delay_percent);
+    GF_OPTION_INIT("delay-percentage", delay_percent, percent, out);
+    GF_OPTION_INIT("enable", delay_enable_fops, str, out);
+    GF_OPTION_INIT("delay-duration", dg->delay_duration, int32, out);
 
-        ret = delay_gen_parse_fill_fops (dg, delay_enable_fops);
-        if (ret)
-                goto out;
+    delay_gen_set_delay_ppm(dg, delay_percent);
 
-        this->private = dg;
+    ret = delay_gen_parse_fill_fops(dg, delay_enable_fops);
+    if (ret)
+        goto out;
 
-        ret = 0;
+    this->private = dg;
+
+    ret = 0;
 out:
-        if (ret)
-                GF_FREE (dg);
-        return ret;
+    if (ret)
+        GF_FREE(dg);
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        GF_FREE (this->private);
+    GF_FREE(this->private);
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_delay_gen_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_delay_gen_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               " failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int32_t
-reconfigure (xlator_t *this, dict_t *dict)
+reconfigure(xlator_t *this, dict_t *dict)
 {
-        /*At the moment I don't see any need to implement this. In future
-         *if this is needed we can add code here.
-         */
-        return 0;
+    /*At the moment I don't see any need to implement this. In future
+     *if this is needed we can add code here.
+     */
+    return 0;
 }
 
 int
-notify (xlator_t *this, int event, void *data, ...)
+notify(xlator_t *this, int event, void *data, ...)
 {
-        return default_notify (this, event, data);
+    return default_notify(this, event, data);
 }
 
-
 struct xlator_fops fops = {
-        .rename               = dg_rename,
-        .ipc                  = dg_ipc,
-        .setactivelk          = dg_setactivelk,
-        .flush                = dg_flush,
-        .readdir              = dg_readdir,
-        .setxattr             = dg_setxattr,
-        .mknod                = dg_mknod,
-        .fsetxattr            = dg_fsetxattr,
-        .readv                = dg_readv,
-        .inodelk              = dg_inodelk,
-        .fremovexattr         = dg_fremovexattr,
-        .open                 = dg_open,
-        .xattrop              = dg_xattrop,
-        .entrylk              = dg_entrylk,
-        .getactivelk          = dg_getactivelk,
-        .finodelk             = dg_finodelk,
-        .create               = dg_create,
-        .discard              = dg_discard,
-        .mkdir                = dg_mkdir,
-        .lk                   = dg_lk,
-        .writev               = dg_writev,
-        .access               = dg_access,
-        .lookup               = dg_lookup,
-        .rmdir                = dg_rmdir,
-        .fallocate            = dg_fallocate,
-        .fstat                = dg_fstat,
-        .lease                = dg_lease,
-        .stat                 = dg_stat,
-        .truncate             = dg_truncate,
-        .getxattr             = dg_getxattr,
-        .symlink              = dg_symlink,
-        .zerofill             = dg_zerofill,
-        .fsyncdir             = dg_fsyncdir,
-        .fgetxattr            = dg_fgetxattr,
-        .readdirp             = dg_readdirp,
-        .link                 = dg_link,
-        .fxattrop             = dg_fxattrop,
-        .ftruncate            = dg_ftruncate,
-        .rchecksum            = dg_rchecksum,
-        .unlink               = dg_unlink,
-        .fentrylk             = dg_fentrylk,
-        .getspec              = dg_getspec,
-        .setattr              = dg_setattr,
-        .fsync                = dg_fsync,
-        .statfs               = dg_statfs,
-        .seek                 = dg_seek,
-        .fsetattr             = dg_fsetattr,
-        .opendir              = dg_opendir,
-        .readlink             = dg_readlink,
-        .removexattr          = dg_removexattr,
+    .rename = dg_rename,
+    .ipc = dg_ipc,
+    .setactivelk = dg_setactivelk,
+    .flush = dg_flush,
+    .readdir = dg_readdir,
+    .setxattr = dg_setxattr,
+    .mknod = dg_mknod,
+    .fsetxattr = dg_fsetxattr,
+    .readv = dg_readv,
+    .inodelk = dg_inodelk,
+    .fremovexattr = dg_fremovexattr,
+    .open = dg_open,
+    .xattrop = dg_xattrop,
+    .entrylk = dg_entrylk,
+    .getactivelk = dg_getactivelk,
+    .finodelk = dg_finodelk,
+    .create = dg_create,
+    .discard = dg_discard,
+    .mkdir = dg_mkdir,
+    .lk = dg_lk,
+    .writev = dg_writev,
+    .access = dg_access,
+    .lookup = dg_lookup,
+    .rmdir = dg_rmdir,
+    .fallocate = dg_fallocate,
+    .fstat = dg_fstat,
+    .lease = dg_lease,
+    .stat = dg_stat,
+    .truncate = dg_truncate,
+    .getxattr = dg_getxattr,
+    .symlink = dg_symlink,
+    .zerofill = dg_zerofill,
+    .fsyncdir = dg_fsyncdir,
+    .fgetxattr = dg_fgetxattr,
+    .readdirp = dg_readdirp,
+    .link = dg_link,
+    .fxattrop = dg_fxattrop,
+    .ftruncate = dg_ftruncate,
+    .rchecksum = dg_rchecksum,
+    .unlink = dg_unlink,
+    .fentrylk = dg_fentrylk,
+    .getspec = dg_getspec,
+    .setattr = dg_setattr,
+    .fsync = dg_fsync,
+    .statfs = dg_statfs,
+    .seek = dg_seek,
+    .fsetattr = dg_fsetattr,
+    .opendir = dg_opendir,
+    .readlink = dg_readlink,
+    .removexattr = dg_removexattr,
 };
 
 struct xlator_cbks cbks = {
-        .forget               = dg_forget,
-        .release              = dg_release,
-        .releasedir           = dg_releasedir,
+    .forget = dg_forget,
+    .release = dg_release,
+    .releasedir = dg_releasedir,
 };
 
 struct volume_options options[] = {
-        { .key  = {"delay-percentage"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .default_value = "10%",
-          .description = "Percentage delay of operations when enabled.",
-          .op_version  = {GD_OP_VERSION_3_13_0},
-          .flags       = OPT_FLAG_SETTABLE,
-          .tags        = {"delay-gen"},
-        },
-
-        { .key  = {"delay-duration"},
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Delay duration in micro seconds",
-          .default_value = "100000",
-          .op_version = {GD_OP_VERSION_3_13_0},
-          .flags = OPT_FLAG_SETTABLE,
-          .tags  = {"delay-gen"},
-        },
-
-        { .key  = {"enable"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Accepts a string which takes ',' separated fop "
-                         "strings to denote which fops are enabled for delay",
-          .op_version = {GD_OP_VERSION_3_13_0},
-          .flags = OPT_FLAG_SETTABLE,
-          .tags  = {"delay-gen"},
-          .default_value = "",
-        },
-
-        { .key  = {NULL} }
+    {
+        .key = {"delay-percentage"},
+        .type = GF_OPTION_TYPE_PERCENT,
+        .default_value = "10%",
+        .description = "Percentage delay of operations when enabled.",
+        .op_version = {GD_OP_VERSION_3_13_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"delay-gen"},
+    },
+
+    {
+        .key = {"delay-duration"},
+        .type = GF_OPTION_TYPE_INT,
+        .description = "Delay duration in micro seconds",
+        .default_value = "100000",
+        .op_version = {GD_OP_VERSION_3_13_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"delay-gen"},
+    },
+
+    {
+        .key = {"enable"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Accepts a string which takes ',' separated fop "
+                       "strings to denote which fops are enabled for delay",
+        .op_version = {GD_OP_VERSION_3_13_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"delay-gen"},
+        .default_value = "",
+    },
+
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {GD_OP_VERSION_3_12_0},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "delay-gen",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/debug/delay-gen/src/delay-gen.h b/xlators/debug/delay-gen/src/delay-gen.h
index e1111c782e2..afa95e5eb2d 100644
--- a/xlators/debug/delay-gen/src/delay-gen.h
+++ b/xlators/debug/delay-gen/src/delay-gen.h
@@ -8,21 +8,20 @@
  *   cases as published by the Free Software Foundation.
  */
 
-
 #ifndef __DELAY_GEN_H__
 #define __DELAY_GEN_H__
 
 #include "delay-gen-mem-types.h"
 #include "delay-gen-messages.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 typedef struct {
-        int enable[GF_FOP_MAXVALUE];
-        int op_count;
-        int delay_ppm;
-        int delay_duration;
+    int enable[GF_FOP_MAXVALUE];
+    int op_count;
+    int delay_ppm;
+    int delay_duration;
 } dg_t;
 
 #endif /* __DELAY_GEN_H__ */
diff --git a/xlators/debug/error-gen/src/error-gen-mem-types.h b/xlators/debug/error-gen/src/error-gen-mem-types.h
index f02280535df..b9b713af8fc 100644
--- a/xlators/debug/error-gen/src/error-gen-mem-types.h
+++ b/xlators/debug/error-gen/src/error-gen-mem-types.h
@@ -11,10 +11,10 @@
 #ifndef __ERROR_GEN_MEM_TYPES_H__
 #define __ERROR_GEN_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_error_gen_mem_types_ {
-        gf_error_gen_mt_eg_t = gf_common_mt_end + 1,
-        gf_error_gen_mt_end
+    gf_error_gen_mt_eg_t = gf_common_mt_end + 1,
+    gf_error_gen_mt_end
 };
 #endif
diff --git a/xlators/debug/error-gen/src/error-gen.c b/xlators/debug/error-gen/src/error-gen.c
index 6724fcf4c22..d45655ef4c3 100644
--- a/xlators/debug/error-gen/src/error-gen.c
+++ b/xlators/debug/error-gen/src/error-gen.c
@@ -7,10 +7,10 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "error-gen.h"
-#include "statedump.h"
-#include "defaults.h"
+#include <glusterfs/statedump.h>
+#include <glusterfs/defaults.h>
 
 /*
  * The user can specify an error probability as a float percentage, but we
@@ -26,1688 +26,1638 @@
  * something completely different and this number is irrelevant.  See error_gen
  * for the legacy code.)
  */
-#define FAILURE_GRANULARITY     (1 << 20)
+#define FAILURE_GRANULARITY (1 << 20)
 
 sys_error_t error_no_list[] = {
-        [GF_FOP_LOOKUP]            = { .error_no_count = 4,
-                                    .error_no = {ENOENT,ENOTDIR,
-                                                 ENAMETOOLONG,EAGAIN}},
-        [GF_FOP_STAT]              = { .error_no_count = 7,
-                                    .error_no = {EACCES,EBADF,EFAULT,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOTDIR}},
-        [GF_FOP_READLINK]          = { .error_no_count = 8,
-                                    .error_no = {EACCES,EFAULT,EINVAL,EIO,
-                                                 ENAMETOOLONG,ENOENT,ENOMEM,
-                                                 ENOTDIR}},
-        [GF_FOP_MKNOD]             = { .error_no_count = 11,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 EINVAL,ENAMETOOLONG,
-                                                 ENOENT,ENOMEM,ENOSPC,
-                                                 ENOTDIR,EPERM,EROFS}},
-        [GF_FOP_MKDIR]             = { .error_no_count = 10,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOSPC,ENOTDIR,
-						 EPERM,EROFS}},
-        [GF_FOP_UNLINK]            = { .error_no_count = 10,
-                                    .error_no = {EACCES,EBUSY,EFAULT,EIO,
-                                                 EISDIR,ENAMETOOLONG,
-                                                 ENOENT,ENOMEM,ENOTDIR,
-                                                 EPERM,EROFS}},
-        [GF_FOP_RMDIR]             = { .error_no_count = 8,
-                                    .error_no = {EACCES,EBUSY,EFAULT,
-                                                 ENOMEM,ENOTDIR,ENOTEMPTY,
-                                                 EPERM,EROFS}},
-        [GF_FOP_SYMLINK]           = { .error_no_count = 11,
-                                    .error_no = {EACCES,EEXIST,EFAULT,EIO,
-                                                 ENAMETOOLONG,ENOENT,ENOMEM,
-                                                 ENOSPC,ENOTDIR,EPERM,
-                                                 EROFS}},
-        [GF_FOP_RENAME]            = { .error_no_count = 13,
-                                    .error_no = {EACCES,EBUSY,EFAULT,
-                                                 EINVAL,EISDIR,EMLINK,
-                                                 ENAMETOOLONG,ENOENT,ENOMEM,
-                                                 ENOSPC,ENOTDIR,EEXIST,
-                                                 EXDEV}},
-        [GF_FOP_LINK]              = { .error_no_count = 13,
-                                    .error_no = {EACCES,EFAULT,EEXIST,EIO,
-                                                 EMLINK,ENAMETOOLONG,
-                                                 ENOENT,ENOMEM,ENOSPC,
-                                                 ENOTDIR,EPERM,EROFS,
-                                                 EXDEV}},
-        [GF_FOP_TRUNCATE]          = { .error_no_count = 10,
-                                    .error_no = {EACCES,EFAULT,EFBIG,
-                                                 EINTR,EINVAL,EIO,EISDIR,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 EISDIR}},
-        [GF_FOP_CREATE]            = {.error_no_count = 10,
-                                   .error_no = {EACCES,EEXIST,EFAULT,
-                                                EISDIR,EMFILE,ENAMETOOLONG,
-                                                ENFILE,ENODEV,ENOENT,
-                                                ENODEV}},
-        [GF_FOP_OPEN]              = { .error_no_count = 10,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 EISDIR,EMFILE,
-                                                 ENAMETOOLONG,ENFILE,
-                                                 ENODEV,ENOENT,ENOMEM}},
-        [GF_FOP_READ]              = { .error_no_count = 5,
-                                    .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
-                                                 ENAMETOOLONG}},
-        [GF_FOP_WRITE]             = { .error_no_count = 7,
-                                    .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
-                                                 ENAMETOOLONG,ENOSPC,
-						 GF_ERROR_SHORT_WRITE}},
-        [GF_FOP_STATFS]            = {.error_no_count = 10,
-                                   .error_no = {EACCES,EBADF,EFAULT,EINTR,
-                                                EIO,ENAMETOOLONG,ENOENT,
-                                                ENOMEM,ENOSYS,ENOTDIR}},
-        [GF_FOP_FLUSH]             = { .error_no_count = 5,
-                                    .error_no = {EACCES,EFAULT,
-                                                 ENAMETOOLONG,ENOSYS,
-                                                 ENOENT}},
-        [GF_FOP_FSYNC]             = { .error_no_count = 4,
-                                    .error_no = {EBADF,EIO,EROFS,EINVAL}},
-        [GF_FOP_SETXATTR]          = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,EINTR,
-                                                 ENAMETOOLONG}},
-        [GF_FOP_GETXATTR]          = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                 EINTR}},
-        [GF_FOP_REMOVEXATTR]       = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                 EINTR}},
-        [GF_FOP_FSETXATTR]          = { .error_no_count = 4,
-                                        .error_no = {EACCES,EBADF,EINTR,
-                                                     ENAMETOOLONG}},
-        [GF_FOP_FGETXATTR]          = { .error_no_count = 4,
-                                        .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                     EINTR}},
-        [GF_FOP_FREMOVEXATTR]       = { .error_no_count = 4,
-                                        .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                     EINTR}},
-        [GF_FOP_OPENDIR]           = { .error_no_count = 8,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 EISDIR,EMFILE,
-                                                 ENAMETOOLONG,ENFILE,
-                                                 ENODEV}},
-        [GF_FOP_READDIR]           = { .error_no_count = 5,
-                                    .error_no = {EINVAL,EACCES,EBADF,
-                                                 EMFILE,ENOENT}},
-        [GF_FOP_READDIRP]          = { .error_no_count = 5,
-                                    .error_no = {EINVAL,EACCES,EBADF,
-                                                 EMFILE,ENOENT}},
-        [GF_FOP_FSYNCDIR]          = { .error_no_count = 4,
-                                    .error_no = {EBADF,EIO,EROFS,EINVAL}},
-        [GF_FOP_ACCESS]            = { .error_no_count = 8,
-                                    .error_no = {EACCES,ENAMETOOLONG,
-                                                 ENOENT,ENOTDIR,EROFS,
-                                                 EFAULT,EINVAL,EIO}},
-        [GF_FOP_FTRUNCATE]         = { .error_no_count = 9,
-                                    .error_no = {EACCES,EFAULT,EFBIG,
-                                                 EINTR,EINVAL,EIO,EISDIR,
-                                                 ENAMETOOLONG,ENOENT}},
-        [GF_FOP_FSTAT]             = { .error_no_count = 7,
-                                    .error_no = {EACCES,EBADF,EFAULT,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOTDIR}},
-        [GF_FOP_LK]                = { .error_no_count = 4,
-                                    .error_no = {EACCES,EFAULT,ENOENT,
-                                                 EINTR}},
-        [GF_FOP_XATTROP]           = { .error_no_count = 5,
-                                    .error_no = {EACCES,EFAULT,
-                                                 ENAMETOOLONG,ENOSYS,
-                                                 ENOENT}},
-        [GF_FOP_FXATTROP]          = { .error_no_count = 4,
-                                    .error_no = {EBADF,EIO,EROFS,EINVAL}},
-        [GF_FOP_INODELK]           = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,EINTR,
-                                                 ENAMETOOLONG}},
-        [GF_FOP_FINODELK]          = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,EINTR,
-                                                 ENAMETOOLONG}},
-        [GF_FOP_ENTRYLK]           = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,
-                                                 ENAMETOOLONG,EINTR}},
-        [GF_FOP_FENTRYLK]          = { .error_no_count = 10,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 EISDIR,EMFILE,
-                                                 ENAMETOOLONG,ENFILE,
-                                                 ENODEV,ENOENT,ENOMEM}},
-        [GF_FOP_SETATTR]           = {.error_no_count = 11,
-                                    .error_no = {EACCES,EFAULT,EIO,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOTDIR,EPERM,
-                                                 EROFS,EBADF,EIO}},
-        [GF_FOP_FSETATTR]          = { .error_no_count = 11,
-                                    .error_no = {EACCES,EFAULT,EIO,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOTDIR,EPERM,
-                                                 EROFS,EBADF,EIO}},
-        [GF_FOP_GETSPEC]           = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                 EINTR}}
-};
+    [GF_FOP_LOOKUP] = {.error_no_count = 4,
+                       .error_no = {ENOENT, ENOTDIR, ENAMETOOLONG, EAGAIN}},
+    [GF_FOP_STAT] = {.error_no_count = 6,
+                     .error_no = {EACCES, EFAULT, ENAMETOOLONG, ENOENT, ENOMEM,
+                                  ENOTDIR}},
+    [GF_FOP_READLINK] = {.error_no_count = 8,
+                         .error_no = {EACCES, EFAULT, EINVAL, EIO, ENAMETOOLONG,
+                                      ENOENT, ENOMEM, ENOTDIR}},
+    [GF_FOP_MKNOD] = {.error_no_count = 11,
+                      .error_no = {EACCES, EEXIST, EFAULT, EINVAL, ENAMETOOLONG,
+                                   ENOENT, ENOMEM, ENOSPC, ENOTDIR, EPERM,
+                                   EROFS}},
+    [GF_FOP_MKDIR] = {.error_no_count = 10,
+                      .error_no = {EACCES, EEXIST, EFAULT, ENAMETOOLONG, ENOENT,
+                                   ENOMEM, ENOSPC, ENOTDIR, EPERM, EROFS}},
+    [GF_FOP_UNLINK] = {.error_no_count = 10,
+                       .error_no = {EACCES, EBUSY, EFAULT, EIO, EISDIR,
+                                    ENAMETOOLONG, ENOENT, ENOMEM, ENOTDIR,
+                                    EPERM, EROFS}},
+    [GF_FOP_RMDIR] = {.error_no_count = 8,
+                      .error_no = {EACCES, EBUSY, EFAULT, ENOMEM, ENOTDIR,
+                                   ENOTEMPTY, EPERM, EROFS}},
+    [GF_FOP_SYMLINK] = {.error_no_count = 11,
+                        .error_no = {EACCES, EEXIST, EFAULT, EIO, ENAMETOOLONG,
+                                     ENOENT, ENOMEM, ENOSPC, ENOTDIR, EPERM,
+                                     EROFS}},
+    [GF_FOP_RENAME] = {.error_no_count = 13,
+                       .error_no = {EACCES, EBUSY, EFAULT, EINVAL, EISDIR,
+                                    EMLINK, ENAMETOOLONG, ENOENT, ENOMEM,
+                                    ENOSPC, ENOTDIR, EEXIST, EXDEV}},
+    [GF_FOP_LINK] = {.error_no_count = 13,
+                     .error_no = {EACCES, EFAULT, EEXIST, EIO, EMLINK,
+                                  ENAMETOOLONG, ENOENT, ENOMEM, ENOSPC, ENOTDIR,
+                                  EPERM, EROFS, EXDEV}},
+    [GF_FOP_TRUNCATE] = {.error_no_count = 10,
+                         .error_no = {EACCES, EFAULT, EFBIG, EINTR, EINVAL, EIO,
+                                      EISDIR, ENAMETOOLONG, ENOENT, EISDIR}},
+    [GF_FOP_CREATE] = {.error_no_count = 10,
+                       .error_no = {EACCES, EEXIST, EFAULT, EISDIR, EMFILE,
+                                    ENAMETOOLONG, ENFILE, ENODEV, ENOENT,
+                                    ENODEV}},
+    [GF_FOP_OPEN] = {.error_no_count = 10,
+                     .error_no = {EACCES, EEXIST, EFAULT, EISDIR, EMFILE,
+                                  ENAMETOOLONG, ENFILE, ENODEV, ENOENT,
+                                  ENOMEM}},
+    [GF_FOP_READ] = {.error_no_count = 5,
+                     .error_no = {EINVAL, EBADF, EFAULT, EISDIR, ENAMETOOLONG}},
+    [GF_FOP_WRITE] = {.error_no_count = 7,
+                      .error_no = {EINVAL, EBADF, EFAULT, EISDIR, ENAMETOOLONG,
+                                   ENOSPC, GF_ERROR_SHORT_WRITE}},
+    [GF_FOP_STATFS] = {.error_no_count = 9,
+                       .error_no = {EACCES, EFAULT, EINTR, EIO, ENAMETOOLONG,
+                                    ENOENT, ENOMEM, ENOSYS, ENOTDIR}},
+    [GF_FOP_FLUSH] = {.error_no_count = 5,
+                      .error_no = {EACCES, EFAULT, ENAMETOOLONG, ENOSYS,
+                                   ENOENT}},
+    [GF_FOP_FSYNC] = {.error_no_count = 4,
+                      .error_no = {EBADF, EIO, EROFS, EINVAL}},
+    [GF_FOP_SETXATTR] = {.error_no_count = 3,
+                         .error_no = {EACCES, EINTR, ENAMETOOLONG}},
+    [GF_FOP_GETXATTR] = {.error_no_count = 3,
+                         .error_no = {EACCES, ENAMETOOLONG, EINTR}},
+    [GF_FOP_REMOVEXATTR] = {.error_no_count = 3,
+                            .error_no = {EACCES, ENAMETOOLONG, EINTR}},
+    [GF_FOP_FSETXATTR] = {.error_no_count = 4,
+                          .error_no = {EACCES, EBADF, EINTR, ENAMETOOLONG}},
+    [GF_FOP_FGETXATTR] = {.error_no_count = 4,
+                          .error_no = {EACCES, EBADF, ENAMETOOLONG, EINTR}},
+    [GF_FOP_FREMOVEXATTR] = {.error_no_count = 4,
+                             .error_no = {EACCES, EBADF, ENAMETOOLONG, EINTR}},
+    [GF_FOP_OPENDIR] = {.error_no_count = 8,
+                        .error_no = {EACCES, EEXIST, EFAULT, EISDIR, EMFILE,
+                                     ENAMETOOLONG, ENFILE, ENODEV}},
+    [GF_FOP_READDIR] = {.error_no_count = 5,
+                        .error_no = {EINVAL, EACCES, EBADF, EMFILE, ENOENT}},
+    [GF_FOP_READDIRP] = {.error_no_count = 5,
+                         .error_no = {EINVAL, EACCES, EBADF, EMFILE, ENOENT}},
+    [GF_FOP_FSYNCDIR] = {.error_no_count = 4,
+                         .error_no = {EBADF, EIO, EROFS, EINVAL}},
+    [GF_FOP_ACCESS] = {.error_no_count = 8,
+                       .error_no = {EACCES, ENAMETOOLONG, ENOENT, ENOTDIR,
+                                    EROFS, EFAULT, EINVAL, EIO}},
+    [GF_FOP_FTRUNCATE] = {.error_no_count = 9,
+                          .error_no = {EACCES, EFAULT, EFBIG, EINTR, EINVAL,
+                                       EIO, EISDIR, ENAMETOOLONG, ENOENT}},
+    [GF_FOP_FSTAT] = {.error_no_count = 7,
+                      .error_no = {EACCES, EBADF, EFAULT, ENAMETOOLONG, ENOENT,
+                                   ENOMEM, ENOTDIR}},
+    [GF_FOP_LK] = {.error_no_count = 4,
+                   .error_no = {EACCES, EFAULT, ENOENT, EINTR}},
+    [GF_FOP_XATTROP] = {.error_no_count = 5,
+                        .error_no = {EACCES, EFAULT, ENAMETOOLONG, ENOSYS,
+                                     ENOENT}},
+    [GF_FOP_FXATTROP] = {.error_no_count = 4,
+                         .error_no = {EBADF, EIO, EROFS, EINVAL}},
+    [GF_FOP_INODELK] = {.error_no_count = 3,
+                        .error_no = {EACCES, EINTR, ENAMETOOLONG}},
+    [GF_FOP_FINODELK] = {.error_no_count = 4,
+                         .error_no = {EACCES, EBADF, EINTR, ENAMETOOLONG}},
+    [GF_FOP_ENTRYLK] = {.error_no_count = 3,
+                        .error_no = {EACCES, ENAMETOOLONG, EINTR}},
+    [GF_FOP_FENTRYLK] = {.error_no_count = 10,
+                         .error_no = {EACCES, EEXIST, EFAULT, EISDIR, EMFILE,
+                                      ENAMETOOLONG, ENFILE, ENODEV, ENOENT,
+                                      ENOMEM}},
+    [GF_FOP_SETATTR] = {.error_no_count = 10,
+                        .error_no = {EACCES, EFAULT, EIO, ENAMETOOLONG, ENOENT,
+                                     ENOMEM, ENOTDIR, EPERM, EROFS, EIO}},
+    [GF_FOP_FSETATTR] = {.error_no_count = 11,
+                         .error_no = {EACCES, EFAULT, EIO, ENAMETOOLONG, ENOENT,
+                                      ENOMEM, ENOTDIR, EPERM, EROFS, EBADF,
+                                      EIO}},
+    [GF_FOP_GETSPEC] = {.error_no_count = 3,
+                        .error_no = {EACCES, ENAMETOOLONG, EINTR}}};
 
 int
-generate_rand_no (int op_no)
+generate_rand_no(int op_no)
 {
-        int             rand_no = 0;
+    int rand_no = 0;
+    int error_no_list_size = 0;
+
+    error_no_list_size = sizeof(error_no_list) / sizeof(error_no_list[0]);
 
-        if (op_no < GF_FOP_MAXVALUE)
-                /* coverty[DC.WEAK_CRYPTO] */
-                rand_no = rand () % error_no_list[op_no].error_no_count;
-        return rand_no;
+    if (op_no < error_no_list_size)
+        /* coverity[DC.WEAK_CRYPTO] */
+        rand_no = rand() % error_no_list[op_no].error_no_count;
+    return rand_no;
 }
 
 int
-conv_errno_to_int (char **error_no)
+conv_errno_to_int(char **error_no)
 {
-        if (!strcmp ((*error_no), "ENOENT"))
-                return ENOENT;
-        else if (!strcmp ((*error_no), "ENOTDIR"))
-                return ENOTDIR;
-        else if (!strcmp ((*error_no), "ENAMETOOLONG"))
-                return ENAMETOOLONG;
-        else if (!strcmp ((*error_no), "EACCES"))
-                return EACCES;
-        else if (!strcmp ((*error_no), "EBADF"))
-                return EBADF;
-        else if (!strcmp ((*error_no), "EFAULT"))
-                return EFAULT;
-        else if (!strcmp ((*error_no), "ENOMEM"))
-                return ENOMEM;
-        else if (!strcmp ((*error_no), "EINVAL"))
-                return EINVAL;
-        else if (!strcmp ((*error_no), "EIO"))
-                return EIO;
-        else if (!strcmp ((*error_no), "EEXIST"))
-                return EEXIST;
-        else if (!strcmp ((*error_no), "ENOSPC"))
-                return ENOSPC;
-        else if (!strcmp ((*error_no), "EPERM"))
-                return EPERM;
-        else if (!strcmp ((*error_no), "EROFS"))
-                return EROFS;
-        else if (!strcmp ((*error_no), "EBUSY"))
-                return EBUSY;
-        else if (!strcmp ((*error_no), "EISDIR"))
-                return EISDIR;
-        else if (!strcmp ((*error_no), "ENOTEMPTY"))
-                return ENOTEMPTY;
-        else if (!strcmp ((*error_no), "EMLINK"))
-                return EMLINK;
-        else if (!strcmp ((*error_no), "ENODEV"))
-                return ENODEV;
-        else if (!strcmp ((*error_no), "EXDEV"))
-                return EXDEV;
-        else if (!strcmp ((*error_no), "EMFILE"))
-                return EMFILE;
-        else if (!strcmp ((*error_no), "ENFILE"))
-                return ENFILE;
-        else if (!strcmp ((*error_no), "ENOSYS"))
-                return ENOSYS;
-        else if (!strcmp ((*error_no), "EINTR"))
-                return EINTR;
-        else if (!strcmp ((*error_no), "EFBIG"))
-                return EFBIG;
-	else if (!strcmp((*error_no), "GF_ERROR_SHORT_WRITE"))
-		return GF_ERROR_SHORT_WRITE;
-        else
-                return EAGAIN;
+    if (!strcmp((*error_no), "ENOENT"))
+        return ENOENT;
+    else if (!strcmp((*error_no), "ENOTDIR"))
+        return ENOTDIR;
+    else if (!strcmp((*error_no), "ENAMETOOLONG"))
+        return ENAMETOOLONG;
+    else if (!strcmp((*error_no), "EACCES"))
+        return EACCES;
+    else if (!strcmp((*error_no), "EBADF"))
+        return EBADF;
+    else if (!strcmp((*error_no), "EFAULT"))
+        return EFAULT;
+    else if (!strcmp((*error_no), "ENOMEM"))
+        return ENOMEM;
+    else if (!strcmp((*error_no), "EINVAL"))
+        return EINVAL;
+    else if (!strcmp((*error_no), "EIO"))
+        return EIO;
+    else if (!strcmp((*error_no), "EEXIST"))
+        return EEXIST;
+    else if (!strcmp((*error_no), "ENOSPC"))
+        return ENOSPC;
+    else if (!strcmp((*error_no), "EPERM"))
+        return EPERM;
+    else if (!strcmp((*error_no), "EROFS"))
+        return EROFS;
+    else if (!strcmp((*error_no), "EBUSY"))
+        return EBUSY;
+    else if (!strcmp((*error_no), "EISDIR"))
+        return EISDIR;
+    else if (!strcmp((*error_no), "ENOTEMPTY"))
+        return ENOTEMPTY;
+    else if (!strcmp((*error_no), "EMLINK"))
+        return EMLINK;
+    else if (!strcmp((*error_no), "ENODEV"))
+        return ENODEV;
+    else if (!strcmp((*error_no), "EXDEV"))
+        return EXDEV;
+    else if (!strcmp((*error_no), "EMFILE"))
+        return EMFILE;
+    else if (!strcmp((*error_no), "ENFILE"))
+        return ENFILE;
+    else if (!strcmp((*error_no), "ENOSYS"))
+        return ENOSYS;
+    else if (!strcmp((*error_no), "EINTR"))
+        return EINTR;
+    else if (!strcmp((*error_no), "EFBIG"))
+        return EFBIG;
+    else if (!strcmp((*error_no), "GF_ERROR_SHORT_WRITE"))
+        return GF_ERROR_SHORT_WRITE;
+    else
+        return EAGAIN;
 }
 
 int
-error_gen (xlator_t *this, int op_no)
+error_gen(xlator_t *this, int op_no)
 {
-        eg_t             *egp = NULL;
-        int              count = 0;
-        int              error_no_int = 0;
-        int              rand_no = 0;
-        int              ret = 0;
-        gf_boolean_t     should_err = _gf_false;
-
-        egp = this->private;
-
-        if (egp->random_failure) {
-                /*
-                 * I honestly don't know why anyone would use this "feature"
-                 * but I'll try to preserve its functionality anyway.  Without
-                 * locking twice to update failure_iter_no and egp->op_count
-                 * separately, then not locking at all to update
-                 * egp->failure_iter_no.  That's not needed for compatibility,
-                 * and it's abhorrently wrong.  I have *some* standards.
-                 */
-                LOCK (&egp->lock);
-                {
-                        count = ++(egp->op_count);
-                        error_no_int = egp->error_no_int;
-                        if ((count % egp->failure_iter_no) == 0) {
-                                egp->op_count = 0;
-                                /* coverty[DC.WEAK_CRYPTO] */
-                                egp->failure_iter_no = 3
-                                        + (rand () % GF_UNIVERSAL_ANSWER);
-                                should_err = _gf_true;
-                        }
-                }
-                UNLOCK (&egp->lock);
-        } else {
-                /*
-                 * It turns out that rand() is almost universally implemented
-                 * as a linear congruential PRNG, which is about as cheap as
-                 * it gets.  This gets us real random behavior, including
-                 * phenomena like streaks and dry spells, with controllable
-                 * long-term probability, cheaply.
-                 */
-                if ((rand () % FAILURE_GRANULARITY) < egp->failure_iter_no) {
-                        should_err = _gf_true;
-                }
+    eg_t *egp = NULL;
+    int count = 0;
+    int error_no_int = 0;
+    int rand_no = 0;
+    int ret = 0;
+    gf_boolean_t should_err = _gf_false;
+    int error_no_list_size = 0;
+
+    egp = this->private;
+
+    if (egp->random_failure) {
+        /*
+         * I honestly don't know why anyone would use this "feature"
+         * but I'll try to preserve its functionality anyway.  Without
+         * locking twice to update failure_iter_no and egp->op_count
+         * separately, then not locking at all to update
+         * egp->failure_iter_no.  That's not needed for compatibility,
+         * and it's abhorrently wrong.  I have *some* standards.
+         */
+        LOCK(&egp->lock);
+        {
+            count = ++(egp->op_count);
+            error_no_int = egp->error_no_int;
+            if ((count % egp->failure_iter_no) == 0) {
+                egp->op_count = 0;
+                /* coverity[DC.WEAK_CRYPTO] */
+                egp->failure_iter_no = 3 + (rand() % GF_UNIVERSAL_ANSWER);
+                should_err = _gf_true;
+            }
         }
-
-        if (should_err) {
-                if (error_no_int)
-                        ret = error_no_int;
-                else {
-                        rand_no = generate_rand_no (op_no);
-                        if (op_no >= GF_FOP_MAXVALUE)
-                                op_no = 0;
-                        if (rand_no >= error_no_list[op_no].error_no_count)
-                                rand_no = 0;
-                        ret = error_no_list[op_no].error_no[rand_no];
-                }
+        UNLOCK(&egp->lock);
+    } else {
+        /*
+         * It turns out that rand() is almost universally implemented
+         * as a linear congruential PRNG, which is about as cheap as
+         * it gets.  This gets us real random behavior, including
+         * phenomena like streaks and dry spells, with controllable
+         * long-term probability, cheaply.
+         */
+        if ((rand() % FAILURE_GRANULARITY) < egp->failure_iter_no) {
+            should_err = _gf_true;
         }
+    }
+
+    error_no_list_size = sizeof(error_no_list) / sizeof(error_no_list[0]);
+    if (should_err) {
+        if (error_no_int)
+            ret = error_no_int;
+        else {
+            rand_no = generate_rand_no(op_no);
+            if (op_no >= error_no_list_size)
+                op_no = 0;
+            if (rand_no >= error_no_list[op_no].error_no_count)
+                rand_no = 0;
+            ret = error_no_list[op_no].error_no[rand_no];
+        }
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-error_gen_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  dict_t *xdata)
+error_gen_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_LOOKUP];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_LOOKUP);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                                     NULL);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->lookup,
-		    loc, xdata);
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_LOOKUP];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_LOOKUP);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                            NULL);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup,
+                    loc, xdata);
+    return 0;
 }
 
 int
-error_gen_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+error_gen_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_STAT];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_STAT];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_STAT);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_STAT);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL, xdata);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                   FIRST_CHILD(this)->fops->stat, loc, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(stat, frame, -1, op_errno, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->stat,
+                    loc, xdata);
+    return 0;
 }
 
 int
-error_gen_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
+error_gen_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_SETATTR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_SETATTR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_SETATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_SETATTR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL, xdata);
-	        return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setattr,
-		    loc, stbuf, valid, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(setattr, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-}
+    }
 
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setattr,
+                    loc, stbuf, valid, xdata);
+    return 0;
+}
 
 int
-error_gen_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
+error_gen_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-	int             op_errno = 0;
-        eg_t            *egp = NULL;
-        int             enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSETATTR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSETATTR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSETATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSETATTR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL, xdata);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsetattr,
-		    fd, stbuf, valid, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fsetattr, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetattr,
+                    fd, stbuf, valid, xdata);
+    return 0;
 }
 
 int
-error_gen_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    off_t offset, dict_t *xdata)
+error_gen_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   off_t offset, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_TRUNCATE];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_TRUNCATE);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (truncate, frame, -1, op_errno,
-                                     NULL, NULL, xdata);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->truncate,
-		    loc, offset, xdata);
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_TRUNCATE];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_TRUNCATE);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(truncate, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
+                    loc, offset, xdata);
+    return 0;
 }
 
 int
-error_gen_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		     off_t offset, dict_t *xdata)
+error_gen_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                    dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp =NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FTRUNCATE];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FTRUNCATE);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno,
-                                     NULL, NULL, xdata);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->ftruncate,
-		    fd, offset, xdata);
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FTRUNCATE];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FTRUNCATE);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(ftruncate, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
 int
-error_gen_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		  int32_t mask, dict_t *xdata)
+error_gen_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+                 dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_ACCESS];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_ACCESS);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_ACCESS];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (access, frame, -1, op_errno, xdata);
-		return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_ACCESS);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->access,
-		    loc, mask, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(access, frame, -1, op_errno, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->access,
+                    loc, mask, xdata);
+    return 0;
 }
 
 int
-error_gen_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    size_t size, dict_t *xdata)
+error_gen_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                   dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_READLINK];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_READLINK);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_READLINK];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readlink, frame, -1, op_errno, NULL, NULL, xdata);
-		return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_READLINK);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readlink,
-		    loc, size, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(readlink, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readlink,
+                    loc, size, xdata);
+    return 0;
 }
 
 int
-error_gen_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		 mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
+error_gen_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                dev_t rdev, mode_t umask, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_MKNOD];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_MKNOD);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, xdata);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->mknod,
-		    loc, mode, rdev, umask, xdata);
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_MKNOD];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_MKNOD);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
+                    loc, mode, rdev, umask, xdata);
+    return 0;
 }
 
 int
-error_gen_mkdir (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+error_gen_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                mode_t umask, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_MKDIR];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_MKDIR);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, xdata);
-                return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->mkdir,
-		    loc, mode, umask, xdata);
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_MKDIR];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_MKDIR);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                    loc, mode, umask, xdata);
+    return 0;
 }
 
 int
-error_gen_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-                  dict_t *xdata)
+error_gen_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                 dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_UNLINK];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_UNLINK);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL,
-                                     xdata);
-                return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->unlink,
-		    loc, xflag, xdata);
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_UNLINK];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_UNLINK);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(unlink, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                    loc, xflag, xdata);
+    return 0;
 }
 
 int
-error_gen_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-                 dict_t *xdata)
+error_gen_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+                dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_RMDIR];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_RMDIR);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_RMDIR];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL, NULL, xdata);
-		return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_RMDIR);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->rmdir,
-		    loc, flags, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(rmdir, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
+                    loc, flags, xdata);
+    return 0;
 }
 
 int
-error_gen_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-		   loc_t *loc, mode_t umask, dict_t *xdata)
+error_gen_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+                  loc_t *loc, mode_t umask, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_SYMLINK];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_SYMLINK);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL); /* pre & post parent attr */
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->symlink,
-		    linkpath, loc, umask, xdata);
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_SYMLINK];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_SYMLINK);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(symlink, frame, -1, op_errno, NULL, NULL, NULL,
+                            NULL, NULL); /* pre & post parent attr */
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                    linkpath, loc, umask, xdata);
+    return 0;
 }
 
 int
-error_gen_rename (call_frame_t *frame, xlator_t *this,
-		  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+error_gen_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                 loc_t *newloc, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_RENAME];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_RENAME);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
-                                     NULL, NULL, NULL, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->rename,
-		    oldloc, newloc, xdata);
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_RENAME];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_RENAME);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
+    return 0;
 }
 
 int
-error_gen_link (call_frame_t *frame, xlator_t *this,
-		loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+error_gen_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
 {
-	int             op_errno = 0;
-        eg_t            *egp = NULL;
-        int             enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_LINK];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_LINK);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->link,
-		    oldloc, newloc, xdata);
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_LINK];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_LINK);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            NULL);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+    return 0;
 }
 
 int
-error_gen_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		  int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
-                  dict_t *xdata)
+error_gen_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                 mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-	int             op_errno = 0;
-        eg_t            *egp = NULL;
-        int             enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_CREATE];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_CREATE);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->create,
-		    loc, flags, mode, umask, fd, xdata);
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_CREATE];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_CREATE);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+    return 0;
 }
 
 int
-error_gen_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		int32_t flags, fd_t *fd, dict_t *xdata)
+error_gen_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+               fd_t *fd, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_OPEN];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_OPEN];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_OPEN);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_OPEN);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL, xdata);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->open,
-		    loc, flags, fd, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(open, frame, -1, op_errno, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
+                    loc, flags, fd, xdata);
+    return 0;
 }
 
 int
-error_gen_readv (call_frame_t *frame, xlator_t *this,
-		 fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+error_gen_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, uint32_t flags, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_READ];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_READ);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0,
-                                     NULL, NULL, xdata);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readv,
-		    fd, size, offset, flags, xdata);
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_READ];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_READ);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(readv, frame, -1, op_errno, NULL, 0, NULL, NULL,
+                            xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+    return 0;
 }
 
 int
-error_gen_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		  struct iovec *vector, int32_t count,
-		  off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata)
+error_gen_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+                 struct iobref *iobref, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-	struct iovec *shortvec = NULL;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_WRITE];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_WRITE);
-
-	if (op_errno == GF_ERROR_SHORT_WRITE) {
-
-		/*
-		 * A short write error returns some value less than what was
-		 * requested from a write. To simulate this, replace the vector
-		 * with one half the size;
-		 */
-		shortvec = iov_dup(vector, 1);
-		shortvec->iov_len /= 2;
-		goto wind;
-	} else if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, xdata);
-        	return 0;
-	}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+    struct iovec *shortvec = NULL;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_WRITE];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_WRITE);
+
+    if (op_errno == GF_ERROR_SHORT_WRITE) {
+        /*
+         * A short write error returns some value less than what was
+         * requested from a write. To simulate this, replace the vector
+         * with one half the size;
+         */
+        shortvec = iov_dup(vector, 1);
+        shortvec->iov_len /= 2;
+        count = 1;
+        goto wind;
+    } else if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(writev, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
+    }
 wind:
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                   FIRST_CHILD(this)->fops->writev,
-                   fd, shortvec?shortvec:vector,
-                   count, off, flags, iobref, xdata);
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
+                    fd, shortvec ? shortvec : vector, count, off, flags, iobref,
+                    xdata);
 
-	if (shortvec)
-		GF_FREE (shortvec);
-        return 0;
+    if (shortvec)
+        GF_FREE(shortvec);
+    return 0;
 }
 
 int
-error_gen_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+error_gen_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FLUSH];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FLUSH);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FLUSH];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (flush, frame, -1, op_errno, xdata);
-		return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FLUSH);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->flush,
-		    fd, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(flush, frame, -1, op_errno, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->flush,
+                    fd, xdata);
+    return 0;
 }
 
 int
-error_gen_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
+error_gen_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+                dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSYNC];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSYNC);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSYNC];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL, xdata);
-		return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSYNC);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsync,
-		    fd, flags, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fsync, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
+                    fd, flags, xdata);
+    return 0;
 }
 
 int
-error_gen_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+error_gen_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSTAT];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSTAT];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSTAT);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSTAT);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL, xdata);
-	        return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fstat,
-		    fd, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fstat, frame, -1, op_errno, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat,
+                    fd, xdata);
+    return 0;
 }
 
 int
-error_gen_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
+error_gen_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                  dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_OPENDIR];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_OPENDIR);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_OPENDIR];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL, xdata);
-		return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_OPENDIR);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->opendir,
-		    loc, fd, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(opendir, frame, -1, op_errno, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->opendir,
+                    loc, fd, xdata);
+    return 0;
 }
 
 int
-error_gen_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    int32_t flags, dict_t *xdata)
+error_gen_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+                   dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSYNCDIR];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSYNCDIR);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSYNCDIR];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsyncdir, frame, -1, op_errno, xdata);
-		return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSYNCDIR);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsyncdir,
-		    fd, flags, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fsyncdir, frame, -1, op_errno, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsyncdir,
+                    fd, flags, xdata);
+    return 0;
 }
 
 int
-error_gen_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+error_gen_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_STATFS];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_STATFS);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_STATFS];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (statfs, frame, -1, op_errno, NULL, xdata);
-		return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_STATFS);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->statfs,
-		    loc, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(statfs, frame, -1, op_errno, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->statfs,
+                    loc, xdata);
+    return 0;
 }
 
 int
-error_gen_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    dict_t *dict, int32_t flags, dict_t *xdata)
+error_gen_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *dict, int32_t flags, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_SETXATTR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_SETXATTR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_SETXATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_SETXATTR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setxattr,
-		    loc, dict, flags, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(setxattr, frame, -1, op_errno, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+    return 0;
 }
 
 int
-error_gen_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    const char *name, dict_t *xdata)
+error_gen_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   const char *name, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_GETXATTR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_GETXATTR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_GETXATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_GETXATTR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (getxattr, frame, -1, op_errno, NULL, xdata);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->getxattr,
-		    loc, name, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(getxattr, frame, -1, op_errno, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->getxattr,
+                    loc, name, xdata);
+    return 0;
 }
 
 int
-error_gen_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     dict_t *dict, int32_t flags, dict_t *xdata)
+error_gen_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                    int32_t flags, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSETXATTR];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSETXATTR);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSETXATTR];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsetxattr, frame, -1, op_errno, xdata);
-		return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSETXATTR);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsetxattr,
-		    fd, dict, flags, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fsetxattr, frame, -1, op_errno, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
 int
-error_gen_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     const char *name, dict_t *xdata)
+error_gen_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    const char *name, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FGETXATTR];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FGETXATTR);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FGETXATTR];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, xdata);
-		return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FGETXATTR);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fgetxattr,
-		    fd, name, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fgetxattr, frame, -1, op_errno, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 }
 
 int
-error_gen_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+error_gen_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_XATTROP];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_XATTROP];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_XATTROP);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_XATTROP);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (xattrop, frame, -1, op_errno, NULL, xdata);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->xattrop,
-		    loc, flags, dict, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(xattrop, frame, -1, op_errno, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->xattrop,
+                    loc, flags, dict, xdata);
+    return 0;
 }
 
 int
-error_gen_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+error_gen_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FXATTROP];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FXATTROP];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FXATTROP);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FXATTROP);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fxattrop, frame, -1, op_errno, NULL, xdata);
-	        return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fxattrop,
-		    fd, flags, dict, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fxattrop, frame, -1, op_errno, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fxattrop,
+                    fd, flags, dict, xdata);
+    return 0;
 }
 
 int
-error_gen_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       const char *name, dict_t *xdata)
+error_gen_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      const char *name, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_REMOVEXATTR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_REMOVEXATTR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_REMOVEXATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_REMOVEXATTR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (removexattr, frame, -1, op_errno, xdata);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->removexattr,
-		    loc, name, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(removexattr, frame, -1, op_errno, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 }
 
 int
-error_gen_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                        const char *name, dict_t *xdata)
+error_gen_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       const char *name, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FREMOVEXATTR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FREMOVEXATTR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FREMOVEXATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FREMOVEXATTR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fremovexattr, frame, -1, op_errno, xdata);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fremovexattr,
-		    fd, name, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fremovexattr, frame, -1, op_errno, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
 int
-error_gen_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-	      struct gf_flock *lock, dict_t *xdata)
+error_gen_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+             struct gf_flock *lock, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_LK];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_LK);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_LK];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (lk, frame, -1, op_errno, NULL, xdata);
-		return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_LK);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->lk,
-		    fd, cmd, lock, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(lk, frame, -1, op_errno, NULL, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->lk, fd,
+                    cmd, lock, xdata);
+    return 0;
 }
 
 int
-error_gen_inodelk (call_frame_t *frame, xlator_t *this,
-		   const char *volume, loc_t *loc, int32_t cmd,
-                   struct gf_flock *lock, dict_t *xdata)
+error_gen_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_INODELK];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_INODELK);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_INODELK];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (inodelk, frame, -1, op_errno, xdata);
-		return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_INODELK);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->inodelk,
-		    volume, loc, cmd, lock, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(inodelk, frame, -1, op_errno, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->inodelk,
+                    volume, loc, cmd, lock, xdata);
+    return 0;
 }
 
 int
-error_gen_finodelk (call_frame_t *frame, xlator_t *this,
-		    const char *volume, fd_t *fd, int32_t cmd,
-                    struct gf_flock *lock, dict_t *xdata)
+error_gen_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                   fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FINODELK];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FINODELK);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FINODELK];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (finodelk, frame, -1, op_errno, xdata);
-		return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FINODELK);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->finodelk,
-		    volume, fd, cmd, lock, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(finodelk, frame, -1, op_errno, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->finodelk,
+                    volume, fd, cmd, lock, xdata);
+    return 0;
 }
 
 int
-error_gen_entrylk (call_frame_t *frame, xlator_t *this,
-		   const char *volume, loc_t *loc, const char *basename,
-		   entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+error_gen_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  loc_t *loc, const char *basename, entrylk_cmd cmd,
+                  entrylk_type type, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_ENTRYLK];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_ENTRYLK);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_ENTRYLK];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (entrylk, frame, -1, op_errno, xdata);
-		return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_ENTRYLK);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->entrylk,
-		    volume, loc, basename, cmd, type, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(entrylk, frame, -1, op_errno, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->entrylk,
+                    volume, loc, basename, cmd, type, xdata);
+    return 0;
 }
 
 int
-error_gen_fentrylk (call_frame_t *frame, xlator_t *this,
-		    const char *volume, fd_t *fd, const char *basename,
-		    entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+error_gen_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                   fd_t *fd, const char *basename, entrylk_cmd cmd,
+                   entrylk_type type, dict_t *xdata)
 {
-	int             op_errno = 0;
-        eg_t            *egp = NULL;
-        int             enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FENTRYLK];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FENTRYLK);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FENTRYLK];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fentrylk, frame, -1, op_errno, xdata);
-                return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FENTRYLK);
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fentrylk,
-		    volume, fd, basename, cmd, type, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fentrylk, frame, -1, op_errno, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fentrylk,
+                    volume, fd, basename, cmd, type, xdata);
+    return 0;
 }
 
 int
-error_gen_getspec (call_frame_t *frame, xlator_t *this, const char *key,
-		   int32_t flags)
+error_gen_getspec(call_frame_t *frame, xlator_t *this, const char *key,
+                  int32_t flags)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_GETSPEC];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_GETSPEC);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (getspec, frame, -1, op_errno, NULL);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->getspec,
-		    key, flags);
-	return 0;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_GETSPEC];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_GETSPEC);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(getspec, frame, -1, op_errno, NULL);
+        return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->getspec,
+                    key, flags);
+    return 0;
 }
 
 int
-error_gen_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		   size_t size, off_t off, dict_t *xdata)
+error_gen_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                  off_t off, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_READDIR];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_READDIR);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readdir, frame, -1, op_errno, NULL, xdata);
-		return 0;
-	}
-
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readdir,
-		    fd, size, off, xdata);
-	return 0;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_READDIR];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_READDIR);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(readdir, frame, -1, op_errno, NULL, xdata);
+        return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdir,
+                    fd, size, off, xdata);
+    return 0;
 }
 
 int
-error_gen_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                    off_t off, dict_t *dict)
+error_gen_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                   off_t off, dict_t *dict)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_READDIRP];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_READDIRP];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_READDIRP);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_READDIRP);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL, NULL);
-		return 0;
-	}
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(readdirp, frame, -1, op_errno, NULL, NULL);
+        return 0;
+    }
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp,
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
                     fd, size, off, dict);
-	return 0;
+    return 0;
 }
 
 static void
-error_gen_set_failure (eg_t *pvt, double percent)
+error_gen_set_failure(eg_t *pvt, double percent)
 {
-        double  ppm;
+    double ppm;
 
-        GF_ASSERT (pvt);
+    GF_ASSERT(pvt);
 
-        ppm = (percent / 100.0) * (double)FAILURE_GRANULARITY;
-        pvt->failure_iter_no = (int)ppm;
+    ppm = (percent / 100.0) * (double)FAILURE_GRANULARITY;
+    pvt->failure_iter_no = (int)ppm;
 }
 
 static void
-error_gen_parse_fill_fops (eg_t *pvt, char *enable_fops)
+error_gen_parse_fill_fops(eg_t *pvt, char *enable_fops)
 {
-        char            *op_no_str = NULL;
-        int              op_no = -1;
-        int              i = 0;
-        xlator_t        *this = THIS;
-        char            *saveptr = NULL;
+    char *op_no_str = NULL;
+    int op_no = -1;
+    int i = 0;
+    xlator_t *this = THIS;
+    char *saveptr = NULL;
+
+    GF_ASSERT(pvt);
+    GF_ASSERT(this);
 
-        GF_ASSERT (pvt);
-        GF_ASSERT (this);
+    for (i = 0; i < GF_FOP_MAXVALUE; i++)
+        pvt->enable[i] = 0;
 
+    if (!enable_fops) {
+        gf_log(this->name, GF_LOG_WARNING, "All fops are enabled.");
         for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                pvt->enable[i] = 0;
-
-        if (!enable_fops) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "All fops are enabled.");
-                for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                        pvt->enable[i] = 1;
-        } else {
-                op_no_str = strtok_r (enable_fops, ",", &saveptr);
-                while (op_no_str) {
-                        op_no = gf_fop_int (op_no_str);
-                        if (op_no == -1) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Wrong option value %s", op_no_str);
-                        } else
-                                pvt->enable[op_no] = 1;
-
-                        op_no_str = strtok_r (NULL, ",", &saveptr);
-                }
+            pvt->enable[i] = 1;
+    } else {
+        op_no_str = strtok_r(enable_fops, ",", &saveptr);
+        while (op_no_str) {
+            op_no = gf_fop_int(op_no_str);
+            if (op_no == -1) {
+                gf_log(this->name, GF_LOG_WARNING, "Wrong option value %s",
+                       op_no_str);
+            } else
+                pvt->enable[op_no] = 1;
+
+            op_no_str = strtok_r(NULL, ",", &saveptr);
         }
+    }
 }
 
 int32_t
-error_gen_priv_dump (xlator_t *this)
+error_gen_priv_dump(xlator_t *this)
 {
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
-        int             ret = -1;
-        eg_t            *conf = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    int ret = -1;
+    eg_t *conf = NULL;
 
-        if (!this)
-            goto out;
+    if (!this)
+        goto out;
 
-        conf = this->private;
-        if (!conf)
-            goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        ret = TRY_LOCK(&conf->lock);
-        if (ret != 0) {
-                return ret;
-        }
+    ret = TRY_LOCK(&conf->lock);
+    if (ret != 0) {
+        return ret;
+    }
 
-        gf_proc_dump_add_section("xlator.debug.error-gen.%s.priv", this->name);
-        gf_proc_dump_build_key(key_prefix,"xlator.debug.error-gen","%s.priv",
-                               this->name);
+    gf_proc_dump_add_section("xlator.debug.error-gen.%s.priv", this->name);
+    gf_proc_dump_build_key(key_prefix, "xlator.debug.error-gen", "%s.priv",
+                           this->name);
 
-        gf_proc_dump_write("op_count", "%d", conf->op_count);
-        gf_proc_dump_write("failure_iter_no", "%d", conf->failure_iter_no);
-        gf_proc_dump_write("error_no", "%s", conf->error_no);
-        gf_proc_dump_write("random_failure", "%d", conf->random_failure);
+    gf_proc_dump_write("op_count", "%d", conf->op_count);
+    gf_proc_dump_write("failure_iter_no", "%d", conf->failure_iter_no);
+    gf_proc_dump_write("error_no", "%d", conf->error_no_int);
+    gf_proc_dump_write("random_failure", "%d", conf->random_failure);
 
-        UNLOCK(&conf->lock);
+    UNLOCK(&conf->lock);
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_error_gen_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_error_gen_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               " failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        eg_t            *pvt = NULL;
-        int32_t          ret = 0;
-        char            *error_enable_fops = NULL;
-        double           failure_percent_dbl = 0.0;
+    eg_t *pvt = NULL;
+    int32_t ret = 0;
+    char *error_enable_fops = NULL;
+    char *error_no = NULL;
+    double failure_percent_dbl = 0.0;
 
-        if (!this || !this->private)
-                goto out;
+    if (!this || !this->private)
+        goto out;
 
-        pvt = this->private;
+    pvt = this->private;
 
-        ret = -1;
+    ret = -1;
 
-        GF_OPTION_RECONF ("error-no", pvt->error_no, options, str, out);
+    GF_OPTION_RECONF("error-no", error_no, options, str, out);
 
-        if (pvt->error_no)
-                pvt->error_no_int = conv_errno_to_int (&pvt->error_no);
+    if (error_no)
+        pvt->error_no_int = conv_errno_to_int(&error_no);
 
-        GF_OPTION_RECONF ("failure", failure_percent_dbl, options, percent,
-                          out);
+    GF_OPTION_RECONF("failure", failure_percent_dbl, options, percent, out);
 
-        GF_OPTION_RECONF ("enable", error_enable_fops, options, str, out);
+    GF_OPTION_RECONF("enable", error_enable_fops, options, str, out);
 
-        GF_OPTION_RECONF ("random-failure", pvt->random_failure, options,
-                          bool, out);
+    GF_OPTION_RECONF("random-failure", pvt->random_failure, options, bool, out);
 
-        error_gen_parse_fill_fops (pvt, error_enable_fops);
-        error_gen_set_failure (pvt, failure_percent_dbl);
+    error_gen_parse_fill_fops(pvt, error_enable_fops);
+    error_gen_set_failure(pvt, failure_percent_dbl);
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "reconfigure returning %d", ret);
-        return ret;
+    gf_log(this ? this->name : "error-gen", GF_LOG_DEBUG,
+           "reconfigure returning %d", ret);
+    return ret;
 }
 
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        eg_t            *pvt = NULL;
-        int32_t          ret = 0;
-        char            *error_enable_fops = NULL;
-        double          failure_percent_dbl = 0.0;
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error-gen not configured with one subvolume");
-                ret = -1;
-                goto out;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
-
-        pvt = GF_CALLOC (1, sizeof (eg_t), gf_error_gen_mt_eg_t);
+    eg_t *pvt = NULL;
+    int32_t ret = 0;
+    char *error_enable_fops = NULL;
+    char *error_no = NULL;
+    double failure_percent_dbl = 0.0;
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "error-gen not configured with one subvolume");
+        ret = -1;
+        goto out;
+    }
 
-        if (!pvt) {
-                ret = -1;
-                goto out;
-        }
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
 
-        LOCK_INIT (&pvt->lock);
+    pvt = GF_CALLOC(1, sizeof(eg_t), gf_error_gen_mt_eg_t);
 
+    if (!pvt) {
         ret = -1;
+        goto out;
+    }
+
+    LOCK_INIT(&pvt->lock);
 
-        GF_OPTION_INIT ("error-no", pvt->error_no, str, out);
+    ret = -1;
 
-        if (pvt->error_no)
-                pvt->error_no_int = conv_errno_to_int (&pvt->error_no);
+    GF_OPTION_INIT("error-no", error_no, str, out);
 
-        GF_OPTION_INIT ("failure", failure_percent_dbl, percent, out);
+    if (error_no)
+        pvt->error_no_int = conv_errno_to_int(&error_no);
 
-        GF_OPTION_INIT ("enable", error_enable_fops, str, out);
+    GF_OPTION_INIT("failure", failure_percent_dbl, percent, out);
 
-        GF_OPTION_INIT ("random-failure", pvt->random_failure, bool, out);
+    GF_OPTION_INIT("enable", error_enable_fops, str, out);
 
+    GF_OPTION_INIT("random-failure", pvt->random_failure, bool, out);
 
-        error_gen_parse_fill_fops (pvt, error_enable_fops);
-        error_gen_set_failure (pvt, failure_percent_dbl);
+    error_gen_parse_fill_fops(pvt, error_enable_fops);
+    error_gen_set_failure(pvt, failure_percent_dbl);
 
-        this->private = pvt;
+    this->private = pvt;
 
-        /* Give some seed value here */
-        srand (time(NULL));
+    /* Give some seed value here. */
+    srand(gf_time());
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret)
-                GF_FREE (pvt);
-        return ret;
+    if (ret)
+        GF_FREE(pvt);
+    return ret;
 }
 
-
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        eg_t            *pvt = NULL;
+    eg_t *pvt = NULL;
 
-        if (!this)
-                 return;
-        pvt = this->private;
-
-        if (pvt) {
-                LOCK_DESTROY (&pvt->lock);
-                GF_FREE (pvt);
-                gf_log (this->name, GF_LOG_DEBUG, "fini called");
-        }
+    if (!this)
         return;
+    pvt = this->private;
+
+    if (pvt) {
+        LOCK_DESTROY(&pvt->lock);
+        GF_FREE(pvt);
+        gf_log(this->name, GF_LOG_DEBUG, "fini called");
+    }
+    return;
 }
 
 struct xlator_dumpops dumpops = {
-        .priv = error_gen_priv_dump,
+    .priv = error_gen_priv_dump,
 };
 
 struct xlator_cbks cbks;
 
 struct xlator_fops fops = {
-	.lookup      = error_gen_lookup,
-	.stat        = error_gen_stat,
-	.readlink    = error_gen_readlink,
-	.mknod       = error_gen_mknod,
-	.mkdir       = error_gen_mkdir,
-	.unlink      = error_gen_unlink,
-	.rmdir       = error_gen_rmdir,
-	.symlink     = error_gen_symlink,
-	.rename      = error_gen_rename,
-	.link        = error_gen_link,
-	.truncate    = error_gen_truncate,
-	.create      = error_gen_create,
-	.open        = error_gen_open,
-	.readv       = error_gen_readv,
-	.writev      = error_gen_writev,
-	.statfs      = error_gen_statfs,
-	.flush       = error_gen_flush,
-	.fsync       = error_gen_fsync,
-	.setxattr    = error_gen_setxattr,
-	.getxattr    = error_gen_getxattr,
-	.removexattr = error_gen_removexattr,
-	.fsetxattr    = error_gen_fsetxattr,
-	.fgetxattr    = error_gen_fgetxattr,
-	.fremovexattr = error_gen_fremovexattr,
-	.opendir     = error_gen_opendir,
-	.readdir     = error_gen_readdir,
-	.readdirp    = error_gen_readdirp,
-	.fsyncdir    = error_gen_fsyncdir,
-	.access      = error_gen_access,
-	.ftruncate   = error_gen_ftruncate,
-	.fstat       = error_gen_fstat,
-	.lk          = error_gen_lk,
-	.xattrop     = error_gen_xattrop,
-	.fxattrop    = error_gen_fxattrop,
-	.inodelk     = error_gen_inodelk,
-	.finodelk    = error_gen_finodelk,
-	.entrylk     = error_gen_entrylk,
-	.fentrylk    = error_gen_fentrylk,
-        .setattr     = error_gen_setattr,
-        .fsetattr    = error_gen_fsetattr,
-	.getspec     = error_gen_getspec,
+    .lookup = error_gen_lookup,
+    .stat = error_gen_stat,
+    .readlink = error_gen_readlink,
+    .mknod = error_gen_mknod,
+    .mkdir = error_gen_mkdir,
+    .unlink = error_gen_unlink,
+    .rmdir = error_gen_rmdir,
+    .symlink = error_gen_symlink,
+    .rename = error_gen_rename,
+    .link = error_gen_link,
+    .truncate = error_gen_truncate,
+    .create = error_gen_create,
+    .open = error_gen_open,
+    .readv = error_gen_readv,
+    .writev = error_gen_writev,
+    .statfs = error_gen_statfs,
+    .flush = error_gen_flush,
+    .fsync = error_gen_fsync,
+    .setxattr = error_gen_setxattr,
+    .getxattr = error_gen_getxattr,
+    .removexattr = error_gen_removexattr,
+    .fsetxattr = error_gen_fsetxattr,
+    .fgetxattr = error_gen_fgetxattr,
+    .fremovexattr = error_gen_fremovexattr,
+    .opendir = error_gen_opendir,
+    .readdir = error_gen_readdir,
+    .readdirp = error_gen_readdirp,
+    .fsyncdir = error_gen_fsyncdir,
+    .access = error_gen_access,
+    .ftruncate = error_gen_ftruncate,
+    .fstat = error_gen_fstat,
+    .lk = error_gen_lk,
+    .xattrop = error_gen_xattrop,
+    .fxattrop = error_gen_fxattrop,
+    .inodelk = error_gen_inodelk,
+    .finodelk = error_gen_finodelk,
+    .entrylk = error_gen_entrylk,
+    .fentrylk = error_gen_fentrylk,
+    .setattr = error_gen_setattr,
+    .fsetattr = error_gen_fsetattr,
+    .getspec = error_gen_getspec,
 };
 
 struct volume_options options[] = {
-        { .key  = {"failure"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .description = "Percentage failure of operations when enabled.",
-        },
-
-        { .key  = {"error-no"},
-          .value = {"ENOENT","ENOTDIR","ENAMETOOLONG","EACCES","EBADF",
-                    "EFAULT","ENOMEM","EINVAL","EIO","EEXIST","ENOSPC",
-                    "EPERM","EROFS","EBUSY","EISDIR","ENOTEMPTY","EMLINK"
-                    "ENODEV","EXDEV","EMFILE","ENFILE","ENOSYS","EINTR",
-                    "EFBIG","EAGAIN","GF_ERROR_SHORT_WRITE"},
-          .type = GF_OPTION_TYPE_STR,
-          .op_version = {3},
-          .tags = {"error-gen"},
-          .flags = OPT_FLAG_SETTABLE,
-
-        },
-
-        { .key = {"random-failure"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .op_version = {3},
-          .tags = {"error-gen"},
-          .flags = OPT_FLAG_SETTABLE,
-        },
-
-        { .key  = {"enable", "error-fops"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Accepts a string which takes ',' separated fop "
-                         "strings to denote which fops are enabled for error",
-          .op_version = {3},
-          .tags = {"error-gen"},
-          .flags = OPT_FLAG_SETTABLE,
-        },
-
-        { .key  = {NULL} }
+    {
+        .key = {"failure"},
+        .type = GF_OPTION_TYPE_PERCENT,
+        .description = "Percentage failure of operations when enabled.",
+    },
+
+    {
+        .key = {"error-no"},
+        .value = {"ENOENT",
+                  "ENOTDIR",
+                  "ENAMETOOLONG",
+                  "EACCES",
+                  "EBADF",
+                  "EFAULT",
+                  "ENOMEM",
+                  "EINVAL",
+                  "EIO",
+                  "EEXIST",
+                  "ENOSPC",
+                  "EPERM",
+                  "EROFS",
+                  "EBUSY",
+                  "EISDIR",
+                  "ENOTEMPTY",
+                  "EMLINK"
+                  "ENODEV",
+                  "EXDEV",
+                  "EMFILE",
+                  "ENFILE",
+                  "ENOSYS",
+                  "EINTR",
+                  "EFBIG",
+                  "EAGAIN",
+                  "GF_ERROR_SHORT_WRITE"},
+        .type = GF_OPTION_TYPE_STR,
+        .op_version = {3},
+        .tags = {"error-gen"},
+        .flags = OPT_FLAG_SETTABLE,
+
+    },
+
+    {
+        .key = {"random-failure"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .op_version = {3},
+        .tags = {"error-gen"},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+
+    {
+        .key = {"enable", "error-fops"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Accepts a string which takes ',' separated fop "
+                       "strings to denote which fops are enabled for error",
+        .op_version = {3},
+        .tags = {"error-gen"},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "error-gen",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/debug/error-gen/src/error-gen.h b/xlators/debug/error-gen/src/error-gen.h
index 2cd95de335a..2478cd5b21c 100644
--- a/xlators/debug/error-gen/src/error-gen.h
+++ b/xlators/debug/error-gen/src/error-gen.h
@@ -22,29 +22,28 @@
  * range.
  */
 enum GF_PSEUDO_ERRORS {
-	GF_ERROR_SHORT_WRITE = 1000,	/* short writev return value */
-	GF_ERROR_MAX
+    GF_ERROR_SHORT_WRITE = 1000, /* short writev return value */
+    GF_ERROR_MAX
 };
 
 typedef struct {
-        int enable[GF_FOP_MAXVALUE];
-        int op_count;
-        /*
-         * This is only an iteration number in the random-failure case.  For
-         * the normal controlled-probability case, it's actually a numerator
-         * for the failure probability (see FAILURE_GRANULARITY declaration).
-         * It's just not worth blowing up the diff by changing it.
-         */
-        int failure_iter_no;
-        char *error_no;
-        int error_no_int;
-        gf_boolean_t random_failure;
-        gf_lock_t lock;
+    int enable[GF_FOP_MAXVALUE];
+    int op_count;
+    /*
+     * This is only an iteration number in the random-failure case.  For
+     * the normal controlled-probability case, it's actually a numerator
+     * for the failure probability (see FAILURE_GRANULARITY declaration).
+     * It's just not worth blowing up the diff by changing it.
+     */
+    int failure_iter_no;
+    int error_no_int;
+    gf_boolean_t random_failure;
+    gf_lock_t lock;
 } eg_t;
 
 typedef struct {
-        int error_no_count;
-	int error_no[20];
+    int error_no_count;
+    int error_no[20];
 } sys_error_t;
 
 #endif
diff --git a/xlators/debug/io-stats/src/io-stats-mem-types.h b/xlators/debug/io-stats/src/io-stats-mem-types.h
index 9dde9373264..51d38d8b97c 100644
--- a/xlators/debug/io-stats/src/io-stats-mem-types.h
+++ b/xlators/debug/io-stats/src/io-stats-mem-types.h
@@ -11,18 +11,17 @@
 #ifndef __IO_STATS_MEM_TYPES_H__
 #define __IO_STATS_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 extern const char *__progname;
 
 enum gf_io_stats_mem_types_ {
-        gf_io_stats_mt_ios_conf = gf_common_mt_end + 1,
-        gf_io_stats_mt_ios_fd,
-        gf_io_stats_mt_ios_stat,
-        gf_io_stats_mt_ios_stat_list,
-        gf_io_stats_mt_ios_sample_buf,
-        gf_io_stats_mt_ios_sample,
-        gf_io_stats_mt_end
+    gf_io_stats_mt_ios_conf = gf_common_mt_end + 1,
+    gf_io_stats_mt_ios_fd,
+    gf_io_stats_mt_ios_stat,
+    gf_io_stats_mt_ios_stat_list,
+    gf_io_stats_mt_ios_sample_buf,
+    gf_io_stats_mt_ios_sample,
+    gf_io_stats_mt_end
 };
 #endif
-
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index f5d86e1bc6a..aa00c446e5a 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -7,8 +7,8 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "xlator.h"
-#include "syscall.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/syscall.h>
 
 /**
  * xlators/debug/io_stats :
@@ -17,9 +17,9 @@
  *
  *  a) total read data - since process start, last interval and per fd
  *  b) total write data - since process start, last interval and per fd
- *  c) counts of read IO block size - since process start, last interval and per fd
- *  d) counts of write IO block size - since process start, last interval and per fd
- *  e) counts of all FOP types passing through it
+ *  c) counts of read IO block size - since process start, last interval and per
+ * fd d) counts of write IO block size - since process start, last interval and
+ * per fd e) counts of all FOP types passing through it
  *
  *  Usage: setfattr -n trusted.io-stats-dump /tmp/filename /mnt/gluster
  *      output is written to /tmp/filename.<iostats xlator instance name>
@@ -28,187 +28,199 @@
 
 #include <fnmatch.h>
 #include <errno.h>
-#include "glusterfs.h"
-#include "xlator.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
 #include "io-stats-mem-types.h"
 #include <stdarg.h>
-#include "defaults.h"
-#include "logging.h"
-#include "cli1-xdr.h"
-#include "statedump.h"
-#include "syncop.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/syncop.h>
 #include <pwd.h>
 #include <grp.h>
-#include "upcall-utils.h"
+#include <glusterfs/upcall-utils.h>
+#include <glusterfs/async.h>
 
 #define MAX_LIST_MEMBERS 100
 #define DEFAULT_PWD_BUF_SZ 16384
 #define DEFAULT_GRP_BUF_SZ 16384
 #define IOS_BLOCK_COUNT_SIZE 32
 
+#define IOS_STATS_DUMP_DIR DEFAULT_VAR_RUN_DIRECTORY
+
 typedef enum {
-        IOS_STATS_TYPE_NONE,
-        IOS_STATS_TYPE_OPEN,
-        IOS_STATS_TYPE_READ,
-        IOS_STATS_TYPE_WRITE,
-        IOS_STATS_TYPE_OPENDIR,
-        IOS_STATS_TYPE_READDIRP,
-        IOS_STATS_TYPE_READ_THROUGHPUT,
-        IOS_STATS_TYPE_WRITE_THROUGHPUT,
-        IOS_STATS_TYPE_MAX
-}ios_stats_type_t;
+    IOS_STATS_TYPE_NONE,
+    IOS_STATS_TYPE_OPEN,
+    IOS_STATS_TYPE_READ,
+    IOS_STATS_TYPE_WRITE,
+    IOS_STATS_TYPE_OPENDIR,
+    IOS_STATS_TYPE_READDIRP,
+    IOS_STATS_TYPE_READ_THROUGHPUT,
+    IOS_STATS_TYPE_WRITE_THROUGHPUT,
+    IOS_STATS_TYPE_MAX
+} ios_stats_type_t;
 
 typedef enum {
-        IOS_STATS_THRU_READ,
-        IOS_STATS_THRU_WRITE,
-        IOS_STATS_THRU_MAX,
-}ios_stats_thru_t;
+    IOS_STATS_THRU_READ,
+    IOS_STATS_THRU_WRITE,
+    IOS_STATS_THRU_MAX,
+} ios_stats_thru_t;
+
+/* This is same as gf1_cli_info_op */
+/* had to be defined here again, so we have modularity between
+ xdr, xlator, and library functions */
+typedef enum ios_info_op {
+    GF_IOS_INFO_NONE = 0,
+    GF_IOS_INFO_ALL = 1,
+    GF_IOS_INFO_INCREMENTAL = 2,
+    GF_IOS_INFO_CUMULATIVE = 3,
+    GF_IOS_INFO_CLEAR = 4,
+} ios_info_op_t;
 
 struct ios_stat_lat {
-        struct timeval  time;
-        double          throughput;
+    struct timeval time;
+    double throughput;
 };
 
 struct ios_stat {
-        gf_lock_t       lock;
-        uuid_t          gfid;
-        char           *filename;
-        gf_atomic_t     counters[IOS_STATS_TYPE_MAX];
-        struct ios_stat_lat thru_counters [IOS_STATS_THRU_MAX];
-        gf_atomic_t     refcnt;
+    gf_lock_t lock;
+    uuid_t gfid;
+    char *filename;
+    gf_atomic_t counters[IOS_STATS_TYPE_MAX];
+    struct ios_stat_lat thru_counters[IOS_STATS_THRU_MAX];
+    gf_atomic_t refcnt;
 };
 
 struct ios_stat_list {
-        struct list_head  list;
-        struct ios_stat  *iosstat;
-        double            value;
+    struct list_head list;
+    struct ios_stat *iosstat;
+    double value;
 };
 
 struct ios_stat_head {
-       gf_lock_t                lock;
-       double                   min_cnt;
-       uint64_t                 members;
-       struct ios_stat_list    *iosstats;
+    gf_lock_t lock;
+    double min_cnt;
+    uint64_t members;
+    struct ios_stat_list *iosstats;
 };
 
 typedef struct _ios_sample_t {
-        uid_t  uid;
-        gid_t  gid;
-        char   identifier[UNIX_PATH_MAX];
-        glusterfs_fop_t fop_type;
-        struct timeval timestamp;
-        double elapsed;
+    uid_t uid;
+    gid_t gid;
+    char identifier[UNIX_PATH_MAX];
+    glusterfs_fop_t fop_type;
+    struct timeval timestamp;
+    double elapsed;
 } ios_sample_t;
 
-
 typedef struct _ios_sample_buf_t {
-        uint64_t        pos;  /* Position in write buffer */
-        uint64_t        size;  /* Size of ring buffer */
-        uint64_t        collected;  /* Number of samples we've collected */
-        uint64_t        observed;  /* Number of FOPs we've observed */
-        ios_sample_t    *ios_samples;  /* Our list of samples */
+    uint64_t pos;              /* Position in write buffer */
+    uint64_t size;             /* Size of ring buffer */
+    uint64_t collected;        /* Number of samples we've collected */
+    uint64_t observed;         /* Number of FOPs we've observed */
+    ios_sample_t *ios_samples; /* Our list of samples */
 } ios_sample_buf_t;
 
-
 struct ios_lat {
-        double      min;
-        double      max;
-        double      avg;
-        uint64_t    total;
+    double min;
+    double max;
+    double avg;
+    uint64_t total;
 };
 
 struct ios_global_stats {
-        gf_atomic_t     data_written;
-        gf_atomic_t     data_read;
-        gf_atomic_t     block_count_write[IOS_BLOCK_COUNT_SIZE];
-        gf_atomic_t     block_count_read[IOS_BLOCK_COUNT_SIZE];
-        gf_atomic_t     fop_hits[GF_FOP_MAXVALUE];
-        gf_atomic_t     upcall_hits[GF_UPCALL_FLAGS_MAXVALUE];
-        struct timeval  started_at;
-        struct ios_lat  latency[GF_FOP_MAXVALUE];
-        uint64_t        nr_opens;
-        uint64_t        max_nr_opens;
-        struct timeval  max_openfd_time;
+    gf_atomic_t data_written;
+    gf_atomic_t data_read;
+    gf_atomic_t block_count_write[IOS_BLOCK_COUNT_SIZE];
+    gf_atomic_t block_count_read[IOS_BLOCK_COUNT_SIZE];
+    gf_atomic_t fop_hits[GF_FOP_MAXVALUE];
+    gf_atomic_t upcall_hits[GF_UPCALL_FLAGS_MAXVALUE];
+    time_t started_at;
+    struct ios_lat latency[GF_FOP_MAXVALUE];
+    uint64_t nr_opens;
+    uint64_t max_nr_opens;
+    struct timeval max_openfd_time;
 };
 
 typedef enum {
-        IOS_DUMP_TYPE_NONE      = 0,
-        IOS_DUMP_TYPE_FILE      = 1,
-        IOS_DUMP_TYPE_DICT      = 2,
-        IOS_DUMP_TYPE_JSON_FILE = 3,
-        IOS_DUMP_TYPE_SAMPLES   = 4,
-        IOS_DUMP_TYPE_MAX       = 5
+    IOS_DUMP_TYPE_NONE = 0,
+    IOS_DUMP_TYPE_FILE = 1,
+    IOS_DUMP_TYPE_DICT = 2,
+    IOS_DUMP_TYPE_JSON_FILE = 3,
+    IOS_DUMP_TYPE_SAMPLES = 4,
+    IOS_DUMP_TYPE_MAX = 5
 } ios_dump_type_t;
 
 struct ios_conf {
-        gf_lock_t                 lock;
-        struct ios_global_stats   cumulative;
-        uint64_t                  increment;
-        struct ios_global_stats   incremental;
-        gf_boolean_t              dump_fd_stats;
-        gf_boolean_t              count_fop_hits;
-        gf_boolean_t              measure_latency;
-        struct ios_stat_head      list[IOS_STATS_TYPE_MAX];
-        struct ios_stat_head      thru_list[IOS_STATS_THRU_MAX];
-        int32_t                   ios_dump_interval;
-        pthread_t                 dump_thread;
-        gf_boolean_t              dump_thread_should_die;
-        gf_lock_t                 ios_sampling_lock;
-        int32_t                   ios_sample_interval;
-        int32_t                   ios_sample_buf_size;
-        ios_sample_buf_t          *ios_sample_buf;
-        struct dnscache           *dnscache;
-        int32_t                   ios_dnscache_ttl_sec;
-        /*
-         * What we really need here is just a unique value to keep files
-         * created by this instance distinct from those created by any other.
-         * On the client side this isn't a problem, so we just use the
-         * translator name.  On the server side conflicts can occur, so the
-         * volfile-generation code automatically sets this (via an option)
-         * to be the brick path.
-         *
-         * NB While the *field* name has changed, it didn't seem worth changing
-         * all of the cases where "xlator_name" is used as a *variable* name.
-         */
-        char                      *unique_id;
-        ios_dump_type_t            dump_format;
-        char                      *dump_format_str;
+    gf_lock_t lock;
+    struct ios_global_stats cumulative;
+    uint64_t increment;
+    struct ios_global_stats incremental;
+    gf_boolean_t dump_fd_stats;
+    gf_boolean_t count_fop_hits;
+    gf_boolean_t measure_latency;
+    struct ios_stat_head list[IOS_STATS_TYPE_MAX];
+    struct ios_stat_head thru_list[IOS_STATS_THRU_MAX];
+    int32_t ios_dump_interval;
+    pthread_t dump_thread;
+    gf_boolean_t dump_thread_should_die;
+    gf_boolean_t dump_thread_running;
+    gf_lock_t ios_sampling_lock;
+    int32_t ios_sample_interval;
+    int32_t ios_sample_buf_size;
+    ios_sample_buf_t *ios_sample_buf;
+    struct dnscache *dnscache;
+    int32_t ios_dnscache_ttl_sec;
+    /*
+     * What we really need here is just a unique value to keep files
+     * created by this instance distinct from those created by any other.
+     * On the client side this isn't a problem, so we just use the
+     * translator name.  On the server side conflicts can occur, so the
+     * volfile-generation code automatically sets this (via an option)
+     * to be the brick path.
+     *
+     * NB While the *field* name has changed, it didn't seem worth changing
+     * all of the cases where "xlator_name" is used as a *variable* name.
+     */
+    char *unique_id;
+    ios_dump_type_t dump_format;
 };
 
-
 struct ios_fd {
-        char           *filename;
-        gf_atomic_t     data_written;
-        gf_atomic_t     data_read;
-        gf_atomic_t     block_count_write[IOS_BLOCK_COUNT_SIZE];
-        gf_atomic_t     block_count_read[IOS_BLOCK_COUNT_SIZE];
-        struct timeval  opened_at;
+    char *filename;
+    gf_atomic_t data_written;
+    gf_atomic_t data_read;
+    gf_atomic_t block_count_write[IOS_BLOCK_COUNT_SIZE];
+    gf_atomic_t block_count_read[IOS_BLOCK_COUNT_SIZE];
+    struct timeval opened_at;
 };
 
 struct ios_dump_args {
-        ios_dump_type_t type;
-        union {
-                FILE *logfp;
-                dict_t *dict;
-        } u;
+    ios_dump_type_t type;
+    union {
+        FILE *logfp;
+        dict_t *dict;
+    } u;
 };
 
-typedef int (*block_dump_func) (xlator_t *, struct ios_dump_args*,
-                                    int , int , uint64_t ) ;
+typedef int (*block_dump_func)(xlator_t *, struct ios_dump_args *, int, int,
+                               uint64_t);
 
 struct ios_local {
-        struct timeval  wind_at;
-        struct timeval  unwind_at;
+    struct timeval wind_at;
+    struct timeval unwind_at;
 };
 
 struct volume_options options[];
 
 static int
-is_fop_latency_started (call_frame_t *frame)
+is_fop_latency_started(call_frame_t *frame)
 {
-        GF_ASSERT (frame);
-        struct timeval epoch = {0,};
-        return memcmp (&frame->begin, &epoch, sizeof (epoch));
+    GF_ASSERT(frame);
+    struct timeval epoch = {
+        0,
+    };
+    return memcmp(&frame->begin, &epoch, sizeof(epoch));
 }
 
 #define _IOS_SAMP_DIR DEFAULT_LOG_FILE_DIRECTORY "/samples"
@@ -218,354 +230,337 @@ is_fop_latency_started (call_frame_t *frame)
 #define _IOS_DUMP_DIR DATADIR "/db/glusterd/stats"
 #endif
 
-#define END_FOP_LATENCY(frame, op)                                      \
-        do {                                                            \
-                struct ios_conf  *conf = NULL;                          \
-                                                                        \
-                conf = this->private;                                   \
-                if (conf && conf->measure_latency) {                    \
-                        timespec_now (&frame->end);                     \
-                        update_ios_latency (conf, frame, GF_FOP_##op);  \
-                }                                                       \
-        } while (0)
-
-#define START_FOP_LATENCY(frame)                                         \
-        do {                                                             \
-                struct ios_conf  *conf = NULL;                           \
-                                                                         \
-                conf = this->private;                                    \
-                if (conf && conf->measure_latency) {                     \
-                        timespec_now (&frame->begin);                    \
-                } else {                                                 \
-                        memset (&frame->begin, 0, sizeof (frame->begin));\
-                }                                                        \
-        } while (0)
-
-
-#define BUMP_FOP(op)                                                    \
-        do {                                                            \
-                struct ios_conf  *conf = NULL;                          \
-                                                                        \
-                conf = this->private;                                   \
-                if (!conf)                                              \
-                        break;                                          \
-                GF_ATOMIC_INC (conf->cumulative.fop_hits[GF_FOP_##op]); \
-                GF_ATOMIC_INC (conf->incremental.fop_hits[GF_FOP_##op]);\
-        } while (0)
-
-#define UPDATE_PROFILE_STATS(frame, op)                                       \
-        do {                                                                  \
-                struct ios_conf  *conf = NULL;                                \
-                                                                              \
-                if (!is_fop_latency_started (frame))                          \
-                        break;                                                \
-                conf = this->private;                                         \
-                if (conf && conf->measure_latency &&                          \
-                    conf->count_fop_hits) {                                   \
-                        BUMP_FOP (op);                                        \
-                        timespec_now (&frame->end);                           \
-                        update_ios_latency (conf, frame, GF_FOP_##op);        \
-                }                                                             \
-        } while (0)
+#define END_FOP_LATENCY(frame, op)                                             \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
+                                                                               \
+        conf = this->private;                                                  \
+        if (conf && conf->measure_latency) {                                   \
+            timespec_now(&frame->end);                                         \
+            update_ios_latency(conf, frame, GF_FOP_##op);                      \
+        }                                                                      \
+    } while (0)
+
+#define START_FOP_LATENCY(frame)                                               \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
+                                                                               \
+        conf = this->private;                                                  \
+        if (conf && conf->measure_latency) {                                   \
+            timespec_now(&frame->begin);                                       \
+        } else {                                                               \
+            memset(&frame->begin, 0, sizeof(frame->begin));                    \
+        }                                                                      \
+    } while (0)
+
+#define BUMP_FOP(op)                                                           \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
+                                                                               \
+        conf = this->private;                                                  \
+        if (!conf)                                                             \
+            break;                                                             \
+        GF_ATOMIC_INC(conf->cumulative.fop_hits[GF_FOP_##op]);                 \
+        GF_ATOMIC_INC(conf->incremental.fop_hits[GF_FOP_##op]);                \
+    } while (0)
+
+#define UPDATE_PROFILE_STATS(frame, op)                                        \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
+                                                                               \
+        if (!is_fop_latency_started(frame))                                    \
+            break;                                                             \
+        conf = this->private;                                                  \
+        if (conf && conf->measure_latency && conf->count_fop_hits) {           \
+            BUMP_FOP(op);                                                      \
+            timespec_now(&frame->end);                                         \
+            update_ios_latency(conf, frame, GF_FOP_##op);                      \
+        }                                                                      \
+    } while (0)
 
 #define BUMP_THROUGHPUT(iosstat, type)                                         \
-        do {                                                                   \
-                struct ios_conf         *conf = NULL;                          \
-                double                   elapsed;                              \
-                struct timespec         *begin, *end;                          \
-                double                   throughput;                           \
-                int                      flag = 0;                             \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
+        double elapsed;                                                        \
+        struct timespec *begin, *end;                                          \
+        double throughput;                                                     \
+        int flag = 0;                                                          \
+        struct timeval tv = {                                                  \
+            0,                                                                 \
+        };                                                                     \
                                                                                \
-                begin = &frame->begin;                                         \
-                end   = &frame->end;                                           \
+        begin = &frame->begin;                                                 \
+        end = &frame->end;                                                     \
                                                                                \
-                elapsed = ((end->tv_sec - begin->tv_sec) * 1e9                 \
-                           + (end->tv_nsec - begin->tv_nsec)) / 1000;          \
-                throughput = op_ret / elapsed;                                 \
+        elapsed = gf_tsdiff(begin, end) / 1000.0;                              \
+        throughput = op_ret / elapsed;                                         \
                                                                                \
-                conf = this->private;                                          \
-                LOCK (&iosstat->lock);                                         \
-                {                                                              \
-                        if (iosstat->thru_counters[type].throughput            \
-                                <= throughput) {                               \
-                                iosstat->thru_counters[type].throughput =      \
-                                                                throughput;    \
-                                gettimeofday (&iosstat->                       \
-                                             thru_counters[type].time, NULL);  \
-                               flag = 1;                                       \
-                        }                                                      \
-                }                                                              \
-                UNLOCK (&iosstat->lock);                                       \
-                if (flag)                                                      \
-                       ios_stat_add_to_list (&conf->thru_list[type],           \
-                                               throughput, iosstat);           \
-        } while (0)
+        conf = this->private;                                                  \
+        gettimeofday(&tv, NULL);                                               \
+        LOCK(&iosstat->lock);                                                  \
+        {                                                                      \
+            if (iosstat->thru_counters[type].throughput <= throughput) {       \
+                iosstat->thru_counters[type].throughput = throughput;          \
+                memcpy(&iosstat->thru_counters[type].time, &tv,                \
+                       sizeof(struct timeval));                                \
+                flag = 1;                                                      \
+            }                                                                  \
+        }                                                                      \
+        UNLOCK(&iosstat->lock);                                                \
+        if (flag)                                                              \
+            ios_stat_add_to_list(&conf->thru_list[type], throughput, iosstat); \
+    } while (0)
 
 static int
-ios_fd_ctx_get (fd_t *fd, xlator_t *this, struct ios_fd **iosfd)
+ios_fd_ctx_get(fd_t *fd, xlator_t *this, struct ios_fd **iosfd)
 {
-        uint64_t      iosfd64 = 0;
-        unsigned long iosfdlong = 0;
-        int           ret = 0;
+    uint64_t iosfd64 = 0;
+    unsigned long iosfdlong = 0;
+    int ret = 0;
 
-        ret = fd_ctx_get (fd, this, &iosfd64);
-        iosfdlong = iosfd64;
-        if (ret != -1)
-                *iosfd = (void *) iosfdlong;
+    ret = fd_ctx_get(fd, this, &iosfd64);
+    iosfdlong = iosfd64;
+    if (ret != -1)
+        *iosfd = (void *)iosfdlong;
 
-        return ret;
+    return ret;
 }
 
-
 static int
-ios_fd_ctx_set (fd_t *fd, xlator_t *this, struct ios_fd *iosfd)
+ios_fd_ctx_set(fd_t *fd, xlator_t *this, struct ios_fd *iosfd)
 {
-        uint64_t   iosfd64 = 0;
-        int        ret = 0;
+    uint64_t iosfd64 = 0;
+    int ret = 0;
 
-        iosfd64 = (unsigned long) iosfd;
-        ret = fd_ctx_set (fd, this, iosfd64);
+    iosfd64 = (unsigned long)iosfd;
+    ret = fd_ctx_set(fd, this, iosfd64);
 
-        return ret;
+    return ret;
 }
 
-
 static int
-ios_stat_ref (struct ios_stat *iosstat)
+ios_stat_ref(struct ios_stat *iosstat)
 {
-        uint64_t refcnt = 0;
-        refcnt = GF_ATOMIC_INC (iosstat->refcnt);
+    uint64_t refcnt = 0;
+    refcnt = GF_ATOMIC_INC(iosstat->refcnt);
 
-        return refcnt;
+    return refcnt;
 }
 
-
 static int
-ios_stat_unref (struct ios_stat *iosstat)
+ios_stat_unref(struct ios_stat *iosstat)
 {
-        int cleanup = 0;
-        uint64_t refcnt = 0;
+    int cleanup = 0;
+    uint64_t refcnt = 0;
 
-        refcnt = GF_ATOMIC_DEC (iosstat->refcnt);
-        if (refcnt == 0) {
-                if (iosstat->filename) {
-                        GF_FREE (iosstat->filename);
-                        iosstat->filename = NULL;
-                }
-                cleanup = 1;
+    refcnt = GF_ATOMIC_DEC(iosstat->refcnt);
+    if (refcnt == 0) {
+        if (iosstat->filename) {
+            GF_FREE(iosstat->filename);
+            iosstat->filename = NULL;
         }
+        cleanup = 1;
+    }
 
-        if (cleanup) {
-                LOCK_DESTROY (&iosstat->lock);
-                GF_FREE (iosstat);
-                iosstat = NULL;
-        }
+    if (cleanup) {
+        LOCK_DESTROY(&iosstat->lock);
+        GF_FREE(iosstat);
+        iosstat = NULL;
+    }
 
-        return 0;
+    return 0;
 }
 
-
 static int
-ios_stat_add_to_list (struct ios_stat_head *list_head, uint64_t value,
-                            struct ios_stat *iosstat)
-{
-        struct ios_stat_list *new = NULL;
-        struct ios_stat_list *entry = NULL;
-        struct ios_stat_list *t = NULL;
-        struct ios_stat_list *list_entry = NULL;
-        struct ios_stat_list *tmp = NULL;
-        struct ios_stat_list *last = NULL;
-        struct ios_stat      *stat = NULL;
-        int                   cnt = 0;
-        int                   found = 0;
-        int                   reposition = 0;
-        double                min_count = 0;
-
-        LOCK (&list_head->lock);
+ios_stat_add_to_list(struct ios_stat_head *list_head, uint64_t value,
+                     struct ios_stat *iosstat)
+{
+    struct ios_stat_list *new = NULL;
+    struct ios_stat_list *entry = NULL;
+    struct ios_stat_list *t = NULL;
+    struct ios_stat_list *list_entry = NULL;
+    struct ios_stat_list *tmp = NULL;
+    struct ios_stat_list *last = NULL;
+    struct ios_stat *stat = NULL;
+    int cnt = 0;
+    int found = 0;
+    int reposition = 0;
+    double min_count = 0;
+
+    LOCK(&list_head->lock);
+    {
+        if (list_head->min_cnt == 0)
+            list_head->min_cnt = value;
+        if ((list_head->members == MAX_LIST_MEMBERS) &&
+            (list_head->min_cnt > value))
+            goto out;
+
+        list_for_each_entry_safe(entry, t, &list_head->iosstats->list, list)
         {
-
-                if (list_head->min_cnt == 0)
+            cnt++;
+            if (cnt == list_head->members)
+                last = entry;
+
+            if (!gf_uuid_compare(iosstat->gfid, entry->iosstat->gfid)) {
+                list_entry = entry;
+                found = cnt;
+                entry->value = value;
+                if (!reposition) {
+                    if (cnt == list_head->members)
                         list_head->min_cnt = value;
-                if ((list_head->members == MAX_LIST_MEMBERS) &&
-                     (list_head->min_cnt > value))
-                        goto out;
-
-                list_for_each_entry_safe (entry, t,
-                              &list_head->iosstats->list, list) {
-                        cnt++;
-                        if (cnt == list_head->members)
-                                last = entry;
-
-                        if (!gf_uuid_compare (iosstat->gfid,
-                            entry->iosstat->gfid)) {
-                                list_entry = entry;
-                                found = cnt;
-                                entry->value = value;
-                                if (!reposition) {
-                                        if (cnt == list_head->members)
-                                                list_head->min_cnt = value;
-                                        goto out;
-                                }
-                                break;
-                        } else if (entry->value <= value && !reposition) {
-                                reposition = cnt;
-                                tmp = entry;
-                                if (cnt == list_head->members - 1)
-                                        min_count = entry->value;
-                        }
-                }
-                if (found) {
-                        list_del (&list_entry->list);
-                        list_add_tail (&list_entry->list, &tmp->list);
-                        if (min_count)
-                                list_head->min_cnt = min_count;
-                        goto out;
-                } else if (list_head->members == MAX_LIST_MEMBERS && reposition) {
-                        new = GF_CALLOC (1, sizeof (*new),
-                                        gf_io_stats_mt_ios_stat_list);
-                        new->iosstat = iosstat;
-                        new->value = value;
-                        ios_stat_ref (iosstat);
-                        list_add_tail (&new->list, &tmp->list);
-                        if (last) {
-                                stat = last->iosstat;
-                                last->iosstat = NULL;
-                                ios_stat_unref (stat);
-                                list_del (&last->list);
-                                GF_FREE (last);
-                        }
-                        if (reposition == MAX_LIST_MEMBERS)
-                                list_head->min_cnt = value;
-                        else if (min_count) {
-                                list_head->min_cnt = min_count;
-                        }
-                } else if (list_head->members < MAX_LIST_MEMBERS) {
-                        new  = GF_CALLOC (1, sizeof (*new),
-                                          gf_io_stats_mt_ios_stat_list);
-                        new->iosstat = iosstat;
-                        new->value = value;
-                        ios_stat_ref (iosstat);
-                        if (reposition) {
-                                list_add_tail (&new->list, &tmp->list);
-                        } else {
-                                list_add_tail (&new->list, &entry->list);
-                        }
-                        list_head->members++;
-                        if (list_head->min_cnt > value)
-                                list_head->min_cnt = value;
+                    goto out;
                 }
+                break;
+            } else if (entry->value <= value && !reposition) {
+                reposition = cnt;
+                tmp = entry;
+                if (cnt == list_head->members - 1)
+                    min_count = entry->value;
+            }
+        }
+        if (found) {
+            list_del(&list_entry->list);
+            list_add_tail(&list_entry->list, &tmp->list);
+            if (min_count)
+                list_head->min_cnt = min_count;
+            goto out;
+        } else if (list_head->members == MAX_LIST_MEMBERS && reposition) {
+            new = GF_CALLOC(1, sizeof(*new), gf_io_stats_mt_ios_stat_list);
+            new->iosstat = iosstat;
+            new->value = value;
+            ios_stat_ref(iosstat);
+            list_add_tail(&new->list, &tmp->list);
+            if (last) {
+                stat = last->iosstat;
+                last->iosstat = NULL;
+                ios_stat_unref(stat);
+                list_del(&last->list);
+                GF_FREE(last);
+            }
+            if (reposition == MAX_LIST_MEMBERS)
+                list_head->min_cnt = value;
+            else if (min_count) {
+                list_head->min_cnt = min_count;
+            }
+        } else if (list_head->members < MAX_LIST_MEMBERS) {
+            new = GF_CALLOC(1, sizeof(*new), gf_io_stats_mt_ios_stat_list);
+            new->iosstat = iosstat;
+            new->value = value;
+            ios_stat_ref(iosstat);
+            if (reposition) {
+                list_add_tail(&new->list, &tmp->list);
+            } else {
+                list_add_tail(&new->list, &entry->list);
+            }
+            list_head->members++;
+            if (list_head->min_cnt > value)
+                list_head->min_cnt = value;
         }
+    }
 out:
-        UNLOCK (&list_head->lock);
-        return 0;
+    UNLOCK(&list_head->lock);
+    return 0;
 }
 
-
 static void
-ios_bump_read (xlator_t *this, fd_t *fd, size_t len)
-{
-        struct ios_conf  *conf = NULL;
-        struct ios_fd    *iosfd = NULL;
-        int               lb2 = 0;
-
-        conf = this->private;
-        lb2 = log_base2 (len);
-        ios_fd_ctx_get (fd, this, &iosfd);
-        if (!conf)
-                return;
-
-        GF_ATOMIC_ADD (conf->cumulative.data_read, len);
-        GF_ATOMIC_ADD (conf->incremental.data_read, len);
-        GF_ATOMIC_INC (conf->cumulative.block_count_read[lb2]);
-        GF_ATOMIC_INC (conf->incremental.block_count_read[lb2]);
-
-        if (iosfd) {
-                GF_ATOMIC_ADD (iosfd->data_read, len);
-                GF_ATOMIC_INC (iosfd->block_count_read[lb2]);
-        }
-}
+ios_bump_read(xlator_t *this, fd_t *fd, size_t len)
+{
+    struct ios_conf *conf = NULL;
+    struct ios_fd *iosfd = NULL;
+    int lb2 = 0;
 
+    conf = this->private;
+    lb2 = log_base2(len);
+    ios_fd_ctx_get(fd, this, &iosfd);
+    if (!conf)
+        return;
 
-static void
-ios_bump_write (xlator_t *this, fd_t *fd, size_t len)
-{
-        struct ios_conf  *conf = NULL;
-        struct ios_fd    *iosfd = NULL;
-        int               lb2 = 0;
-
-        conf = this->private;
-        lb2 = log_base2 (len);
-        ios_fd_ctx_get (fd, this, &iosfd);
-        if (!conf)
-                return;
-
-        GF_ATOMIC_ADD (conf->cumulative.data_written, len);
-        GF_ATOMIC_ADD (conf->incremental.data_written, len);
-        GF_ATOMIC_INC (conf->cumulative.block_count_write[lb2]);
-        GF_ATOMIC_INC (conf->incremental.block_count_write[lb2]);
-
-        if (iosfd) {
-                GF_ATOMIC_ADD (iosfd->data_written, len);
-                GF_ATOMIC_INC (iosfd->block_count_write[lb2]);
-        }
-}
+    GF_ATOMIC_ADD(conf->cumulative.data_read, len);
+    GF_ATOMIC_ADD(conf->incremental.data_read, len);
+    GF_ATOMIC_INC(conf->cumulative.block_count_read[lb2]);
+    GF_ATOMIC_INC(conf->incremental.block_count_read[lb2]);
 
+    if (iosfd) {
+        GF_ATOMIC_ADD(iosfd->data_read, len);
+        GF_ATOMIC_INC(iosfd->block_count_read[lb2]);
+    }
+}
 
 static void
-ios_bump_upcall (xlator_t *this, gf_upcall_flags_t event)
+ios_bump_write(xlator_t *this, fd_t *fd, size_t len)
 {
-        struct ios_conf  *conf = NULL;
+    struct ios_conf *conf = NULL;
+    struct ios_fd *iosfd = NULL;
+    int lb2 = 0;
 
-        conf = this->private;
-        if (!conf)
-                return;
-        if (conf->count_fop_hits) {
-                GF_ATOMIC_INC (conf->cumulative.upcall_hits[event]);
-                GF_ATOMIC_INC (conf->incremental.upcall_hits[event]);
-        }
+    conf = this->private;
+    lb2 = log_base2(len);
+    ios_fd_ctx_get(fd, this, &iosfd);
+    if (!conf)
+        return;
+
+    GF_ATOMIC_ADD(conf->cumulative.data_written, len);
+    GF_ATOMIC_ADD(conf->incremental.data_written, len);
+    GF_ATOMIC_INC(conf->cumulative.block_count_write[lb2]);
+    GF_ATOMIC_INC(conf->incremental.block_count_write[lb2]);
+
+    if (iosfd) {
+        GF_ATOMIC_ADD(iosfd->data_written, len);
+        GF_ATOMIC_INC(iosfd->block_count_write[lb2]);
+    }
 }
 
+static void
+ios_bump_upcall(xlator_t *this, gf_upcall_flags_t event)
+{
+    struct ios_conf *conf = NULL;
+
+    conf = this->private;
+    if (!conf)
+        return;
+    if (conf->count_fop_hits) {
+        GF_ATOMIC_INC(conf->cumulative.upcall_hits[event]);
+        GF_ATOMIC_INC(conf->incremental.upcall_hits[event]);
+    }
+}
 
 static void
-ios_bump_stats (xlator_t *this, struct ios_stat *iosstat,
-                ios_stats_type_t type)
+ios_bump_stats(xlator_t *this, struct ios_stat *iosstat, ios_stats_type_t type)
 {
-        struct ios_conf         *conf = NULL;
-        uint64_t                 value = 0;
+    struct ios_conf *conf = NULL;
+    uint64_t value = 0;
 
-        conf = this->private;
+    conf = this->private;
 
-        value = GF_ATOMIC_INC (iosstat->counters[type]);
-        ios_stat_add_to_list (&conf->list[type], value, iosstat);
+    value = GF_ATOMIC_INC(iosstat->counters[type]);
+    ios_stat_add_to_list(&conf->list[type], value, iosstat);
 }
 
-
 int
-ios_inode_ctx_set (inode_t *inode, xlator_t *this, struct ios_stat *iosstat)
+ios_inode_ctx_set(inode_t *inode, xlator_t *this, struct ios_stat *iosstat)
 {
-        uint64_t   iosstat64 = 0;
-        int        ret     = 0;
+    uint64_t iosstat64 = 0;
+    int ret = 0;
 
-        ios_stat_ref (iosstat);
-        iosstat64 = (unsigned long) iosstat;
-        ret = inode_ctx_put (inode, this, iosstat64);
-        return ret;
+    ios_stat_ref(iosstat);
+    iosstat64 = (unsigned long)iosstat;
+    ret = inode_ctx_put(inode, this, iosstat64);
+    return ret;
 }
 
 int
-ios_inode_ctx_get (inode_t *inode, xlator_t *this, struct ios_stat **iosstat)
+ios_inode_ctx_get(inode_t *inode, xlator_t *this, struct ios_stat **iosstat)
 {
-        uint64_t      iosstat64 = 0;
-        unsigned long iosstatlong = 0;
-        int           ret = 0;
+    uint64_t iosstat64 = 0;
+    unsigned long iosstatlong = 0;
+    int ret = 0;
 
-        ret = inode_ctx_get (inode, this, &iosstat64);
-        iosstatlong = iosstat64;
-        if (ret != -1)
-                *iosstat = (void *) iosstatlong;
-
-        return ret;
+    ret = inode_ctx_get(inode, this, &iosstat64);
+    iosstatlong = iosstat64;
+    if (ret != -1)
+        *iosstat = (void *)iosstatlong;
 
+    return ret;
 }
 
 /*
@@ -580,598 +575,561 @@ ios_inode_ctx_get (inode_t *inode, xlator_t *this, struct ios_stat **iosstat)
  *
  */
 ios_sample_buf_t *
-ios_create_sample_buf (size_t buf_size)
+ios_create_sample_buf(size_t buf_size)
 {
-        ios_sample_buf_t *ios_sample_buf = NULL;
-        ios_sample_t     *ios_samples = NULL;
+    ios_sample_buf_t *ios_sample_buf = NULL;
+    ios_sample_t *ios_samples = NULL;
 
-        ios_sample_buf = GF_CALLOC (1,
-                sizeof (*ios_sample_buf),
-                gf_io_stats_mt_ios_sample_buf);
-        if (!ios_sample_buf)
-                goto err;
+    ios_sample_buf = GF_CALLOC(1, sizeof(*ios_sample_buf),
+                               gf_io_stats_mt_ios_sample_buf);
+    if (!ios_sample_buf)
+        goto err;
 
-        ios_samples = GF_CALLOC (buf_size,
-                sizeof (*ios_samples),
-                gf_io_stats_mt_ios_sample);
+    ios_samples = GF_CALLOC(buf_size, sizeof(*ios_samples),
+                            gf_io_stats_mt_ios_sample);
 
-        if (!ios_samples)
-                goto err;
+    if (!ios_samples)
+        goto err;
 
-        ios_sample_buf->ios_samples = ios_samples;
-        ios_sample_buf->size = buf_size;
-        ios_sample_buf->pos = 0;
-        ios_sample_buf->observed = 0;
-        ios_sample_buf->collected = 0;
+    ios_sample_buf->ios_samples = ios_samples;
+    ios_sample_buf->size = buf_size;
+    ios_sample_buf->pos = 0;
+    ios_sample_buf->observed = 0;
+    ios_sample_buf->collected = 0;
 
-        return ios_sample_buf;
+    return ios_sample_buf;
 err:
-        GF_FREE (ios_sample_buf);
-        return NULL;
+    GF_FREE(ios_sample_buf);
+    return NULL;
 }
 
 void
-ios_destroy_sample_buf (ios_sample_buf_t *ios_sample_buf)
+ios_destroy_sample_buf(ios_sample_buf_t *ios_sample_buf)
 {
-        GF_FREE (ios_sample_buf->ios_samples);
-        GF_FREE (ios_sample_buf);
+    GF_FREE(ios_sample_buf->ios_samples);
+    GF_FREE(ios_sample_buf);
 }
 
 static int
-ios_init_sample_buf (struct ios_conf *conf)
+ios_init_sample_buf(struct ios_conf *conf)
 {
-        int32_t        ret = -1;
+    int32_t ret = -1;
 
-        GF_ASSERT (conf);
-        LOCK (&conf->lock);
-        conf->ios_sample_buf = ios_create_sample_buf (
-                conf->ios_sample_buf_size);
-        if (!conf->ios_sample_buf)
-                goto out;
-        ret = 0;
+    GF_ASSERT(conf);
+    LOCK(&conf->lock);
+    conf->ios_sample_buf = ios_create_sample_buf(conf->ios_sample_buf_size);
+    if (!conf->ios_sample_buf)
+        goto out;
+    ret = 0;
 out:
-        UNLOCK (&conf->lock);
-        return ret;
+    UNLOCK(&conf->lock);
+    return ret;
 }
 
 static int
-ios_stats_cleanup (xlator_t *this, inode_t *inode)
+ios_stats_cleanup(xlator_t *this, inode_t *inode)
 {
+    struct ios_stat *iosstat = NULL;
+    uint64_t iosstat64 = 0;
 
-        struct ios_stat *iosstat = NULL;
-        uint64_t         iosstat64 = 0;
-
-        inode_ctx_del (inode, this, &iosstat64);
-        if (!iosstat64) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "could not get inode ctx");
-                return 0;
-        }
-        iosstat = (void *) (long)iosstat64;
-        if (iosstat) {
-                ios_stat_unref (iosstat);
-        }
+    inode_ctx_del(inode, this, &iosstat64);
+    if (!iosstat64) {
+        gf_log(this->name, GF_LOG_WARNING, "could not get inode ctx");
         return 0;
-}
-
-#define ios_log(this, logfp, fmt ...)                           \
-        do {                                                    \
-                if (logfp) {                                    \
-                        fprintf (logfp, fmt);                   \
-                        fprintf (logfp, "\n");                  \
-                }                                               \
-                gf_log (this->name, GF_LOG_DEBUG, fmt);        \
-        } while (0)
+    }
+    iosstat = (void *)(long)iosstat64;
+    if (iosstat) {
+        ios_stat_unref(iosstat);
+    }
+    return 0;
+}
+
+#define ios_log(this, logfp, fmt...)                                           \
+    do {                                                                       \
+        if (logfp) {                                                           \
+            fprintf(logfp, fmt);                                               \
+            fprintf(logfp, "\n");                                              \
+        }                                                                      \
+        gf_log(this->name, GF_LOG_DEBUG, fmt);                                 \
+    } while (0)
 
 int
-ios_dump_file_stats (struct ios_stat_head *list_head, xlator_t *this,
-                     FILE *logfp)
+ios_dump_file_stats(struct ios_stat_head *list_head, xlator_t *this,
+                    FILE *logfp)
 {
-        struct ios_stat_list *entry = NULL;
+    struct ios_stat_list *entry = NULL;
 
-        LOCK (&list_head->lock);
+    LOCK(&list_head->lock);
+    {
+        list_for_each_entry(entry, &list_head->iosstats->list, list)
         {
-                list_for_each_entry (entry, &list_head->iosstats->list, list) {
-                        ios_log (this, logfp, "%-12.0f %s",
-                                entry->value, entry->iosstat->filename);
-                }
+            ios_log(this, logfp, "%-12.0f %s", entry->value,
+                    entry->iosstat->filename);
         }
-        UNLOCK (&list_head->lock);
-        return 0;
+    }
+    UNLOCK(&list_head->lock);
+    return 0;
 }
 
 int
-ios_dump_throughput_stats (struct ios_stat_head *list_head, xlator_t *this,
-                           FILE *logfp, ios_stats_thru_t type)
+ios_dump_throughput_stats(struct ios_stat_head *list_head, xlator_t *this,
+                          FILE *logfp, ios_stats_thru_t type)
 {
-        struct ios_stat_list *entry = NULL;
-        struct timeval        time  = {0, };
-        char                  timestr[256] = {0, };
+    struct ios_stat_list *entry = NULL;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
 
-        LOCK (&list_head->lock);
+    LOCK(&list_head->lock);
+    {
+        list_for_each_entry(entry, &list_head->iosstats->list, list)
         {
-                list_for_each_entry (entry, &list_head->iosstats->list, list) {
-                        gf_time_fmt (timestr, sizeof timestr,
-                                     entry->iosstat->thru_counters[type].time.tv_sec,
-                                     gf_timefmt_FT);
-                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
-                          ".%"GF_PRI_SUSECONDS, time.tv_usec);
-
-                        ios_log (this, logfp, "%s \t %-10.2f  \t  %s",
-                                 timestr, entry->value, entry->iosstat->filename);
-                }
+            gf_time_fmt_tv(timestr, sizeof timestr,
+                           &entry->iosstat->thru_counters[type].time,
+                           gf_timefmt_FT);
+
+            ios_log(this, logfp, "%s \t %-10.2f  \t  %s", timestr, entry->value,
+                    entry->iosstat->filename);
         }
-        UNLOCK (&list_head->lock);
-        return 0;
+    }
+    UNLOCK(&list_head->lock);
+    return 0;
 }
 
 int
-_io_stats_get_key_prefix (xlator_t *this, char **key_prefix) {
-        char                  *key_root = "gluster";
-        char                  *xlator_name = NULL;
-        char                  *instance_name = NULL;
-        size_t                key_len = 0;
-        int                   bytes_written = 0;
-        int                   i = 0;
-        int                   ret = 0;
-        struct ios_conf       *conf = this->private;
-
-        xlator_name = strdupa (conf->unique_id);
-        for (i = 0; i < strlen (xlator_name); i++) {
-                if (xlator_name[i] == '/')
-                        xlator_name[i] = '_';
+_io_stats_get_key_prefix(xlator_t *this, char **key_prefix)
+{
+    char *key_root = "gluster";
+    char *xlator_name = NULL;
+    char *instance_name = NULL;
+    size_t key_len = 0;
+    int bytes_written = 0;
+    int i = 0;
+    int ret = 0;
+    struct ios_conf *conf = this->private;
+
+    xlator_name = strdupa(conf->unique_id);
+    for (i = 0; i < strlen(xlator_name); i++) {
+        if (xlator_name[i] == '/')
+            xlator_name[i] = '_';
+    }
+
+    instance_name = this->instance_name;
+    if (this->name && strcmp(this->name, "glustershd") == 0) {
+        xlator_name = "shd";
+    } else if (this->prev && strcmp(this->prev->name, "nfs-server") == 0) {
+        xlator_name = "nfsd";
+        if (this->prev->instance_name)
+            instance_name = strdupa(this->prev->instance_name);
+    }
+
+    if (strcmp(__progname, "glusterfsd") == 0)
+        key_root = "gluster.brick";
+
+    if (instance_name) {
+        /* +3 for 2 x "." + NULL */
+        key_len = strlen(key_root) + strlen(xlator_name) +
+                  strlen(instance_name) + 3;
+        *key_prefix = GF_CALLOC(key_len, sizeof(char), gf_common_mt_char);
+        if (!*key_prefix) {
+            ret = -ENOMEM;
+            goto err;
         }
-
-        instance_name = this->instance_name;
-        if (this->name && strcmp (this->name, "glustershd") == 0) {
-                xlator_name = "shd";
-        } else if (this->prev &&
-                   strcmp (this->prev->name, "nfs-server") == 0) {
-                xlator_name = "nfsd";
-                if (this->prev->instance_name)
-                        instance_name = strdupa (this->prev->instance_name);
+        bytes_written = snprintf(*key_prefix, key_len, "%s.%s.%s", key_root,
+                                 xlator_name, instance_name);
+        if (bytes_written != key_len - 1) {
+            ret = -EINVAL;
+            goto err;
         }
-
-        if (strcmp (__progname, "glusterfsd") == 0)
-                key_root = "gluster.brick";
-
-        if (instance_name) {
-                /* +3 for 2 x "." + NULL */
-                key_len = strlen (key_root) + strlen (xlator_name) +
-                        strlen (instance_name) + 3;
-                *key_prefix = GF_CALLOC (key_len, sizeof (char),
-                        gf_common_mt_char);
-                if (!*key_prefix) {
-                        ret = -ENOMEM;
-                        goto err;
-                }
-                bytes_written = snprintf (*key_prefix, key_len, "%s.%s.%s",
-                        key_root, xlator_name, instance_name);
-                if (bytes_written != key_len - 1) {
-                        ret = -EINVAL;
-                        goto err;
-                }
-        } else {
-                /* +2 for 1 x "." + NULL */
-                key_len = strlen (key_root) + strlen (xlator_name) + 2;
-                *key_prefix = GF_CALLOC (key_len, sizeof (char),
-                        gf_common_mt_char);
-                if (!*key_prefix) {
-                        ret = -ENOMEM;
-                        goto err;
-                }
-                bytes_written = snprintf (*key_prefix, key_len, "%s.%s",
-                        key_root, xlator_name);
-                if (bytes_written != key_len - 1) {
-                        ret = -EINVAL;
-                        goto err;
-                }
+    } else {
+        /* +2 for 1 x "." + NULL */
+        key_len = strlen(key_root) + strlen(xlator_name) + 2;
+        *key_prefix = GF_CALLOC(key_len, sizeof(char), gf_common_mt_char);
+        if (!*key_prefix) {
+            ret = -ENOMEM;
+            goto err;
         }
-        return 0;
+        bytes_written = snprintf(*key_prefix, key_len, "%s.%s", key_root,
+                                 xlator_name);
+        if (bytes_written != key_len - 1) {
+            ret = -EINVAL;
+            goto err;
+        }
+    }
+    return 0;
 err:
-        GF_FREE (*key_prefix);
-        *key_prefix = NULL;
-        return ret;
+    GF_FREE(*key_prefix);
+    *key_prefix = NULL;
+    return ret;
 }
 
 int
-io_stats_dump_global_to_json_logfp (xlator_t *this,
-    struct ios_global_stats *stats, struct timeval *now, int interval,
-    FILE *logfp)
-{
-        int                   i = 0;
-        int                   j = 0;
-        struct ios_conf       *conf = NULL;
-        char                  *key_prefix = NULL;
-        char                  *str_prefix = NULL;
-        char                  *lc_fop_name = NULL;
-        int                   ret = 1;  /* Default to error */
-        int                   rw_size;
-        char                  *rw_unit = NULL;
-        uint64_t              fop_hits;
-        float                 fop_lat_ave;
-        float                 fop_lat_min;
-        float                 fop_lat_max;
-        double                interval_sec;
-        double                fop_ave_usec = 0.0;
-        double                fop_ave_usec_sum = 0.0;
-        double                weighted_fop_ave_usec = 0.0;
-        double                weighted_fop_ave_usec_sum = 0.0;
-        long                  total_fop_hits = 0;
-        loc_t                 unused_loc = {0, };
-        dict_t                *xattr = NULL;
-
-        interval_sec = ((now->tv_sec * 1000000.0 + now->tv_usec) -
-                (stats->started_at.tv_sec * 1000000.0 +
-                 stats->started_at.tv_usec)) / 1000000.0;
-
-        conf = this->private;
-
-        ret = _io_stats_get_key_prefix (this, &key_prefix);
-        if (ret) {
-                goto out;
+io_stats_dump_global_to_json_logfp(xlator_t *this,
+                                   struct ios_global_stats *stats, time_t now,
+                                   int interval, FILE *logfp)
+{
+    int i = 0;
+    int j = 0;
+    struct ios_conf *conf = NULL;
+    char *key_prefix = NULL;
+    char *str_prefix = NULL;
+    char *lc_fop_name = NULL;
+    int ret = 1; /* Default to error */
+    int rw_size;
+    char *rw_unit = NULL;
+    uint64_t fop_hits;
+    float fop_lat_ave;
+    float fop_lat_min;
+    float fop_lat_max;
+    double interval_sec;
+    double fop_ave_usec = 0.0;
+    double fop_ave_usec_sum = 0.0;
+    double weighted_fop_ave_usec = 0.0;
+    double weighted_fop_ave_usec_sum = 0.0;
+    long total_fop_hits = 0;
+    loc_t unused_loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+
+    interval_sec = (double)(now - stats->started_at);
+
+    conf = this->private;
+
+    ret = _io_stats_get_key_prefix(this, &key_prefix);
+    if (ret) {
+        goto out;
+    }
+
+    if (interval == -1) {
+        str_prefix = "aggr";
+
+    } else {
+        str_prefix = "inter";
+    }
+    ios_log(this, logfp, "{");
+
+    for (i = 0; i < 31; i++) {
+        rw_size = (1 << i);
+        if (rw_size >= 1024 * 1024) {
+            rw_size = rw_size / (1024 * 1024);
+            rw_unit = "mb";
+        } else if (rw_size >= 1024) {
+            rw_size = rw_size / 1024;
+            rw_unit = "kb";
+        } else {
+            rw_unit = "b";
         }
 
         if (interval == -1) {
-                str_prefix = "aggr";
-
+            ios_log(this, logfp, "\"%s.%s.read_%d%s\": %" GF_PRI_ATOMIC ",",
+                    key_prefix, str_prefix, rw_size, rw_unit,
+                    GF_ATOMIC_GET(stats->block_count_read[i]));
+            ios_log(this, logfp, "\"%s.%s.write_%d%s\": %" GF_PRI_ATOMIC ",",
+                    key_prefix, str_prefix, rw_size, rw_unit,
+                    GF_ATOMIC_GET(stats->block_count_write[i]));
         } else {
-                str_prefix = "inter";
+            ios_log(this, logfp, "\"%s.%s.read_%d%s_per_sec\": %0.2lf,",
+                    key_prefix, str_prefix, rw_size, rw_unit,
+                    (double)(GF_ATOMIC_GET(stats->block_count_read[i]) /
+                             interval_sec));
+            ios_log(this, logfp, "\"%s.%s.write_%d%s_per_sec\": %0.2lf,",
+                    key_prefix, str_prefix, rw_size, rw_unit,
+                    (double)(GF_ATOMIC_GET(stats->block_count_write[i]) /
+                             interval_sec));
         }
-        ios_log (this, logfp, "{");
-
-        for (i = 0; i < 31; i++) {
-                rw_size = (1 << i);
-                if (rw_size >= 1024 * 1024) {
-                        rw_size = rw_size / (1024 * 1024);
-                        rw_unit = "mb";
-                } else if (rw_size >= 1024) {
-                        rw_size = rw_size / 1024;
-                        rw_unit = "kb";
-                } else {
-                        rw_unit = "b";
-                }
-
-                if (interval == -1) {
-                        ios_log (this, logfp,
-                                "\"%s.%s.read_%d%s\": \"%"GF_PRI_ATOMIC"\",",
-                                key_prefix, str_prefix, rw_size, rw_unit,
-                                GF_ATOMIC_GET (stats->block_count_read[i]));
-                        ios_log (this, logfp,
-                                "\"%s.%s.write_%d%s\": \"%"GF_PRI_ATOMIC"\",",
-                                key_prefix, str_prefix, rw_size, rw_unit,
-                                GF_ATOMIC_GET (stats->block_count_write[i]));
-                } else {
-                        ios_log (this, logfp,
-                                "\"%s.%s.read_%d%s_per_sec\": \"%0.2lf\",",
-                                key_prefix, str_prefix, rw_size, rw_unit,
-                                (double)
-                                (GF_ATOMIC_GET (stats->block_count_read[i]) /
-                                 interval_sec));
-                        ios_log (this, logfp,
-                                "\"%s.%s.write_%d%s_per_sec\": \"%0.2lf\",",
-                                key_prefix, str_prefix, rw_size, rw_unit,
-                                (double)
-                                (GF_ATOMIC_GET (stats->block_count_write[i]) /
-                                 interval_sec));
-                }
+    }
+
+    if (interval == -1) {
+        ios_log(this, logfp, "\"%s.%s.fds.open_count\": %" PRId64 ",",
+                key_prefix, str_prefix, conf->cumulative.nr_opens);
+        ios_log(this, logfp, "\"%s.%s.fds.max_open_count\": %" PRId64 ",",
+                key_prefix, str_prefix, conf->cumulative.max_nr_opens);
+    }
+
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        lc_fop_name = strdupa(gf_fop_list[i]);
+        for (j = 0; lc_fop_name[j]; j++) {
+            lc_fop_name[j] = tolower(lc_fop_name[j]);
         }
 
-        if (interval == -1) {
-                ios_log (this, logfp, "\"%s.%s.fds.open_count\": \"%"PRId64
-                        "\",", key_prefix, str_prefix,
-                        conf->cumulative.nr_opens);
-                ios_log (this, logfp,
-                        "\"%s.%s.fds.max_open_count\": \"%"PRId64"\",",
-                        key_prefix, str_prefix, conf->cumulative.max_nr_opens);
+        fop_hits = GF_ATOMIC_GET(stats->fop_hits[i]);
+        fop_lat_ave = 0.0;
+        fop_lat_min = 0.0;
+        fop_lat_max = 0.0;
+        if (fop_hits) {
+            if (stats->latency[i].avg) {
+                fop_lat_ave = stats->latency[i].avg;
+                fop_lat_min = stats->latency[i].min;
+                fop_lat_max = stats->latency[i].max;
+            }
         }
-
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                lc_fop_name = strdupa (gf_fop_list[i]);
-                for (j = 0; lc_fop_name[j]; j++) {
-                        lc_fop_name[j] = tolower (lc_fop_name[j]);
-                }
-
-                fop_hits = GF_ATOMIC_GET (stats->fop_hits[i]);
-                fop_lat_ave = 0.0;
-                fop_lat_min = 0.0;
-                fop_lat_max = 0.0;
-                if (fop_hits) {
-                        if (stats->latency[i].avg) {
-                                fop_lat_ave = stats->latency[i].avg;
-                                fop_lat_min = stats->latency[i].min;
-                                fop_lat_max = stats->latency[i].max;
-                        }
-                }
-                if (interval == -1) {
-                        ios_log (this, logfp,
-                                "\"%s.%s.fop.%s.count\": \"%"GF_PRI_ATOMIC"\",",
-                                key_prefix, str_prefix, lc_fop_name,
-                                fop_hits);
-                } else {
-                        ios_log (this, logfp,
-                                "\"%s.%s.fop.%s.per_sec\": \"%0.2lf\",",
-                                key_prefix, str_prefix, lc_fop_name,
-                                (double)(fop_hits / interval_sec));
-                }
-
-                ios_log (this, logfp,
-                        "\"%s.%s.fop.%s.latency_ave_usec\": \"%0.2lf\",",
-                         key_prefix, str_prefix, lc_fop_name, fop_lat_ave);
-                ios_log (this, logfp,
-                        "\"%s.%s.fop.%s.latency_min_usec\": \"%0.2lf\",",
-                         key_prefix, str_prefix, lc_fop_name, fop_lat_min);
-                ios_log (this, logfp,
-                        "\"%s.%s.fop.%s.latency_max_usec\": \"%0.2lf\",",
-                        key_prefix, str_prefix, lc_fop_name, fop_lat_max);
-
-                fop_ave_usec_sum += fop_lat_ave;
-                weighted_fop_ave_usec_sum += fop_hits * fop_lat_ave;
-                total_fop_hits += fop_hits;
+        if (interval == -1) {
+            ios_log(this, logfp, "\"%s.%s.fop.%s.count\": %" GF_PRI_ATOMIC ",",
+                    key_prefix, str_prefix, lc_fop_name, fop_hits);
+        } else {
+            ios_log(this, logfp, "\"%s.%s.fop.%s.per_sec\": %0.2lf,",
+                    key_prefix, str_prefix, lc_fop_name,
+                    (double)(fop_hits / interval_sec));
         }
 
-        if (total_fop_hits) {
-                weighted_fop_ave_usec =
-                        weighted_fop_ave_usec_sum/total_fop_hits;
-                /* Extra key that does not print out an entry w/ 0.00 for
-                 * intervals with no data
-                 */
-                ios_log (this, logfp,
-                         "\"%s.%s.fop.weighted_latency_ave_usec_nozerofill\": "
-                         "\"%0.4lf\",",
-                         key_prefix, str_prefix, weighted_fop_ave_usec);
-        }
-        ios_log (this, logfp,
-                 "\"%s.%s.fop.weighted_latency_ave_usec\": \"%0.4lf\",",
-                 key_prefix, str_prefix, weighted_fop_ave_usec);
-        ios_log (this, logfp,
-                 "\"%s.%s.fop.weighted_fop_count\": \"%ld\",",
-                 key_prefix, str_prefix, total_fop_hits);
-
-        fop_ave_usec = fop_ave_usec_sum/GF_FOP_MAXVALUE;
-        ios_log (this, logfp,
-                 "\"%s.%s.fop.unweighted_latency_ave_usec\":\"%0.4lf\",",
-                 key_prefix, str_prefix, fop_ave_usec);
-
-        for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
-                lc_fop_name = strdupa (gf_upcall_list[i]);
-                for (j = 0; lc_fop_name[j]; j++) {
-                        lc_fop_name[j] = tolower (lc_fop_name[j]);
-                }
-                fop_hits = GF_ATOMIC_GET (stats->upcall_hits[i]);
-                if (interval == -1) {
-                        ios_log (this, logfp,
-                                "\"%s.%s.fop.%s.count\": \"%"GF_PRI_ATOMIC"\",",
-                                key_prefix, str_prefix, lc_fop_name,
-                                fop_hits);
-                } else {
-                        ios_log (this, logfp,
-                                "\"%s.%s.fop.%s.per_sec\": \"%0.2lf\",",
-                                key_prefix, str_prefix, lc_fop_name,
-                                (double)(fop_hits / interval_sec));
-                }
+        ios_log(this, logfp, "\"%s.%s.fop.%s.latency_ave_usec\": %0.2lf,",
+                key_prefix, str_prefix, lc_fop_name, fop_lat_ave);
+        ios_log(this, logfp, "\"%s.%s.fop.%s.latency_min_usec\": %0.2lf,",
+                key_prefix, str_prefix, lc_fop_name, fop_lat_min);
+        ios_log(this, logfp, "\"%s.%s.fop.%s.latency_max_usec\": %0.2lf,",
+                key_prefix, str_prefix, lc_fop_name, fop_lat_max);
+
+        fop_ave_usec_sum += fop_lat_ave;
+        weighted_fop_ave_usec_sum += fop_hits * fop_lat_ave;
+        total_fop_hits += fop_hits;
+    }
+
+    if (total_fop_hits) {
+        weighted_fop_ave_usec = weighted_fop_ave_usec_sum / total_fop_hits;
+        /* Extra key that does not print out an entry w/ 0.00 for
+         * intervals with no data
+         */
+        ios_log(this, logfp,
+                "\"%s.%s.fop.weighted_latency_ave_usec_nozerofill\": "
+                "%0.4lf,",
+                key_prefix, str_prefix, weighted_fop_ave_usec);
+    }
+    ios_log(this, logfp, "\"%s.%s.fop.weighted_latency_ave_usec\": %0.4lf,",
+            key_prefix, str_prefix, weighted_fop_ave_usec);
+    ios_log(this, logfp, "\"%s.%s.fop.weighted_fop_count\": %ld,", key_prefix,
+            str_prefix, total_fop_hits);
+
+    fop_ave_usec = fop_ave_usec_sum / GF_FOP_MAXVALUE;
+    ios_log(this, logfp, "\"%s.%s.fop.unweighted_latency_ave_usec\":%0.4lf,",
+            key_prefix, str_prefix, fop_ave_usec);
+
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        lc_fop_name = strdupa(gf_upcall_list[i]);
+        for (j = 0; lc_fop_name[j]; j++) {
+            lc_fop_name[j] = tolower(lc_fop_name[j]);
         }
-
-        ret = syncop_getxattr (this, &unused_loc, &xattr,
-                               IO_THREADS_QUEUE_SIZE_KEY, NULL, NULL);
-        if (xattr) {
-                /*
-                 * Iterate over the dictionary returned to us by io-threads and
-                 * dump the results to the stats file.
-                 */
-                data_pair_t *curr = NULL;
-
-                dict_foreach_inline (xattr, curr) {
-                        ios_log (this, logfp,
-                                 "\"%s.%s.%s.queue_size\": \"%d\",",
-                                 key_prefix, str_prefix, curr->key,
-                                 data_to_int32 (curr->value));
-                }
-
-                /* Free the dictionary */
-                dict_unref (xattr);
+        fop_hits = GF_ATOMIC_GET(stats->upcall_hits[i]);
+        if (interval == -1) {
+            ios_log(this, logfp, "\"%s.%s.fop.%s.count\": %" GF_PRI_ATOMIC ",",
+                    key_prefix, str_prefix, lc_fop_name, fop_hits);
         } else {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Unable to get queue size counts from "
-                        "the io-threads translator!");
+            ios_log(this, logfp, "\"%s.%s.fop.%s.per_sec\": %0.2lf,",
+                    key_prefix, str_prefix, lc_fop_name,
+                    (double)(fop_hits / interval_sec));
         }
+    }
 
-        if (interval == -1) {
-                ios_log (this, logfp, "\"%s.%s.uptime\": \"%"PRId64"\",",
-                         key_prefix, str_prefix,
-                         (uint64_t) (now->tv_sec - stats->started_at.tv_sec));
-                ios_log (this, logfp, "\"%s.%s.bytes_read\": \""
-                         "%"GF_PRI_ATOMIC"\",", key_prefix, str_prefix,
-                         GF_ATOMIC_GET (stats->data_read));
-                ios_log (this, logfp, "\"%s.%s.bytes_written\": \""
-                         "%"GF_PRI_ATOMIC"\"", key_prefix, str_prefix,
-                         GF_ATOMIC_GET (stats->data_written));
-        } else {
-                ios_log (this, logfp,
-                         "\"%s.%s.sample_interval_sec\": \"%0.2lf\",",
-                         key_prefix, str_prefix,
-                         interval_sec);
-                ios_log (this, logfp,
-                         "\"%s.%s.bytes_read_per_sec\": \"%0.2lf\",",
-                         key_prefix, str_prefix, (double)
-                         (GF_ATOMIC_GET (stats->data_read) / interval_sec));
-                ios_log (this, logfp,
-                         "\"%s.%s.bytes_written_per_sec\": \"%0.2lf\"",
-                         key_prefix, str_prefix, (double)
-                         (GF_ATOMIC_GET (stats->data_written) / interval_sec));
+    ret = syncop_getxattr(this, &unused_loc, &xattr, IO_THREADS_QUEUE_SIZE_KEY,
+                          NULL, NULL);
+    if (xattr) {
+        /*
+         * Iterate over the dictionary returned to us by io-threads and
+         * dump the results to the stats file.
+         */
+        data_pair_t *curr = NULL;
+
+        dict_foreach_inline(xattr, curr)
+        {
+            ios_log(this, logfp, "\"%s.%s.%s.queue_size\": %d,", key_prefix,
+                    str_prefix, curr->key, data_to_int32(curr->value));
         }
 
-        ios_log (this, logfp, "}");
-        ret = 0;
+        /* Free the dictionary */
+        dict_unref(xattr);
+    } else {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Unable to get queue size counts from "
+               "the io-threads translator!");
+    }
+
+    if (interval == -1) {
+        ios_log(this, logfp, "\"%s.%s.uptime\": %" PRIu64 ",", key_prefix,
+                str_prefix, (uint64_t)(now - stats->started_at));
+        ios_log(this, logfp,
+                "\"%s.%s.bytes_read\": "
+                "%" GF_PRI_ATOMIC ",",
+                key_prefix, str_prefix, GF_ATOMIC_GET(stats->data_read));
+        ios_log(this, logfp,
+                "\"%s.%s.bytes_written\": "
+                "%" GF_PRI_ATOMIC "",
+                key_prefix, str_prefix, GF_ATOMIC_GET(stats->data_written));
+    } else {
+        ios_log(this, logfp, "\"%s.%s.sample_interval_sec\": %0.2lf,",
+                key_prefix, str_prefix, interval_sec);
+        ios_log(this, logfp, "\"%s.%s.bytes_read_per_sec\": %0.2lf,",
+                key_prefix, str_prefix,
+                (double)(GF_ATOMIC_GET(stats->data_read) / interval_sec));
+        ios_log(this, logfp, "\"%s.%s.bytes_written_per_sec\": %0.2lf",
+                key_prefix, str_prefix,
+                (double)(GF_ATOMIC_GET(stats->data_written) / interval_sec));
+    }
+
+    ios_log(this, logfp, "}");
+    ret = 0;
 out:
-        GF_FREE (key_prefix);
-        return ret;
+    GF_FREE(key_prefix);
+    return ret;
 }
 
 char *
-_resolve_username (xlator_t *this, uid_t uid)
+_resolve_username(xlator_t *this, uid_t uid)
 {
-        struct passwd pwd;
-        struct passwd *pwd_result = NULL;
-        size_t pwd_buf_len;
-        char   *pwd_buf = NULL;
-        char   *ret = NULL;
+    struct passwd pwd;
+    struct passwd *pwd_result = NULL;
+    size_t pwd_buf_len;
+    char *pwd_buf = NULL;
+    char *ret = NULL;
 
-        /* Prepare our buffer for the uid->username translation */
+    /* Prepare our buffer for the uid->username translation */
 #ifdef _SC_GETGR_R_SIZE_MAX
-        pwd_buf_len = sysconf (_SC_GETGR_R_SIZE_MAX);
+    pwd_buf_len = sysconf(_SC_GETGR_R_SIZE_MAX);
 #else
-        pwd_buf_len = -1;
+    pwd_buf_len = -1;
 #endif
-        if (pwd_buf_len == -1) {
-                pwd_buf_len = DEFAULT_PWD_BUF_SZ;  /* per the man page */
-        }
+    if (pwd_buf_len == -1) {
+        pwd_buf_len = DEFAULT_PWD_BUF_SZ; /* per the man page */
+    }
 
-        pwd_buf = alloca (pwd_buf_len);
-        if (!pwd_buf)
-                goto err;
+    pwd_buf = alloca(pwd_buf_len);
+    if (!pwd_buf)
+        goto err;
 
-        getpwuid_r (uid, &pwd, pwd_buf, pwd_buf_len,
-                   &pwd_result);
-        if (!pwd_result)
-                goto err;
+    getpwuid_r(uid, &pwd, pwd_buf, pwd_buf_len, &pwd_result);
+    if (!pwd_result)
+        goto err;
 
-        ret = gf_strdup (pwd.pw_name);
-        if (ret)
-                return ret;
-        else
-                gf_log (this->name, GF_LOG_ERROR,
-                        "gf_strdup failed, failing username "
-                        "resolution.");
-err:
+    ret = gf_strdup(pwd.pw_name);
+    if (ret)
         return ret;
+    else
+        gf_log(this->name, GF_LOG_ERROR,
+               "gf_strdup failed, failing username "
+               "resolution.");
+err:
+    return ret;
 }
 
 char *
-_resolve_group_name (xlator_t *this, gid_t gid)
+_resolve_group_name(xlator_t *this, gid_t gid)
 {
-        struct group grp;
-        struct group *grp_result = NULL;
-        size_t grp_buf_len;
-        char   *grp_buf = NULL;
-        char   *ret = NULL;
+    struct group grp;
+    struct group *grp_result = NULL;
+    size_t grp_buf_len;
+    char *grp_buf = NULL;
+    char *ret = NULL;
 
-        /* Prepare our buffer for the gid->group name translation */
+    /* Prepare our buffer for the gid->group name translation */
 #ifdef _SC_GETGR_R_SIZE_MAX
-        grp_buf_len = sysconf (_SC_GETGR_R_SIZE_MAX);
+    grp_buf_len = sysconf(_SC_GETGR_R_SIZE_MAX);
 #else
-        grp_buf_len = -1;
+    grp_buf_len = -1;
 #endif
-        if (grp_buf_len == -1) {
-                grp_buf_len = DEFAULT_GRP_BUF_SZ;  /* per the man page */
-        }
+    if (grp_buf_len == -1) {
+        grp_buf_len = DEFAULT_GRP_BUF_SZ; /* per the man page */
+    }
 
-        grp_buf = alloca (grp_buf_len);
-        if (!grp_buf) {
-                goto err;
-        }
+    grp_buf = alloca(grp_buf_len);
+    if (!grp_buf) {
+        goto err;
+    }
 
-        if (getgrgid_r (gid, &grp, grp_buf, grp_buf_len, &grp_result) != 0)
-                goto err;
+    if (getgrgid_r(gid, &grp, grp_buf, grp_buf_len, &grp_result) != 0)
+        goto err;
 
-        if (!grp_result)
-                goto err;
+    if (!grp_result)
+        goto err;
 
-        ret = gf_strdup (grp.gr_name);
-        if (ret)
-                return ret;
-        else
-                gf_log (this->name, GF_LOG_ERROR,
-                        "gf_strdup failed, failing username "
-                        "resolution.");
-err:
+    ret = gf_strdup(grp.gr_name);
+    if (ret)
         return ret;
+    else
+        gf_log(this->name, GF_LOG_ERROR,
+               "gf_strdup failed, failing username "
+               "resolution.");
+err:
+    return ret;
 }
 
-
 /*
  * This function writes out a latency sample to a given file descriptor
  * and beautifies the output in the process.
  */
 void
-_io_stats_write_latency_sample (xlator_t *this, ios_sample_t *sample,
-                                FILE *logfp)
-{
-        double epoch_time = 0.00;
-        char   *xlator_name = NULL;
-        char   *instance_name = NULL;
-        char   *hostname = NULL;
-        char   *identifier = NULL;
-        char   *port = NULL;
-        char   *port_pos = NULL;
-        char   *group_name = NULL;
-        char   *username = NULL;
-        struct ios_conf *conf = NULL;
-
-        conf = this->private;
-
-        epoch_time = (sample->timestamp).tv_sec +
-          ((sample->timestamp).tv_usec / 1000000.0);
-
-        if (strlen (sample->identifier) == 0) {
-                hostname = "Unknown";
-                port = "Unknown";
-        } else {
-                identifier = strdupa (sample->identifier);
-                port_pos = strrchr (identifier, ':');
-                if (!port_pos || strlen(port_pos) < 2)
-                        goto err;
-                port = strdupa (port_pos + 1);
-                if (!port)
-                        goto err;
-                *port_pos = '\0';
-                hostname = gf_rev_dns_lookup_cached (identifier,
-                                                     conf->dnscache);
-                if (!hostname)
-                        hostname = "Unknown";
-        }
-
-        xlator_name = conf->unique_id;
-        if (!xlator_name || strlen (xlator_name) == 0)
-                xlator_name = "Unknown";
-
-        instance_name = this->instance_name;
-        if (!instance_name || strlen (instance_name) == 0)
-                instance_name = "N/A";
-
-        /* Resolve the UID to a string username */
-        username = _resolve_username (this, sample->uid);
+_io_stats_write_latency_sample(xlator_t *this, ios_sample_t *sample,
+                               FILE *logfp)
+{
+    double epoch_time = 0.00;
+    char *xlator_name = NULL;
+    char *instance_name = NULL;
+    char *hostname = NULL;
+    char *identifier = NULL;
+    char *port = NULL;
+    char *port_pos = NULL;
+    char *group_name = NULL;
+    char *username = NULL;
+    struct ios_conf *conf = NULL;
+
+    conf = this->private;
+
+    epoch_time = (sample->timestamp).tv_sec +
+                 ((sample->timestamp).tv_usec / 1000000.0);
+
+    if (strlen(sample->identifier) == 0) {
+        hostname = "Unknown";
+        port = "Unknown";
+    } else {
+        identifier = strdupa(sample->identifier);
+        port_pos = strrchr(identifier, ':');
+        if (!port_pos || strlen(port_pos) < 2)
+            goto err;
+        port = strdupa(port_pos + 1);
+        if (!port)
+            goto err;
+        *port_pos = '\0';
+        hostname = gf_rev_dns_lookup_cached(identifier, conf->dnscache);
+        if (!hostname)
+            hostname = "Unknown";
+    }
+
+    xlator_name = conf->unique_id;
+    if (!xlator_name || strlen(xlator_name) == 0)
+        xlator_name = "Unknown";
+
+    instance_name = this->instance_name;
+    if (!instance_name || strlen(instance_name) == 0)
+        instance_name = "N/A";
+
+    /* Resolve the UID to a string username */
+    username = _resolve_username(this, sample->uid);
+    if (!username) {
+        username = GF_MALLOC(30, gf_common_mt_char);
         if (!username) {
-                username = GF_MALLOC (30, gf_common_mt_char);
-                if (!username) {
-                        goto out;
-                }
-                sprintf (username, "%d", (int32_t)sample->uid);
+            goto out;
         }
+        sprintf(username, "%d", (int32_t)sample->uid);
+    }
 
-        /* Resolve the GID to a string group name */
-        group_name = _resolve_group_name (this, sample->gid);
+    /* Resolve the GID to a string group name */
+    group_name = _resolve_group_name(this, sample->gid);
+    if (!group_name) {
+        group_name = GF_MALLOC(30, gf_common_mt_char);
         if (!group_name) {
-                group_name = GF_MALLOC (30, gf_common_mt_char);
-                if (!group_name) {
-                        goto out;
-                }
-                sprintf (group_name, "%d", (int32_t)sample->gid);
+            goto out;
         }
-
-        ios_log (this, logfp,
-                 "%0.6lf,%s,%s,%0.4lf,%s,%s,%s,%s,%s,%s",
-                 epoch_time, fop_enum_to_pri_string (sample->fop_type),
-                 gf_fop_string (sample->fop_type),
-                 sample->elapsed, xlator_name, instance_name, username,
-                 group_name, hostname, port);
-        goto out;
+        sprintf(group_name, "%d", (int32_t)sample->gid);
+    }
+
+    ios_log(this, logfp, "%0.6lf,%s,%s,%0.4lf,%s,%s,%s,%s,%s,%s", epoch_time,
+            fop_enum_to_pri_string(sample->fop_type),
+            gf_fop_string(sample->fop_type), sample->elapsed, xlator_name,
+            instance_name, username, group_name, hostname, port);
+    goto out;
 err:
-        gf_log (this->name, GF_LOG_ERROR,
-                "Error parsing socket identifier");
+    gf_log(this->name, GF_LOG_ERROR, "Error parsing socket identifier");
 out:
-        GF_FREE (group_name);
-        GF_FREE (username);
+    GF_FREE(group_name);
+    GF_FREE(username);
 }
 
 /*
@@ -1180,3409 +1138,3343 @@ out:
  * contents of the saved reference.
  */
 int
-io_stats_dump_latency_samples_logfp (xlator_t *this, FILE *logfp)
-{
-        uint64_t              i = 0;
-        struct ios_conf       *conf = NULL;
-        ios_sample_buf_t      *sample_buf = NULL;
-        int                   ret = 1;  /* Default to error */
-
-        conf = this->private;
-
-        /* Save pointer to old buffer; the CS equivalent of
-         * Indiana Jones: https://www.youtube.com/watch?v=Pr-8AP0To4k,
-         * though ours will end better I hope!
-         */
-        sample_buf = conf->ios_sample_buf;
-        if (!sample_buf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Sampling buffer is null, bailing!");
-                goto out;
-        }
-
-        /* Empty case, nothing to do, exit. */
-        if (sample_buf->collected == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No samples, dump not required.");
-                ret = 0;
-                goto out;
-        }
-
-        /* Init a new buffer, so we are free to work on the one we saved a
-         * reference to above.
-         */
-        if (ios_init_sample_buf (conf) != 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Failed to init new sampling buffer, out of memory?");
-                goto out;
-        }
+io_stats_dump_latency_samples_logfp(xlator_t *this, FILE *logfp)
+{
+    uint64_t i = 0;
+    struct ios_conf *conf = NULL;
+    ios_sample_buf_t *sample_buf = NULL;
+    int ret = 1; /* Default to error */
+
+    conf = this->private;
+
+    /* Save pointer to old buffer; the CS equivalent of
+     * Indiana Jones: https://www.youtube.com/watch?v=Pr-8AP0To4k,
+     * though ours will end better I hope!
+     */
+    sample_buf = conf->ios_sample_buf;
+    if (!sample_buf) {
+        gf_log(this->name, GF_LOG_WARNING, "Sampling buffer is null, bailing!");
+        goto out;
+    }
 
-        /* Wrap-around case, dump from pos to sample_buf->size -1
-         * and then from 0 to sample_buf->pos (covered off by
-         * "simple case")
-         */
-        if (sample_buf->collected > sample_buf->pos + 1) {
-                for (i = sample_buf->pos; i < sample_buf->size; i++) {
-                        _io_stats_write_latency_sample (this,
-                                &(sample_buf->ios_samples[i]), logfp);
-                }
+    /* Empty case, nothing to do, exit. */
+    if (sample_buf->collected == 0) {
+        gf_log(this->name, GF_LOG_DEBUG, "No samples, dump not required.");
+        ret = 0;
+        goto out;
+    }
+
+    /* Init a new buffer, so we are free to work on the one we saved a
+     * reference to above.
+     */
+    if (ios_init_sample_buf(conf) != 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Failed to init new sampling buffer, out of memory?");
+        goto out;
+    }
+
+    /* Wrap-around case, dump from pos to sample_buf->size -1
+     * and then from 0 to sample_buf->pos (covered off by
+     * "simple case")
+     */
+    if (sample_buf->collected > sample_buf->pos + 1) {
+        for (i = sample_buf->pos; i < sample_buf->size; i++) {
+            _io_stats_write_latency_sample(this, &(sample_buf->ios_samples[i]),
+                                           logfp);
         }
+    }
 
-        /* Simple case: Dump from 0 to sample_buf->pos */
-        for (i = 0; i < sample_buf->pos; i++) {
-                _io_stats_write_latency_sample (this,
-                        &(sample_buf->ios_samples[i]), logfp);
-        }
-        ios_destroy_sample_buf (sample_buf);
+    /* Simple case: Dump from 0 to sample_buf->pos */
+    for (i = 0; i < sample_buf->pos; i++) {
+        _io_stats_write_latency_sample(this, &(sample_buf->ios_samples[i]),
+                                       logfp);
+    }
+    ios_destroy_sample_buf(sample_buf);
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,
-                               struct timeval *now, int interval, FILE *logfp)
-{
-        int                   i = 0;
-        int                   per_line = 0;
-        int                   index = 0;
-        struct ios_stat_head *list_head = NULL;
-        struct ios_conf      *conf = NULL;
-        char                  timestr[256] = {0, };
-        char                  str_header[128] = {0};
-        char                  str_read[128] = {0};
-        char                  str_write[128] = {0};
-        uint64_t              fop_hits = 0;
-        uint64_t              block_count_read = 0;
-        uint64_t              block_count_write = 0;
-
-        conf = this->private;
-
-        if (interval == -1)
-                ios_log (this, logfp, "\n=== Cumulative stats ===");
+io_stats_dump_global_to_logfp(xlator_t *this, struct ios_global_stats *stats,
+                              time_t now, int interval, FILE *logfp)
+{
+    int i = 0;
+    int per_line = 0;
+    int index = 0;
+    struct ios_stat_head *list_head = NULL;
+    struct ios_conf *conf = NULL;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char str_header[128] = {0};
+    char str_read[128] = {0};
+    char str_write[128] = {0};
+    uint64_t fop_hits = 0;
+    uint64_t block_count_read = 0;
+    uint64_t block_count_write = 0;
+
+    conf = this->private;
+
+    if (interval == -1)
+        ios_log(this, logfp, "\n=== Cumulative stats ===");
+    else
+        ios_log(this, logfp, "\n=== Interval %d stats ===", interval);
+    ios_log(this, logfp, "      Duration : %" PRIu64 " secs",
+            (uint64_t)(now - stats->started_at));
+    ios_log(this, logfp, "     BytesRead : %" GF_PRI_ATOMIC,
+            GF_ATOMIC_GET(stats->data_read));
+    ios_log(this, logfp, "  BytesWritten : %" GF_PRI_ATOMIC "\n",
+            GF_ATOMIC_GET(stats->data_written));
+
+    snprintf(str_header, sizeof(str_header), "%-12s %c", "Block Size", ':');
+    snprintf(str_read, sizeof(str_read), "%-12s %c", "Read Count", ':');
+    snprintf(str_write, sizeof(str_write), "%-12s %c", "Write Count", ':');
+    index = 14;
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        block_count_read = GF_ATOMIC_GET(stats->block_count_read[i]);
+        block_count_write = GF_ATOMIC_GET(stats->block_count_write[i]);
+        if ((block_count_read == 0) && (block_count_write == 0))
+            continue;
+        per_line++;
+
+        snprintf(str_header + index, sizeof(str_header) - index, "%16dB+",
+                 (1 << i));
+        if (block_count_read)
+            snprintf(str_read + index, sizeof(str_read) - index, "%18" PRId64,
+                     block_count_read);
         else
-                ios_log (this, logfp, "\n=== Interval %d stats ===",
-                         interval);
-        ios_log (this, logfp, "      Duration : %"PRId64" secs",
-                 (uint64_t) (now->tv_sec - stats->started_at.tv_sec));
-        ios_log (this, logfp, "     BytesRead : %"GF_PRI_ATOMIC,
-                 GF_ATOMIC_GET (stats->data_read));
-        ios_log (this, logfp, "  BytesWritten : %"GF_PRI_ATOMIC"\n",
-                 GF_ATOMIC_GET (stats->data_written));
-
-        snprintf (str_header, sizeof (str_header), "%-12s %c", "Block Size", ':');
-        snprintf (str_read, sizeof (str_read), "%-12s %c", "Read Count", ':');
-        snprintf (str_write, sizeof (str_write), "%-12s %c", "Write Count", ':');
-        index = 14;
-        for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
-                block_count_read = GF_ATOMIC_GET (stats->block_count_read[i]);
-                block_count_write = GF_ATOMIC_GET (stats->block_count_write[i]);
-                if ((block_count_read == 0) && (block_count_write == 0))
-                        continue;
-                per_line++;
-
-                snprintf (str_header+index, sizeof (str_header)-index,
-                          "%16dB+", (1<<i));
-                if (block_count_read)
-                        snprintf (str_read+index, sizeof (str_read)-index,
-                                  "%18"PRId64, block_count_read);
-                else    snprintf (str_read+index, sizeof (str_read)-index,
-                                  "%18s", "0");
-                if (block_count_write)
-                        snprintf (str_write+index, sizeof (str_write)-index,
-                                  "%18"GF_PRI_ATOMIC, block_count_write);
-                else    snprintf (str_write+index, sizeof (str_write)-index,
-                                  "%18s", "0");
-
-                index += 18;
-                if (per_line == 3) {
-                        ios_log (this, logfp, "%s", str_header);
-                        ios_log (this, logfp, "%s", str_read);
-                        ios_log (this, logfp, "%s\n", str_write);
-
-                        memset (str_header, 0, sizeof (str_header));
-                        memset (str_read, 0, sizeof (str_read));
-                        memset (str_write, 0, sizeof (str_write));
-
-                        snprintf (str_header, sizeof (str_header), "%-12s %c",
-                                  "Block Size", ':');
-                        snprintf (str_read, sizeof (str_read), "%-12s %c",
-                                  "Read Count", ':');
-                        snprintf (str_write, sizeof (str_write), "%-12s %c",
-                                  "Write Count", ':');
-
-                        index = 14;
-                        per_line = 0;
-                }
-        }
-
-        if (per_line != 0) {
-                ios_log (this, logfp, "%s", str_header);
-                ios_log (this, logfp, "%s", str_read);
-                ios_log (this, logfp, "%s\n", str_write);
-        }
-
-        ios_log (this, logfp, "%-13s %10s %14s %14s %14s", "Fop",
-                 "Call Count", "Avg-Latency", "Min-Latency",
-                 "Max-Latency");
-        ios_log (this, logfp, "%-13s %10s %14s %14s %14s", "---", "----------",
-                 "-----------", "-----------", "-----------");
-
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                fop_hits = GF_ATOMIC_GET (stats->fop_hits[i]);
-                if (fop_hits && !stats->latency[i].avg)
-                        ios_log (this, logfp, "%-13s %10"GF_PRI_ATOMIC" %11s "
-                                 "us %11s us %11s us", gf_fop_list[i],
-                                 fop_hits, "0", "0", "0");
-                else if (fop_hits && stats->latency[i].avg)
-                        ios_log (this, logfp, "%-13s %10"GF_PRI_ATOMIC" "
-                                 "%11.2lf us %11.2lf us %11.2lf us",
-                                 gf_fop_list[i], fop_hits,
-                                 stats->latency[i].avg, stats->latency[i].min,
-                                 stats->latency[i].max);
-        }
-
-        for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
-                fop_hits = GF_ATOMIC_GET (stats->upcall_hits[i]);
-                if (fop_hits)
-                        ios_log (this, logfp, "%-13s %10"PRId64" %11s "
-                                 "us %11s us %11s us", gf_upcall_list[i],
-                                 fop_hits, "0", "0", "0");
+            snprintf(str_read + index, sizeof(str_read) - index, "%18s", "0");
+        if (block_count_write)
+            snprintf(str_write + index, sizeof(str_write) - index,
+                     "%18" GF_PRI_ATOMIC, block_count_write);
+        else
+            snprintf(str_write + index, sizeof(str_write) - index, "%18s", "0");
+
+        index += 18;
+        if (per_line == 3) {
+            ios_log(this, logfp, "%s", str_header);
+            ios_log(this, logfp, "%s", str_read);
+            ios_log(this, logfp, "%s\n", str_write);
+
+            snprintf(str_header, sizeof(str_header), "%-12s %c", "Block Size",
+                     ':');
+            snprintf(str_read, sizeof(str_read), "%-12s %c", "Read Count", ':');
+            snprintf(str_write, sizeof(str_write), "%-12s %c", "Write Count",
+                     ':');
+
+            index = 14;
+            per_line = 0;
         }
-
-        ios_log (this, logfp, "------ ----- ----- ----- ----- ----- ----- ----- "
-                 " ----- ----- ----- -----\n");
-
-        if (interval == -1) {
-                LOCK (&conf->lock);
-                {
-                        gf_time_fmt (timestr, sizeof timestr,
-                                     conf->cumulative.max_openfd_time.tv_sec,
-                                     gf_timefmt_FT);
-                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
-                                  ".%"GF_PRI_SUSECONDS,
-                                  conf->cumulative.max_openfd_time.tv_usec);
-                        ios_log (this, logfp, "Current open fd's: %"PRId64
-                                 " Max open fd's: %"PRId64" time %s",
-                                 conf->cumulative.nr_opens,
-                                 conf->cumulative.max_nr_opens, timestr);
-                }
-                UNLOCK (&conf->lock);
-                ios_log (this, logfp, "\n==========Open File Stats========");
-                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_OPEN];
-                ios_dump_file_stats (list_head, this, logfp);
-
-
-                ios_log (this, logfp, "\n==========Read File Stats========");
-                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_READ];
-                ios_dump_file_stats (list_head, this, logfp);
-
-                ios_log (this, logfp, "\n==========Write File Stats========");
-                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_WRITE];
-                ios_dump_file_stats (list_head, this, logfp);
-
-                ios_log (this, logfp, "\n==========Directory open stats========");
-                ios_log (this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
-                ios_dump_file_stats (list_head, this, logfp);
-
-                ios_log (this, logfp, "\n========Directory readdirp Stats=======");
-                ios_log (this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
-                ios_dump_file_stats (list_head, this, logfp);
-
-                ios_log (this, logfp, "\n========Read Throughput File Stats=====");
-                ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
-                         "\tFILE NAME");
-                list_head = &conf->thru_list[IOS_STATS_THRU_READ];
-                ios_dump_throughput_stats(list_head, this, logfp,
-                                          IOS_STATS_THRU_READ);
-
-                ios_log (this, logfp, "\n======Write Throughput File Stats======");
-                ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
-                         "\tFILE NAME");
-                list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
-                ios_dump_throughput_stats (list_head, this, logfp,
-                                           IOS_STATS_THRU_WRITE);
+    }
+
+    if (per_line != 0) {
+        ios_log(this, logfp, "%s", str_header);
+        ios_log(this, logfp, "%s", str_read);
+        ios_log(this, logfp, "%s\n", str_write);
+    }
+
+    ios_log(this, logfp, "%-13s %10s %14s %14s %14s", "Fop", "Call Count",
+            "Avg-Latency", "Min-Latency", "Max-Latency");
+    ios_log(this, logfp, "%-13s %10s %14s %14s %14s", "---", "----------",
+            "-----------", "-----------", "-----------");
+
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        fop_hits = GF_ATOMIC_GET(stats->fop_hits[i]);
+        if (fop_hits && !stats->latency[i].avg)
+            ios_log(this, logfp,
+                    "%-13s %10" GF_PRI_ATOMIC
+                    " %11s "
+                    "us %11s us %11s us",
+                    gf_fop_list[i], fop_hits, "0", "0", "0");
+        else if (fop_hits && stats->latency[i].avg)
+            ios_log(this, logfp,
+                    "%-13s %10" GF_PRI_ATOMIC
+                    " "
+                    "%11.2lf us %11.2lf us %11.2lf us",
+                    gf_fop_list[i], fop_hits, stats->latency[i].avg,
+                    stats->latency[i].min, stats->latency[i].max);
+    }
+
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        fop_hits = GF_ATOMIC_GET(stats->upcall_hits[i]);
+        if (fop_hits)
+            ios_log(this, logfp,
+                    "%-13s %10" PRId64
+                    " %11s "
+                    "us %11s us %11s us",
+                    gf_upcall_list[i], fop_hits, "0", "0", "0");
+    }
+
+    ios_log(this, logfp,
+            "------ ----- ----- ----- ----- ----- ----- ----- "
+            " ----- ----- ----- -----\n");
+
+    if (interval == -1) {
+        LOCK(&conf->lock);
+        {
+            gf_time_fmt_tv(timestr, sizeof timestr,
+                           &conf->cumulative.max_openfd_time, gf_timefmt_FT);
+            ios_log(this, logfp,
+                    "Current open fd's: %" PRId64 " Max open fd's: %" PRId64
+                    " time %s",
+                    conf->cumulative.nr_opens, conf->cumulative.max_nr_opens,
+                    timestr);
         }
-        return 0;
+        UNLOCK(&conf->lock);
+        ios_log(this, logfp, "\n==========Open File Stats========");
+        ios_log(this, logfp, "\nCOUNT:  \t  FILE NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_OPEN];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n==========Read File Stats========");
+        ios_log(this, logfp, "\nCOUNT:  \t  FILE NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_READ];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n==========Write File Stats========");
+        ios_log(this, logfp, "\nCOUNT:  \t  FILE NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_WRITE];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n==========Directory open stats========");
+        ios_log(this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n========Directory readdirp Stats=======");
+        ios_log(this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n========Read Throughput File Stats=====");
+        ios_log(this, logfp,
+                "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
+                "\tFILE NAME");
+        list_head = &conf->thru_list[IOS_STATS_THRU_READ];
+        ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_THRU_READ);
+
+        ios_log(this, logfp, "\n======Write Throughput File Stats======");
+        ios_log(this, logfp,
+                "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
+                "\tFILE NAME");
+        list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
+        ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_THRU_WRITE);
+    }
+    return 0;
 }
 
 int
-io_stats_dump_global_to_dict (xlator_t *this, struct ios_global_stats *stats,
-                              struct timeval *now, int interval, dict_t *dict)
-{
-        int             ret = 0;
-        char            key[256] = {0};
-        uint64_t        sec = 0;
-        int             i = 0;
-        uint64_t        count = 0;
-        uint64_t        fop_hits = 0;
-
-        GF_ASSERT (stats);
-        GF_ASSERT (now);
-        GF_ASSERT (dict);
-        GF_ASSERT (this);
-
-        if (interval == -1)
-                snprintf (key, sizeof (key), "cumulative");
-        else
-                snprintf (key, sizeof (key), "interval");
-        ret = dict_set_int32 (dict, key, interval);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set "
-                        "interval %d", interval);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-duration", interval);
-        sec = (uint64_t) (now->tv_sec - stats->started_at.tv_sec);
-        ret = dict_set_uint64 (dict, key, sec);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set "
-                        "duration(%d) - %"PRId64, interval, sec);
+io_stats_dump_global_to_dict(xlator_t *this, struct ios_global_stats *stats,
+                             time_t now, int interval, dict_t *dict)
+{
+    int ret = 0;
+    char key[64] = {0};
+    uint64_t sec = 0;
+    int i = 0;
+    uint64_t count = 0;
+    uint64_t fop_hits = 0;
+
+    GF_ASSERT(stats);
+    GF_ASSERT(now);
+    GF_ASSERT(dict);
+    GF_ASSERT(this);
+
+    if (interval == -1)
+        snprintf(key, sizeof(key), "cumulative");
+    else
+        snprintf(key, sizeof(key), "interval");
+    ret = dict_set_int32(dict, key, interval);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set "
+               "interval %d",
+               interval);
+
+    snprintf(key, sizeof(key), "%d-duration", interval);
+    sec = now - stats->started_at;
+    ret = dict_set_uint64(dict, key, sec);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set "
+               "duration(%d) - %" PRId64,
+               interval, sec);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%d-total-read", interval);
+    ret = dict_set_uint64(dict, key, GF_ATOMIC_GET(stats->data_read));
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set total "
+               "read(%d) - %" GF_PRI_ATOMIC,
+               interval, GF_ATOMIC_GET(stats->data_read));
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%d-total-write", interval);
+    ret = dict_set_uint64(dict, key, GF_ATOMIC_GET(stats->data_written));
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set total "
+               "write(%d) - %" GF_PRI_ATOMIC,
+               interval, GF_ATOMIC_GET(stats->data_written));
+        goto out;
+    }
+    for (i = 0; i < 32; i++) {
+        count = GF_ATOMIC_GET(stats->block_count_read[i]);
+        if (count) {
+            snprintf(key, sizeof(key), "%d-read-%d", interval, (1 << i));
+            ret = dict_set_uint64(dict, key, count);
+            if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "failed to "
+                       "set read-%db+, with: %" PRId64,
+                       (1 << i), count);
                 goto out;
+            }
         }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-total-read", interval);
-        ret = dict_set_uint64 (dict, key, GF_ATOMIC_GET (stats->data_read));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set total "
-                       "read(%d) - %"GF_PRI_ATOMIC, interval,
-                       GF_ATOMIC_GET (stats->data_read));
+    }
+
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        count = GF_ATOMIC_GET(stats->block_count_write[i]);
+        if (count) {
+            snprintf(key, sizeof(key), "%d-write-%d", interval, (1 << i));
+            ret = dict_set_uint64(dict, key, count);
+            if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "failed to "
+                       "set write-%db+, with: %" PRId64,
+                       (1 << i), count);
                 goto out;
+            }
         }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-total-write", interval);
-        ret = dict_set_uint64 (dict, key, GF_ATOMIC_GET (stats->data_written));
+    }
+
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        fop_hits = GF_ATOMIC_GET(stats->fop_hits[i]);
+        if (fop_hits == 0)
+            continue;
+        snprintf(key, sizeof(key), "%d-%d-hits", interval, i);
+        ret = dict_set_uint64(dict, key, fop_hits);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set total "
-                        "write(%d) - %"GF_PRI_ATOMIC, interval,
-                        GF_ATOMIC_GET (stats->data_written));
-                goto out;
-        }
-        for (i = 0; i < 32; i++) {
-                count = GF_ATOMIC_GET (stats->block_count_read[i]);
-                if (count) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-read-%d", interval,
-                                  (1 << i));
-                        ret = dict_set_uint64 (dict, key, count);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "set read-%db+, with: %"PRId64,
-                                        (1<<i), count);
-                                goto out;
-                        }
-                }
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set "
+                   "%s-fop-hits: %" GF_PRI_ATOMIC,
+                   gf_fop_list[i], fop_hits);
+            goto out;
         }
 
-        for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
-                count = GF_ATOMIC_GET (stats->block_count_write[i]);
-                if (count) {
-                        snprintf (key, sizeof (key), "%d-write-%d", interval,
-                                  (1<<i));
-                        ret = dict_set_uint64 (dict, key, count);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "set write-%db+, with: %"PRId64,
-                                        (1<<i), count);
-                                goto out;
-                        }
-                }
+        if (stats->latency[i].avg == 0)
+            continue;
+        snprintf(key, sizeof(key), "%d-%d-avglatency", interval, i);
+        ret = dict_set_double(dict, key, stats->latency[i].avg);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set %s "
+                   "avglatency(%d) with %f",
+                   gf_fop_list[i], interval, stats->latency[i].avg);
+            goto out;
         }
-
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                fop_hits = GF_ATOMIC_GET (stats->fop_hits[i]);
-                if (fop_hits == 0)
-                        continue;
-                snprintf (key, sizeof (key), "%d-%d-hits", interval, i);
-                ret = dict_set_uint64 (dict, key, fop_hits);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to set "
-                                "%s-fop-hits: %"GF_PRI_ATOMIC, gf_fop_list[i],
-                                fop_hits);
-                        goto out;
-                }
-
-                if (stats->latency[i].avg == 0)
-                        continue;
-                snprintf (key, sizeof (key), "%d-%d-avglatency", interval, i);
-                ret = dict_set_double (dict, key, stats->latency[i].avg);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
-                                "avglatency(%d) with %f", gf_fop_list[i],
-                                interval, stats->latency[i].avg);
-                        goto out;
-                }
-                snprintf (key, sizeof (key), "%d-%d-minlatency", interval, i);
-                ret = dict_set_double (dict, key, stats->latency[i].min);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
-                                "minlatency(%d) with %f", gf_fop_list[i],
-                                interval, stats->latency[i].min);
-                        goto out;
-                }
-                snprintf (key, sizeof (key), "%d-%d-maxlatency", interval, i);
-                ret = dict_set_double (dict, key, stats->latency[i].max);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
-                                "maxlatency(%d) with %f", gf_fop_list[i],
-                                interval, stats->latency[i].max);
-                        goto out;
-                }
+        snprintf(key, sizeof(key), "%d-%d-minlatency", interval, i);
+        ret = dict_set_double(dict, key, stats->latency[i].min);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set %s "
+                   "minlatency(%d) with %f",
+                   gf_fop_list[i], interval, stats->latency[i].min);
+            goto out;
         }
-        for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
-                fop_hits = GF_ATOMIC_GET (stats->upcall_hits[i]);
-                if (fop_hits == 0)
-                        continue;
-                snprintf (key, sizeof (key), "%d-%d-upcall-hits", interval, i);
-                ret = dict_set_uint64 (dict, key, fop_hits);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "set %s-upcall-hits: %"PRIu64,
-                                gf_upcall_list[i], fop_hits);
-                        goto out;
-                }
+        snprintf(key, sizeof(key), "%d-%d-maxlatency", interval, i);
+        ret = dict_set_double(dict, key, stats->latency[i].max);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set %s "
+                   "maxlatency(%d) with %f",
+                   gf_fop_list[i], interval, stats->latency[i].max);
+            goto out;
         }
+    }
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        fop_hits = GF_ATOMIC_GET(stats->upcall_hits[i]);
+        if (fop_hits == 0)
+            continue;
+        snprintf(key, sizeof(key), "%d-%d-upcall-hits", interval, i);
+        ret = dict_set_uint64(dict, key, fop_hits);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to "
+                   "set %s-upcall-hits: %" PRIu64,
+                   gf_upcall_list[i], fop_hits);
+            goto out;
+        }
+    }
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_log(this->name, GF_LOG_DEBUG, "returning %d", ret);
+    return ret;
 }
 
 int
-io_stats_dump_global (xlator_t *this, struct ios_global_stats *stats,
-                      struct timeval *now, int interval,
-                      struct ios_dump_args *args)
+io_stats_dump_global(xlator_t *this, struct ios_global_stats *stats, time_t now,
+                     int interval, struct ios_dump_args *args)
 {
-        int     ret = -1;
-
-        GF_ASSERT (args);
-        GF_ASSERT (now);
-        GF_ASSERT (stats);
-        GF_ASSERT (this);
+    int ret = -1;
 
+    GF_ASSERT(args);
+    GF_ASSERT(now);
+    GF_ASSERT(stats);
+    GF_ASSERT(this);
 
-
-        switch (args->type) {
+    switch (args->type) {
         case IOS_DUMP_TYPE_JSON_FILE:
-                ret = io_stats_dump_global_to_json_logfp (
-                    this, stats, now, interval, args->u.logfp);
-        break;
+            ret = io_stats_dump_global_to_json_logfp(this, stats, now, interval,
+                                                     args->u.logfp);
+            break;
         case IOS_DUMP_TYPE_FILE:
-                ret = io_stats_dump_global_to_logfp (this, stats, now,
-                                                     interval, args->u.logfp);
-        break;
+            ret = io_stats_dump_global_to_logfp(this, stats, now, interval,
+                                                args->u.logfp);
+            break;
         case IOS_DUMP_TYPE_DICT:
-                ret = io_stats_dump_global_to_dict (this, stats, now,
-                                                    interval, args->u.dict);
-        break;
+            ret = io_stats_dump_global_to_dict(this, stats, now, interval,
+                                               args->u.dict);
+            break;
         default:
-                GF_ASSERT (0);
-                ret = -1;
-        break;
-        }
-        return ret;
+            GF_ASSERT(0);
+            ret = -1;
+            break;
+    }
+    return ret;
 }
 
 int
-ios_dump_args_init (struct ios_dump_args *args, ios_dump_type_t type,
-                    void *output)
+ios_dump_args_init(struct ios_dump_args *args, ios_dump_type_t type,
+                   void *output)
 {
-        int             ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (args);
-        GF_ASSERT (type > IOS_DUMP_TYPE_NONE && type < IOS_DUMP_TYPE_MAX);
-        GF_ASSERT (output);
+    GF_ASSERT(args);
+    GF_ASSERT(type > IOS_DUMP_TYPE_NONE && type < IOS_DUMP_TYPE_MAX);
+    GF_ASSERT(output);
 
-        args->type = type;
-        switch (args->type) {
+    args->type = type;
+    switch (args->type) {
         case IOS_DUMP_TYPE_JSON_FILE:
         case IOS_DUMP_TYPE_FILE:
-                args->u.logfp = output;
-                break;
+            args->u.logfp = output;
+            break;
         case IOS_DUMP_TYPE_DICT:
-                args->u.dict = output;
-                break;
+            args->u.dict = output;
+            break;
         default:
-                GF_ASSERT (0);
-                ret = -1;
-        }
+            GF_ASSERT(0);
+            ret = -1;
+    }
 
-        return ret;
+    return ret;
 }
 
 static void
-ios_global_stats_clear (struct ios_global_stats *stats, struct timeval *now)
+ios_global_stats_clear(struct ios_global_stats *stats, time_t now)
 {
-        GF_ASSERT (stats);
-        GF_ASSERT (now);
+    GF_ASSERT(stats);
+    GF_ASSERT(now);
 
-        memset (stats, 0, sizeof (*stats));
-        stats->started_at = *now;
+    memset(stats, 0, sizeof(*stats));
+    stats->started_at = now;
 }
 
 int
-io_stats_dump (xlator_t *this, struct ios_dump_args *args,
-               gf1_cli_info_op op, gf_boolean_t is_peek)
+io_stats_dump(xlator_t *this, struct ios_dump_args *args, ios_info_op_t op,
+              gf_boolean_t is_peek)
 {
-        struct ios_conf         *conf = NULL;
-        struct ios_global_stats  cumulative = { };
-        struct ios_global_stats  incremental = { };
-        int                      increment = 0;
-        struct timeval           now;
+    struct ios_conf *conf = NULL;
+    struct ios_global_stats cumulative = {};
+    struct ios_global_stats incremental = {};
+    int increment = 0;
+    time_t now = 0;
 
-        GF_ASSERT (this);
-        GF_ASSERT (args);
-        GF_ASSERT (args->type > IOS_DUMP_TYPE_NONE);
-        GF_ASSERT (args->type < IOS_DUMP_TYPE_MAX);
+    GF_ASSERT(this);
+    GF_ASSERT(args);
+    GF_ASSERT(args->type > IOS_DUMP_TYPE_NONE);
+    GF_ASSERT(args->type < IOS_DUMP_TYPE_MAX);
 
-        conf = this->private;
+    conf = this->private;
+    now = gf_time();
 
-        gettimeofday (&now, NULL);
-        LOCK (&conf->lock);
-        {
-                if (op == GF_CLI_INFO_ALL ||
-                    op == GF_CLI_INFO_CUMULATIVE)
-                        cumulative  = conf->cumulative;
+    LOCK(&conf->lock);
+    {
+        if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_CUMULATIVE)
+            cumulative = conf->cumulative;
 
-                if (op == GF_CLI_INFO_ALL ||
-                    op == GF_CLI_INFO_INCREMENTAL) {
-                        incremental = conf->incremental;
-                        increment = conf->increment;
+        if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_INCREMENTAL) {
+            incremental = conf->incremental;
+            increment = conf->increment;
 
-                        if (!is_peek) {
-                                increment = conf->increment++;
+            if (!is_peek) {
+                increment = conf->increment++;
 
-                                ios_global_stats_clear (&conf->incremental,
-                                                        &now);
-                        }
-                }
+                ios_global_stats_clear(&conf->incremental, now);
+            }
         }
-        UNLOCK (&conf->lock);
+    }
+    UNLOCK(&conf->lock);
 
-        if (op == GF_CLI_INFO_ALL ||
-            op == GF_CLI_INFO_CUMULATIVE)
-                io_stats_dump_global (this, &cumulative, &now, -1, args);
+    if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_CUMULATIVE)
+        io_stats_dump_global(this, &cumulative, now, -1, args);
 
-        if (op == GF_CLI_INFO_ALL ||
-            op == GF_CLI_INFO_INCREMENTAL)
-                io_stats_dump_global (this, &incremental, &now, increment, args);
+    if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_INCREMENTAL)
+        io_stats_dump_global(this, &incremental, now, increment, args);
 
-        return 0;
+    return 0;
 }
 
-
 int
-io_stats_dump_fd (xlator_t *this, struct ios_fd *iosfd)
+io_stats_dump_fd(xlator_t *this, struct ios_fd *iosfd)
 {
-        struct ios_conf         *conf = NULL;
-        struct timeval           now;
-        uint64_t                 sec = 0;
-        uint64_t                 usec = 0;
-        int                      i = 0;
-        uint64_t                 data_read = 0;
-        uint64_t                 data_written = 0;
-        uint64_t                 block_count_read = 0;
-        uint64_t                 block_count_write = 0;
+    struct ios_conf *conf = NULL;
+    struct timeval now;
+    int i = 0;
+    double usecs = 0;
+    uint64_t data_read = 0;
+    uint64_t data_written = 0;
+    uint64_t block_count_read = 0;
+    uint64_t block_count_write = 0;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->dump_fd_stats)
-                return 0;
+    if (!conf->dump_fd_stats)
+        return 0;
 
-        if (!iosfd)
-                return 0;
+    if (!iosfd)
+        return 0;
 
-        gettimeofday (&now, NULL);
+    gettimeofday(&now, NULL);
+    usecs = gf_tvdiff(&iosfd->opened_at, &now);
 
-        if (iosfd->opened_at.tv_usec > now.tv_usec) {
-                now.tv_usec += 1000000;
-                now.tv_usec--;
-        }
+    gf_log(this->name, GF_LOG_INFO, "--- fd stats ---");
 
-        sec = now.tv_sec - iosfd->opened_at.tv_sec;
-        usec = now.tv_usec - iosfd->opened_at.tv_usec;
-
-        gf_log (this->name, GF_LOG_INFO,
-                "--- fd stats ---");
-
-        if (iosfd->filename)
-                gf_log (this->name, GF_LOG_INFO,
-                        "      Filename : %s",
-                        iosfd->filename);
-
-        if (sec)
-                gf_log (this->name, GF_LOG_INFO,
-                        "      Lifetime : %"PRId64"secs, %"PRId64"usecs",
-                        sec, usec);
-
-        data_read = GF_ATOMIC_GET (iosfd->data_read);
-        if (data_read)
-                gf_log (this->name, GF_LOG_INFO,
-                        "     BytesRead : %"PRId64" bytes", data_read);
-
-        data_written = GF_ATOMIC_GET (iosfd->data_written);
-        if (data_written)
-                gf_log (this->name, GF_LOG_INFO,
-                        "  BytesWritten : %"PRId64" bytes",
-                        data_written);
-
-        for (i = 0; i < 32; i++) {
-                block_count_read = GF_ATOMIC_GET (iosfd->block_count_read[i]);
-                if (block_count_read)
-                        gf_log (this->name, GF_LOG_INFO, " Read %06db+ :"
-                                "%"PRId64, (1 << i), block_count_read);
-        }
-        for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
-                block_count_write = GF_ATOMIC_GET (iosfd->block_count_write[i]);
-                if (block_count_write)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Write %06db+ : %"PRId64,
-                                (1 << i), block_count_write);
-        }
-        return 0;
-}
+    if (iosfd->filename)
+        gf_log(this->name, GF_LOG_INFO, "      Filename : %s", iosfd->filename);
 
-void collect_ios_latency_sample (struct ios_conf *conf,
-                glusterfs_fop_t fop_type, double elapsed,
-                call_frame_t *frame)
-{
-        ios_sample_buf_t *ios_sample_buf = NULL;
-        ios_sample_t     *ios_sample = NULL;
-        struct timespec  *timestamp = NULL;
-        call_stack_t     *root = NULL;
+    if (usecs)
+        gf_log(this->name, GF_LOG_INFO, "      Lifetime : %lf secs", usecs);
 
+    data_read = GF_ATOMIC_GET(iosfd->data_read);
+    if (data_read)
+        gf_log(this->name, GF_LOG_INFO, "     BytesRead : %" PRId64 " bytes",
+               data_read);
 
-        ios_sample_buf = conf->ios_sample_buf;
-        LOCK (&conf->ios_sampling_lock);
-        if (conf->ios_sample_interval == 0 ||
-            ios_sample_buf->observed % conf->ios_sample_interval != 0)
-                goto out;
+    data_written = GF_ATOMIC_GET(iosfd->data_written);
+    if (data_written)
+        gf_log(this->name, GF_LOG_INFO, "  BytesWritten : %" PRId64 " bytes",
+               data_written);
 
-        timestamp = &frame->begin;
-        root = frame->root;
-
-        ios_sample = &(ios_sample_buf->ios_samples[ios_sample_buf->pos]);
-        ios_sample->elapsed = elapsed;
-        ios_sample->fop_type = fop_type;
-        ios_sample->uid = root->uid;
-        ios_sample->gid = root->gid;
-        (ios_sample->timestamp).tv_sec = timestamp->tv_sec;
-        (ios_sample->timestamp).tv_usec = timestamp->tv_nsec / 1000;
-        memcpy (&ios_sample->identifier, &root->identifier,
-                sizeof (root->identifier));
-
-        /* We've reached the end of the circular buffer, start from the
-         * beginning. */
-        if (ios_sample_buf->pos == (ios_sample_buf->size - 1))
-                ios_sample_buf->pos = 0;
-        else
-                ios_sample_buf->pos++;
-        ios_sample_buf->collected++;
+    for (i = 0; i < 32; i++) {
+        block_count_read = GF_ATOMIC_GET(iosfd->block_count_read[i]);
+        if (block_count_read)
+            gf_log(this->name, GF_LOG_INFO,
+                   " Read %06db+ :"
+                   "%" PRId64,
+                   (1 << i), block_count_read);
+    }
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        block_count_write = GF_ATOMIC_GET(iosfd->block_count_write[i]);
+        if (block_count_write)
+            gf_log(this->name, GF_LOG_INFO, "Write %06db+ : %" PRId64, (1 << i),
+                   block_count_write);
+    }
+    return 0;
+}
+
+void
+collect_ios_latency_sample(struct ios_conf *conf, glusterfs_fop_t fop_type,
+                           double elapsed, call_frame_t *frame)
+{
+    ios_sample_buf_t *ios_sample_buf = NULL;
+    ios_sample_t *ios_sample = NULL;
+    struct timespec *timestamp = NULL;
+    call_stack_t *root = NULL;
+
+    ios_sample_buf = conf->ios_sample_buf;
+    LOCK(&conf->ios_sampling_lock);
+    if (conf->ios_sample_interval == 0 ||
+        ios_sample_buf->observed % conf->ios_sample_interval != 0)
+        goto out;
+
+    timestamp = &frame->begin;
+    root = frame->root;
+
+    ios_sample = &(ios_sample_buf->ios_samples[ios_sample_buf->pos]);
+    ios_sample->elapsed = elapsed;
+    ios_sample->fop_type = fop_type;
+    ios_sample->uid = root->uid;
+    ios_sample->gid = root->gid;
+    (ios_sample->timestamp).tv_sec = timestamp->tv_sec;
+    (ios_sample->timestamp).tv_usec = timestamp->tv_nsec / 1000;
+    memcpy(&ios_sample->identifier, &root->identifier,
+           sizeof(root->identifier));
+
+    /* We've reached the end of the circular buffer, start from the
+     * beginning. */
+    if (ios_sample_buf->pos == (ios_sample_buf->size - 1))
+        ios_sample_buf->pos = 0;
+    else
+        ios_sample_buf->pos++;
+    ios_sample_buf->collected++;
 out:
-        ios_sample_buf->observed++;
-        UNLOCK (&conf->ios_sampling_lock);
-        return;
+    ios_sample_buf->observed++;
+    UNLOCK(&conf->ios_sampling_lock);
+    return;
 }
 
 static void
-update_ios_latency_stats (struct ios_global_stats   *stats, double elapsed,
-                          glusterfs_fop_t op)
+update_ios_latency_stats(struct ios_global_stats *stats, double elapsed,
+                         glusterfs_fop_t op)
 {
-        double avg;
+    double avg;
 
-        GF_ASSERT (stats);
+    GF_ASSERT(stats);
 
-        stats->latency[op].total += elapsed;
+    stats->latency[op].total += elapsed;
 
-        if (!stats->latency[op].min)
-                stats->latency[op].min = elapsed;
-        if (stats->latency[op].min > elapsed)
-                stats->latency[op].min = elapsed;
-        if (stats->latency[op].max < elapsed)
-                stats->latency[op].max = elapsed;
+    if (!stats->latency[op].min)
+        stats->latency[op].min = elapsed;
+    if (stats->latency[op].min > elapsed)
+        stats->latency[op].min = elapsed;
+    if (stats->latency[op].max < elapsed)
+        stats->latency[op].max = elapsed;
 
-        avg = stats->latency[op].avg;
+    avg = stats->latency[op].avg;
 
-        stats->latency[op].avg = avg + (elapsed - avg) /
-                                       GF_ATOMIC_GET (stats->fop_hits[op]);
+    stats->latency[op].avg = avg + (elapsed - avg) /
+                                       GF_ATOMIC_GET(stats->fop_hits[op]);
 }
 
 int
-update_ios_latency (struct ios_conf *conf, call_frame_t *frame,
-                    glusterfs_fop_t op)
+update_ios_latency(struct ios_conf *conf, call_frame_t *frame,
+                   glusterfs_fop_t op)
 {
-        double elapsed;
-        struct timespec *begin, *end;
+    double elapsed;
+    struct timespec *begin, *end;
 
-        begin = &frame->begin;
-        end   = &frame->end;
+    begin = &frame->begin;
+    end = &frame->end;
 
-        elapsed = ((end->tv_sec - begin->tv_sec) * 1e9
-                   + (end->tv_nsec - begin->tv_nsec)) / 1000;
+    elapsed = gf_tsdiff(begin, end) / 1000.0;
 
-        update_ios_latency_stats (&conf->cumulative, elapsed, op);
-        update_ios_latency_stats (&conf->incremental, elapsed, op);
-        collect_ios_latency_sample (conf, op, elapsed, frame);
+    update_ios_latency_stats(&conf->cumulative, elapsed, op);
+    update_ios_latency_stats(&conf->incremental, elapsed, op);
+    collect_ios_latency_sample(conf, op, elapsed, frame);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-io_stats_dump_stats_to_dict (xlator_t *this, dict_t *resp,
-                             ios_stats_type_t flags, int32_t list_cnt)
-{
-        struct ios_conf         *conf = NULL;
-        int                      cnt  = 0;
-        char                     key[256];
-        struct ios_stat_head    *list_head = NULL;
-        struct ios_stat_list    *entry = NULL;
-        int                      ret = -1;
-        ios_stats_thru_t         index = IOS_STATS_THRU_MAX;
-        char                     timestr[256] = {0, };
-        char                    *dict_timestr = NULL;
-
-        conf = this->private;
-
-        switch (flags) {
-                case IOS_STATS_TYPE_OPEN:
-                        list_head = &conf->list[IOS_STATS_TYPE_OPEN];
-                        LOCK (&conf->lock);
-                        {
-                                ret = dict_set_uint64 (resp, "current-open",
-                                                     conf->cumulative.nr_opens);
-                                if (ret)
-                                        goto unlock;
-                                ret = dict_set_uint64 (resp, "max-open",
-                                                       conf->cumulative.max_nr_opens);
-
-                                gf_time_fmt (timestr, sizeof timestr,
-                                             conf->cumulative.max_openfd_time.tv_sec,
-                                             gf_timefmt_FT);
-                                if (conf->cumulative.max_openfd_time.tv_sec)
-                                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
-                                                  ".%"GF_PRI_SUSECONDS,
-                                                  conf->cumulative.max_openfd_time.tv_usec);
-
-                                dict_timestr = gf_strdup (timestr);
-                                if (!dict_timestr)
-                                        goto unlock;
-                                ret = dict_set_dynstr (resp, "max-openfd-time",
-                                                       dict_timestr);
-                                if (ret)
-                                        goto unlock;
-                        }
+io_stats_dump_stats_to_dict(xlator_t *this, dict_t *resp,
+                            ios_stats_type_t flags, int32_t list_cnt)
+{
+    struct ios_conf *conf = NULL;
+    int cnt = 0;
+    char key[32];
+    int keylen;
+    struct ios_stat_head *list_head = NULL;
+    struct ios_stat_list *entry = NULL;
+    int ret = -1;
+    ios_stats_thru_t index = IOS_STATS_THRU_MAX;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char *dict_timestr = NULL;
+
+    conf = this->private;
+
+    switch (flags) {
+        case IOS_STATS_TYPE_OPEN:
+            list_head = &conf->list[IOS_STATS_TYPE_OPEN];
+            LOCK(&conf->lock);
+            {
+                ret = dict_set_uint64(resp, "current-open",
+                                      conf->cumulative.nr_opens);
+                if (ret)
+                    goto unlock;
+                ret = dict_set_uint64(resp, "max-open",
+                                      conf->cumulative.max_nr_opens);
+
+                gf_time_fmt_tv(timestr, sizeof timestr,
+                               &conf->cumulative.max_openfd_time,
+                               gf_timefmt_FT);
+
+                dict_timestr = gf_strdup(timestr);
+                if (!dict_timestr)
+                    goto unlock;
+                ret = dict_set_dynstr(resp, "max-openfd-time", dict_timestr);
+                if (ret)
+                    goto unlock;
+            }
         unlock:
-                        UNLOCK (&conf->lock);
-                        /* Do not proceed if we came here because of some error
-                         * during the dict operation */
-                        if (ret)
-                                goto out;
-                        break;
-                case IOS_STATS_TYPE_READ:
-                        list_head = &conf->list[IOS_STATS_TYPE_READ];
-                        break;
-                case IOS_STATS_TYPE_WRITE:
-                        list_head = &conf->list[IOS_STATS_TYPE_WRITE];
-                        break;
-                case IOS_STATS_TYPE_OPENDIR:
-                        list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
-                        break;
-                case IOS_STATS_TYPE_READDIRP:
-                        list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
-                        break;
-                case IOS_STATS_TYPE_READ_THROUGHPUT:
-                        list_head = &conf->thru_list[IOS_STATS_THRU_READ];
-                        index = IOS_STATS_THRU_READ;
-                        break;
-                case IOS_STATS_TYPE_WRITE_THROUGHPUT:
-                        list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
-                        index = IOS_STATS_THRU_WRITE;
-                        break;
-
-                default:
-                       goto out;
-        }
-        ret = dict_set_int32 (resp, "top-op", flags);
-        if (!list_cnt)
+            UNLOCK(&conf->lock);
+            /* Do not proceed if we came here because of some error
+             * during the dict operation */
+            if (ret)
                 goto out;
-        LOCK (&list_head->lock);
-        {
-                list_for_each_entry (entry, &list_head->iosstats->list, list) {
-
-                        cnt++;
-                        snprintf (key, 256, "%s-%d", "filename", cnt);
-                        ret = dict_set_str (resp, key, entry->iosstat->filename);
-                        if (ret)
-                                goto unlock_list_head;
-                         snprintf (key, 256, "%s-%d", "value",cnt);
-                         ret = dict_set_uint64 (resp, key, entry->value);
-                         if (ret)
-                                 goto unlock_list_head;
-                         if (index != IOS_STATS_THRU_MAX) {
-                                 snprintf (key, 256, "%s-%d", "time-sec", cnt);
-                                 ret = dict_set_int32 (resp, key,
-                                         entry->iosstat->thru_counters[index].time.tv_sec);
-                                 if (ret)
-                                         goto unlock_list_head;
-                                 snprintf (key, 256, "%s-%d", "time-usec", cnt);
-                                 ret = dict_set_int32 (resp, key,
-                                         entry->iosstat->thru_counters[index].time.tv_usec);
-                                 if (ret)
-                                         goto unlock_list_head;
-                         }
-                         if (cnt == list_cnt)
-                                 break;
+            break;
+        case IOS_STATS_TYPE_READ:
+            list_head = &conf->list[IOS_STATS_TYPE_READ];
+            break;
+        case IOS_STATS_TYPE_WRITE:
+            list_head = &conf->list[IOS_STATS_TYPE_WRITE];
+            break;
+        case IOS_STATS_TYPE_OPENDIR:
+            list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
+            break;
+        case IOS_STATS_TYPE_READDIRP:
+            list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
+            break;
+        case IOS_STATS_TYPE_READ_THROUGHPUT:
+            list_head = &conf->thru_list[IOS_STATS_THRU_READ];
+            index = IOS_STATS_THRU_READ;
+            break;
+        case IOS_STATS_TYPE_WRITE_THROUGHPUT:
+            list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
+            index = IOS_STATS_THRU_WRITE;
+            break;
 
-                }
+        default:
+            goto out;
+    }
+    ret = dict_set_int32_sizen(resp, "top-op", flags);
+    if (!list_cnt)
+        goto out;
+    LOCK(&list_head->lock);
+    {
+        list_for_each_entry(entry, &list_head->iosstats->list, list)
+        {
+            cnt++;
+            keylen = snprintf(key, sizeof(key), "filename-%d", cnt);
+            ret = dict_set_strn(resp, key, keylen, entry->iosstat->filename);
+            if (ret)
+                goto unlock_list_head;
+            snprintf(key, sizeof(key), "value-%d", cnt);
+            ret = dict_set_uint64(resp, key, entry->value);
+            if (ret)
+                goto unlock_list_head;
+            if (index != IOS_STATS_THRU_MAX) {
+                keylen = snprintf(key, sizeof(key), "time-sec-%d", cnt);
+                ret = dict_set_int32n(
+                    resp, key, keylen,
+                    entry->iosstat->thru_counters[index].time.tv_sec);
+                if (ret)
+                    goto unlock_list_head;
+                keylen = snprintf(key, sizeof(key), "time-usec-%d", cnt);
+                ret = dict_set_int32n(
+                    resp, key, keylen,
+                    entry->iosstat->thru_counters[index].time.tv_usec);
+                if (ret)
+                    goto unlock_list_head;
+            }
+            if (cnt == list_cnt)
+                break;
         }
+    }
 unlock_list_head:
-        UNLOCK (&list_head->lock);
-        /* ret is !=0 if some dict operation in the above critical region
-         * failed. */
-        if (ret)
-                goto out;
-        ret = dict_set_int32 (resp, "members", cnt);
- out:
-        return ret;
+    UNLOCK(&list_head->lock);
+    /* ret is !=0 if some dict operation in the above critical region
+     * failed. */
+    if (ret)
+        goto out;
+    ret = dict_set_int32_sizen(resp, "members", cnt);
+out:
+    return ret;
 }
 
-static struct ios_stat*
-ios_init_iosstat (xlator_t *this, char *path, uuid_t gfid, inode_t *inode)
+static struct ios_stat *
+ios_init_iosstat(xlator_t *this, char *path, uuid_t gfid, inode_t *inode)
 {
-        struct ios_stat *iosstat = NULL;
-        int              i       = 0;
+    struct ios_stat *iosstat = NULL;
+    int i = 0;
 
-        iosstat = GF_CALLOC (1, sizeof (*iosstat), gf_io_stats_mt_ios_stat);
-        if (!iosstat)
-                goto out;
+    iosstat = GF_CALLOC(1, sizeof(*iosstat), gf_io_stats_mt_ios_stat);
+    if (!iosstat)
+        goto out;
 
-        iosstat->filename = gf_strdup (path);
-        gf_uuid_copy (iosstat->gfid, gfid);
-        LOCK_INIT (&iosstat->lock);
+    iosstat->filename = gf_strdup(path);
+    gf_uuid_copy(iosstat->gfid, gfid);
+    LOCK_INIT(&iosstat->lock);
 
-        for (i = 0; i < IOS_STATS_TYPE_MAX; i++)
-                GF_ATOMIC_INIT (iosstat->counters[i], 0);
+    for (i = 0; i < IOS_STATS_TYPE_MAX; i++)
+        GF_ATOMIC_INIT(iosstat->counters[i], 0);
 
-        ios_inode_ctx_set (inode, this, iosstat);
+    ios_inode_ctx_set(inode, this, iosstat);
 
 out:
-        return iosstat;
+    return iosstat;
 }
 
 int
-io_stats_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd,
-                     inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        struct ios_fd *iosfd = NULL;
-        char          *path = NULL;
-        struct ios_stat *iosstat = NULL;
-        struct ios_conf   *conf = NULL;
-
-        conf = this->private;
-
-        path = frame->local;
-        frame->local = NULL;
-
-        if (!path)
-                goto unwind;
-
-        if (op_ret < 0) {
-                GF_FREE (path);
-                goto unwind;
+io_stats_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    struct ios_fd *iosfd = NULL;
+    char *path = NULL;
+    struct ios_stat *iosstat = NULL;
+    struct ios_conf *conf = NULL;
+
+    conf = this->private;
+
+    path = frame->local;
+    frame->local = NULL;
+
+    if (!path)
+        goto unwind;
+
+    if (op_ret < 0) {
+        GF_FREE(path);
+        goto unwind;
+    }
+
+    iosfd = GF_CALLOC(1, sizeof(*iosfd), gf_io_stats_mt_ios_fd);
+    if (!iosfd) {
+        GF_FREE(path);
+        goto unwind;
+    }
+
+    iosfd->filename = path;
+    gettimeofday(&iosfd->opened_at, NULL);
+
+    ios_fd_ctx_set(fd, this, iosfd);
+    LOCK(&conf->lock);
+    {
+        conf->cumulative.nr_opens++;
+        if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
+            conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
+            conf->cumulative.max_openfd_time = iosfd->opened_at;
         }
+    }
+    UNLOCK(&conf->lock);
 
-        iosfd = GF_CALLOC (1, sizeof (*iosfd), gf_io_stats_mt_ios_fd);
-        if (!iosfd) {
-                GF_FREE (path);
-                goto unwind;
-        }
-
-        iosfd->filename = path;
-        gettimeofday (&iosfd->opened_at, NULL);
-
-        ios_fd_ctx_set (fd, this, iosfd);
-        LOCK (&conf->lock);
-        {
-                conf->cumulative.nr_opens++;
-                if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
-                        conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
-                        conf->cumulative.max_openfd_time = iosfd->opened_at;
-                }
-        }
-        UNLOCK (&conf->lock);
-
-        iosstat = ios_init_iosstat (this, path, buf->ia_gfid, inode);
-        if (!iosstat)
-                GF_FREE (path);
+    iosstat = ios_init_iosstat(this, path, buf->ia_gfid, inode);
+    if (!iosstat)
+        GF_FREE(path);
 
 unwind:
-        UPDATE_PROFILE_STATS (frame, CREATE);
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, CREATE);
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                        preparent, postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        struct ios_fd     *iosfd = NULL;
-        char              *path = NULL;
-        struct ios_stat   *iosstat = NULL;
-        struct ios_conf   *conf = NULL;
-        int                i = 0;
-
-        conf = this->private;
-        path = frame->local;
-        frame->local = NULL;
-
-        if (!path)
-                goto unwind;
-
-        if (op_ret < 0) {
-                GF_FREE (path);
-                goto unwind;
-        }
-
-        iosfd = GF_CALLOC (1, sizeof (*iosfd), gf_io_stats_mt_ios_fd);
-        if (!iosfd) {
-                GF_FREE (path);
-                goto unwind;
-        }
-
-        iosfd->filename = path;
-        GF_ATOMIC_INIT (iosfd->data_read, 0);
-        GF_ATOMIC_INIT (iosfd->data_written, 0);
-        for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
-                GF_ATOMIC_INIT (iosfd->block_count_write[i], 0);
-                GF_ATOMIC_INIT (iosfd->block_count_read[i], 0);
-        }
-        gettimeofday (&iosfd->opened_at, NULL);
-
-        ios_fd_ctx_set (fd, this, iosfd);
-
-        ios_inode_ctx_get (fd->inode, this, &iosstat);
-        if (!iosstat) {
-                iosstat = ios_init_iosstat (this, path, fd->inode->gfid,
-                                            fd->inode);
-        }
-
-        LOCK (&conf->lock);
-        {
-                conf->cumulative.nr_opens++;
-                if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
-                        conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
-                        conf->cumulative.max_openfd_time = iosfd->opened_at;
-                }
-        }
-        UNLOCK (&conf->lock);
-        if (iosstat) {
-                ios_bump_stats (this, iosstat, IOS_STATS_TYPE_OPEN);
-                iosstat = NULL;
+io_stats_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    struct ios_fd *iosfd = NULL;
+    char *path = NULL;
+    struct ios_stat *iosstat = NULL;
+    struct ios_conf *conf = NULL;
+    int i = 0;
+
+    conf = this->private;
+    path = frame->local;
+    frame->local = NULL;
+
+    if (!path)
+        goto unwind;
+
+    if (op_ret < 0) {
+        GF_FREE(path);
+        goto unwind;
+    }
+
+    iosfd = GF_CALLOC(1, sizeof(*iosfd), gf_io_stats_mt_ios_fd);
+    if (!iosfd) {
+        GF_FREE(path);
+        goto unwind;
+    }
+
+    iosfd->filename = path;
+    GF_ATOMIC_INIT(iosfd->data_read, 0);
+    GF_ATOMIC_INIT(iosfd->data_written, 0);
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        GF_ATOMIC_INIT(iosfd->block_count_write[i], 0);
+        GF_ATOMIC_INIT(iosfd->block_count_read[i], 0);
+    }
+    gettimeofday(&iosfd->opened_at, NULL);
+
+    ios_fd_ctx_set(fd, this, iosfd);
+
+    ios_inode_ctx_get(fd->inode, this, &iosstat);
+    if (!iosstat) {
+        iosstat = ios_init_iosstat(this, path, fd->inode->gfid, fd->inode);
+    }
+
+    LOCK(&conf->lock);
+    {
+        conf->cumulative.nr_opens++;
+        if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
+            conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
+            conf->cumulative.max_openfd_time = iosfd->opened_at;
         }
+    }
+    UNLOCK(&conf->lock);
+    if (iosstat) {
+        ios_bump_stats(this, iosstat, IOS_STATS_TYPE_OPEN);
+        iosstat = NULL;
+    }
 unwind:
-        UPDATE_PROFILE_STATS (frame, OPEN);
-
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, OPEN);
 
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
-
 int
-io_stats_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+io_stats_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, STAT);
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, STAT);
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iovec *vector, int32_t count,
-                    struct iatt *buf, struct iobref *iobref, dict_t *xdata)
+io_stats_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                   int32_t count, struct iatt *buf, struct iobref *iobref,
+                   dict_t *xdata)
 {
-        int              len = 0;
-        fd_t            *fd = NULL;
-        struct ios_stat *iosstat = NULL;
-
-        fd = frame->local;
-        frame->local = NULL;
+    int len = 0;
+    fd_t *fd = NULL;
+    struct ios_stat *iosstat = NULL;
 
-        if (op_ret > 0) {
-                len = iov_length (vector, count);
-                ios_bump_read (this, fd, len);
-        }
+    fd = frame->local;
+    frame->local = NULL;
 
-        UPDATE_PROFILE_STATS (frame, READ);
-        ios_inode_ctx_get (fd->inode, this, &iosstat);
+    if (op_ret > 0) {
+        len = iov_length(vector, count);
+        ios_bump_read(this, fd, len);
+    }
 
-        if (iosstat) {
-                ios_bump_stats (this, iosstat, IOS_STATS_TYPE_READ);
-                BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_READ);
-                iosstat = NULL;
-        }
+    UPDATE_PROFILE_STATS(frame, READ);
+    ios_inode_ctx_get(fd->inode, this, &iosstat);
 
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, buf, iobref, xdata);
-        return 0;
+    if (iosstat) {
+        ios_bump_stats(this, iosstat, IOS_STATS_TYPE_READ);
+        BUMP_THROUGHPUT(iosstat, IOS_STATS_THRU_READ);
+        iosstat = NULL;
+    }
 
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, vector, count, buf,
+                        iobref, xdata);
+    return 0;
 }
 
-
 int
-io_stats_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
-{
-        struct ios_stat *iosstat = NULL;
-        inode_t         *inode   = NULL;
-
-        UPDATE_PROFILE_STATS (frame, WRITE);
-        if (frame->local){
-                inode = frame->local;
-                frame->local = NULL;
-                ios_inode_ctx_get (inode, this, &iosstat);
-                if (iosstat) {
-                        ios_bump_stats (this, iosstat, IOS_STATS_TYPE_WRITE);
-                        BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_WRITE);
-                        inode = NULL;
-                        iosstat = NULL;
-                }
-        }
+io_stats_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
+{
+    struct ios_stat *iosstat = NULL;
+    inode_t *inode = NULL;
 
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, WRITE);
+    if (frame->local) {
+        inode = frame->local;
+        frame->local = NULL;
+        ios_inode_ctx_get(inode, this, &iosstat);
+        if (iosstat) {
+            ios_bump_stats(this, iosstat, IOS_STATS_TYPE_WRITE);
+            BUMP_THROUGHPUT(iosstat, IOS_STATS_THRU_WRITE);
+            inode = NULL;
+            iosstat = NULL;
+        }
+    }
 
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
+int
+io_stats_copy_file_range_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             struct iatt *stbuf, struct iatt *prebuf_dst,
+                             struct iatt *postbuf_dst, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, COPY_FILE_RANGE);
 
-
+    STACK_UNWIND_STRICT(copy_file_range, frame, op_ret, op_errno, stbuf,
+                        prebuf_dst, postbuf_dst, xdata);
+    return 0;
+}
 
 int
-io_stats_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
+io_stats_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                      dict_t *xdata)
 {
-        struct ios_stat *iosstat = NULL;
-        inode_t         *inode   = frame->local;
+    struct ios_stat *iosstat = NULL;
+    inode_t *inode = frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        UPDATE_PROFILE_STATS (frame, READDIRP);
+    UPDATE_PROFILE_STATS(frame, READDIRP);
 
-        ios_inode_ctx_get (inode, this, &iosstat);
+    ios_inode_ctx_get(inode, this, &iosstat);
 
-        if (iosstat) {
-                ios_bump_stats (this, iosstat, IOS_STATS_TYPE_READDIRP);
-                iosstat = NULL;
-        }
+    if (iosstat) {
+        ios_bump_stats(this, iosstat, IOS_STATS_TYPE_READDIRP);
+        iosstat = NULL;
+    }
 
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
+io_stats_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                     dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, READDIR);
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, READDIR);
+    STACK_UNWIND_STRICT(readdir, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+io_stats_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FSYNC);
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FSYNC);
+    STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
+io_stats_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *preop,
+                     struct iatt *postop, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, SETATTR);
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, SETATTR);
+    STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, preop, postop, xdata);
+    return 0;
 }
 
-
 int
-io_stats_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+io_stats_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, UNLINK);
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
-        return 0;
-
+    UPDATE_PROFILE_STATS(frame, UNLINK);
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
-
 int
-io_stats_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                     struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent, dict_t *xdata)
-{
-        UPDATE_PROFILE_STATS (frame, RENAME);
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
-                             preoldparent, postoldparent,
-                             prenewparent, postnewparent, xdata);
-        return 0;
+io_stats_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, RENAME);
+    STACK_UNWIND_STRICT(rename, frame, op_ret, op_errno, buf, preoldparent,
+                        postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, const char *buf,
-                       struct iatt *sbuf, dict_t *xdata)
+io_stats_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, const char *buf,
+                      struct iatt *sbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, READLINK);
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, READLINK);
+    STACK_UNWIND_STRICT(readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     inode_t *inode, struct iatt *buf,
-                     dict_t *xdata, struct iatt *postparent)
-{
-        UPDATE_PROFILE_STATS (frame, LOOKUP);
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xdata,
-                             postparent);
-        return 0;
+io_stats_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+    UPDATE_PROFILE_STATS(frame, LOOKUP);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    return 0;
 }
 
-
 int
-io_stats_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        UPDATE_PROFILE_STATS (frame, SYMLINK);
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+io_stats_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, SYMLINK);
+    STACK_UNWIND_STRICT(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        UPDATE_PROFILE_STATS (frame, MKNOD);
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+io_stats_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, MKNOD);
+    STACK_UNWIND_STRICT(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent,
-                    dict_t *xdata)
+io_stats_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        char   *path = frame->local;
+    char *path = frame->local;
 
-        if (!path)
-                goto unwind;
+    if (!path)
+        goto unwind;
 
-        UPDATE_PROFILE_STATS (frame, MKDIR);
-        if (op_ret < 0)
-                goto unwind;
+    UPDATE_PROFILE_STATS(frame, MKDIR);
+    if (op_ret < 0)
+        goto unwind;
 
-        /* allocate a struct ios_stat and set the inode ctx */
-        ios_init_iosstat (this, path, buf->ia_gfid, inode);
+    /* allocate a struct ios_stat and set the inode ctx */
+    ios_init_iosstat(this, path, buf->ia_gfid, inode);
 
 unwind:
-        /* local is assigned with path */
-        GF_FREE (frame->local);
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+    /* local is assigned with path */
+    GF_FREE(frame->local);
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        UPDATE_PROFILE_STATS (frame, LINK);
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+io_stats_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, LINK);
+    STACK_UNWIND_STRICT(link, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FLUSH);
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FLUSH);
+    STACK_UNWIND_STRICT(flush, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+io_stats_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        struct ios_stat *iosstat = NULL;
-        int              ret     = -1;
+    struct ios_stat *iosstat = NULL;
+    int ret = -1;
 
-        UPDATE_PROFILE_STATS (frame, OPENDIR);
-        if (op_ret < 0)
-                goto unwind;
+    UPDATE_PROFILE_STATS(frame, OPENDIR);
+    if (op_ret < 0)
+        goto unwind;
 
-        ios_fd_ctx_set (fd, this, 0);
+    ios_fd_ctx_set(fd, this, 0);
 
-        ret = ios_inode_ctx_get (fd->inode, this, &iosstat);
-        if (!ret)
-                ios_bump_stats (this, iosstat, IOS_STATS_TYPE_OPENDIR);
+    ret = ios_inode_ctx_get(fd->inode, this, &iosstat);
+    if (!ret)
+        ios_bump_stats(this, iosstat, IOS_STATS_TYPE_OPENDIR);
 
 unwind:
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
-
 int
-io_stats_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+io_stats_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
+    UPDATE_PROFILE_STATS(frame, RMDIR);
 
-        UPDATE_PROFILE_STATS (frame, RMDIR);
-
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
-
 int
-io_stats_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+io_stats_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, TRUNCATE);
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, TRUNCATE);
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 int
-io_stats_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+io_stats_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                    dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, STATFS);
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, STATFS);
+    STACK_UNWIND_STRICT(statfs, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, SETXATTR);
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, SETXATTR);
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+io_stats_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, GETXATTR);
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, GETXATTR);
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int
-io_stats_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, REMOVEXATTR);
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, REMOVEXATTR);
+    STACK_UNWIND_STRICT(removexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-io_stats_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FSETXATTR);
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FSETXATTR);
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+io_stats_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FGETXATTR);
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FGETXATTR);
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FREMOVEXATTR);
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FREMOVEXATTR);
+    STACK_UNWIND_STRICT(fremovexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FSYNCDIR);
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FSYNCDIR);
+    STACK_UNWIND_STRICT(fsyncdir, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, ACCESS);
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, ACCESS);
+    STACK_UNWIND_STRICT(access, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno,
-                        struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+io_stats_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FTRUNCATE);
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FTRUNCATE);
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 int
-io_stats_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+io_stats_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FSTAT);
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FSTAT);
+    STACK_UNWIND_STRICT(fstat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
 io_stats_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-		       struct iatt *postbuf, dict_t *xdata)
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-	UPDATE_PROFILE_STATS(frame, FALLOCATE);
-	STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
-			    xdata);
-	return 0;
+    UPDATE_PROFILE_STATS(frame, FALLOCATE);
+    STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 int
 io_stats_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-		     struct iatt *postbuf, dict_t *xdata)
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-	UPDATE_PROFILE_STATS(frame, DISCARD);
-	STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, prebuf, postbuf,
-			    xdata);
-	return 0;
+    UPDATE_PROFILE_STATS(frame, DISCARD);
+    STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int
 io_stats_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS(frame, ZEROFILL);
-        STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, ZEROFILL);
+    STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int32_t
-io_stats_ipc_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS(frame, IPC);
-        STACK_UNWIND_STRICT (ipc, frame, op_ret, op_errno,
-                             xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, IPC);
+    STACK_UNWIND_STRICT(ipc, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-io_stats_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+io_stats_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, LK);
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, LK);
+    STACK_UNWIND_STRICT(lk, frame, op_ret, op_errno, lock, xdata);
+    return 0;
 }
 
-
 int
-io_stats_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, ENTRYLK);
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, ENTRYLK);
+    STACK_UNWIND_STRICT(entrylk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-io_stats_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FENTRYLK);
-        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FENTRYLK);
+    STACK_UNWIND_STRICT(fentrylk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-io_stats_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+io_stats_rchecksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
                        uint8_t *strong_checksum, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, RCHECKSUM);
-        STACK_UNWIND_STRICT (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                            strong_checksum, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, RCHECKSUM);
+    STACK_UNWIND_STRICT(rchecksum, frame, op_ret, op_errno, weak_checksum,
+                        strong_checksum, xdata);
+    return 0;
 }
 
 int
-io_stats_seek_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, off_t offset,
-                   dict_t *xdata)
+io_stats_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, off_t offset, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, SEEK);
-        STACK_UNWIND_STRICT (seek, frame, op_ret, op_errno, offset, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, SEEK);
+    STACK_UNWIND_STRICT(seek, frame, op_ret, op_errno, offset, xdata);
+    return 0;
 }
 
 int
-io_stats_lease_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   struct gf_lease *lease, dict_t *xdata)
+io_stats_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct gf_lease *lease,
+                   dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, LEASE);
-        STACK_UNWIND_STRICT (lease, frame, op_ret, op_errno, lease, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, LEASE);
+    STACK_UNWIND_STRICT(lease, frame, op_ret, op_errno, lease, xdata);
+    return 0;
 }
 
 int
-io_stats_getactivelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          lock_migration_info_t *locklist,
-                          dict_t *xdata)
-{
-        UPDATE_PROFILE_STATS (frame, GETACTIVELK);
-        STACK_UNWIND_STRICT (getactivelk, frame, op_ret, op_errno,
-                             locklist, xdata);
-        return 0;
+io_stats_getactivelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno,
+                         lock_migration_info_t *locklist, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, GETACTIVELK);
+    STACK_UNWIND_STRICT(getactivelk, frame, op_ret, op_errno, locklist, xdata);
+    return 0;
 }
 
 int
-io_stats_setactivelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_setactivelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, SETACTIVELK);
-        STACK_UNWIND_STRICT (setactivelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, SETACTIVELK);
+    STACK_UNWIND_STRICT(setactivelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-io_stats_compound_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, void *data,
-                     dict_t *xdata)
+io_stats_compound_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, void *data,
+                      dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, COMPOUND);
-        STACK_UNWIND_STRICT (compound, frame, op_ret, op_errno, data, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, COMPOUND);
+    STACK_UNWIND_STRICT(compound, frame, op_ret, op_errno, data, xdata);
+    return 0;
 }
 
 int
-io_stats_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+io_stats_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, XATTROP);
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, XATTROP);
+    STACK_UNWIND_STRICT(xattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+io_stats_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FXATTROP);
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FXATTROP);
+    STACK_UNWIND_STRICT(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int
-io_stats_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, INODELK);
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, INODELK);
+    STACK_UNWIND_STRICT(inodelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-io_stats_entrylk (call_frame_t *frame, xlator_t *this,
-                  const char *volume, loc_t *loc, const char *basename,
-                  entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+io_stats_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 loc_t *loc, const char *basename, entrylk_cmd cmd,
+                 entrylk_type type, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_entrylk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->entrylk,
-                    volume, loc, basename, cmd, type, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, volume, loc, basename, cmd,
+               type, xdata);
+    return 0;
 }
 
 int
-io_stats_fentrylk (call_frame_t *frame, xlator_t *this,
-		    const char *volume, fd_t *fd, const char *basename,
-		    entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+io_stats_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  fd_t *fd, const char *basename, entrylk_cmd cmd,
+                  entrylk_type type, dict_t *xdata)
 {
-       START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-       STACK_WIND (frame, io_stats_fentrylk_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fentrylk,
-		    volume, fd, basename, cmd, type, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fentrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fentrylk, volume, fd, basename, cmd,
+               type, xdata);
+    return 0;
 }
 
-
 int
-io_stats_inodelk (call_frame_t *frame, xlator_t *this,
-                  const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+io_stats_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
+    START_FOP_LATENCY(frame);
 
-        START_FOP_LATENCY (frame);
-
-        STACK_WIND (frame, io_stats_inodelk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->inodelk,
-                    volume, loc, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, volume, loc, cmd, flock,
+               xdata);
+    return 0;
 }
 
-
 int
-io_stats_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-
-        UPDATE_PROFILE_STATS (frame, FINODELK);
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FINODELK);
+    STACK_UNWIND_STRICT(finodelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                   fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+io_stats_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_finodelk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->finodelk,
-                    volume, fd, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_finodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->finodelk, volume, fd, cmd, flock,
+               xdata);
+    return 0;
 }
 
-
 int
-io_stats_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+io_stats_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_xattrop_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop,
-                    loc, flags, dict, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+io_stats_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fxattrop_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fxattrop,
-                    fd, flags, dict, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
+    return 0;
 }
 
-
 int
-io_stats_lookup (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, dict_t *xdata)
+io_stats_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_lookup_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup,
-                    loc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+io_stats_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_stat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat,
-                    loc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_readlink (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, size_t size, dict_t *xdata)
+io_stats_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                  dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_readlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink,
-                    loc, size, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
+    return 0;
 }
 
-
 int
-io_stats_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+io_stats_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+               dev_t dev, mode_t umask, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_mknod_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, dev, umask, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, dev, umask, xdata);
+    return 0;
 }
 
-
 int
-io_stats_mkdir (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+io_stats_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+               mode_t umask, dict_t *xdata)
 {
-        if (loc->path)
-                frame->local = gf_strdup (loc->path);
+    if (loc->path)
+        frame->local = gf_strdup(loc->path);
 
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_mkdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, umask, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    return 0;
 }
 
-
 int
-io_stats_unlink (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int xflag, dict_t *xdata)
+io_stats_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_unlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink,
-                    loc, xflag, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
 }
 
-
 int
-io_stats_rmdir (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int flags, dict_t *xdata)
+io_stats_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+               dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_rmdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir,
-                    loc, flags, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+    return 0;
 }
 
-
 int
-io_stats_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                  loc_t *loc, mode_t umask, dict_t *xdata)
+io_stats_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+                 loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_symlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, umask, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask, xdata);
+    return 0;
 }
 
-
 int
-io_stats_rename (call_frame_t *frame, xlator_t *this,
-                 loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+io_stats_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_rename_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_link (call_frame_t *frame, xlator_t *this,
-               loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+io_stats_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_link_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_setattr (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, struct iatt *stbuf, int32_t valid, dict_t *xdata)
+io_stats_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_setattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int
-io_stats_truncate (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, off_t offset, dict_t *xdata)
+io_stats_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                  dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_truncate_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
-
 int
-io_stats_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int32_t flags, fd_t *fd, dict_t *xdata)
+io_stats_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              fd_t *fd, dict_t *xdata)
 {
-        if (loc->path)
-                frame->local = gf_strdup (loc->path);
+    if (loc->path)
+        frame->local = gf_strdup(loc->path);
 
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_open_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
-
 int
-io_stats_create (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int32_t flags, mode_t mode,
-                 mode_t umask, fd_t *fd, dict_t *xdata)
+io_stats_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        if (loc->path)
-                frame->local = gf_strdup (loc->path);
+    if (loc->path)
+        frame->local = gf_strdup(loc->path);
 
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_create_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 }
 
-
 int
-io_stats_readv (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+io_stats_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, uint32_t flags, dict_t *xdata)
 {
-        frame->local = fd;
+    frame->local = fd;
 
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_readv_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
 }
 
-
 int
-io_stats_writev (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, struct iovec *vector,
-                 int32_t count, off_t offset,
-                 uint32_t flags, struct iobref *iobref, dict_t *xdata)
+io_stats_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                struct iovec *vector, int32_t count, off_t offset,
+                uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        int                 len = 0;
+    int len = 0;
 
-        if (fd->inode)
-                frame->local = fd->inode;
-        len = iov_length (vector, count);
-
-        ios_bump_write (this, fd, len);
-        START_FOP_LATENCY (frame);
+    if (fd->inode)
+        frame->local = fd->inode;
+    len = iov_length(vector, count);
 
-        STACK_WIND (frame, io_stats_writev_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref, xdata);
-        return 0;
+    ios_bump_write(this, fd, len);
+    START_FOP_LATENCY(frame);
 
+    STACK_WIND(frame, io_stats_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 }
 
-
 int
-io_stats_statfs (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, dict_t *xdata)
+io_stats_copy_file_range(call_frame_t *frame, xlator_t *this, fd_t *fd_in,
+                         off_t off_in, fd_t *fd_out, off_t off_out, size_t len,
+                         uint32_t flags, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_statfs_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->statfs,
-                    loc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_copy_file_range_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->copy_file_range, fd_in, off_in, fd_out,
+               off_out, len, flags, xdata);
+    return 0;
 }
 
-
 int
-io_stats_flush (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, dict_t *xdata)
+io_stats_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_flush_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush,
-                    fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsync (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, int32_t flags, dict_t *xdata)
+io_stats_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fsync_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync,
-                    fd, flags, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
 }
 
-
 int
-conditional_dump (dict_t *dict, char *key, data_t *value, void *data)
-{
-        struct {
-                xlator_t       *this;
-                inode_t        *inode;
-                const char     *path;
-        } *stub;
-        xlator_t             *this = NULL;
-        char                 *filename = NULL;
-        FILE                 *logfp = NULL;
-        struct ios_dump_args args = {0};
-        int                   pid, namelen;
-        char                  dump_key[100];
-        char                 *slash_ptr = NULL;
-
-        stub  = data;
-        this  = stub->this;
-
-        /* Create a file name that is appended with the io-stats instance
-        name as well. This helps when there is more than a single io-stats
-        instance in the graph, or the client and server processes are running
-        on the same node */
-        /* hmmm... no check for this */
-        /* name format: <passed in path/filename>.<xlator name slashes to -> */
-        namelen = value->len + strlen (this->name) + 2; /* '.' and '\0' */
-        filename = alloca0 (namelen);
-        memcpy (filename, data_to_str (value), value->len);
-        memcpy (filename + value->len, ".", 1);
-        memcpy (filename + value->len + 1, this->name, strlen(this->name));
-
-        /* convert any slashes to '-' so that fopen works correctly */
-        slash_ptr = strchr (filename + value->len + 1, '/');
-        while (slash_ptr) {
-                *slash_ptr = '-';
-                slash_ptr = strchr (slash_ptr, '/');
-        }
+io_stats_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+               dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
 
-        pid = getpid ();
+    STACK_WIND(frame, io_stats_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
+}
 
-        if (!strncmp (filename, "", 1)) {
-                gf_log (this->name, GF_LOG_ERROR, "No filename given");
-                return -1;
-        }
-        logfp = fopen (filename, "w+");
-        if (!logfp) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to open %s "
-                                "for writing", filename);
-                return -1;
-        }
-        sprintf (dump_key, "*io*stat*%d_json_dump", pid);
-        if (fnmatch (dump_key, key, 0) == 0) {
-                (void) ios_dump_args_init (
-                                &args, IOS_DUMP_TYPE_JSON_FILE,
-                                logfp);
-        } else {
-                (void) ios_dump_args_init (&args, IOS_DUMP_TYPE_FILE,
-                                logfp);
-        }
-        io_stats_dump (this, &args, GF_CLI_INFO_ALL, _gf_false);
-        fclose (logfp);
-        return 0;
+int
+conditional_dump(dict_t *dict, char *key, data_t *value, void *data)
+{
+    struct {
+        xlator_t *this;
+        inode_t *inode;
+        const char *path;
+    } * stub;
+    xlator_t *this = NULL;
+    char *filename = NULL;
+    FILE *logfp = NULL;
+    struct ios_dump_args args = {0};
+    int pid, namelen, dirlen;
+    char dump_key[100];
+    char *slash_ptr = NULL;
+    char *path_in_value = NULL;
+    char *identifier = NULL;
+    struct ios_conf *conf = NULL;
+
+    stub = data;
+    this = stub->this;
+    conf = this->private;
+
+    /* Don't do this on 'brick-side', only do this on client side */
+    /* Addresses CVE-2018-14659 */
+    if (this->ctx->process_mode != GF_CLIENT_PROCESS) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "taking io-stats dump using setxattr not permitted on brick."
+               " Use 'gluster profile' instead");
+        return -1;
+    }
+
+    /* Create a file name that is appended with the io-stats instance
+    name as well. This helps when there is more than a single io-stats
+    instance in the graph, or the client and server processes are running
+    on the same node */
+    /* For the sanity of where the file should be located, we should make
+       sure file is written only inside RUNDIR (ie, /var/run/gluster) */
+    /* TODO: provide an option to dump it to different directory of
+       choice, based on options */
+    /* name format: /var/run/gluster/<passed in path/filename>.<xlator name
+     * slashes to -> */
+
+    path_in_value = alloca0(value->len + 1);
+
+    /* We need a memcpy here because of the way dict_unserialize works */
+
+    memcpy(path_in_value, data_to_str(value), value->len);
+    path_in_value[value->len] = '\0';
+
+    if (strstr(path_in_value, "../")) {
+        gf_log(this->name, GF_LOG_ERROR, "%s: no \"../\" allowed in path",
+               path_in_value);
+        return -1;
+    }
+
+    if (path_in_value[0] == '/') {
+        path_in_value = path_in_value + 1;
+    }
+
+    dirlen = strlen(IOS_STATS_DUMP_DIR);
+    if (conf->unique_id) {
+        /* this->name will be the same for all bricks of the volume */
+        identifier = conf->unique_id;
+    } else {
+        identifier = this->name;
+    }
+
+    namelen = (dirlen + value->len + strlen(identifier) + 3);
+    /* +3 for '/', '.' and '\0' added in snprintf below*/
+
+    filename = alloca0(namelen);
+    snprintf(filename, namelen, "%s/%s.%s", IOS_STATS_DUMP_DIR, path_in_value,
+             identifier);
+
+    /* convert any slashes to '-' so that fopen works correctly */
+    slash_ptr = strchr(filename + dirlen + 1, '/');
+    while (slash_ptr) {
+        *slash_ptr = '-';
+        slash_ptr = strchr(slash_ptr, '/');
+    }
+
+    pid = getpid();
+
+    if (!strncmp(filename, "", 1)) {
+        gf_log(this->name, GF_LOG_ERROR, "No filename given");
+        return -1;
+    }
+    logfp = fopen(filename, "w+");
+    if (!logfp) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to open %s "
+               "for writing",
+               filename);
+        return -1;
+    }
+    sprintf(dump_key, "*io*stat*%d_json_dump", pid);
+    if (fnmatch(dump_key, key, 0) == 0) {
+        (void)ios_dump_args_init(&args, IOS_DUMP_TYPE_JSON_FILE, logfp);
+    } else {
+        (void)ios_dump_args_init(&args, IOS_DUMP_TYPE_FILE, logfp);
+    }
+    io_stats_dump(this, &args, GF_IOS_INFO_ALL, _gf_false);
+    fclose(logfp);
+    return 0;
 }
 
 int
-_ios_destroy_dump_thread (struct ios_conf *conf) {
-        conf->dump_thread_should_die = _gf_true;
-        if (conf->ios_dump_interval > 0) {
-                (void) pthread_cancel (conf->dump_thread);
-                (void) pthread_join (conf->dump_thread, NULL);
-        }
-        return 0;
+_ios_destroy_dump_thread(struct ios_conf *conf)
+{
+    conf->dump_thread_should_die = _gf_true;
+    if (conf->dump_thread_running) {
+        (void)pthread_cancel(conf->dump_thread);
+        (void)pthread_join(conf->dump_thread, NULL);
+    }
+    return 0;
 }
 
 void *
-_ios_dump_thread (xlator_t *this) {
-        struct ios_conf         *conf = NULL;
-        FILE                    *stats_logfp = NULL;
-        FILE                    *samples_logfp = NULL;
-        struct ios_dump_args args = {0};
-        int                     i;
-        int                     stats_bytes_written = 0;
-        int                     samples_bytes_written = 0;
-        char                    stats_filename[PATH_MAX];
-        char                    samples_filename[PATH_MAX];
-        char                    *xlator_name;
-        char                    *instance_name;
-        gf_boolean_t            log_stats_fopen_failure = _gf_true;
-        gf_boolean_t            log_samples_fopen_failure = _gf_true;
-        int                     old_cancel_type;
-
-        conf = this->private;
-        gf_log (this->name, GF_LOG_INFO, "IO stats dump thread started, "
-                "polling IO stats every %d seconds", conf->ios_dump_interval);
-        xlator_name = strdupa (conf->unique_id);
-        for (i = 0; i < strlen (xlator_name); i++) {
-                if (xlator_name[i] == '/')
-                        xlator_name[i] = '_';
-        }
-        instance_name = this->instance_name;
-        if (this->name && strcmp (this->name, "glustershd") == 0) {
-                xlator_name = "shd";
-        } else if (this->prev &&
-                   strcmp (this->prev->name, "nfs-server") == 0) {
-                xlator_name = "nfsd";
-                instance_name = this->prev->instance_name;
+_ios_dump_thread(xlator_t *this)
+{
+    struct ios_conf *conf = NULL;
+    FILE *stats_logfp = NULL;
+    FILE *samples_logfp = NULL;
+    struct ios_dump_args args = {0};
+    int i;
+    int stats_bytes_written = 0;
+    int samples_bytes_written = 0;
+    char stats_filename[PATH_MAX];
+    char samples_filename[PATH_MAX];
+    char *xlator_name;
+    char *instance_name;
+    gf_boolean_t log_stats_fopen_failure = _gf_true;
+    gf_boolean_t log_samples_fopen_failure = _gf_true;
+    int old_cancel_type;
+
+    conf = this->private;
+    gf_log(this->name, GF_LOG_INFO,
+           "IO stats dump thread started, "
+           "polling IO stats every %d seconds",
+           conf->ios_dump_interval);
+    xlator_name = strdupa(conf->unique_id);
+    for (i = 0; i < strlen(xlator_name); i++) {
+        if (xlator_name[i] == '/')
+            xlator_name[i] = '_';
+    }
+    instance_name = this->instance_name;
+    if (this->name && strcmp(this->name, "glustershd") == 0) {
+        xlator_name = "shd";
+    } else if (this->prev && strcmp(this->prev->name, "nfs-server") == 0) {
+        xlator_name = "nfsd";
+        instance_name = this->prev->instance_name;
+    }
+    if (sys_mkdir(_IOS_DUMP_DIR, S_IRWXU | S_IRWXO | S_IRWXG) == (-1)) {
+        if (errno != EEXIST) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "could not create stats-dump directory %s", _IOS_DUMP_DIR);
+            goto out;
         }
-        if (sys_mkdir (_IOS_DUMP_DIR, S_IRWXU | S_IRWXO | S_IRWXG) == (-1)) {
-                if (errno != EEXIST) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not create stats-dump directory %s",
-                                _IOS_DUMP_DIR);
-                        goto out;
-                }
-        }
-        if (sys_mkdir (_IOS_SAMP_DIR, S_IRWXU | S_IRWXO | S_IRWXG) == (-1)) {
-                if (errno != EEXIST) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not create stats-sample directory %s",
-                                _IOS_SAMP_DIR);
-                        goto out;
-                }
+    }
+    if (sys_mkdir(_IOS_SAMP_DIR, S_IRWXU | S_IRWXO | S_IRWXG) == (-1)) {
+        if (errno != EEXIST) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "could not create stats-sample directory %s", _IOS_SAMP_DIR);
+            goto out;
         }
-        if (instance_name) {
-                stats_bytes_written = snprintf (stats_filename, PATH_MAX,
-                        "%s/%s_%s_%s.dump", _IOS_DUMP_DIR,
-                        __progname, xlator_name, instance_name);
-                samples_bytes_written = snprintf (samples_filename, PATH_MAX,
-                        "%s/%s_%s_%s.samp", _IOS_SAMP_DIR,
-                        __progname, xlator_name, instance_name);
-        } else {
-                stats_bytes_written = snprintf (stats_filename, PATH_MAX,
-                        "%s/%s_%s.dump", _IOS_DUMP_DIR, __progname,
-                        xlator_name);
-                samples_bytes_written = snprintf (samples_filename, PATH_MAX,
-                        "%s/%s_%s.samp", _IOS_SAMP_DIR, __progname,
-                        xlator_name);
-        }
-        if ((stats_bytes_written >= PATH_MAX) ||
-            (samples_bytes_written >= PATH_MAX)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Invalid path for stats dump (%s) and/or latency "
-                        "samples (%s)", stats_filename, samples_filename);
-                goto out;
+    }
+    if (instance_name) {
+        stats_bytes_written = snprintf(stats_filename, PATH_MAX,
+                                       "%s/%s_%s_%s.dump", _IOS_DUMP_DIR,
+                                       __progname, xlator_name, instance_name);
+        samples_bytes_written = snprintf(
+            samples_filename, PATH_MAX, "%s/%s_%s_%s.samp", _IOS_SAMP_DIR,
+            __progname, xlator_name, instance_name);
+    } else {
+        stats_bytes_written = snprintf(stats_filename, PATH_MAX,
+                                       "%s/%s_%s.dump", _IOS_DUMP_DIR,
+                                       __progname, xlator_name);
+        samples_bytes_written = snprintf(samples_filename, PATH_MAX,
+                                         "%s/%s_%s.samp", _IOS_SAMP_DIR,
+                                         __progname, xlator_name);
+    }
+    if ((stats_bytes_written >= PATH_MAX) ||
+        (samples_bytes_written >= PATH_MAX)) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Invalid path for stats dump (%s) and/or latency "
+               "samples (%s)",
+               stats_filename, samples_filename);
+        goto out;
+    }
+    while (1) {
+        if (conf->dump_thread_should_die)
+            break;
+        (void)pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS,
+                                    &old_cancel_type);
+        sleep(conf->ios_dump_interval);
+        (void)pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, &old_cancel_type);
+        /*
+         * It's not clear whether we should reopen this each time, or
+         * just hold it open and rewind/truncate on each iteration.
+         * Leaving it alone for now.
+         */
+        stats_logfp = fopen(stats_filename, "w+");
+        if (stats_logfp) {
+            (void)ios_dump_args_init(&args, conf->dump_format, stats_logfp);
+            io_stats_dump(this, &args, GF_IOS_INFO_ALL, _gf_false);
+            fclose(stats_logfp);
+            log_stats_fopen_failure = _gf_true;
+        } else if (log_stats_fopen_failure) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "could not open stats-dump file %s (%s)", stats_filename,
+                   strerror(errno));
+            log_stats_fopen_failure = _gf_false;
         }
-        while (1) {
-                if (conf->dump_thread_should_die)
-                        break;
-                (void) pthread_setcanceltype (PTHREAD_CANCEL_ASYNCHRONOUS,
-                                              &old_cancel_type);
-                sleep (conf->ios_dump_interval);
-                (void) pthread_setcanceltype (PTHREAD_CANCEL_DEFERRED,
-                                              &old_cancel_type);
-                /*
-                 * It's not clear whether we should reopen this each time, or
-                 * just hold it open and rewind/truncate on each iteration.
-                 * Leaving it alone for now.
-                 */
-                stats_logfp = fopen (stats_filename, "w+");
-                if (stats_logfp) {
-                        (void) ios_dump_args_init (&args,
-                                                   conf->dump_format,
-                                                   stats_logfp);
-                        io_stats_dump (this, &args, GF_CLI_INFO_ALL, _gf_false);
-                        fclose (stats_logfp);
-                        log_stats_fopen_failure = _gf_true;
-                } else if (log_stats_fopen_failure) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not open stats-dump file %s (%s)",
-                                stats_filename, strerror(errno));
-                        log_stats_fopen_failure = _gf_false;
-                }
-                samples_logfp = fopen (samples_filename, "w+");
-                if (samples_logfp) {
-                        io_stats_dump_latency_samples_logfp (this,
-                                                             samples_logfp);
-                        fclose (samples_logfp);
-                        log_samples_fopen_failure = _gf_true;
-                } else if (log_samples_fopen_failure) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not open samples-dump file %s (%s)",
-                                samples_filename, strerror(errno));
-                        log_samples_fopen_failure = _gf_false;
-                }
+        samples_logfp = fopen(samples_filename, "w+");
+        if (samples_logfp) {
+            io_stats_dump_latency_samples_logfp(this, samples_logfp);
+            fclose(samples_logfp);
+            log_samples_fopen_failure = _gf_true;
+        } else if (log_samples_fopen_failure) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "could not open samples-dump file %s (%s)", samples_filename,
+                   strerror(errno));
+            log_samples_fopen_failure = _gf_false;
         }
+    }
 out:
-        gf_log (this->name, GF_LOG_INFO, "IO stats dump thread terminated");
-        return NULL;
+    conf->dump_thread_running = _gf_false;
+    gf_log(this->name, GF_LOG_INFO, "IO stats dump thread terminated");
+    return NULL;
 }
 
 static gf_boolean_t
-match_special_xattr (dict_t *d, char *k, data_t *val, void *mdata)
+match_special_xattr(dict_t *d, char *k, data_t *val, void *mdata)
 {
-        gf_boolean_t ret = _gf_false;
-        if (fnmatch ("*io*stat*dump", k, 0) == 0) {
-                ret = _gf_true;
-        }
+    gf_boolean_t ret = _gf_false;
+    if (fnmatch("*io*stat*dump", k, 0) == 0) {
+        ret = _gf_true;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-io_stats_setxattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, dict_t *dict,
-                   int32_t flags, dict_t *xdata)
+io_stats_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                  int32_t flags, dict_t *xdata)
 {
-        struct {
-                xlator_t     *this;
-                inode_t      *inode;
-                const char   *path;
-        } stub;
+    struct {
+        xlator_t *this;
+        inode_t *inode;
+        const char *path;
+    } stub;
 
-        stub.this  = this;
-        stub.inode = loc->inode;
-        stub.path  = loc->path;
+    stub.this = this;
+    stub.inode = loc->inode;
+    stub.path = loc->path;
 
-        (void) dict_foreach_match (dict, match_special_xattr, NULL,
-                                   conditional_dump, &stub);
+    (void)dict_foreach_match(dict, match_special_xattr, NULL, conditional_dump,
+                             &stub);
 
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_setxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
 }
 
-
 int
-io_stats_getxattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name, dict_t *xdata)
+io_stats_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_getxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr,
-                    loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 }
 
-
 int
-io_stats_removexattr (call_frame_t *frame, xlator_t *this,
-                      loc_t *loc, const char *name, dict_t *xdata)
+io_stats_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     const char *name, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_removexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsetxattr (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, dict_t *dict,
-                    int32_t flags, dict_t *xdata)
+io_stats_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                   int32_t flags, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fsetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fgetxattr (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, const char *name, dict_t *xdata)
+io_stats_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fgetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fremovexattr (call_frame_t *frame, xlator_t *this,
-                       fd_t *fd, const char *name, dict_t *xdata)
+io_stats_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      const char *name, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fremovexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
-
 int
-io_stats_opendir (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, fd_t *fd, dict_t *xdata)
+io_stats_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                 dict_t *xdata)
 {
+    START_FOP_LATENCY(frame);
 
-        START_FOP_LATENCY (frame);
-
-        STACK_WIND (frame, io_stats_opendir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir,
-                    loc, fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 }
 
 int
-io_stats_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                   off_t offset, dict_t *dict)
+io_stats_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                  off_t offset, dict_t *dict)
 {
-        frame->local = fd->inode;
-        START_FOP_LATENCY (frame);
+    frame->local = fd->inode;
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_readdirp_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, offset, dict);
-        return 0;
+    STACK_WIND(frame, io_stats_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
+    return 0;
 }
 
-
 int
-io_stats_readdir (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, size_t size, off_t offset, dict_t *xdata)
+io_stats_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t offset, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_readdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir,
-                    fd, size, offset, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, offset, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsyncdir (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, int32_t datasync, dict_t *xdata)
+io_stats_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  int32_t datasync, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fsyncdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsyncdir,
-                    fd, datasync, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fsyncdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsyncdir, fd, datasync, xdata);
+    return 0;
 }
 
-
 int
-io_stats_access (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int32_t mask, dict_t *xdata)
+io_stats_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+                dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_access_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->access,
-                    loc, mask, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_access_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->access, loc, mask, xdata);
+    return 0;
 }
 
-
 int
-io_stats_ftruncate (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, off_t offset, dict_t *xdata)
+io_stats_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                   dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_ftruncate_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsetattr (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, struct iatt *stbuf, int32_t valid, dict_t *xdata)
+io_stats_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_setattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fstat (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, dict_t *xdata)
+io_stats_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fstat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat,
-                    fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
 }
 
-
 int
 io_stats_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-		   off_t offset, size_t len, dict_t *xdata)
+                   off_t offset, size_t len, dict_t *xdata)
 {
-	START_FOP_LATENCY(frame);
+    START_FOP_LATENCY(frame);
 
-	STACK_WIND(frame, io_stats_fallocate_cbk, FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
-		   xdata);
+    STACK_WIND(frame, io_stats_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+               xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
 io_stats_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-		 size_t len, dict_t *xdata)
+                 size_t len, dict_t *xdata)
 {
-	START_FOP_LATENCY(frame);
+    START_FOP_LATENCY(frame);
 
-	STACK_WIND(frame, io_stats_discard_cbk, FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+    STACK_WIND(frame, io_stats_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
 
-	return 0;
+    return 0;
 }
 
 int
 io_stats_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 off_t len, dict_t *xdata)
+                  off_t len, dict_t *xdata)
 {
-        START_FOP_LATENCY(frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND(frame, io_stats_zerofill_cbk, FIRST_CHILD(this),
-                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+    STACK_WIND(frame, io_stats_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-io_stats_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+io_stats_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        START_FOP_LATENCY(frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_ipc_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ipc, op, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
 }
 
 int
-io_stats_lk (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+io_stats_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+            struct gf_flock *lock, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_lk_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, lock, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_lk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lk, fd, cmd, lock, xdata);
+    return 0;
 }
 
 int
-io_stats_rchecksum (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, off_t offset, int32_t len, dict_t *xdata)
+io_stats_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                   int32_t len, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_rchecksum_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rchecksum,
-                    fd, offset, len, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_rchecksum_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rchecksum, fd, offset, len, xdata);
+    return 0;
 }
 
 int
-io_stats_seek (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                off_t offset, gf_seek_what_t what, dict_t *xdata)
+io_stats_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              gf_seek_what_t what, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_seek_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->seek,
-                    fd, offset, what, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_seek_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->seek, fd, offset, what, xdata);
+    return 0;
 }
 
 int
-io_stats_lease (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct gf_lease *lease, dict_t *xdata)
+io_stats_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               struct gf_lease *lease, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_lease_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lease,
-                    loc, lease, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_lease_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lease, loc, lease, xdata);
+    return 0;
 }
 
 int
-io_stats_getactivelk (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      dict_t *xdata)
+io_stats_getactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_getactivelk_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getactivelk,
-                    loc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_getactivelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getactivelk, loc, xdata);
+    return 0;
 }
 
 int
-io_stats_setactivelk (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      lock_migration_info_t *locklist, dict_t *xdata)
+io_stats_setactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     lock_migration_info_t *locklist, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_setactivelk_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setactivelk,
-                    loc, locklist, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_setactivelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setactivelk, loc, locklist, xdata);
+    return 0;
 }
 
 int
-io_stats_compound (call_frame_t *frame, xlator_t *this,
-                   void *args, dict_t *xdata)
+io_stats_compound(call_frame_t *frame, xlator_t *this, void *args,
+                  dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_compound_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->compound,
-                    args, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_compound_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->compound, args, xdata);
+    return 0;
 }
 
 int
-io_stats_release (xlator_t *this, fd_t *fd)
+io_stats_release(xlator_t *this, fd_t *fd)
 {
-        struct ios_fd  *iosfd = NULL;
-        struct ios_conf *conf = NULL;
-
-        BUMP_FOP (RELEASE);
+    struct ios_fd *iosfd = NULL;
+    struct ios_conf *conf = NULL;
 
-        conf = this->private;
+    BUMP_FOP(RELEASE);
 
-        LOCK (&conf->lock);
+    conf = this->private;
+    if (conf) {
+        LOCK(&conf->lock);
         {
-                conf->cumulative.nr_opens--;
+            conf->cumulative.nr_opens--;
         }
-        UNLOCK (&conf->lock);
+        UNLOCK(&conf->lock);
+    }
 
-        ios_fd_ctx_get (fd, this, &iosfd);
-        if (iosfd) {
-                io_stats_dump_fd (this, iosfd);
+    ios_fd_ctx_get(fd, this, &iosfd);
+    if (iosfd) {
+        io_stats_dump_fd(this, iosfd);
 
-                GF_FREE (iosfd->filename);
-                GF_FREE (iosfd);
-        }
+        GF_FREE(iosfd->filename);
+        GF_FREE(iosfd);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-io_stats_releasedir (xlator_t *this, fd_t *fd)
+io_stats_releasedir(xlator_t *this, fd_t *fd)
 {
-        BUMP_FOP (RELEASEDIR);
+    BUMP_FOP(RELEASEDIR);
 
-        return 0;
+    return 0;
 }
 
-
 int
-io_stats_forget (xlator_t *this, inode_t *inode)
+io_stats_forget(xlator_t *this, inode_t *inode)
 {
-        BUMP_FOP (FORGET);
-        ios_stats_cleanup (this, inode);
-        return 0;
+    BUMP_FOP(FORGET);
+    ios_stats_cleanup(this, inode);
+    return 0;
 }
 
 static int
-ios_init_top_stats (struct ios_conf *conf)
+ios_init_top_stats(struct ios_conf *conf)
 {
-        int     i = 0;
+    int i = 0;
 
-        GF_ASSERT (conf);
+    GF_ASSERT(conf);
 
-        for (i = 0; i <IOS_STATS_TYPE_MAX; i++) {
-                conf->list[i].iosstats = GF_CALLOC (1,
-                                         sizeof(*conf->list[i].iosstats),
-                                         gf_io_stats_mt_ios_stat);
+    for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
+        conf->list[i].iosstats = GF_CALLOC(1, sizeof(*conf->list[i].iosstats),
+                                           gf_io_stats_mt_ios_stat);
 
-                if (!conf->list[i].iosstats)
-                        return -1;
+        if (!conf->list[i].iosstats)
+            return -1;
 
-                INIT_LIST_HEAD(&conf->list[i].iosstats->list);
-                LOCK_INIT (&conf->list[i].lock);
-        }
+        INIT_LIST_HEAD(&conf->list[i].iosstats->list);
+        LOCK_INIT(&conf->list[i].lock);
+    }
 
-        for (i = 0; i < IOS_STATS_THRU_MAX; i ++) {
-                conf->thru_list[i].iosstats = GF_CALLOC (1,
-                                 sizeof (*conf->thru_list[i].iosstats),
-                                 gf_io_stats_mt_ios_stat);
+    for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
+        conf->thru_list[i].iosstats = GF_CALLOC(
+            1, sizeof(*conf->thru_list[i].iosstats), gf_io_stats_mt_ios_stat);
 
-                if (!conf->thru_list[i].iosstats)
-                        return -1;
+        if (!conf->thru_list[i].iosstats)
+            return -1;
 
-                INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
-                LOCK_INIT (&conf->thru_list[i].lock);
-        }
+        INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
+        LOCK_INIT(&conf->thru_list[i].lock);
+    }
 
-        return 0;
+    return 0;
 }
 
 static void
-ios_destroy_top_stats (struct ios_conf *conf)
-{
-        int                     i = 0;
-        struct ios_stat_head    *list_head = NULL;
-        struct ios_stat_list    *entry     = NULL;
-        struct ios_stat_list    *tmp       = NULL;
-        struct ios_stat_list    *list      = NULL;
-        struct ios_stat         *stat      = NULL;
-
-        GF_ASSERT (conf);
-
-        LOCK (&conf->lock);
-
-        conf->cumulative.nr_opens = 0;
-        conf->cumulative.max_nr_opens = 0;
-        conf->cumulative.max_openfd_time.tv_sec = 0;
-        conf->cumulative.max_openfd_time.tv_usec = 0;
-
-        for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
-                list_head = &conf->list[i];
-                if (!list_head)
-                        continue;
-                list_for_each_entry_safe (entry, tmp,
-                                          &list_head->iosstats->list, list) {
-                        list = entry;
-                        stat = list->iosstat;
-                        ios_stat_unref (stat);
-                        list_del (&list->list);
-                        GF_FREE (list);
-                        list_head->members--;
-                }
-                GF_FREE (list_head->iosstats);
-        }
+ios_destroy_top_stats(struct ios_conf *conf)
+{
+    int i = 0;
+    struct ios_stat_head *list_head = NULL;
+    struct ios_stat_list *entry = NULL;
+    struct ios_stat_list *tmp = NULL;
+    struct ios_stat_list *list = NULL;
+    struct ios_stat *stat = NULL;
 
-        for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
-                list_head = &conf->thru_list[i];
-                if (!list_head)
-                        continue;
-                list_for_each_entry_safe (entry, tmp,
-                                          &list_head->iosstats->list, list) {
-                        list = entry;
-                        stat = list->iosstat;
-                        ios_stat_unref (stat);
-                        list_del (&list->list);
-                        GF_FREE (list);
-                        list_head->members--;
-                }
-                GF_FREE (list_head->iosstats);
+    GF_ASSERT(conf);
+
+    LOCK(&conf->lock);
+
+    conf->cumulative.nr_opens = 0;
+    conf->cumulative.max_nr_opens = 0;
+    conf->cumulative.max_openfd_time.tv_sec = 0;
+    conf->cumulative.max_openfd_time.tv_usec = 0;
+
+    for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
+        list_head = &conf->list[i];
+        if (!list_head)
+            continue;
+        list_for_each_entry_safe(entry, tmp, &list_head->iosstats->list, list)
+        {
+            list = entry;
+            stat = list->iosstat;
+            ios_stat_unref(stat);
+            list_del(&list->list);
+            GF_FREE(list);
+            list_head->members--;
+        }
+        GF_FREE(list_head->iosstats);
+    }
+
+    for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
+        list_head = &conf->thru_list[i];
+        if (!list_head)
+            continue;
+        list_for_each_entry_safe(entry, tmp, &list_head->iosstats->list, list)
+        {
+            list = entry;
+            stat = list->iosstat;
+            ios_stat_unref(stat);
+            list_del(&list->list);
+            GF_FREE(list);
+            list_head->members--;
         }
+        GF_FREE(list_head->iosstats);
+    }
 
-        UNLOCK (&conf->lock);
+    UNLOCK(&conf->lock);
 
-        return;
+    return;
 }
 
-static int
-io_stats_clear (struct ios_conf *conf)
+static void
+io_stats_clear(struct ios_conf *conf)
 {
-        struct timeval      now;
-        int                 ret = -1;
+    time_t now = 0;
 
-        GF_ASSERT (conf);
+    GF_ASSERT(conf);
+    now = gf_time();
 
-        if (!gettimeofday (&now, NULL))
-        {
-            LOCK (&conf->lock);
-            {
-                    ios_global_stats_clear (&conf->cumulative, &now);
-                    ios_global_stats_clear (&conf->incremental, &now);
-                    conf->increment = 0;
-            }
-            UNLOCK (&conf->lock);
-            ret = 0;
-        }
-
-        return ret;
+    LOCK(&conf->lock);
+    {
+        ios_global_stats_clear(&conf->cumulative, now);
+        ios_global_stats_clear(&conf->incremental, now);
+        conf->increment = 0;
+    }
+    UNLOCK(&conf->lock);
 }
 
 int32_t
-io_priv (xlator_t *this)
-{
-        int                 i;
-        char                key[GF_DUMP_MAX_BUF_LEN];
-        char                key_prefix_cumulative[GF_DUMP_MAX_BUF_LEN];
-        char                key_prefix_incremental[GF_DUMP_MAX_BUF_LEN];
-        double              min, max, avg;
-        uint64_t            count, total;
-        struct ios_conf    *conf = NULL;
-
-        conf = this->private;
-        if (!conf)
-                return -1;
-
-        if(!conf->count_fop_hits || !conf->measure_latency)
-                return -1;
-
-        gf_proc_dump_write ("cumulative.data_read", "%"GF_PRI_ATOMIC,
-                            GF_ATOMIC_GET (conf->cumulative.data_read));
-        gf_proc_dump_write ("cumulative.data_written", "%"GF_PRI_ATOMIC,
-                            GF_ATOMIC_GET (conf->cumulative.data_written));
-
-        gf_proc_dump_write ("incremental.data_read", "%"GF_PRI_ATOMIC,
-                            GF_ATOMIC_GET (conf->incremental.data_read));
-        gf_proc_dump_write ("incremental.data_written", "%"GF_PRI_ATOMIC,
-                            GF_ATOMIC_GET (conf->incremental.data_written));
-
-        snprintf (key_prefix_cumulative, GF_DUMP_MAX_BUF_LEN, "%s.cumulative",
-                  this->name);
-        snprintf (key_prefix_incremental, GF_DUMP_MAX_BUF_LEN, "%s.incremental",
-                  this->name);
-
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                count = GF_ATOMIC_GET (conf->cumulative.fop_hits[i]);
-                total = conf->cumulative.latency[i].total;
-                min = conf->cumulative.latency[i].min;
-                max = conf->cumulative.latency[i].max;
-                avg = conf->cumulative.latency[i].avg;
-
-                gf_proc_dump_build_key (key, key_prefix_cumulative, "%s",
-                                        (char *)gf_fop_list[i]);
-
-                gf_proc_dump_write (key,"%"PRId64",%"PRId64",%.03f,%.03f,%.03f",
-                                    count, total, min, max, avg);
-
-                count = GF_ATOMIC_GET (conf->incremental.fop_hits[i]);
-                total = conf->incremental.latency[i].total;
-                min = conf->incremental.latency[i].min;
-                max = conf->incremental.latency[i].max;
-                avg = conf->incremental.latency[i].avg;
-
-                gf_proc_dump_build_key (key, key_prefix_incremental, "%s",
-                                        (char *)gf_fop_list[i]);
-
-                gf_proc_dump_write (key,"%"PRId64",%"PRId64",%.03f,%.03f,%.03f",
-                                    count, total, min, max, avg);
+io_priv(xlator_t *this)
+{
+    int i;
+    char key[GF_DUMP_MAX_BUF_LEN];
+    char key_prefix_cumulative[GF_DUMP_MAX_BUF_LEN];
+    char key_prefix_incremental[GF_DUMP_MAX_BUF_LEN];
+    double min, max, avg;
+    uint64_t count, total;
+    struct ios_conf *conf = NULL;
 
-        }
+    conf = this->private;
+    if (!conf)
+        return -1;
 
-        return 0;
+    if (!conf->count_fop_hits || !conf->measure_latency)
+        return -1;
+
+    gf_proc_dump_write("cumulative.data_read", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(conf->cumulative.data_read));
+    gf_proc_dump_write("cumulative.data_written", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(conf->cumulative.data_written));
+
+    gf_proc_dump_write("incremental.data_read", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(conf->incremental.data_read));
+    gf_proc_dump_write("incremental.data_written", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(conf->incremental.data_written));
+
+    snprintf(key_prefix_cumulative, GF_DUMP_MAX_BUF_LEN, "%s.cumulative",
+             this->name);
+    snprintf(key_prefix_incremental, GF_DUMP_MAX_BUF_LEN, "%s.incremental",
+             this->name);
+
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        count = GF_ATOMIC_GET(conf->cumulative.fop_hits[i]);
+        total = conf->cumulative.latency[i].total;
+        min = conf->cumulative.latency[i].min;
+        max = conf->cumulative.latency[i].max;
+        avg = conf->cumulative.latency[i].avg;
+
+        gf_proc_dump_build_key(key, key_prefix_cumulative, "%s",
+                               (char *)gf_fop_list[i]);
+
+        gf_proc_dump_write(key, "%" PRId64 ",%" PRId64 ",%.03f,%.03f,%.03f",
+                           count, total, min, max, avg);
+
+        count = GF_ATOMIC_GET(conf->incremental.fop_hits[i]);
+        total = conf->incremental.latency[i].total;
+        min = conf->incremental.latency[i].min;
+        max = conf->incremental.latency[i].max;
+        avg = conf->incremental.latency[i].avg;
+
+        gf_proc_dump_build_key(key, key_prefix_incremental, "%s",
+                               (char *)gf_fop_list[i]);
+
+        gf_proc_dump_write(key, "%" PRId64 ",%" PRId64 ",%.03f,%.03f,%.03f",
+                           count, total, min, max, avg);
+    }
+
+    return 0;
 }
 
 static void
-ios_set_log_format_code (struct ios_conf *conf)
+ios_set_log_format_code(struct ios_conf *conf, char *dump_format_str)
 {
-        if (strcmp (conf->dump_format_str, "json") == 0)
-                conf->dump_format = IOS_DUMP_TYPE_JSON_FILE;
-        else if (strcmp (conf->dump_format_str, "text") == 0)
-                conf->dump_format = IOS_DUMP_TYPE_FILE;
-        else if (strcmp (conf->dump_format_str, "dict") == 0)
-                conf->dump_format = IOS_DUMP_TYPE_DICT;
-        else if (strcmp (conf->dump_format_str, "samples") == 0)
-                conf->dump_format = IOS_DUMP_TYPE_SAMPLES;
+    if (strcmp(dump_format_str, "json") == 0)
+        conf->dump_format = IOS_DUMP_TYPE_JSON_FILE;
+    else if (strcmp(dump_format_str, "text") == 0)
+        conf->dump_format = IOS_DUMP_TYPE_FILE;
+    else if (strcmp(dump_format_str, "dict") == 0)
+        conf->dump_format = IOS_DUMP_TYPE_DICT;
+    else if (strcmp(dump_format_str, "samples") == 0)
+        conf->dump_format = IOS_DUMP_TYPE_SAMPLES;
 }
 
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        struct ios_conf    *conf = NULL;
-        int                 ret = -1;
-        char               *sys_log_str = NULL;
-        char               *log_format_str = NULL;
-        char               *logger_str    = NULL;
-        int                 sys_log_level = -1;
-        char               *log_str = NULL;
-        int                 log_level = -1;
-        int                 log_format = -1;
-        int                 logger = -1;
-        uint32_t            log_buf_size = 0;
-        uint32_t            log_flush_timeout = 0;
-        int32_t             old_dump_interval;
-
-        if (!this || !this->private)
-                goto out;
-
-        conf = this->private;
+void
+xlator_set_loglevel(xlator_t *this, int log_level)
+{
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *top = NULL;
+    xlator_t *trav = this;
 
-        GF_OPTION_RECONF ("dump-fd-stats", conf->dump_fd_stats, options, bool,
-                          out);
+    ctx = this->ctx;
+    GF_ASSERT(ctx);
+    active = ctx->active;
+    top = active->first;
 
-        GF_OPTION_RECONF ("count-fop-hits", conf->count_fop_hits, options, bool,
-                          out);
+    if (log_level == -1)
+        return;
 
-        GF_OPTION_RECONF ("latency-measurement", conf->measure_latency,
-                          options, bool, out);
+    if (ctx->cmd_args.brick_mux) {
+        /* Set log-level for all brick xlators */
+        top->loglevel = log_level;
 
-        old_dump_interval = conf->ios_dump_interval;
-        GF_OPTION_RECONF ("ios-dump-interval", conf->ios_dump_interval, options,
-                         int32, out);
-        if ((old_dump_interval <= 0) && (conf->ios_dump_interval > 0)) {
-                ret = gf_thread_create (&conf->dump_thread, NULL,
-                                  (void *) &_ios_dump_thread, this, "iosdump");
-                if (ret) {
-                        gf_log (this ? this->name : "io-stats",
-                                GF_LOG_ERROR, "Failed to start thread"
-                                "while reconfigure. Returning %d", ret);
-                        goto out;
-                }
-        }
+        /* Set log-level for parent xlator */
+        if (this->parents)
+            this->parents->xlator->loglevel = log_level;
 
-        GF_OPTION_RECONF ("ios-sample-interval", conf->ios_sample_interval,
-                         options, int32, out);
-        GF_OPTION_RECONF ("ios-dump-format", conf->dump_format_str, options,
-                          str, out);
-        ios_set_log_format_code (conf);
-        GF_OPTION_RECONF ("ios-sample-buf-size", conf->ios_sample_buf_size,
-                         options, int32, out);
-        GF_OPTION_RECONF ("sys-log-level", sys_log_str, options, str, out);
-        if (sys_log_str) {
-                sys_log_level = glusterd_check_log_level (sys_log_str);
-                set_sys_log_level (sys_log_level);
+        while (trav) {
+            trav->loglevel = log_level;
+            trav = trav->next;
         }
+    } else {
+        gf_log_set_loglevel(this->ctx, log_level);
+    }
+}
 
-        GF_OPTION_RECONF ("log-level", log_str, options, str, out);
-        if (log_str) {
-                log_level = glusterd_check_log_level (log_str);
-                gf_log_set_loglevel (this->ctx, log_level);
-        }
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    struct ios_conf *conf = NULL;
+    int ret = -1;
+    char *sys_log_str = NULL;
+    char *log_format_str = NULL;
+    char *logger_str = NULL;
+    char *dump_format_str = NULL;
+    int sys_log_level = -1;
+    char *log_str = NULL;
+    int log_level = -1;
+    int log_format = -1;
+    int logger = -1;
+    uint32_t log_buf_size = 0;
+    uint32_t log_flush_timeout = 0;
+    int32_t old_dump_interval;
+    int32_t threads;
+
+    if (!this || !this->private)
+        goto out;
 
-        GF_OPTION_RECONF ("logger", logger_str, options, str, out);
-        if (logger_str) {
-                logger = gf_check_logger (logger_str);
-                gf_log_set_logger (logger);
-        }
+    conf = this->private;
 
-        GF_OPTION_RECONF ("log-format", log_format_str, options, str, out);
-        if (log_format_str) {
-                log_format = gf_check_log_format (log_format_str);
-                gf_log_set_logformat (log_format);
-        }
+    GF_OPTION_RECONF("dump-fd-stats", conf->dump_fd_stats, options, bool, out);
 
-        GF_OPTION_RECONF ("log-buf-size", log_buf_size, options, uint32, out);
-        gf_log_set_log_buf_size (log_buf_size);
+    GF_OPTION_RECONF("count-fop-hits", conf->count_fop_hits, options, bool,
+                     out);
 
-        GF_OPTION_RECONF ("log-flush-timeout", log_flush_timeout, options,
-                          time, out);
-        gf_log_set_log_flush_timeout (log_flush_timeout);
+    GF_OPTION_RECONF("latency-measurement", conf->measure_latency, options,
+                     bool, out);
 
-        ret = 0;
+    old_dump_interval = conf->ios_dump_interval;
+    GF_OPTION_RECONF("ios-dump-interval", conf->ios_dump_interval, options,
+                     int32, out);
+    if ((old_dump_interval <= 0) && (conf->ios_dump_interval > 0)) {
+        conf->dump_thread_running = _gf_true;
+        conf->dump_thread_should_die = _gf_false;
+        ret = gf_thread_create(&conf->dump_thread, NULL,
+                               (void *)&_ios_dump_thread, this, "iosdump");
+        if (ret) {
+            conf->dump_thread_running = _gf_false;
+            gf_log(this ? this->name : "io-stats", GF_LOG_ERROR,
+                   "Failed to start thread"
+                   "while reconfigure. Returning %d",
+                   ret);
+            goto out;
+        }
+    } else if ((old_dump_interval > 0) && (conf->ios_dump_interval == 0)) {
+        _ios_destroy_dump_thread(conf);
+    }
+
+    GF_OPTION_RECONF("ios-sample-interval", conf->ios_sample_interval, options,
+                     int32, out);
+    GF_OPTION_RECONF("ios-dump-format", dump_format_str, options, str, out);
+    ios_set_log_format_code(conf, dump_format_str);
+    GF_OPTION_RECONF("ios-sample-buf-size", conf->ios_sample_buf_size, options,
+                     int32, out);
+    GF_OPTION_RECONF("sys-log-level", sys_log_str, options, str, out);
+    if (sys_log_str) {
+        sys_log_level = glusterd_check_log_level(sys_log_str);
+        set_sys_log_level(sys_log_level);
+    }
+
+    GF_OPTION_RECONF("log-level", log_str, options, str, out);
+    if (log_str) {
+        log_level = glusterd_check_log_level(log_str);
+        /* Set loglevel for all children and server xlators */
+        xlator_set_loglevel(this, log_level);
+    }
+
+    GF_OPTION_RECONF("logger", logger_str, options, str, out);
+    if (logger_str) {
+        logger = gf_check_logger(logger_str);
+        gf_log_set_logger(logger);
+    }
+
+    GF_OPTION_RECONF("log-format", log_format_str, options, str, out);
+    if (log_format_str) {
+        log_format = gf_check_log_format(log_format_str);
+        gf_log_set_logformat(log_format);
+    }
+
+    GF_OPTION_RECONF("log-buf-size", log_buf_size, options, uint32, out);
+    gf_log_set_log_buf_size(log_buf_size);
+
+    GF_OPTION_RECONF("log-flush-timeout", log_flush_timeout, options, time,
+                     out);
+    gf_log_set_log_flush_timeout(log_flush_timeout);
+
+    GF_OPTION_RECONF("threads", threads, options, int32, out);
+    gf_async_adjust_threads(threads);
+
+    ret = 0;
 out:
-        gf_log (this ? this->name : "io-stats",
-                        GF_LOG_DEBUG, "reconfigure returning %d", ret);
-        return ret;
+    gf_log(this ? this->name : "io-stats", GF_LOG_DEBUG,
+           "reconfigure returning %d", ret);
+    return ret;
 }
 
-
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_io_stats_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_io_stats_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               " failed");
         return ret;
+    }
+
+    return ret;
 }
 
 void
-ios_conf_destroy (struct ios_conf *conf)
+ios_conf_destroy(struct ios_conf *conf)
 {
-        if (!conf)
-                return;
+    if (!conf)
+        return;
 
-        ios_destroy_top_stats (conf);
-        _ios_destroy_dump_thread (conf);
-        ios_destroy_sample_buf (conf->ios_sample_buf);
-        LOCK_DESTROY (&conf->lock);
-        GF_FREE(conf->dnscache);
-        GF_FREE(conf);
+    ios_destroy_top_stats(conf);
+    _ios_destroy_dump_thread(conf);
+    ios_destroy_sample_buf(conf->ios_sample_buf);
+    LOCK_DESTROY(&conf->lock);
+    gf_dnscache_deinit(conf->dnscache);
+    GF_FREE(conf);
 }
 
 static void
-ios_init_stats (struct ios_global_stats *stats)
+ios_init_stats(struct ios_global_stats *stats)
 {
-        int i = 0;
+    int i = 0;
 
-        GF_ATOMIC_INIT (stats->data_read, 0);
-        GF_ATOMIC_INIT (stats->data_written, 0);
+    GF_ATOMIC_INIT(stats->data_read, 0);
+    GF_ATOMIC_INIT(stats->data_written, 0);
 
-        for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
-                GF_ATOMIC_INIT (stats->block_count_write[i], 0);
-                GF_ATOMIC_INIT (stats->block_count_read[i], 0);
-        }
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        GF_ATOMIC_INIT(stats->block_count_write[i], 0);
+        GF_ATOMIC_INIT(stats->block_count_read[i], 0);
+    }
 
-        for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                GF_ATOMIC_INIT (stats->fop_hits[i], 0);
+    for (i = 0; i < GF_FOP_MAXVALUE; i++)
+        GF_ATOMIC_INIT(stats->fop_hits[i], 0);
 
-        for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++)
-                GF_ATOMIC_INIT (stats->upcall_hits[i], 0);
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++)
+        GF_ATOMIC_INIT(stats->upcall_hits[i], 0);
 
-        gettimeofday (&stats->started_at, NULL);
+    stats->started_at = gf_time();
 }
 
 int
-init (xlator_t *this)
-{
-        struct ios_conf    *conf = NULL;
-        char               *sys_log_str = NULL;
-        char               *logger_str  = NULL;
-        char               *log_format_str  = NULL;
-        int                 logger = -1;
-        int                 log_format = -1;
-        int                 sys_log_level = -1;
-        char               *log_str = NULL;
-        int                 log_level = -1;
-        int                 ret = -1;
-        uint32_t            log_buf_size = 0;
-        uint32_t            log_flush_timeout = 0;
-
-        if (!this)
-                return -1;
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "io_stats translator requires atleast one subvolume");
-                return -1;
-        }
-
-        if (!this->parents) {
-                /* This is very much valid as io-stats currently is loaded
-                 * on top of volumes on both client and server, hence this is
-                 * not an warning message */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dangling volume. check volfile ");
-        }
-
-        conf = GF_CALLOC (1, sizeof(*conf), gf_io_stats_mt_ios_conf);
-
-        if (!conf)
-                goto out;
-
-        if (dict_get_str (this->options, "unique-id", &conf->unique_id) != 0) {
-                /* This is always set on servers, so we must be a client. */
-                conf->unique_id = this->name;
-        }
-
-        /*
-         * Init it just after calloc, so that we are sure the lock is inited
-         * in case of error paths.
-         */
-        LOCK_INIT (&conf->lock);
-        LOCK_INIT (&conf->ios_sampling_lock);
+init(xlator_t *this)
+{
+    struct ios_conf *conf = NULL;
+    char *volume_id = NULL;
+    char *sys_log_str = NULL;
+    char *logger_str = NULL;
+    char *log_format_str = NULL;
+    char *dump_format_str = NULL;
+    int logger = -1;
+    int log_format = -1;
+    int sys_log_level = -1;
+    char *log_str = NULL;
+    int log_level = -1;
+    int ret = -1;
+    uint32_t log_buf_size = 0;
+    uint32_t log_flush_timeout = 0;
+    int32_t threads;
+
+    if (!this)
+        return -1;
+
+    if (!this->children) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "io_stats translator requires at least one subvolume");
+        return -1;
+    }
+
+    if (!this->parents) {
+        /* This is very much valid as io-stats currently is loaded
+         * on top of volumes on both client and server, hence this is
+         * not an warning message */
+        gf_log(this->name, GF_LOG_DEBUG, "dangling volume. check volfile ");
+    }
+
+    conf = GF_CALLOC(1, sizeof(*conf), gf_io_stats_mt_ios_conf);
+
+    if (!conf)
+        goto out;
 
-        ios_init_stats (&conf->cumulative);
-        ios_init_stats (&conf->incremental);
+    if (dict_get_str(this->options, "unique-id", &conf->unique_id) != 0) {
+        /* This is always set on servers, so we must be a client. */
+        conf->unique_id = this->name;
+    }
+
+    ret = dict_get_strn(this->options, "volume-id", SLEN("volume-id"),
+                        &volume_id);
+    if (!ret) {
+        strncpy(this->graph->volume_id, volume_id, GF_UUID_BUF_SIZE);
+    }
+    /*
+     * Init it just after calloc, so that we are sure the lock is inited
+     * in case of error paths.
+     */
+    LOCK_INIT(&conf->lock);
+    LOCK_INIT(&conf->ios_sampling_lock);
+
+    ios_init_stats(&conf->cumulative);
+    ios_init_stats(&conf->incremental);
+
+    ret = ios_init_top_stats(conf);
+    if (ret)
+        goto out;
 
-        ret = ios_init_top_stats (conf);
-        if (ret)
-                goto out;
+    GF_OPTION_INIT("dump-fd-stats", conf->dump_fd_stats, bool, out);
 
-        GF_OPTION_INIT ("dump-fd-stats", conf->dump_fd_stats, bool, out);
+    GF_OPTION_INIT("count-fop-hits", conf->count_fop_hits, bool, out);
 
-        GF_OPTION_INIT ("count-fop-hits", conf->count_fop_hits, bool, out);
+    GF_OPTION_INIT("latency-measurement", conf->measure_latency, bool, out);
 
-        GF_OPTION_INIT ("latency-measurement", conf->measure_latency,
-                        bool, out);
+    GF_OPTION_INIT("ios-dump-interval", conf->ios_dump_interval, int32, out);
 
-        GF_OPTION_INIT ("ios-dump-interval", conf->ios_dump_interval,
-                        int32, out);
+    GF_OPTION_INIT("ios-sample-interval", conf->ios_sample_interval, int32,
+                   out);
 
-        GF_OPTION_INIT ("ios-sample-interval", conf->ios_sample_interval,
-                        int32, out);
+    GF_OPTION_INIT("ios-dump-format", dump_format_str, str, out);
+    ios_set_log_format_code(conf, dump_format_str);
 
-        GF_OPTION_INIT ("ios-dump-format", conf->dump_format_str, str, out);
-        ios_set_log_format_code (conf);
+    GF_OPTION_INIT("ios-sample-buf-size", conf->ios_sample_buf_size, int32,
+                   out);
 
-        GF_OPTION_INIT ("ios-sample-buf-size", conf->ios_sample_buf_size,
-                        int32, out);
+    ret = ios_init_sample_buf(conf);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Out of memory.");
+        goto out;
+    }
 
-        ret = ios_init_sample_buf (conf);
+    GF_OPTION_INIT("ios-dnscache-ttl-sec", conf->ios_dnscache_ttl_sec, int32,
+                   out);
+    conf->dnscache = gf_dnscache_init(conf->ios_dnscache_ttl_sec);
+    if (!conf->dnscache) {
+        ret = -1;
+        goto out;
+    }
+
+    GF_OPTION_INIT("sys-log-level", sys_log_str, str, out);
+    if (sys_log_str) {
+        sys_log_level = glusterd_check_log_level(sys_log_str);
+        set_sys_log_level(sys_log_level);
+    }
+
+    GF_OPTION_INIT("log-level", log_str, str, out);
+    if (log_str) {
+        log_level = glusterd_check_log_level(log_str);
+        if (DEFAULT_LOG_LEVEL != log_level)
+            gf_log_set_loglevel(this->ctx, log_level);
+    }
+
+    GF_OPTION_INIT("logger", logger_str, str, out);
+    if (logger_str) {
+        logger = gf_check_logger(logger_str);
+        gf_log_set_logger(logger);
+    }
+
+    GF_OPTION_INIT("log-format", log_format_str, str, out);
+    if (log_format_str) {
+        log_format = gf_check_log_format(log_format_str);
+        gf_log_set_logformat(log_format);
+    }
+
+    GF_OPTION_INIT("log-buf-size", log_buf_size, uint32, out);
+    gf_log_set_log_buf_size(log_buf_size);
+
+    GF_OPTION_INIT("log-flush-timeout", log_flush_timeout, time, out);
+    gf_log_set_log_flush_timeout(log_flush_timeout);
+
+    GF_OPTION_INIT("threads", threads, int32, out);
+    gf_async_adjust_threads(threads);
+
+    this->private = conf;
+    if (conf->ios_dump_interval > 0) {
+        conf->dump_thread_running = _gf_true;
+        conf->dump_thread_should_die = _gf_false;
+        ret = gf_thread_create(&conf->dump_thread, NULL,
+                               (void *)&_ios_dump_thread, this, "iosdump");
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                goto out;
-        }
-
-        GF_OPTION_INIT ("ios-dnscache-ttl-sec", conf->ios_dnscache_ttl_sec,
-                        int32, out);
-        conf->dnscache = gf_dnscache_init (conf->ios_dnscache_ttl_sec);
-
-        GF_OPTION_INIT ("sys-log-level", sys_log_str, str, out);
-        if (sys_log_str) {
-                sys_log_level = glusterd_check_log_level (sys_log_str);
-                set_sys_log_level (sys_log_level);
-        }
-
-        GF_OPTION_INIT ("log-level", log_str, str, out);
-        if (log_str) {
-                log_level = glusterd_check_log_level (log_str);
-                if (DEFAULT_LOG_LEVEL != log_level)
-                        gf_log_set_loglevel (this->ctx, log_level);
-        }
-
-        GF_OPTION_INIT ("logger", logger_str, str, out);
-        if (logger_str) {
-                logger = gf_check_logger (logger_str);
-                gf_log_set_logger (logger);
-        }
-
-        GF_OPTION_INIT ("log-format", log_format_str, str, out);
-        if (log_format_str) {
-                log_format = gf_check_log_format (log_format_str);
-                gf_log_set_logformat (log_format);
+            conf->dump_thread_running = _gf_false;
+            gf_log(this ? this->name : "io-stats", GF_LOG_ERROR,
+                   "Failed to start thread"
+                   "in init. Returning %d",
+                   ret);
+            goto out;
         }
-
-        GF_OPTION_INIT ("log-buf-size", log_buf_size, uint32, out);
-        gf_log_set_log_buf_size (log_buf_size);
-
-        GF_OPTION_INIT ("log-flush-timeout", log_flush_timeout, time, out);
-        gf_log_set_log_flush_timeout (log_flush_timeout);
-
-        this->private = conf;
-        if (conf->ios_dump_interval > 0) {
-                ret = gf_thread_create (&conf->dump_thread, NULL,
-                                        (void *) &_ios_dump_thread,
-                                        this, "iosdump");
-                if (ret) {
-                        gf_log (this ? this->name : "io-stats",
-                                GF_LOG_ERROR, "Failed to start thread"
-                                "in init. Returning %d", ret);
-                        goto out;
-                }
-        }
-        return 0;
+    }
+    return 0;
 out:
-        ios_conf_destroy (conf);
-        return ret;
+    ios_conf_destroy(conf);
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        struct ios_conf *conf = NULL;
+    struct ios_conf *conf = NULL;
 
-        if (!this)
-                return;
+    if (!this)
+        return;
 
-        conf = this->private;
+    conf = this->private;
 
-        ios_conf_destroy (conf);
-        this->private = NULL;
-        gf_log (this->name, GF_LOG_INFO,
-                "io-stats translator unloaded");
-        return;
+    ios_conf_destroy(conf);
+    this->private = NULL;
+    gf_log(this->name, GF_LOG_INFO, "io-stats translator unloaded");
+    return;
 }
 
 int
-notify (xlator_t *this, int32_t event, void *data, ...)
-{
-        int          ret = 0;
-        struct ios_dump_args args = {0};
-        dict_t       *output = NULL;
-        dict_t       *dict = NULL;
-        int32_t       op = 0;
-        int32_t       list_cnt = 0;
-        double        throughput = 0;
-        double        time = 0;
-        gf_boolean_t  is_peek = _gf_false;
-        va_list ap;
-        struct gf_upcall *up_data = NULL;
-        struct gf_upcall_cache_invalidation *up_ci = NULL;
-
-        dict = data;
-        va_start (ap, data);
-        output = va_arg (ap, dict_t*);
-        va_end (ap);
-        switch (event) {
+notify(xlator_t *this, int32_t event, void *data, ...)
+{
+    int ret = 0;
+    struct ios_dump_args args = {0};
+    dict_t *output = NULL;
+    dict_t *dict = NULL;
+    int32_t op = 0;
+    int32_t list_cnt = 0;
+    double throughput = 0;
+    double time = 0;
+    gf_boolean_t is_peek = _gf_false;
+    va_list ap;
+    struct gf_upcall *up_data = NULL;
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+
+    dict = data;
+    va_start(ap, data);
+    output = va_arg(ap, dict_t *);
+    va_end(ap);
+    switch (event) {
         case GF_EVENT_TRANSLATOR_INFO:
-                ret = dict_get_str_boolean (dict, "clear-stats", _gf_false);
+            ret = dict_get_str_boolean(dict, "clear-stats", _gf_false);
+            if (ret) {
+                ret = dict_set_int32(output, "top-op", op);
                 if (ret) {
-                        ret = dict_set_int32 (output, "top-op", op);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set top-op in dict");
-                                goto out;
-                        }
-                        ios_destroy_top_stats (this->private);
-                        ret = ios_init_top_stats (this->private);
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Failed to set top-op in dict");
+                    goto out;
+                }
+                ios_destroy_top_stats(this->private);
+                ret = ios_init_top_stats(this->private);
+                if (ret)
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Failed to reset top stats");
+                ret = dict_set_int32(output, "stats-cleared", ret ? 0 : 1);
+                if (ret)
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Failed to set stats-cleared"
+                           " in dict");
+                goto out;
+            }
+
+            ret = dict_get_int32(dict, "top-op", &op);
+            if (!ret) {
+                ret = dict_get_int32(dict, "list-cnt", &list_cnt);
+                if (op > IOS_STATS_TYPE_NONE && op < IOS_STATS_TYPE_MAX)
+                    ret = io_stats_dump_stats_to_dict(this, output, op,
+                                                      list_cnt);
+                if (op == IOS_STATS_TYPE_READ_THROUGHPUT ||
+                    op == IOS_STATS_TYPE_WRITE_THROUGHPUT) {
+                    ret = dict_get_double(dict, "throughput", &throughput);
+                    if (!ret) {
+                        ret = dict_get_double(dict, "time", &time);
+                        if (ret)
+                            goto out;
+                        ret = dict_set_double(output, "throughput", throughput);
                         if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to reset top stats");
-                        ret = dict_set_int32 (output, "stats-cleared",
-                                              ret ? 0 : 1);
+                            goto out;
+                        ret = dict_set_double(output, "time", time);
                         if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set stats-cleared"
-                                        " in dict");
-                        goto out;
+                            goto out;
+                    }
+                    ret = 0;
                 }
+            } else {
+                ret = dict_get_int32(dict, "info-op", &op);
+                if (ret || op < GF_IOS_INFO_ALL || GF_IOS_INFO_CLEAR < op)
+                    op = GF_IOS_INFO_ALL;
 
-                ret = dict_get_int32 (dict, "top-op", &op);
-                if (!ret) {
-                        ret = dict_get_int32 (dict, "list-cnt", &list_cnt);
-                        if (op > IOS_STATS_TYPE_NONE &&
-                            op < IOS_STATS_TYPE_MAX)
-                                ret = io_stats_dump_stats_to_dict (this, output,
-                                                             op, list_cnt);
-                        if (op == IOS_STATS_TYPE_READ_THROUGHPUT ||
-                                op == IOS_STATS_TYPE_WRITE_THROUGHPUT) {
-                                ret = dict_get_double (dict, "throughput",
-                                                        &throughput);
-                                if (!ret) {
-                                        ret = dict_get_double (dict, "time",
-                                                                &time);
-                                        if (ret)
-                                                goto out;
-                                        ret = dict_set_double (output,
-                                                "throughput", throughput);
-                                        if (ret)
-                                                goto out;
-                                        ret = dict_set_double (output, "time",
-                                                        time);
-                                        if (ret)
-                                                goto out;
-                                }
-                                ret = 0;
-
-                        }
+                ret = dict_set_int32(output, "info-op", op);
+                if (ret) {
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Failed to set info-op in dict");
+                    goto out;
+                }
+
+                if (GF_IOS_INFO_CLEAR == op) {
+                    io_stats_clear(this->private);
+
+                    ret = dict_set_int32(output, "stats-cleared", 1);
+                    if (ret)
+                        gf_log(this->name, GF_LOG_ERROR,
+                               "Failed to set stats-cleared"
+                               " in dict");
                 } else {
-                        ret = dict_get_int32 (dict, "info-op", &op);
-                        if (ret || op < GF_CLI_INFO_ALL ||
-                            GF_CLI_INFO_CLEAR < op)
-                            op = GF_CLI_INFO_ALL;
-
-                        ret = dict_set_int32 (output, "info-op", op);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set info-op in dict");
-                                goto out;
-                        }
-
-                        if (GF_CLI_INFO_CLEAR == op) {
-                                ret = io_stats_clear (this->private);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to clear info stats");
-
-                                ret = dict_set_int32 (output, "stats-cleared",
-                                              ret ? 0 : 1);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to set stats-cleared"
-                                                " in dict");
-                        }
-                        else {
-                                ret = dict_get_str_boolean (dict, "peek",
-                                                            _gf_false);
-                                if (-1 != ret)
-                                        is_peek = ret;
-
-                                (void) ios_dump_args_init (&args,
-                                                IOS_DUMP_TYPE_DICT, output);
-                                ret = io_stats_dump (this, &args, op, is_peek);
-                        }
+                    ret = dict_get_str_boolean(dict, "peek", _gf_false);
+                    if (-1 != ret)
+                        is_peek = ret;
+
+                    (void)ios_dump_args_init(&args, IOS_DUMP_TYPE_DICT, output);
+                    ret = io_stats_dump(this, &args, op, is_peek);
                 }
-                break;
+            }
+            break;
         case GF_EVENT_UPCALL:
-                up_data = (struct gf_upcall *)data;
-                ios_bump_upcall (this, GF_UPCALL);
+            up_data = (struct gf_upcall *)data;
+            ios_bump_upcall(this, GF_UPCALL);
 
-                switch (up_data->event_type) {
+            switch (up_data->event_type) {
                 case GF_UPCALL_RECALL_LEASE:
-                        ios_bump_upcall (this, GF_UPCALL_LEASE_RECALL);
-                        break;
+                    ios_bump_upcall(this, GF_UPCALL_LEASE_RECALL);
+                    break;
                 case GF_UPCALL_CACHE_INVALIDATION:
-                        up_ci = (struct gf_upcall_cache_invalidation *)up_data->data;
-                        if (up_ci->flags & (UP_XATTR | UP_XATTR_RM))
-                                ios_bump_upcall (this, GF_UPCALL_CI_XATTR);
-                        if (up_ci->flags & IATT_UPDATE_FLAGS)
-                                ios_bump_upcall (this, GF_UPCALL_CI_STAT);
-                        if (up_ci->flags & UP_RENAME_FLAGS)
-                                ios_bump_upcall (this, GF_UPCALL_CI_RENAME);
-                        if (up_ci->flags & UP_FORGET)
-                                ios_bump_upcall (this, GF_UPCALL_CI_FORGET);
-                        if (up_ci->flags & UP_NLINK)
-                                ios_bump_upcall (this, GF_UPCALL_CI_NLINK);
-                        break;
+                    up_ci = (struct gf_upcall_cache_invalidation *)
+                                up_data->data;
+                    if (up_ci->flags & (UP_XATTR | UP_XATTR_RM))
+                        ios_bump_upcall(this, GF_UPCALL_CI_XATTR);
+                    if (up_ci->flags & IATT_UPDATE_FLAGS)
+                        ios_bump_upcall(this, GF_UPCALL_CI_STAT);
+                    if (up_ci->flags & UP_RENAME_FLAGS)
+                        ios_bump_upcall(this, GF_UPCALL_CI_RENAME);
+                    if (up_ci->flags & UP_FORGET)
+                        ios_bump_upcall(this, GF_UPCALL_CI_FORGET);
+                    if (up_ci->flags & UP_NLINK)
+                        ios_bump_upcall(this, GF_UPCALL_CI_NLINK);
+                    break;
                 default:
-                        gf_msg_debug (this->name, 0, "Unknown upcall event "
-                                      "type :%d", up_data->event_type);
-                        break;
-                }
+                    gf_msg_debug(this->name, 0,
+                                 "Unknown upcall event "
+                                 "type :%d",
+                                 up_data->event_type);
+                    break;
+            }
 
-                default_notify (this, event, data);
-                break;
+            default_notify(this, event, data);
+            break;
         default:
-                default_notify (this, event, data);
-                break;
-
-        }
+            default_notify(this, event, data);
+            break;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-struct xlator_dumpops dumpops = {
-        .priv    = io_priv
-};
+struct xlator_dumpops dumpops = {.priv = io_priv};
 
 struct xlator_fops fops = {
-        .stat        = io_stats_stat,
-        .readlink    = io_stats_readlink,
-        .mknod       = io_stats_mknod,
-        .mkdir       = io_stats_mkdir,
-        .unlink      = io_stats_unlink,
-        .rmdir       = io_stats_rmdir,
-        .symlink     = io_stats_symlink,
-        .rename      = io_stats_rename,
-        .link        = io_stats_link,
-        .truncate    = io_stats_truncate,
-        .open        = io_stats_open,
-        .readv       = io_stats_readv,
-        .writev      = io_stats_writev,
-        .statfs      = io_stats_statfs,
-        .flush       = io_stats_flush,
-        .fsync       = io_stats_fsync,
-        .setxattr    = io_stats_setxattr,
-        .getxattr    = io_stats_getxattr,
-        .removexattr = io_stats_removexattr,
-        .fsetxattr    = io_stats_fsetxattr,
-        .fgetxattr    = io_stats_fgetxattr,
-        .fremovexattr = io_stats_fremovexattr,
-        .opendir     = io_stats_opendir,
-        .readdir     = io_stats_readdir,
-        .readdirp    = io_stats_readdirp,
-        .fsyncdir    = io_stats_fsyncdir,
-        .access      = io_stats_access,
-        .ftruncate   = io_stats_ftruncate,
-        .fstat       = io_stats_fstat,
-        .create      = io_stats_create,
-        .lk          = io_stats_lk,
-        .inodelk     = io_stats_inodelk,
-        .finodelk    = io_stats_finodelk,
-        .entrylk     = io_stats_entrylk,
-        .fentrylk    = io_stats_fentrylk,
-        .lookup      = io_stats_lookup,
-        .xattrop     = io_stats_xattrop,
-        .fxattrop    = io_stats_fxattrop,
-        .setattr     = io_stats_setattr,
-        .fsetattr    = io_stats_fsetattr,
-	.fallocate   = io_stats_fallocate,
-	.discard     = io_stats_discard,
-        .zerofill    = io_stats_zerofill,
-        .ipc         = io_stats_ipc,
-        .rchecksum   = io_stats_rchecksum,
-        .seek        = io_stats_seek,
-        .lease       = io_stats_lease,
-        .getactivelk = io_stats_getactivelk,
-        .setactivelk = io_stats_setactivelk,
-        .compound    = io_stats_compound,
+    .stat = io_stats_stat,
+    .readlink = io_stats_readlink,
+    .mknod = io_stats_mknod,
+    .mkdir = io_stats_mkdir,
+    .unlink = io_stats_unlink,
+    .rmdir = io_stats_rmdir,
+    .symlink = io_stats_symlink,
+    .rename = io_stats_rename,
+    .link = io_stats_link,
+    .truncate = io_stats_truncate,
+    .open = io_stats_open,
+    .readv = io_stats_readv,
+    .writev = io_stats_writev,
+    .statfs = io_stats_statfs,
+    .flush = io_stats_flush,
+    .fsync = io_stats_fsync,
+    .setxattr = io_stats_setxattr,
+    .getxattr = io_stats_getxattr,
+    .removexattr = io_stats_removexattr,
+    .fsetxattr = io_stats_fsetxattr,
+    .fgetxattr = io_stats_fgetxattr,
+    .fremovexattr = io_stats_fremovexattr,
+    .opendir = io_stats_opendir,
+    .readdir = io_stats_readdir,
+    .readdirp = io_stats_readdirp,
+    .fsyncdir = io_stats_fsyncdir,
+    .access = io_stats_access,
+    .ftruncate = io_stats_ftruncate,
+    .fstat = io_stats_fstat,
+    .create = io_stats_create,
+    .lk = io_stats_lk,
+    .inodelk = io_stats_inodelk,
+    .finodelk = io_stats_finodelk,
+    .entrylk = io_stats_entrylk,
+    .fentrylk = io_stats_fentrylk,
+    .lookup = io_stats_lookup,
+    .xattrop = io_stats_xattrop,
+    .fxattrop = io_stats_fxattrop,
+    .setattr = io_stats_setattr,
+    .fsetattr = io_stats_fsetattr,
+    .fallocate = io_stats_fallocate,
+    .discard = io_stats_discard,
+    .zerofill = io_stats_zerofill,
+    .ipc = io_stats_ipc,
+    .rchecksum = io_stats_rchecksum,
+    .seek = io_stats_seek,
+    .lease = io_stats_lease,
+    .getactivelk = io_stats_getactivelk,
+    .setactivelk = io_stats_setactivelk,
+    .compound = io_stats_compound,
+    .copy_file_range = io_stats_copy_file_range,
 };
 
 struct xlator_cbks cbks = {
-        .release     = io_stats_release,
-        .releasedir  = io_stats_releasedir,
-        .forget      = io_stats_forget,
+    .release = io_stats_release,
+    .releasedir = io_stats_releasedir,
+    .forget = io_stats_forget,
 };
 
 struct volume_options options[] = {
-        { .key  = {"dump-fd-stats"},
-          .op_version = {1},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "If on stats related to file-operations would be "
-                         "tracked inside GlusterFS data-structures."
-        },
-        { .key  = { "ios-dump-interval" },
-          .type = GF_OPTION_TYPE_INT,
-          .op_version = {1},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .min = 0,
-          .max = 3600,
-          .default_value = "0",
-          .description = "Interval (in seconds) at which to auto-dump "
-                         "statistics. Zero disables automatic dumping."
-        },
-        { .key  = { "ios-sample-interval" },
-          .type = GF_OPTION_TYPE_INT,
-          .op_version = {1},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .min = 0,
-          .max = 65535,
-          .default_value = "0",
-          .description = "Interval in which we want to collect FOP latency "
-                         "samples.  2 means collect a sample every 2nd FOP."
-        },
-        { .key = {"ios-dump-format"},
-          .type = GF_OPTION_TYPE_STR,
-          .op_version = {GD_OP_VERSION_3_12_0},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .default_value = "json",
-          .description = " The dump-format option specifies the format in which"
-                         " to dump the statistics. Select between \"text\", "
-                         "\"json\", \"dict\" and \"samples\". Default is "
-                         "\"json\".",
-          .value       = { "text", "json", "dict", "samples"}
-        },
-        { .key  = { "ios-sample-buf-size" },
-          .type = GF_OPTION_TYPE_INT,
-          .op_version = {1},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .min = 1024,
-          .max = 1024*1024,
-          .default_value = "65535",
-          .description = "The maximum size of our FOP sampling ring buffer."
-        },
-        { .key  = { "ios-dnscache-ttl-sec" },
-          .type = GF_OPTION_TYPE_INT,
-          .op_version = {1},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .min = 1,
-          .max = 3600 * 72,
-          .default_value = "86400",
-          .description = "The interval after wish a cached DNS entry will be "
-                         "re-validated.  Default: 24 hrs"
-        },
-        { .key  = { "latency-measurement" },
-          .type = GF_OPTION_TYPE_BOOL,
-          .op_version = {1},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .default_value = "off",
-          .description = "If on stats related to the latency of each operation "
-                         "would be tracked inside GlusterFS data-structures. "
-        },
-        { .key  = {"count-fop-hits"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .op_version = {1},
-          .flags = OPT_FLAG_SETTABLE,
-          .tags = {"io-stats"},
-        },
-        { .key = {"log-level"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
-                     "CRITICAL", "NONE", "TRACE"}
-        },
-
-        /* These are synthetic entries to assist validation of CLI's  *
-         *  volume set  command                                       */
-        { .key = {"client-log-level"},
-          .type = GF_OPTION_TYPE_STR,
-          .op_version = {1},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .default_value = "INFO",
-          .description = "Changes the log-level of the clients",
-          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
-                     "CRITICAL", "NONE", "TRACE"}
-        },
-        { .key = {"sys-log-level"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "CRITICAL",
-          .description = "Gluster's syslog log-level",
-          .value = { "WARNING", "ERROR", "INFO", "CRITICAL"}
-        },
-        { .key = {"brick-log-level"},
-          .type = GF_OPTION_TYPE_STR,
-          .op_version = {1},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .default_value = "INFO",
-          .description = "Changes the log-level of the bricks",
-          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
-                     "CRITICAL", "NONE", "TRACE"}
-        },
-        { .key = {"logger"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}
-        },
-        { .key = {"client-logger"},
-          .type = GF_OPTION_TYPE_STR,
-          .op_version = {GD_OP_VERSION_3_6_0},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .default_value = GF_LOGGER_GLUSTER_LOG,
-          .description = "Changes the logging sub-system to log to, for the "
-                         "clients",
-          .value = { GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}
-        },
-        { .key = {"brick-logger"},
-          .type = GF_OPTION_TYPE_STR,
-          .op_version = {GD_OP_VERSION_3_6_0},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .default_value = GF_LOGGER_GLUSTER_LOG,
-          .description = "Changes the logging sub-system to log to, for the "
-                         "bricks",
-          .value = { GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}
-        },
-        { .key = {"log-format"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}
-        },
-        { .key = {"client-log-format"},
-          .type = GF_OPTION_TYPE_STR,
-          .op_version = {GD_OP_VERSION_3_6_0},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .default_value = GF_LOG_FORMAT_WITH_MSG_ID,
-          .description = "Changes log format for the clients",
-          .value = { GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}
-        },
-        { .key = {"brick-log-format"},
-          .type = GF_OPTION_TYPE_STR,
-          .op_version = {GD_OP_VERSION_3_6_0},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .default_value = GF_LOG_FORMAT_WITH_MSG_ID,
-          .description = "Changes the log format for the bricks",
-          .value = { GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}
-        },
-        { .key  = {"log-buf-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = GF_LOG_LRU_BUFSIZE_MIN,
-          .max  = GF_LOG_LRU_BUFSIZE_MAX,
-          .default_value = "5",
-        },
-        { .key  = {"client-log-buf-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .op_version = {GD_OP_VERSION_3_6_0},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .min  = GF_LOG_LRU_BUFSIZE_MIN,
-          .max  = GF_LOG_LRU_BUFSIZE_MAX,
-          .default_value = "5",
-          .description = "This option determines the maximum number of unique "
-                         "log messages that can be buffered for a time equal to"
-                         " the value of the option client-log-flush-timeout."
-        },
-        { .key  = {"brick-log-buf-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .op_version = {GD_OP_VERSION_3_6_0},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .min  = GF_LOG_LRU_BUFSIZE_MIN,
-          .max  = GF_LOG_LRU_BUFSIZE_MAX,
-          .default_value = "5",
-          .description = "This option determines the maximum number of unique "
-                         "log messages that can be buffered for a time equal to"
-                         " the value of the option brick-log-flush-timeout."
-        },
-        { .key  = {"log-flush-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .min  = GF_LOG_FLUSH_TIMEOUT_MIN,
-          .max  = GF_LOG_FLUSH_TIMEOUT_MAX,
-          .default_value = "120",
-        },
-        { .key  = {"client-log-flush-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .op_version = {GD_OP_VERSION_3_6_0},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .min  = GF_LOG_FLUSH_TIMEOUT_MIN,
-          .max  = GF_LOG_FLUSH_TIMEOUT_MAX,
-          .default_value = "120",
-          .description = "This option determines the maximum number of unique "
-                         "log messages that can be buffered for a time equal to"
-                         " the value of the option client-log-flush-timeout."
-        },
-        { .key  = {"brick-log-flush-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .op_version = {GD_OP_VERSION_3_6_0},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
-          .tags = {"io-stats"},
-          .min  = GF_LOG_FLUSH_TIMEOUT_MIN,
-          .max  = GF_LOG_FLUSH_TIMEOUT_MAX,
-          .default_value = "120",
-          .description = "This option determines the maximum number of unique "
-                         "log messages that can be buffered for a time equal to"
-                         " the value of the option brick-log-flush-timeout."
-        },
-        { .key = {"unique-id"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "/no/such/path",
-          .description = "Unique ID for our files."
-        },
-        { .key  = {NULL} },
+    {.key = {"dump-fd-stats"},
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "If on stats related to file-operations would be "
+                    "tracked inside GlusterFS data-structures."},
+    {.key = {"ios-dump-interval"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = 0,
+     .max = 3600,
+     .default_value = "0",
+     .description = "Interval (in seconds) at which to auto-dump "
+                    "statistics. Zero disables automatic dumping."},
+    {.key = {"ios-sample-interval"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = 0,
+     .max = 65535,
+     .default_value = "0",
+     .description = "Interval in which we want to collect FOP latency "
+                    "samples.  2 means collect a sample every 2nd FOP."},
+    {.key = {"ios-dump-format"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_12_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = "json",
+     .description = " The dump-format option specifies the format in which"
+                    " to dump the statistics. Select between \"text\", "
+                    "\"json\", \"dict\" and \"samples\". Default is "
+                    "\"json\".",
+     .value = {"text", "json", "dict", "samples"}},
+    {.key = {"ios-sample-buf-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = 1024,
+     .max = 1024 * 1024,
+     .default_value = "65535",
+     .description = "The maximum size of our FOP sampling ring buffer."},
+    {.key = {"ios-dnscache-ttl-sec"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = 1,
+     .max = 3600 * 72,
+     .default_value = "86400",
+     .description = "The interval after wish a cached DNS entry will be "
+                    "re-validated.  Default: 24 hrs"},
+    {.key = {"latency-measurement"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = "off",
+     .description = "If on stats related to the latency of each operation "
+                    "would be tracked inside GlusterFS data-structures. "},
+    {
+        .key = {"count-fop-hits"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .op_version = {1},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"io-stats"},
+    },
+    {.key = {"log-level"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {"DEBUG", "WARNING", "ERROR", "INFO", "CRITICAL", "NONE",
+               "TRACE"}},
+
+    /* These are synthetic entries to assist validation of CLI's  *
+     *  volume set  command                                       */
+    {.key = {"client-log-level"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = "INFO",
+     .description = "Changes the log-level of the clients",
+     .value = {"DEBUG", "WARNING", "ERROR", "INFO", "CRITICAL", "NONE",
+               "TRACE"}},
+    {.key = {"sys-log-level"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "CRITICAL",
+     .description = "Gluster's syslog log-level",
+     .value = {"WARNING", "ERROR", "INFO", "CRITICAL"}},
+    {.key = {"brick-log-level"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = "INFO",
+     .description = "Changes the log-level of the bricks",
+     .value = {"DEBUG", "WARNING", "ERROR", "INFO", "CRITICAL", "NONE",
+               "TRACE"}},
+    {.key = {"logger"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}},
+    {.key = {"client-logger"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = GF_LOGGER_GLUSTER_LOG,
+     .description = "Changes the logging sub-system to log to, for the "
+                    "clients",
+     .value = {GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}},
+    {.key = {"brick-logger"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = GF_LOGGER_GLUSTER_LOG,
+     .description = "Changes the logging sub-system to log to, for the "
+                    "bricks",
+     .value = {GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}},
+    {.key = {"log-format"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}},
+    {.key = {"client-log-format"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = GF_LOG_FORMAT_WITH_MSG_ID,
+     .description = "Changes log format for the clients",
+     .value = {GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}},
+    {.key = {"brick-log-format"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = GF_LOG_FORMAT_WITH_MSG_ID,
+     .description = "Changes the log format for the bricks",
+     .value = {GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}},
+    {
+        .key = {"log-buf-size"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = GF_LOG_LRU_BUFSIZE_MIN,
+        .max = GF_LOG_LRU_BUFSIZE_MAX,
+        .default_value = "5",
+    },
+    {.key = {"client-log-buf-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = GF_LOG_LRU_BUFSIZE_MIN,
+     .max = GF_LOG_LRU_BUFSIZE_MAX,
+     .default_value = "5",
+     .description = "This option determines the maximum number of unique "
+                    "log messages that can be buffered for a time equal to"
+                    " the value of the option client-log-flush-timeout."},
+    {.key = {"brick-log-buf-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = GF_LOG_LRU_BUFSIZE_MIN,
+     .max = GF_LOG_LRU_BUFSIZE_MAX,
+     .default_value = "5",
+     .description = "This option determines the maximum number of unique "
+                    "log messages that can be buffered for a time equal to"
+                    " the value of the option brick-log-flush-timeout."},
+    {
+        .key = {"log-flush-timeout"},
+        .type = GF_OPTION_TYPE_TIME,
+        .min = GF_LOG_FLUSH_TIMEOUT_MIN,
+        .max = GF_LOG_FLUSH_TIMEOUT_MAX,
+        .default_value = "120",
+    },
+    {.key = {"client-log-flush-timeout"},
+     .type = GF_OPTION_TYPE_TIME,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = GF_LOG_FLUSH_TIMEOUT_MIN,
+     .max = GF_LOG_FLUSH_TIMEOUT_MAX,
+     .default_value = "120",
+     .description = "This option determines the maximum number of unique "
+                    "log messages that can be buffered for a time equal to"
+                    " the value of the option client-log-flush-timeout."},
+    {.key = {"brick-log-flush-timeout"},
+     .type = GF_OPTION_TYPE_TIME,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = GF_LOG_FLUSH_TIMEOUT_MIN,
+     .max = GF_LOG_FLUSH_TIMEOUT_MAX,
+     .default_value = "120",
+     .description = "This option determines the maximum number of unique "
+                    "log messages that can be buffered for a time equal to"
+                    " the value of the option brick-log-flush-timeout."},
+    {.key = {"unique-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "/no/such/path",
+     .description = "Unique ID for our files."},
+    {.key = {"global-threading"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {GD_OP_VERSION_6_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .tags = {"io-stats", "threading"},
+     .description = "This option enables the global threading support for "
+                    "bricks. If enabled, it's recommended to also enable "
+                    "'performance.iot-pass-through'"},
+    {.key = {"threads"}, .type = GF_OPTION_TYPE_INT},
+    {.key = {"brick-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "16",
+     .min = 0,
+     .max = GF_ASYNC_MAX_THREADS,
+     .op_version = {GD_OP_VERSION_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats", "threading"},
+     .description = "When global threading is used, this value determines the "
+                    "maximum amount of threads that can be created on bricks"},
+    {.key = {"client-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "16",
+     .min = 0,
+     .max = GF_ASYNC_MAX_THREADS,
+     .op_version = {GD_OP_VERSION_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"io-stats", "threading"},
+     .description = "When global threading is used, this value determines the "
+                    "maximum amount of threads that can be created on clients"},
+    {.key = {"volume-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_7_1},
+     .tags = {"global", "volume-id"},
+     .description =
+         "This option points to the 'unique' UUID particular to this "
+         "volume, which would be set in 'graph->volume_id'"},
+    {.key = {NULL}},
+};
 
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "io-stats",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/debug/sink/src/sink.c b/xlators/debug/sink/src/sink.c
index dfd5685a969..9822bbb732e 100644
--- a/xlators/debug/sink/src/sink.c
+++ b/xlators/debug/sink/src/sink.c
@@ -8,41 +8,41 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        return 0;
+    return 0;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        return;
+    return;
 }
 
 /*
  * notify - when parent sends PARENT_UP, send CHILD_UP event from here
  */
 int32_t
-notify (xlator_t *this, int32_t event, void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        switch (event) {
+    switch (event) {
         case GF_EVENT_PARENT_UP:
-                /* Tell the parent that this xlator is up */
-                default_notify (this, GF_EVENT_CHILD_UP, data);
-                break;
+            /* Tell the parent that this xlator is up */
+            default_notify(this, GF_EVENT_CHILD_UP, data);
+            break;
         case GF_EVENT_PARENT_DOWN:
-                /* Tell the parent that this xlator is down */
-                default_notify (this, GF_EVENT_CHILD_DOWN, data);
-                break;
+            /* Tell the parent that this xlator is down */
+            default_notify(this, GF_EVENT_CHILD_DOWN, data);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        return 0;
+    return 0;
 }
 
 /*
@@ -53,27 +53,42 @@ notify (xlator_t *this, int32_t event, void *data, ...)
  * extended to support different LOOKUPs too.
  */
 static int32_t
-sink_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+sink_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        struct iatt stbuf = { 0, };
-        struct iatt postparent = { 0, };
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
 
-        /* the root of the volume always need to be a directory */
-        stbuf.ia_type = IA_IFDIR;
+    /* the root of the volume always need to be a directory */
+    stbuf.ia_type = IA_IFDIR;
 
-        STACK_UNWIND_STRICT (lookup, frame, 0, 0, loc ? loc->inode : NULL,
-                             &stbuf, xdata, &postparent);
+    STACK_UNWIND_STRICT(lookup, frame, 0, 0, loc ? loc->inode : NULL, &stbuf,
+                        xdata, &postparent);
 
-        return 0;
+    return 0;
 }
 
 struct xlator_fops fops = {
-        .lookup = sink_lookup,
+    .lookup = sink_lookup,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks = {};
 
 struct volume_options options[] = {
-        { .key  = {NULL} },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .op_version = {GD_OP_VERSION_3_12_0},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "sink",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/debug/trace/src/trace-mem-types.h b/xlators/debug/trace/src/trace-mem-types.h
index 9fa7d97c2ca..18a7e0414a6 100644
--- a/xlators/debug/trace/src/trace-mem-types.h
+++ b/xlators/debug/trace/src/trace-mem-types.h
@@ -8,14 +8,13 @@
    cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __TRACE_MEM_TYPES_H__
 #define __TRACE_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_trace_mem_types_ {
-        gf_trace_mt_trace_conf_t = gf_common_mt_end + 1,
-        gf_trace_mt_end
+    gf_trace_mt_trace_conf_t = gf_common_mt_end + 1,
+    gf_trace_mt_end
 };
 #endif
diff --git a/xlators/debug/trace/src/trace.c b/xlators/debug/trace/src/trace.c
index be2f84d936f..6ed0ca00342 100644
--- a/xlators/debug/trace/src/trace.c
+++ b/xlators/debug/trace/src/trace.c
@@ -17,3351 +17,3518 @@
  *    their _cbk functions, which later passes the call to next layer.
  *    Very helpful translator for debugging.
  */
-#define TRACE_STAT_TO_STR(buf, str) trace_stat_to_str (buf, str, sizeof (str))
+#define TRACE_STAT_TO_STR(buf, str) trace_stat_to_str(buf, str, sizeof(str))
 
 static void
 trace_stat_to_str(struct iatt *buf, char *str, size_t len)
 {
-        char     atime_buf[256]    = {0,};
-        char     mtime_buf[256]    = {0,};
-        char     ctime_buf[256]    = {0,};
-
-        if (!buf)
-                return;
+    char atime_buf[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char mtime_buf[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char ctime_buf[GF_TIMESTR_SIZE] = {
+        0,
+    };
+
+    if (!buf)
+        return;
 
-        gf_time_fmt (atime_buf, sizeof atime_buf, buf->ia_atime,
-                     gf_timefmt_dirent);
+    gf_time_fmt(atime_buf, sizeof atime_buf, buf->ia_atime, gf_timefmt_dirent);
 
-        gf_time_fmt (mtime_buf, sizeof mtime_buf, buf->ia_mtime,
-                     gf_timefmt_dirent);
+    gf_time_fmt(mtime_buf, sizeof mtime_buf, buf->ia_mtime, gf_timefmt_dirent);
 
-        gf_time_fmt (ctime_buf, sizeof ctime_buf, buf->ia_ctime,
-                     gf_timefmt_dirent);
+    gf_time_fmt(ctime_buf, sizeof ctime_buf, buf->ia_ctime, gf_timefmt_dirent);
 
-        snprintf (str, len, "gfid=%s ino=%"PRIu64", mode=%o, "
-                  "nlink=%"GF_PRI_NLINK", uid=%u, gid=%u, size=%"PRIu64", "
-                  "blocks=%"PRIu64", atime=%s mtime=%s ctime=%s "
-                  "atime_sec=%"PRId64", atime_nsec=%"PRIu32","
-                  " mtime_sec=%"PRId64", mtime_nsec=%"PRIu32", "
-                  "ctime_sec=%"PRId64", ctime_nsec=%"PRIu32"",
-                  uuid_utoa (buf->ia_gfid), buf->ia_ino,
-                  st_mode_from_ia (buf->ia_prot, buf->ia_type), buf->ia_nlink,
-                  buf->ia_uid, buf->ia_gid, buf->ia_size, buf->ia_blocks,
-                  atime_buf, mtime_buf, ctime_buf,
-                  buf->ia_atime, buf->ia_atime_nsec,
-                  buf->ia_mtime, buf->ia_mtime_nsec,
-                  buf->ia_ctime, buf->ia_ctime_nsec);
+    snprintf(str, len,
+             "gfid=%s ino=%" PRIu64
+             ", mode=%o, "
+             "nlink=%" GF_PRI_NLINK ", uid=%u, gid=%u, size=%" PRIu64
+             ", "
+             "blocks=%" PRIu64
+             ", atime=%s mtime=%s ctime=%s "
+             "atime_sec=%" PRId64 ", atime_nsec=%" PRIu32
+             ","
+             " mtime_sec=%" PRId64 ", mtime_nsec=%" PRIu32
+             ", "
+             "ctime_sec=%" PRId64 ", ctime_nsec=%" PRIu32 "",
+             uuid_utoa(buf->ia_gfid), buf->ia_ino,
+             st_mode_from_ia(buf->ia_prot, buf->ia_type), buf->ia_nlink,
+             buf->ia_uid, buf->ia_gid, buf->ia_size, buf->ia_blocks, atime_buf,
+             mtime_buf, ctime_buf, buf->ia_atime, buf->ia_atime_nsec,
+             buf->ia_mtime, buf->ia_mtime_nsec, buf->ia_ctime,
+             buf->ia_ctime_nsec);
 }
 
-
 int
-dump_history_trace (circular_buffer_t *cb, void *data)
+dump_history_trace(circular_buffer_t *cb, void *data)
 {
-        char     timestr[256] = {0,};
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
 
-        /* Since we are continuing with adding entries to the buffer even when
-           gettimeofday () fails, it's safe to check tm and then dump the time
-           at which the entry was added to the buffer */
+    /* Since we are continuing with adding entries to the buffer even when
+       gettimeofday () fails, it's safe to check tm and then dump the time
+       at which the entry was added to the buffer */
 
-        gf_time_fmt (timestr, sizeof timestr, cb->tv.tv_sec, gf_timefmt_Ymd_T);
-        snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
-                  ".%"GF_PRI_SUSECONDS, cb->tv.tv_usec);
-        gf_proc_dump_write ("TIME", "%s", timestr);
+    gf_time_fmt_tv(timestr, sizeof timestr, &cb->tv, gf_timefmt_Ymd_T);
+    gf_proc_dump_write("TIME", "%s", timestr);
 
-        gf_proc_dump_write ("FOP", "%s\n", cb->data);
+    gf_proc_dump_write("FOP", "%s\n", (char *)cb->data);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd,
-                  inode_t *inode, struct iatt *buf,
-                  struct iatt *preparent, struct iatt *postparent,
-                  dict_t *xdata)
+trace_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_CREATE].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret >= 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d, fd=%p"
-                                  "*stbuf {%s}, *preparent {%s}, "
-                                  "*postparent = {%s})",
-                                  frame->root->unique,
-                                  uuid_utoa (inode->gfid), op_ret, fd,
-                                  statstr, preparentstr, postparentstr);
-
-                        /* for 'release' log */
-                        fd_ctx_set (fd, this, 0);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
-}
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
 
-int
-trace_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        trace_conf_t      *conf = NULL;
+    conf = this->private;
 
-        conf = this->private;
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_CREATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret >= 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPEN].enabled) {
-                char     string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, "
-                          "*fd=%p", frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno,
-                          fd);
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d, fd=%p"
+                     "*stbuf {%s}, *preparent {%s}, "
+                     "*postparent = {%s})",
+                     frame->root->unique, uuid_utoa(inode->gfid), op_ret, fd,
+                     statstr, preparentstr, postparentstr);
 
-                LOG_ELEMENT (conf, string);
+            /* for 'release' log */
+            fd_ctx_set(fd, this, 0);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": (op_ret=%d, op_errno=%d)",
+                     frame->root->unique, op_ret, op_errno);
         }
-
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        /* for 'release' log */
-        if (op_ret >= 0)
-                fd_ctx_set (fd, this, 0);
-
-        TRACE_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, buf,
+                       preparent, postparent, xdata);
+    return 0;
 }
 
 int
-trace_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                dict_t *xdata)
+trace_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        char          statstr[4096] = {0, };
-        trace_conf_t  *conf         = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_STAT].enabled) {
-                char string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  statstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
+    trace_conf_t *conf = NULL;
 
-int
-trace_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                 int32_t count, struct iatt *buf, struct iobref *iobref,
-                 dict_t *xdata)
-{
-        char          statstr[4096] = {0, };
-        trace_conf_t  *conf         = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READ].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret >= 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  statstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
-                            buf, iobref, xdata);
-        return 0;
-}
-
-int
-trace_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
-{
-        char         preopstr[4096]  = {0, };
-        char         postopstr[4096] = {0, };
-        trace_conf_t *conf           = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_WRITE].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret >= 0) {
-                        TRACE_STAT_TO_STR (prebuf, preopstr);
-                        TRACE_STAT_TO_STR (postbuf, postopstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s})",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-        return 0;
-}
-
-int
-trace_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
-                   dict_t *xdata)
-{
-        trace_conf_t  *conf = NULL;
+    conf = this->private;
 
-        conf = this->private;
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPEN].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, op_errno=%d, "
+                 "*fd=%p",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 fd);
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READDIR].enabled) {
-                char    string[4096]  = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique, uuid_utoa (frame->local),
-                          op_ret, op_errno);
+        LOG_ELEMENT(conf, string);
+    }
 
-                LOG_ELEMENT (conf, string);
-        }
 out:
-        TRACE_STACK_UNWIND (readdir, frame, op_ret, op_errno, buf, xdata);
+    /* for 'release' log */
+    if (op_ret >= 0)
+        fd_ctx_set(fd, this, 0);
 
-        return 0;
+    TRACE_STACK_UNWIND(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
 int
-trace_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
-                    dict_t *xdata)
+trace_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               dict_t *xdata)
 {
-        int             count         = 0;
-        char            statstr[4096] = {0,};
-        char            string[4096]  = {0,};
-        trace_conf_t   *conf          = NULL;
-        gf_dirent_t    *entry         = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READDIRP].enabled) {
-                snprintf (string, sizeof (string),
-                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique, uuid_utoa (frame->local),
-                          op_ret, op_errno);
-
-                LOG_ELEMENT (conf, string);
-        }
-        if (op_ret < 0)
-                goto out;
-
-        list_for_each_entry (entry, &buf->list, list) {
-                count++;
-                TRACE_STAT_TO_STR (&entry->d_stat, statstr);
-                snprintf (string, sizeof (string), "entry no. %d, pargfid=%s, "
-                          "bname=%s *buf {%s}", count, uuid_utoa (frame->local),
-                          entry->d_name, statstr);
-                LOG_ELEMENT (conf, string);
-        }
-
-out:
-        TRACE_STACK_UNWIND (readdirp, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
-
-int
-trace_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+    char statstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_STAT].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            (void)snprintf(
+                string, sizeof(string), "%" PRId64 ": gfid=%s op_ret=%d buf=%s",
+                frame->root->unique, uuid_utoa(frame->local), op_ret, statstr);
+        } else {
+            (void)snprintf(string, sizeof(string),
+                           "%" PRId64
+                           ": gfid=%s op_ret=%d, "
+                           "op_errno=%d)",
+                           frame->root->unique, uuid_utoa(frame->local), op_ret,
+                           op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int
+trace_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                int32_t count, struct iatt *buf, struct iobref *iobref,
+                dict_t *xdata)
 {
-        char          preopstr[4096]  = {0, };
-        char          postopstr[4096] = {0, };
-        trace_conf_t  *conf           = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (prebuf, preopstr);
-                        TRACE_STAT_TO_STR (postbuf, postopstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s}",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-
-        return 0;
-}
-
-int
-trace_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+    char statstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READ].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret >= 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            snprintf(
+                string, sizeof(string), "%" PRId64 ": gfid=%s op_ret=%d buf=%s",
+                frame->root->unique, uuid_utoa(frame->local), op_ret, statstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, buf,
+                       iobref, xdata);
+    return 0;
+}
+
+int
+trace_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_WRITE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret >= 0) {
+            TRACE_STAT_TO_STR(prebuf, preopstr);
+            TRACE_STAT_TO_STR(postbuf, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s})",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
+
+int
+trace_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                  dict_t *xdata)
 {
-        char          preopstr[4096]  = {0, };
-        char          postopstr[4096] = {0, };
-        trace_conf_t  *conf           = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
-                char  string[4096]  = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (statpre, preopstr);
-                        TRACE_STAT_TO_STR (statpost, postopstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s})",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (setattr, frame, op_ret, op_errno, statpre,
-                            statpost, xdata);
-        return 0;
-}
-
-int
-trace_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
-{
-        char          preopstr[4096]  = {0, };
-        char          postopstr[4096] = {0, };
-        trace_conf_t  *conf           = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (statpre, preopstr);
-                        TRACE_STAT_TO_STR (statpost, postopstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s})",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                  frame->root->unique, uuid_utoa (frame->local),
-                                  op_ret, op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
-                            statpre, statpost, xdata);
-        return 0;
-}
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 " : gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-int
-trace_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iatt *preparent, struct iatt *postparent,
-                  dict_t *xdata)
-{
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-                char string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  " *preparent = {%s}, "
-                                  "*postparent = {%s})",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, preparentstr,
-                                  postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (unlink, frame, op_ret, op_errno,
-                            preparent, postparent, xdata);
-        return 0;
-}
+    TRACE_STACK_UNWIND(readdir, frame, op_ret, op_errno, buf, xdata);
 
-int
-trace_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                  struct iatt *preoldparent, struct iatt *postoldparent,
-                  struct iatt *prenewparent, struct iatt *postnewparent,
-                  dict_t *xdata)
-{
-        char           statstr[4096]          = {0, };
-        char           preoldparentstr[4096]  = {0, };
-        char           postoldparentstr[4096] = {0, };
-        char           prenewparentstr[4096]  = {0, };
-        char           postnewparentstr[4096] = {0, };
-        trace_conf_t   *conf                  = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RENAME].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preoldparent, preoldparentstr);
-                        TRACE_STAT_TO_STR (postoldparent, postoldparentstr);
-                        TRACE_STAT_TO_STR (prenewparent, prenewparentstr);
-                        TRACE_STAT_TO_STR (postnewparent, postnewparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*stbuf = {%s}, *preoldparent = {%s},"
-                                  " *postoldparent = {%s}"
-                                  " *prenewparent = {%s}, "
-                                  "*postnewparent = {%s})",
-                                  frame->root->unique, op_ret, statstr,
-                                  preoldparentstr, postoldparentstr,
-                                  prenewparentstr, postnewparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, op_errno);
-
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (rename, frame, op_ret, op_errno, buf,
-                            preoldparent, postoldparent,
-                            prenewparent, postnewparent, xdata);
-        return 0;
-}
-
-int
-trace_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    const char *buf, struct iatt *stbuf, dict_t *xdata)
-{
-        char          statstr[4096] = {0, };
-        trace_conf_t  *conf         = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READLINK].enabled) {
-                char string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (stbuf, statstr);
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, op_errno=%d,"
-                                  "buf=%s, stbuf = { %s })",
-                                  frame->root->unique, op_ret, op_errno,
-                                  buf, statstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, stbuf,
-                            xdata);
-        return 0;
-}
-
-int
-trace_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  inode_t *inode, struct iatt *buf,
-                  dict_t *xdata, struct iatt *postparent)
-{
-        char          statstr[4096]       = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-                        /* print buf->ia_gfid instead of inode->gfid,
-                         * since if the inode is not yet linked to the
-                         * inode table (fresh lookup) then null gfid
-                         * will be printed.
-                         */
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d "
-                                  "*buf {%s}, *postparent {%s}",
-                                  frame->root->unique,
-                                  uuid_utoa (buf->ia_gfid),
-                                  op_ret, statstr, postparentstr);
-
-                        /* For 'forget' */
-                        inode_ctx_put (inode, this, 0);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
-                            xdata, postparent);
-        return 0;
+    return 0;
 }
 
 int
-trace_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent,
+trace_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
                    dict_t *xdata)
 {
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d "
-                                  "*stbuf = {%s}, *preparent = {%s}, "
-                                  "*postparent = {%s})",
-                                  frame->root->unique,
-                                  uuid_utoa (inode->gfid),
-                                  op_ret, statstr, preparentstr,
-                                  postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": op_ret=%d, op_errno=%d",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        char string[4096]  = {0,};
-        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d "
-                                  "*stbuf = {%s}, *preparent = {%s}, "
-                                  "*postparent = {%s})",
-                                  frame->root->unique,
-                                  uuid_utoa (inode->gfid),
-                                  op_ret, statstr, preparentstr,
-                                  postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
-                char  string[4096]  = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d "
-                                  ", *stbuf = {%s}, *prebuf = {%s}, "
-                                  "*postbuf = {%s} )",
-                                  frame->root->unique,
-                                  uuid_utoa (inode->gfid),
-                                  op_ret, statstr, preparentstr,
-                                  postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno,
-                inode_t *inode, struct iatt *buf,
-                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        char  string[4096]  = {0,};
-        if (trace_fop_names[GF_FOP_LINK].enabled) {
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*stbuf = {%s},  *prebuf = {%s},"
-                                  " *postbuf = {%s})",
-                                  frame->root->unique, op_ret,
-                                  statstr, preparentstr, postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+    int count = 0;
+    char statstr[1024] = {
+        0,
+    };
+    char string[4096] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+    gf_dirent_t *entry = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READDIRP].enabled) {
+        snprintf(string, sizeof(string),
+                 "%" PRId64 " : gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
+
+        LOG_ELEMENT(conf, string);
+    }
+    if (op_ret < 0)
+        goto out;
+
+    list_for_each_entry(entry, &buf->list, list)
+    {
+        count++;
+        TRACE_STAT_TO_STR(&entry->d_stat, statstr);
+        snprintf(string, sizeof(string),
+                 "entry no. %d, pargfid=%s, "
+                 "bname=%s *buf {%s}",
+                 count, uuid_utoa(frame->local), entry->d_name, statstr);
+        LOG_ELEMENT(conf, string);
+    }
+
+out:
+    TRACE_STACK_UNWIND(readdirp, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int
+trace_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSYNC].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(prebuf, preopstr);
+            TRACE_STAT_TO_STR(postbuf, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s}",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+
+    return 0;
+}
+
+int
+trace_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                  struct iatt *statpost, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(statpre, preopstr);
+            TRACE_STAT_TO_STR(statpost, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s})",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(setattr, frame, op_ret, op_errno, statpre, statpost,
+                       xdata);
+    return 0;
+}
+
+int
+trace_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                   struct iatt *statpost, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(statpre, preopstr);
+            TRACE_STAT_TO_STR(statpost, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s})",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(fsetattr, frame, op_ret, op_errno, statpre, statpost,
+                       xdata);
+    return 0;
+}
+
+int
+trace_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_UNLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     " *preparent = {%s}, "
+                     "*postparent = {%s})",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(unlink, frame, op_ret, op_errno, preparent, postparent,
+                       xdata);
+    return 0;
+}
+
+int
+trace_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 struct iatt *preoldparent, struct iatt *postoldparent,
+                 struct iatt *prenewparent, struct iatt *postnewparent,
+                 dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preoldparentstr[1024] = {
+        0,
+    };
+    char postoldparentstr[1024] = {
+        0,
+    };
+    char prenewparentstr[1024] = {
+        0,
+    };
+    char postnewparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RENAME].enabled) {
+        char string[6044] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preoldparent, preoldparentstr);
+            TRACE_STAT_TO_STR(postoldparent, postoldparentstr);
+            TRACE_STAT_TO_STR(prenewparent, prenewparentstr);
+            TRACE_STAT_TO_STR(postnewparent, postnewparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*stbuf = {%s}, *preoldparent = {%s},"
+                     " *postoldparent = {%s}"
+                     " *prenewparent = {%s}, "
+                     "*postnewparent = {%s})",
+                     frame->root->unique, op_ret, statstr, preoldparentstr,
+                     postoldparentstr, prenewparentstr, postnewparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(rename, frame, op_ret, op_errno, buf, preoldparent,
+                       postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
+}
+
+int
+trace_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, const char *buf,
+                   struct iatt *stbuf, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(stbuf, statstr);
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, op_errno=%d,"
+                     "buf=%s, stbuf = { %s })",
+                     frame->root->unique, op_ret, op_errno, buf, statstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(readlink, frame, op_ret, op_errno, buf, stbuf, xdata);
+    return 0;
+}
+
+int
+trace_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+            /* print buf->ia_gfid instead of inode->gfid,
+             * since if the inode is not yet linked to the
+             * inode table (fresh lookup) then null gfid
+             * will be printed.
+             */
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d "
+                     "*buf {%s}, *postparent {%s}",
+                     frame->root->unique, uuid_utoa(buf->ia_gfid), op_ret,
+                     statstr, postparentstr);
+
+            /* For 'forget' */
+            inode_ctx_put(inode, this, 0);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                       postparent);
+    return 0;
+}
+
+int
+trace_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d "
+                     "*stbuf = {%s}, *preparent = {%s}, "
+                     "*postparent = {%s})",
+                     frame->root->unique, uuid_utoa(inode->gfid), op_ret,
+                     statstr, preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": op_ret=%d, op_errno=%d", frame->root->unique,
+                     op_ret, op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
+}
+
+int
+trace_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    char string[4096] = {
+        0,
+    };
+    if (trace_fop_names[GF_FOP_MKNOD].enabled) {
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d "
+                     "*stbuf = {%s}, *preparent = {%s}, "
+                     "*postparent = {%s})",
+                     frame->root->unique, uuid_utoa(inode->gfid), op_ret,
+                     statstr, preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": (op_ret=%d, op_errno=%d)",
+                     frame->root->unique, op_ret, op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
+}
+
+int
+trace_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_MKDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d "
+                     ", *stbuf = {%s}, *prebuf = {%s}, "
+                     "*postbuf = {%s} )",
+                     frame->root->unique, uuid_utoa(inode->gfid), op_ret,
+                     statstr, preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": (op_ret=%d, op_errno=%d)",
+                     frame->root->unique, op_ret, op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
+}
+
+int
+trace_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    char string[4096] = {
+        0,
+    };
+    if (trace_fop_names[GF_FOP_LINK].enabled) {
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*stbuf = {%s},  *prebuf = {%s},"
+                     " *postbuf = {%s})",
+                     frame->root->unique, op_ret, statstr, preparentstr,
+                     postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(link, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
+}
+
+int
+trace_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    char string[4096] = {
+        0,
+    };
+    if (trace_fop_names[GF_FOP_FLUSH].enabled) {
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(flush, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int
+trace_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    char string[4096] = {
+        0,
+    };
+    if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, op_errno=%d,"
+                 " fd=%p",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 fd);
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    /* for 'releasedir' log */
+    if (op_ret >= 0)
+        fd_ctx_set(fd, this, 0);
+
+    TRACE_STACK_UNWIND(opendir, frame, op_ret, op_errno, fd, xdata);
+    return 0;
+}
+
+int
+trace_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RMDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "*prebuf={%s},  *postbuf={%s}",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                       xdata);
+    return 0;
+}
+
+int
+trace_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(prebuf, preopstr);
+            TRACE_STAT_TO_STR(postbuf, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s} )",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                       xdata);
+    return 0;
+}
+
+int
+trace_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                 dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_STATFS].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": ({f_bsize=%lu, "
+                     "f_frsize=%lu, "
+                     "f_blocks=%" GF_PRI_FSBLK ", f_bfree=%" GF_PRI_FSBLK
+                     ", "
+                     "f_bavail=%" GF_PRI_FSBLK
+                     ", "
+                     "f_files=%" GF_PRI_FSBLK
+                     ", "
+                     "f_ffree=%" GF_PRI_FSBLK
+                     ", "
+                     "f_favail=%" GF_PRI_FSBLK
+                     ", "
+                     "f_fsid=%lu, f_flag=%lu, "
+                     "f_namemax=%lu}) => ret=%d",
+                     frame->root->unique, buf->f_bsize, buf->f_frsize,
+                     buf->f_blocks, buf->f_bfree, buf->f_bavail, buf->f_files,
+                     buf->f_ffree, buf->f_favail, buf->f_fsid, buf->f_flag,
+                     buf->f_namemax, op_ret);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, op_ret, op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(statfs, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int
+trace_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        char      string[4096] = {0,};
-        if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique, uuid_utoa (frame->local),
-                          op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+trace_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        char    string[4096] = {0,};
-        if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
-                          " fd=%p",
-                          frame->root->unique, uuid_utoa (frame->local),
-                          op_ret, op_errno, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, op_errno=%d,"
+                 " dict=%p",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 dict);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        /* for 'releasedir' log */
-        if (op_ret >= 0)
-                fd_ctx_set (fd, this, 0);
-
-        TRACE_STACK_UNWIND (opendir, frame, op_ret, op_errno, fd, xdata);
-        return 0;
-}
-
-int
-trace_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        char           preparentstr[4096]  = {0, };
-        char           postparentstr[4096] = {0, };
-        trace_conf_t   *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "*prebuf={%s},  *postbuf={%s}",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, preparentstr, postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
-{
-        char           preopstr[4096]  = {0, };
-        char           postopstr[4096] = {0, };
-        trace_conf_t   *conf           = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
-                char   string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (prebuf, preopstr);
-                        TRACE_STAT_TO_STR (postbuf, postopstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s} )",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
-                            postbuf, xdata);
-        return 0;
-}
+    TRACE_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
 
-int
-trace_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct statvfs *buf,
-                  dict_t *xdata)
-{
-        trace_conf_t  *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_STATFS].enabled) {
-                char   string[4096] = {0,};
-                if (op_ret == 0) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": ({f_bsize=%lu, "
-                                  "f_frsize=%lu, "
-                                  "f_blocks=%"GF_PRI_FSBLK
-                                  ", f_bfree=%"GF_PRI_FSBLK", "
-                                  "f_bavail=%"GF_PRI_FSBLK", "
-                                  "f_files=%"GF_PRI_FSBLK", "
-                                  "f_ffree=%"GF_PRI_FSBLK", "
-                                  "f_favail=%"GF_PRI_FSBLK", "
-                                  "f_fsid=%lu, f_flag=%lu, "
-                                  "f_namemax=%lu}) => ret=%d",
-                                  frame->root->unique, buf->f_bsize,
-                                  buf->f_frsize, buf->f_blocks,
-                                  buf->f_bfree, buf->f_bavail,
-                                  buf->f_files, buf->f_ffree,
-                                  buf->f_favail, buf->f_fsid,
-                                  buf->f_flag, buf->f_namemax, op_ret);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (statfs, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    return 0;
 }
 
 int
-trace_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t     *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret,
-                          op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, dict_t *dict,
                     dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-                char       string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
-                          " dict=%p", frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno,
-                          dict);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, op_errno=%d,"
+                 " dict=%p",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 dict);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
+    TRACE_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, dict, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+trace_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
-                char    string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
+    TRACE_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
 
-int
-trace_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict,
-                     dict_t *xdata)
-{
-        trace_conf_t    *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
-                char      string[4096]  = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
-                          " dict=%p", frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno,
-                          dict);
-
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
-
-        return 0;
+    return 0;
 }
 
 int
-trace_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+trace_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-                char       string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
-
-        return 0;
+    TRACE_STACK_UNWIND(fsyncdir, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+trace_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t     *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-                char       string[4096]   =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
-
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (fsyncdir, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-trace_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_ACCESS].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, "
+                 "op_errno=%d)",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int
+trace_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
+{
+    char prebufstr[1024] = {
+        0,
+    };
+    char postbufstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(prebuf, prebufstr);
+            TRACE_STAT_TO_STR(postbuf, postbufstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s} )",
+                     frame->root->unique, op_ret, prebufstr, postbufstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                       xdata);
+    return 0;
+}
+
+int
+trace_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSTAT].enabled) {
+        char string[4096] = {0.};
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d "
+                     "buf=%s",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     statstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int
+trace_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "{l_type=%d, l_whence=%d, "
+                     "l_start=%" PRId64
+                     ", "
+                     "l_len=%" PRId64 ", l_pid=%u})",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     lock->l_type, lock->l_whence, lock->l_start, lock->l_len,
+                     lock->l_pid);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(lk, frame, op_ret, op_errno, lock, xdata);
+    return 0;
+}
+
+int
+trace_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
-
-        conf = this->private;
+    trace_conf_t *conf = NULL;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, "
-                          "op_errno=%d)", frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    conf = this->private;
 
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-trace_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
-{
-        char          prebufstr[4096]  = {0, };
-        char          postbufstr[4096] = {0, };
-        trace_conf_t  *conf            = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-                char  string[4096]  = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (prebuf, prebufstr);
-                        TRACE_STAT_TO_STR (postbuf, postbufstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s} )",
-                                  frame->root->unique, op_ret,
-                                  prebufstr, postbufstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-        return 0;
-}
-
-int
-trace_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        char          statstr[4096] = {0, };
-        trace_conf_t  *conf         = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-                char string[4096]  = {0.};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d "
-                                  "buf=%s", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  statstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-int
-trace_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-              dict_t *xdata)
-{
-        trace_conf_t   *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LK].enabled) {
-                char      string[4096] = {0,};
-                if (op_ret == 0) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "{l_type=%d, l_whence=%d, "
-                                  "l_start=%"PRId64", "
-                                  "l_len=%"PRId64", l_pid=%u})",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, lock->l_type, lock->l_whence,
-                                  lock->l_start, lock->l_len,
-                                  lock->l_pid);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (lk, frame, op_ret, op_errno, lock, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(entrylk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
-                char   string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(fentrylk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+trace_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_XATTROP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int
-trace_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, dict_t *dict,
                    dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-                char    string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int
-trace_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict,
-                    dict_t *xdata)
+trace_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_INODELK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(inodelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_INODELK].enabled) {
-                char       string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local),op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FINODELK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-trace_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        trace_conf_t   *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FINODELK].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
-
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(finodelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     uint32_t weak_checksum, uint8_t *strong_checksum,
-                     dict_t *xdata)
+trace_rchecksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
+                    uint8_t *strong_checksum, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        TRACE_STACK_UNWIND (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                            strong_checksum, xdata);
+    TRACE_STACK_UNWIND(rchecksum, frame, op_ret, op_errno, weak_checksum,
+                       strong_checksum, xdata);
 
-        return 0;
+    return 0;
 }
 
 /* *_cbk section over <----------> fop section start */
 
 int
-trace_entrylk (call_frame_t *frame, xlator_t *this,
-               const char *volume, loc_t *loc, const char *basename,
-               entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+trace_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+              loc_t *loc, const char *basename, entrylk_cmd cmd,
+              entrylk_type type, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
-                char     string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s volume=%s, (path=%s "
-                          "basename=%s, cmd=%s, type=%s)",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid),
-                          volume, loc->path, basename,
-                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
-                           "ENTRYLK_UNLOCK"),
-                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
-                           "ENTRYLK_WRLCK"));
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s volume=%s, (path=%s "
+                 "basename=%s, cmd=%s, type=%s)",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), volume,
+                 loc->path, basename,
+                 ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" : "ENTRYLK_UNLOCK"),
+                 ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK"));
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_entrylk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->entrylk,
-                    volume, loc, basename, cmd, type, xdata);
-        return 0;
+    STACK_WIND(frame, trace_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, volume, loc, basename, cmd,
+               type, xdata);
+    return 0;
 }
 
 int
-trace_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-               loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+trace_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+              loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        char         *cmd_str  = NULL;
-        char         *type_str = NULL;
-        trace_conf_t *conf     = NULL;
+    char *cmd_str = NULL;
+    char *type_str = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_INODELK].enabled) {
-                char string[4096]  = {0,};
-                switch (cmd) {
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_INODELK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        switch (cmd) {
 #if F_GETLK != F_GETLK64
-                case F_GETLK64:
+            case F_GETLK64:
 #endif
-                case F_GETLK:
-                        cmd_str = "GETLK";
-                        break;
+            case F_GETLK:
+                cmd_str = "GETLK";
+                break;
 
 #if F_SETLK != F_SETLK64
-                case F_SETLK64:
+            case F_SETLK64:
 #endif
-                case F_SETLK:
-                        cmd_str = "SETLK";
-                        break;
+            case F_SETLK:
+                cmd_str = "SETLK";
+                break;
 
 #if F_SETLKW != F_SETLKW64
-                case F_SETLKW64:
+            case F_SETLKW64:
 #endif
-                case F_SETLKW:
-                        cmd_str = "SETLKW";
-                        break;
-
-                default:
-                        cmd_str = "UNKNOWN";
-                        break;
-                }
-
-                switch (flock->l_type) {
-                case F_RDLCK:
-                        type_str = "READ";
-                        break;
-                case F_WRLCK:
-                        type_str = "WRITE";
-                        break;
-                case F_UNLCK:
-                        type_str = "UNLOCK";
-                        break;
-                default:
-                        type_str = "UNKNOWN";
-                        break;
-                }
-
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s volume=%s, (path=%s "
-                          "cmd=%s, type=%s, start=%llu, len=%llu, "
-                          "pid=%llu)", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), volume,
-                          loc->path, cmd_str, type_str,
-                          (unsigned long long)flock->l_start,
-                          (unsigned long long) flock->l_len,
-                          (unsigned long long) flock->l_pid);
-
-                frame->local = loc->inode->gfid;
-
-                LOG_ELEMENT (conf, string);
+            case F_SETLKW:
+                cmd_str = "SETLKW";
+                break;
+
+            default:
+                cmd_str = "UNKNOWN";
+                break;
+        }
+
+        switch (flock->l_type) {
+            case F_RDLCK:
+                type_str = "READ";
+                break;
+            case F_WRLCK:
+                type_str = "WRITE";
+                break;
+            case F_UNLCK:
+                type_str = "UNLOCK";
+                break;
+            default:
+                type_str = "UNKNOWN";
+                break;
         }
 
+        snprintf(
+            string, sizeof(string),
+            "%" PRId64
+            ": gfid=%s volume=%s, (path=%s "
+            "cmd=%s, type=%s, start=%llu, len=%llu, "
+            "pid=%llu)",
+            frame->root->unique, uuid_utoa(loc->inode->gfid), volume, loc->path,
+            cmd_str, type_str, (unsigned long long)flock->l_start,
+            (unsigned long long)flock->l_len, (unsigned long long)flock->l_pid);
+
+        frame->local = loc->inode->gfid;
+
+        LOG_ELEMENT(conf, string);
+    }
+
 out:
-        STACK_WIND (frame, trace_inodelk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->inodelk,
-                    volume, loc, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, trace_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, volume, loc, cmd, flock,
+               xdata);
+    return 0;
 }
 
 int
-trace_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+trace_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+               fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        char          *cmd_str  = NULL;
-        char          *type_str = NULL;
-        trace_conf_t  *conf     = NULL;
+    char *cmd_str = NULL;
+    char *type_str = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FINODELK].enabled) {
-                char  string[4096] = {0,};
-                switch (cmd) {
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FINODELK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        switch (cmd) {
 #if F_GETLK != F_GETLK64
-                case F_GETLK64:
+            case F_GETLK64:
 #endif
-                case F_GETLK:
-                        cmd_str = "GETLK";
-                        break;
+            case F_GETLK:
+                cmd_str = "GETLK";
+                break;
 
 #if F_SETLK != F_SETLK64
-                case F_SETLK64:
+            case F_SETLK64:
 #endif
-                case F_SETLK:
-                        cmd_str = "SETLK";
-                        break;
+            case F_SETLK:
+                cmd_str = "SETLK";
+                break;
 
 #if F_SETLKW != F_SETLKW64
-                case F_SETLKW64:
+            case F_SETLKW64:
 #endif
-                case F_SETLKW:
-                        cmd_str = "SETLKW";
-                        break;
-
-                default:
-                        cmd_str = "UNKNOWN";
-                        break;
-                }
-
-                switch (flock->l_type) {
-                case F_RDLCK:
-                        type_str = "READ";
-                        break;
-                case F_WRLCK:
-                        type_str = "WRITE";
-                        break;
-                case F_UNLCK:
-                        type_str = "UNLOCK";
-                        break;
-                default:
-                        type_str = "UNKNOWN";
-                        break;
-                }
-
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s volume=%s, (fd =%p "
-                          "cmd=%s, type=%s, start=%llu, len=%llu, "
-                          "pid=%llu)", frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), volume, fd,
-                          cmd_str, type_str,
-                          (unsigned long long) flock->l_start,
-                          (unsigned long long) flock->l_len,
-                          (unsigned long long) flock->l_pid);
-
-                frame->local = fd->inode->gfid;
-
-                LOG_ELEMENT (conf, string);
+            case F_SETLKW:
+                cmd_str = "SETLKW";
+                break;
+
+            default:
+                cmd_str = "UNKNOWN";
+                break;
         }
+
+        switch (flock->l_type) {
+            case F_RDLCK:
+                type_str = "READ";
+                break;
+            case F_WRLCK:
+                type_str = "WRITE";
+                break;
+            case F_UNLCK:
+                type_str = "UNLOCK";
+                break;
+            default:
+                type_str = "UNKNOWN";
+                break;
+        }
+
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s volume=%s, (fd =%p "
+                 "cmd=%s, type=%s, start=%llu, len=%llu, "
+                 "pid=%llu)",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), volume, fd,
+                 cmd_str, type_str, (unsigned long long)flock->l_start,
+                 (unsigned long long)flock->l_len,
+                 (unsigned long long)flock->l_pid);
+
+        frame->local = fd->inode->gfid;
+
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        STACK_WIND (frame, trace_finodelk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->finodelk,
-                    volume, fd, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, trace_finodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->finodelk, volume, fd, cmd, flock,
+               xdata);
+    return 0;
 }
 
 int
-trace_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+trace_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s (path=%s flags=%d)",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_XATTROP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s (path=%s flags=%d)", frame->root->unique,
+                 uuid_utoa(loc->inode->gfid), loc->path, flags);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_xattrop_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop,
-                    loc, flags, dict, xdata);
+    STACK_WIND(frame, trace_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+trace_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-                char    string[4096]  = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, flags=%d",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p, flags=%d",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, flags);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fxattrop_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fxattrop,
-                    fd, flags, dict, xdata);
+    STACK_WIND(frame, trace_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_lookup (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xdata)
+trace_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        trace_conf_t *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                char string[4096] = {0,};
-                /* TODO: print all the keys mentioned in xattr_req */
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        /* TODO: print all the keys mentioned in xattr_req */
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_lookup_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup,
-                    loc, xdata);
+    STACK_WIND(frame, trace_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+trace_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_STAT].enabled) {
-                char  string[4096]  = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_STAT].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_stat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat,
-                    loc, xdata);
+    STACK_WIND(frame, trace_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-                dict_t *xdata)
+trace_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+               dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READLINK].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s, "
-                          "size=%"GF_PRI_SIZET")", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          size);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s, "
+                 "size=%" GF_PRI_SIZET ")",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 size);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_readlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink,
-                    loc, size, xdata);
+    STACK_WIND(frame, trace_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+trace_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            dev_t dev, mode_t umask, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s mode=%d "
-                          "umask=0%o, dev=%"GF_PRI_DEV")",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          mode, umask, dev);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_MKNOD].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s mode=%d "
+                 "umask=0%o, dev=%" GF_PRI_DEV ")",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 mode, umask, dev);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_mknod_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, dev, umask, xdata);
+    STACK_WIND(frame, trace_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, dev, umask, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             mode_t umask, dict_t *xdata)
+trace_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            mode_t umask, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s mode=%d"
-                          " umask=0%o", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          mode, umask);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_MKDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s mode=%d"
+                 " umask=0%o",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 mode, umask);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_mkdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, umask, xdata);
-        return 0;
+    STACK_WIND(frame, trace_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    return 0;
 }
 
 int
-trace_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-              dict_t *xdata)
+trace_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+             dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s flag=%d",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          xflag);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_UNLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s flag=%d",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 xflag);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        STACK_WIND (frame, trace_unlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink,
-                    loc, xflag, xdata);
-        return 0;
+    STACK_WIND(frame, trace_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
 }
 
 int
-trace_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-             dict_t *xdata)
+trace_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+            dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s flags=%d",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RMDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s path=%s flags=%d", frame->root->unique,
+                 uuid_utoa(loc->inode->gfid), loc->path, flags);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_rmdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir,
-                    loc, flags, xdata);
+    STACK_WIND(frame, trace_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-               loc_t *loc, mode_t umask, dict_t *xdata)
+trace_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+              loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        trace_conf_t     *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-                char     string[4096]   =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s linkpath=%s, path=%s"
-                          " umask=0%o", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), linkpath,
-                          loc->path, umask);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s linkpath=%s, path=%s"
+                 " umask=0%o",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), linkpath,
+                 loc->path, umask);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_symlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, umask, xdata);
+    STACK_WIND(frame, trace_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-              dict_t *xdata)
+trace_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
-        char         oldgfid[50] = {0,};
-        char         newgfid[50] = {0,};
-        trace_conf_t *conf       = NULL;
+    char oldgfid[50] = {
+        0,
+    };
+    char newgfid[50] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RENAME].enabled) {
-                char string[4096] = {0,};
-                if (newloc->inode)
-                        uuid_utoa_r (newloc->inode->gfid, newgfid);
-                else
-                        strcpy (newgfid, "0");
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RENAME].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (newloc->inode)
+            uuid_utoa_r(newloc->inode->gfid, newgfid);
+        else
+            strcpy(newgfid, "0");
 
-                uuid_utoa_r (oldloc->inode->gfid, oldgfid);
+        uuid_utoa_r(oldloc->inode->gfid, oldgfid);
 
-                snprintf (string, sizeof (string),
-                          "%"PRId64": oldgfid=%s oldpath=%s --> "
-                          "newgfid=%s newpath=%s",
-                          frame->root->unique, oldgfid,
-                          oldloc->path, newgfid, newloc->path);
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": oldgfid=%s oldpath=%s --> "
+                 "newgfid=%s newpath=%s",
+                 frame->root->unique, oldgfid, oldloc->path, newgfid,
+                 newloc->path);
 
-                frame->local = oldloc->inode->gfid;
+        frame->local = oldloc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_rename_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc, xdata);
+    STACK_WIND(frame, trace_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-            dict_t *xdata)
+trace_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-        char         oldgfid[50] = {0,};
-        char         newgfid[50] = {0,};
-        trace_conf_t *conf       = NULL;
+    char oldgfid[50] = {
+        0,
+    };
+    char newgfid[50] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LINK].enabled) {
-                char string[4096]  = {0,};
-                if (newloc->inode)
-                        uuid_utoa_r (newloc->inode->gfid, newgfid);
-                else
-                        strcpy (newgfid, "0");
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (newloc->inode)
+            uuid_utoa_r(newloc->inode->gfid, newgfid);
+        else
+            strcpy(newgfid, "0");
 
-                uuid_utoa_r (oldloc->inode->gfid, oldgfid);
+        uuid_utoa_r(oldloc->inode->gfid, oldgfid);
 
-                snprintf (string, sizeof (string),
-                          "%"PRId64": oldgfid=%s oldpath=%s --> "
-                          "newgfid=%s newpath=%s", frame->root->unique,
-                          oldgfid, oldloc->path, newgfid,
-                          newloc->path);
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": oldgfid=%s oldpath=%s --> "
+                 "newgfid=%s newpath=%s",
+                 frame->root->unique, oldgfid, oldloc->path, newgfid,
+                 newloc->path);
 
-                frame->local = oldloc->inode->gfid;
+        frame->local = oldloc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_link_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, trace_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 }
 
 int
-trace_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid, dict_t *xdata)
+trace_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        char         actime_str[256]  = {0,};
-        char         modtime_str[256] = {0,};
-        trace_conf_t *conf            = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
-                char     string[4096]  =  {0,};
-                if (valid & GF_SET_ATTR_MODE) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s path=%s mode=%o)",
-                                  frame->root->unique,
-                                  uuid_utoa (loc->inode->gfid),
-                                  loc->path,
-                                  st_mode_from_ia (stbuf->ia_prot,
-                                                   stbuf->ia_type));
-
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0 , sizeof (string));
-                }
-
-                if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s path=%s uid=%o,"
-                                  " gid=%o", frame->root->unique,
-                                  uuid_utoa (loc->inode->gfid),
-                                  loc->path, stbuf->ia_uid,
-                                  stbuf->ia_gid);
-
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0 , sizeof (string));
-                }
-
-                if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
-                        gf_time_fmt (actime_str, sizeof actime_str,
-                                     stbuf->ia_atime, gf_timefmt_bdT);
-
-                        gf_time_fmt (modtime_str, sizeof modtime_str,
-                                     stbuf->ia_mtime, gf_timefmt_bdT);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s path=%s "
-                                  "ia_atime=%s, ia_mtime=%s",
-                                  frame->root->unique,
-                                  uuid_utoa (loc->inode->gfid),
-                                  loc->path, actime_str, modtime_str);
-
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0 , sizeof (string));
-                }
-                frame->local = loc->inode->gfid;
+    char actime_str[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char modtime_str[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (valid & GF_SET_ATTR_MODE) {
+            snprintf(
+                string, sizeof(string), "%" PRId64 ": gfid=%s path=%s mode=%o)",
+                frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                st_mode_from_ia(stbuf->ia_prot, stbuf->ia_type));
+
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
+        }
+
+        if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s path=%s uid=%o,"
+                     " gid=%o",
+                     frame->root->unique, uuid_utoa(loc->inode->gfid),
+                     loc->path, stbuf->ia_uid, stbuf->ia_gid);
+
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
+        }
+
+        if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
+            gf_time_fmt(actime_str, sizeof actime_str, stbuf->ia_atime,
+                        gf_timefmt_bdT);
+
+            gf_time_fmt(modtime_str, sizeof modtime_str, stbuf->ia_mtime,
+                        gf_timefmt_bdT);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s path=%s "
+                     "ia_atime=%s, ia_mtime=%s",
+                     frame->root->unique, uuid_utoa(loc->inode->gfid),
+                     loc->path, actime_str, modtime_str);
+
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
         }
+        frame->local = loc->inode->gfid;
+    }
 
 out:
-        STACK_WIND (frame, trace_setattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid, xdata);
+    STACK_WIND(frame, trace_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+trace_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        char           actime_str[256]  = {0,};
-        char           modtime_str[256] = {0,};
-        trace_conf_t   *conf            = NULL;
+    char actime_str[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char modtime_str[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
-                char     string[4096]  =  {0,};
-                if (valid & GF_SET_ATTR_MODE) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s fd=%p, mode=%o",
-                                  frame->root->unique,
-                                  uuid_utoa (fd->inode->gfid), fd,
-                                  st_mode_from_ia (stbuf->ia_prot,
-                                                   stbuf->ia_type));
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (valid & GF_SET_ATTR_MODE) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": gfid=%s fd=%p, mode=%o", frame->root->unique,
+                     uuid_utoa(fd->inode->gfid), fd,
+                     st_mode_from_ia(stbuf->ia_prot, stbuf->ia_type));
 
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0, sizeof (string));
-                }
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
+        }
 
-                if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s fd=%p, uid=%o, "
-                                  "gid=%o", frame->root->unique,
-                                  uuid_utoa (fd->inode->gfid),
-                                  fd, stbuf->ia_uid, stbuf->ia_gid);
+        if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s fd=%p, uid=%o, "
+                     "gid=%o",
+                     frame->root->unique, uuid_utoa(fd->inode->gfid), fd,
+                     stbuf->ia_uid, stbuf->ia_gid);
 
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0, sizeof (string));
-                }
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
+        }
 
-                if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
-                        gf_time_fmt (actime_str, sizeof actime_str,
-                                     stbuf->ia_atime, gf_timefmt_bdT);
+        if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
+            gf_time_fmt(actime_str, sizeof actime_str, stbuf->ia_atime,
+                        gf_timefmt_bdT);
 
-                        gf_time_fmt (modtime_str, sizeof modtime_str,
-                                     stbuf->ia_mtime, gf_timefmt_bdT);
+            gf_time_fmt(modtime_str, sizeof modtime_str, stbuf->ia_mtime,
+                        gf_timefmt_bdT);
 
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s fd=%p "
-                                  "ia_atime=%s, ia_mtime=%s",
-                                  frame->root->unique,
-                                  uuid_utoa (fd->inode->gfid),
-                                  fd, actime_str, modtime_str);
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s fd=%p "
+                     "ia_atime=%s, ia_mtime=%s",
+                     frame->root->unique, uuid_utoa(fd->inode->gfid), fd,
+                     actime_str, modtime_str);
 
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0, sizeof (string));
-                }
-                frame->local = fd->inode->gfid;
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
         }
+        frame->local = fd->inode->gfid;
+    }
 
 out:
-        STACK_WIND (frame, trace_fsetattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid, xdata);
+    STACK_WIND(frame, trace_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
 
-        return 0;
+    return 0;
 }
 
 static int
-trace_seek_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, off_t offset, dict_t *xdata)
-{
-        trace_conf_t  *conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_SEEK].enabled) {
-                char  string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d op_errno=%d, "
-                          "offset=%"PRId64"", frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno, offset);
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (seek, frame, op_ret, op_errno, offset, xdata);
-        return 0;
+trace_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, off_t offset, dict_t *xdata)
+{
+    trace_conf_t *conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SEEK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d op_errno=%d, "
+                 "offset=%" PRId64 "",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 offset);
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(seek, frame, op_ret, op_errno, offset, xdata);
+    return 0;
 }
 
 static int
-trace_seek (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            off_t offset, gf_seek_what_t what, dict_t *xdata)
+trace_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           gf_seek_what_t what, dict_t *xdata)
 {
-        trace_conf_t   *conf = this->private;
+    trace_conf_t *conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_SEEK].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string), "%"PRId64": gfid=%s fd=%p "
-                          "offset=%"PRId64" what=%d", frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, offset, what);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SEEK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s fd=%p "
+                 "offset=%" PRId64 " what=%d",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, offset,
+                 what);
 
-                frame->local = fd->inode->gfid;
-                LOG_ELEMENT (conf, string);
-        }
+        frame->local = fd->inode->gfid;
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        STACK_WIND (frame, trace_seek_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->seek, fd, offset, what, xdata);
+    STACK_WIND(frame, trace_seek_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->seek, fd, offset, what, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                off_t offset, dict_t *xdata)
+trace_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+               dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s, "
-                          "offset=%"PRId64"", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          offset);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s, "
+                 "offset=%" PRId64 "",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 offset);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_truncate_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate,
-                    loc, offset, xdata);
+    STACK_WIND(frame, trace_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            int32_t flags, fd_t *fd, dict_t *xdata)
+trace_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           fd_t *fd, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPEN].enabled) {
-                char      string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s flags=%d fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          flags, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPEN].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s path=%s flags=%d fd=%p",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 flags, fd);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_open_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
 int
-trace_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
-              dict_t *xdata)
+trace_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_CREATE].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s, fd=%p, "
-                          "flags=0%o mode=0%o umask=0%o",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          fd, flags, mode, umask);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_CREATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s, fd=%p, "
+                 "flags=0%o mode=0%o umask=0%o",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 fd, flags, mode, umask);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_create_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 }
 
 int
-trace_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+trace_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READ].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, size=%"
-                          GF_PRI_SIZET"offset=%"PRId64" flags=0%x)",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, size,
-                          offset, flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READ].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s fd=%p, size=%" GF_PRI_SIZET
+                 "offset=%" PRId64 " flags=0%x)",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, size,
+                 offset, flags);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_readv_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags, xdata);
-        return 0;
+    STACK_WIND(frame, trace_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
 }
 
 int
-trace_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              struct iovec *vector, int32_t count,
-              off_t offset, uint32_t flags, struct iobref *iobref, dict_t *xdata)
+trace_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+             struct iobref *iobref, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
-        int i = 0;
-        size_t total_size = 0;
+    trace_conf_t *conf = NULL;
+    int i = 0;
+    size_t total_size = 0;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_WRITE].enabled) {
-                char     string[4096]  =  {0,};
-                for (i = 0; i < count; i++)
-                        total_size += vector[i].iov_len;
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_WRITE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        for (i = 0; i < count; i++)
+            total_size += vector[i].iov_len;
 
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, count=%d, "
-                          " offset=%"PRId64" flags=0%x write_size=%zu",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, count,
-                          offset, flags, total_size);
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s fd=%p, count=%d, "
+                 " offset=%" PRId64 " flags=0%x write_size=%zu",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, count,
+                 offset, flags, total_size);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_writev_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref, xdata);
-        return 0;
+    STACK_WIND(frame, trace_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 }
 
 int
-trace_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+trace_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_STATFS].enabled) {
-                char  string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s",
-                          frame->root->unique, (loc->inode)?
-                          uuid_utoa (loc->inode->gfid):"0", loc->path);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_STATFS].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s",
+                 frame->root->unique,
+                 (loc->inode) ? uuid_utoa(loc->inode->gfid) : "0", loc->path);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_statfs_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->statfs,
-                    loc, xdata);
-        return 0;
+    STACK_WIND(frame, trace_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
 }
 
 int
-trace_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+trace_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FLUSH].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_flush_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush,
-                    fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
 }
 
 int
-trace_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-             dict_t *xdata)
+trace_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+            dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s flags=%d fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), flags, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSYNC].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s flags=%d fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), flags, fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fsync_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync,
-                    fd, flags, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
 }
 
 int
-trace_setxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
+trace_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s flags=%d",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s path=%s flags=%d", frame->root->unique,
+                 uuid_utoa(loc->inode->gfid), loc->path, flags);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_setxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, trace_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
 }
 
 int
-trace_getxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, const char *name, dict_t *xdata)
+trace_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s name=%s",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          name);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s name=%s",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 name);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_getxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr,
-                    loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, trace_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 }
 
 int
-trace_removexattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name, dict_t *xdata)
+trace_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s name=%s",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          name);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s name=%s",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 name);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_removexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr,
-                    loc, name, xdata);
+    STACK_WIND(frame, trace_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-               dict_t *xdata)
+trace_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+              dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s fd=%p",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 fd);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_opendir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir,
-                    loc, fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 }
 
 int
-trace_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t offset, dict_t *dict)
+trace_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, dict_t *dict)
 {
-        trace_conf_t *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READDIRP].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
-                          ", offset=%"PRId64" dict=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, size,
-                          offset, dict);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READDIRP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s fd=%p, size=%" GF_PRI_SIZET
+                 ", offset=%" PRId64 " dict=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, size,
+                 offset, dict);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_readdirp_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, offset, dict);
+    STACK_WIND(frame, trace_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               size_t size, off_t offset, dict_t *xdata)
+trace_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READDIR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
-                          ", offset=%"PRId64,
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, size,
-                          offset);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s fd=%p, size=%" GF_PRI_SIZET
+                 ", offset=%" PRId64,
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, size,
+                 offset);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_readdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir,
-                    fd, size, offset, xdata);
+    STACK_WIND(frame, trace_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, offset, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_fsyncdir (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, int32_t datasync, dict_t *xdata)
+trace_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+               dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s datasync=%d fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), datasync, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s datasync=%d fd=%p", frame->root->unique,
+                 uuid_utoa(fd->inode->gfid), datasync, fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fsyncdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsyncdir,
-                    fd, datasync, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fsyncdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsyncdir, fd, datasync, xdata);
+    return 0;
 }
 
 int
-trace_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
-              dict_t *xdata)
+trace_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+             dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s mask=0%o",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid),
-                          loc->path, mask);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_ACCESS].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s path=%s mask=0%o", frame->root->unique,
+                 uuid_utoa(loc->inode->gfid), loc->path, mask);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_access_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->access,
-                    loc, mask, xdata);
-        return 0;
+    STACK_WIND(frame, trace_access_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->access, loc, mask, xdata);
+    return 0;
 }
 
 int32_t
-trace_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 int32_t len, dict_t *xdata)
+trace_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                int32_t len, dict_t *xdata)
 {
+    trace_conf_t *conf = NULL;
 
-        trace_conf_t *conf = NULL;
-
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s offset=%"PRId64
-                          "len=%u fd=%p", frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), offset, len, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s offset=%" PRId64 "len=%u fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), offset, len,
+                 fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_rchecksum_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rchecksum,
-                    fd, offset, len, xdata);
-
-        return 0;
+    STACK_WIND(frame, trace_rchecksum_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rchecksum, fd, offset, len, xdata);
 
+    return 0;
 }
 
 int32_t
-trace_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-                fd_t *fd, const char *basename, entrylk_cmd cmd,
-                entrylk_type type, dict_t *xdata)
-{
-        trace_conf_t   *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
-                char      string[4096]   =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s volume=%s, (fd=%p "
-                          "basename=%s, cmd=%s, type=%s)",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), volume, fd,
-                          basename,
-                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
-                           "ENTRYLK_UNLOCK"),
-                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
-                           "ENTRYLK_WRLCK"));
-
-                frame->local = fd->inode->gfid;
-
-                LOG_ELEMENT (conf, string);
-        }
+trace_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+               fd_t *fd, const char *basename, entrylk_cmd cmd,
+               entrylk_type type, dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
 
-out:
-        STACK_WIND (frame, trace_fentrylk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fentrylk,
-                    volume, fd, basename, cmd, type, xdata);
-        return 0;
+    conf = this->private;
 
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s volume=%s, (fd=%p "
+                 "basename=%s, cmd=%s, type=%s)",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), volume, fd,
+                 basename,
+                 ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" : "ENTRYLK_UNLOCK"),
+                 ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK"));
+
+        frame->local = fd->inode->gfid;
+
+        LOG_ELEMENT(conf, string);
+    }
+
+out:
+    STACK_WIND(frame, trace_fentrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fentrylk, volume, fd, basename, cmd,
+               type, xdata);
+    return 0;
 }
 
 int32_t
-trace_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name, dict_t *xdata)
+trace_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p name=%s",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, name);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p name=%s",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, name);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fgetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 }
 
 int32_t
-trace_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 dict_t *dict, int32_t flags, dict_t *xdata)
+trace_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                int32_t flags, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p flags=%d",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p flags=%d",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, flags);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fsetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
 int
-trace_ftruncate (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, off_t offset, dict_t *xdata)
+trace_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s offset=%"PRId64" fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), offset, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s offset=%" PRId64 " fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), offset, fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_ftruncate_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset, xdata);
+    STACK_WIND(frame, trace_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+trace_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSTAT].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fstat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat,
-                    fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
 }
 
 int
-trace_lk (call_frame_t *frame, xlator_t *this, fd_t *fd,
-          int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+trace_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+         struct gf_flock *lock, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LK].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, cmd=%d, "
-                          "lock {l_type=%d, "
-                          "l_whence=%d, l_start=%"PRId64", "
-                          "l_len=%"PRId64", l_pid=%u})",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, cmd,
-                          lock->l_type, lock->l_whence,
-                          lock->l_start, lock->l_len, lock->l_pid);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s fd=%p, cmd=%d, "
+                 "lock {l_type=%d, "
+                 "l_whence=%d, l_start=%" PRId64
+                 ", "
+                 "l_len=%" PRId64 ", l_pid=%u})",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, cmd,
+                 lock->l_type, lock->l_whence, lock->l_start, lock->l_len,
+                 lock->l_pid);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_lk_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, lock, xdata);
-        return 0;
+    STACK_WIND(frame, trace_lk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lk, fd, cmd, lock, xdata);
+    return 0;
 }
 
 int32_t
-trace_forget (xlator_t *this, inode_t *inode)
+trace_forget(xlator_t *this, inode_t *inode)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
-        /* If user want to understand when a lookup happens,
-           he should know about 'forget' too */
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "gfid=%s", uuid_utoa (inode->gfid));
+    conf = this->private;
+    /* If user want to understand when a lookup happens,
+       he should know about 'forget' too */
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "gfid=%s", uuid_utoa(inode->gfid));
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-trace_releasedir (xlator_t *this, fd_t *fd)
+trace_releasedir(xlator_t *this, fd_t *fd)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "gfid=%s fd=%p",
-                          uuid_utoa (fd->inode->gfid), fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "gfid=%s fd=%p",
+                 uuid_utoa(fd->inode->gfid), fd);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-trace_release (xlator_t *this, fd_t *fd)
+trace_release(xlator_t *this, fd_t *fd)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPEN].enabled ||
-            trace_fop_names[GF_FOP_CREATE].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "gfid=%s fd=%p",
-                          uuid_utoa (fd->inode->gfid), fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPEN].enabled ||
+        trace_fop_names[GF_FOP_CREATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "gfid=%s fd=%p",
+                 uuid_utoa(fd->inode->gfid), fd);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-
 void
-enable_all_calls (int enabled)
+enable_all_calls(int enabled)
 {
-        int i;
+    int i;
 
-        for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                trace_fop_names[i].enabled = enabled;
+    for (i = 0; i < GF_FOP_MAXVALUE; i++)
+        trace_fop_names[i].enabled = enabled;
 }
 
 void
-enable_call (const char *name, int enabled)
+enable_call(const char *name, int enabled)
 {
-        int i;
-        for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                if (!strcasecmp(trace_fop_names[i].name, name))
-                        trace_fop_names[i].enabled = enabled;
+    int i;
+    for (i = 0; i < GF_FOP_MAXVALUE; i++)
+        if (!strcasecmp(trace_fop_names[i].name, name))
+            trace_fop_names[i].enabled = enabled;
 }
 
-
 /*
   include = 1 for "include-ops"
   = 0 for "exclude-ops"
 */
 void
-process_call_list (const char *list, int include)
+process_call_list(const char *list, int include)
 {
-        enable_all_calls (include ? 0 : 1);
+    enable_all_calls(include ? 0 : 1);
 
-        char *call = strsep ((char **)&list, ",");
+    char *call = strsep((char **)&list, ",");
 
-        while (call) {
-                enable_call (call, include);
-                call = strsep ((char **)&list, ",");
-        }
+    while (call) {
+        enable_call(call, include);
+        call = strsep((char **)&list, ",");
+    }
 }
 
 int32_t
-trace_dump_history (xlator_t *this)
-{
-        int ret = -1;
-        char key_prefix[GF_DUMP_MAX_BUF_LEN] = {0,};
-        trace_conf_t *conf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("trace", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->history, out);
-
-        conf = this->private;
-        // Is it ok to return silently if log-history option his off?
-        if (conf && conf->log_history == _gf_true) {
-                gf_proc_dump_build_key (key_prefix, "xlator.debug.trace",
-                                        "history");
-                gf_proc_dump_add_section (key_prefix);
-                eh_dump (this->history, NULL, dump_history_trace);
-        }
-        ret = 0;
+trace_dump_history(xlator_t *this)
+{
+    int ret = -1;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("trace", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->history, out);
+
+    conf = this->private;
+    // Is it ok to return silently if log-history option his off?
+    if (conf && conf->log_history == _gf_true) {
+        gf_proc_dump_build_key(key_prefix, "xlator.debug.trace", "history");
+        gf_proc_dump_add_section("%s", key_prefix);
+        eh_dump(this->history, NULL, dump_history_trace);
+    }
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_trace_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_trace_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               " failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int32_t       ret            = -1;
-        trace_conf_t    *conf           = NULL;
-        char            *includes = NULL, *excludes = NULL;
+    int32_t ret = -1;
+    trace_conf_t *conf = NULL;
+    char *includes = NULL, *excludes = NULL;
 
-        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, options, out);
+    GF_VALIDATE_OR_GOTO("quick-read", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, options, out);
 
-        conf = this->private;
+    conf = this->private;
 
-        includes = data_to_str (dict_get (options, "include-ops"));
-        excludes = data_to_str (dict_get (options, "exclude-ops"));
+    includes = data_to_str(dict_get(options, "include-ops"));
+    excludes = data_to_str(dict_get(options, "exclude-ops"));
 
-        {
-                int i;
-                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                        if (gf_fop_list[i])
-                                strncpy (trace_fop_names[i].name,
-                                         gf_fop_list[i],
-                                         sizeof (trace_fop_names[i].name));
-                        else
-                                strncpy (trace_fop_names[i].name, ":0",
-                                         sizeof (trace_fop_names[i].name));
-                        trace_fop_names[i].enabled = 1;
-                        trace_fop_names[i].name[sizeof (
-                                        trace_fop_names[i].name) - 1] = 0;
-                }
+    {
+        int i;
+        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+            if (gf_fop_list[i])
+                strncpy(trace_fop_names[i].name, gf_fop_list[i],
+                        sizeof(trace_fop_names[i].name));
+            else
+                strncpy(trace_fop_names[i].name, ":0",
+                        sizeof(trace_fop_names[i].name));
+            trace_fop_names[i].enabled = 1;
+            trace_fop_names[i].name[sizeof(trace_fop_names[i].name) - 1] = 0;
         }
+    }
 
-        if (includes && excludes) {
-                gf_log (this->name,
-                        GF_LOG_ERROR,
-                        "must specify only one of 'include-ops' and "
-                        "'exclude-ops'");
-                goto out;
-        }
+    if (includes && excludes) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "must specify only one of 'include-ops' and "
+               "'exclude-ops'");
+        goto out;
+    }
 
-        if (includes)
-                process_call_list (includes, 1);
-        if (excludes)
-                process_call_list (excludes, 0);
+    if (includes)
+        process_call_list(includes, 1);
+    if (excludes)
+        process_call_list(excludes, 0);
 
-        /* Should resizing of the event-history be allowed in reconfigure?
-         * for which a new event_history might have to be allocated and the
-         * older history has to be freed.
-         */
-        GF_OPTION_RECONF ("log-file", conf->log_file, options, bool, out);
+    /* Should resizing of the event-history be allowed in reconfigure?
+     * for which a new event_history might have to be allocated and the
+     * older history has to be freed.
+     */
+    GF_OPTION_RECONF("log-file", conf->log_file, options, bool, out);
 
-        GF_OPTION_RECONF ("log-history", conf->log_history, options, bool, out);
+    GF_OPTION_RECONF("log-history", conf->log_history, options, bool, out);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
-{
-        dict_t *options = NULL;
-        char *includes = NULL, *excludes = NULL;
-        char *forced_loglevel = NULL;
-        eh_t *history = NULL;
-        int  ret = -1;
-        size_t  history_size = TRACE_DEFAULT_HISTORY_SIZE;
-        trace_conf_t    *conf = NULL;
-
-        if (!this)
-                return -1;
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "trace translator requires one subvolume");
-                return -1;
-        }
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
-
-        conf = GF_CALLOC (1, sizeof (trace_conf_t), gf_trace_mt_trace_conf_t);
-        if (!conf) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot allocate "
-                        "xl->private");
-                return -1;
-        }
-
-        options = this->options;
-        includes = data_to_str (dict_get (options, "include-ops"));
-        excludes = data_to_str (dict_get (options, "exclude-ops"));
-
-        {
-                int i;
-                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                        if (gf_fop_list[i])
-                                strncpy (trace_fop_names[i].name,
-                                         gf_fop_list[i],
-                                         sizeof (trace_fop_names[i].name));
-                        else
-                                strncpy (trace_fop_names[i].name, ":O",
-                                         sizeof (trace_fop_names[i].name));
-                        trace_fop_names[i].enabled = 1;
-                        trace_fop_names[i].name[sizeof (
-                                        trace_fop_names[i].name) - 1] = 0;
-                }
-        }
-
-        if (includes && excludes) {
-                gf_log (this->name,
-                        GF_LOG_ERROR,
-                        "must specify only one of 'include-ops' and "
-                        "'exclude-ops'");
-                return -1;
-        }
-
-        if (includes)
-                process_call_list (includes, 1);
-        if (excludes)
-                process_call_list (excludes, 0);
-
-
-        GF_OPTION_INIT ("history-size", conf->history_size, size, out);
-
-        gf_log (this->name, GF_LOG_INFO, "history size %"GF_PRI_SIZET,
-                history_size);
-
-        GF_OPTION_INIT ("log-file", conf->log_file, bool, out);
-
-        gf_log (this->name, GF_LOG_INFO, "logging to file %s",
-                (conf->log_file == _gf_true)?"enabled":"disabled");
-
-        GF_OPTION_INIT ("log-history", conf->log_history, bool, out);
-
-        gf_log (this->name, GF_LOG_DEBUG, "logging to history %s",
-                (conf->log_history == _gf_true)?"enabled":"disabled");
-
-        history = eh_new (history_size, _gf_false, NULL);
-        if (!history) {
-                gf_log (this->name, GF_LOG_ERROR, "event history cannot be "
-                        "initialized");
-                return -1;
-        }
-
-        this->history = history;
-
-        conf->trace_log_level = GF_LOG_INFO;
-
-        if (dict_get (options, "force-log-level")) {
-                forced_loglevel = data_to_str (dict_get (options,
-                                                         "force-log-level"));
-                if (!forced_loglevel)
-                        goto setloglevel;
-
-                if (strcmp (forced_loglevel, "INFO") == 0)
-                        conf->trace_log_level = GF_LOG_INFO;
-                else if (strcmp (forced_loglevel, "TRACE") == 0)
-                        conf->trace_log_level = GF_LOG_TRACE;
-                else if (strcmp (forced_loglevel, "ERROR") == 0)
-                        conf->trace_log_level = GF_LOG_ERROR;
-                else if (strcmp (forced_loglevel, "DEBUG") == 0)
-                        conf->trace_log_level = GF_LOG_DEBUG;
-                else if (strcmp (forced_loglevel, "WARNING") == 0)
-                        conf->trace_log_level = GF_LOG_WARNING;
-                else if (strcmp (forced_loglevel, "CRITICAL") == 0)
-                        conf->trace_log_level = GF_LOG_CRITICAL;
-                else if (strcmp (forced_loglevel, "NONE") == 0)
-                        conf->trace_log_level = GF_LOG_NONE;
-        }
+init(xlator_t *this)
+{
+    dict_t *options = NULL;
+    char *includes = NULL, *excludes = NULL;
+    char *forced_loglevel = NULL;
+    eh_t *history = NULL;
+    int ret = -1;
+    uint64_t history_size = TRACE_DEFAULT_HISTORY_SIZE;
+    trace_conf_t *conf = NULL;
+
+    if (!this)
+        return -1;
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "trace translator requires one subvolume");
+        return -1;
+    }
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
+
+    conf = GF_CALLOC(1, sizeof(trace_conf_t), gf_trace_mt_trace_conf_t);
+    if (!conf) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "cannot allocate "
+               "xl->private");
+        return -1;
+    }
+
+    options = this->options;
+    includes = data_to_str(dict_get(options, "include-ops"));
+    excludes = data_to_str(dict_get(options, "exclude-ops"));
+
+    {
+        int i;
+        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+            if (gf_fop_list[i])
+                strncpy(trace_fop_names[i].name, gf_fop_list[i],
+                        sizeof(trace_fop_names[i].name));
+            else
+                strncpy(trace_fop_names[i].name, ":O",
+                        sizeof(trace_fop_names[i].name));
+            trace_fop_names[i].enabled = 1;
+            trace_fop_names[i].name[sizeof(trace_fop_names[i].name) - 1] = 0;
+        }
+    }
+
+    if (includes && excludes) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "must specify only one of 'include-ops' and "
+               "'exclude-ops'");
+        return -1;
+    }
+
+    if (includes)
+        process_call_list(includes, 1);
+    if (excludes)
+        process_call_list(excludes, 0);
+
+    GF_OPTION_INIT("history-size", history_size, size, out);
+    conf->history_size = history_size;
+
+    gf_log(this->name, GF_LOG_INFO, "history size %" PRIu64, history_size);
+
+    GF_OPTION_INIT("log-file", conf->log_file, bool, out);
+
+    gf_log(this->name, GF_LOG_INFO, "logging to file %s",
+           (conf->log_file == _gf_true) ? "enabled" : "disabled");
+
+    GF_OPTION_INIT("log-history", conf->log_history, bool, out);
+
+    gf_log(this->name, GF_LOG_DEBUG, "logging to history %s",
+           (conf->log_history == _gf_true) ? "enabled" : "disabled");
+
+    history = eh_new(history_size, _gf_false, NULL);
+    if (!history) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "event history cannot be "
+               "initialized");
+        return -1;
+    }
+
+    this->history = history;
+
+    conf->trace_log_level = GF_LOG_INFO;
+
+    if (dict_get(options, "force-log-level")) {
+        forced_loglevel = data_to_str(dict_get(options, "force-log-level"));
+        if (!forced_loglevel)
+            goto setloglevel;
+
+        if (strcmp(forced_loglevel, "INFO") == 0)
+            conf->trace_log_level = GF_LOG_INFO;
+        else if (strcmp(forced_loglevel, "TRACE") == 0)
+            conf->trace_log_level = GF_LOG_TRACE;
+        else if (strcmp(forced_loglevel, "ERROR") == 0)
+            conf->trace_log_level = GF_LOG_ERROR;
+        else if (strcmp(forced_loglevel, "DEBUG") == 0)
+            conf->trace_log_level = GF_LOG_DEBUG;
+        else if (strcmp(forced_loglevel, "WARNING") == 0)
+            conf->trace_log_level = GF_LOG_WARNING;
+        else if (strcmp(forced_loglevel, "CRITICAL") == 0)
+            conf->trace_log_level = GF_LOG_CRITICAL;
+        else if (strcmp(forced_loglevel, "NONE") == 0)
+            conf->trace_log_level = GF_LOG_NONE;
+    }
 
 setloglevel:
-        gf_log_set_loglevel (this->ctx, conf->trace_log_level);
-        this->private = conf;
-        ret = 0;
-out:
-        if (ret == -1) {
-                if (history)
-                        GF_FREE (history);
-                if (conf)
-                        GF_FREE (conf);
-        }
+    gf_log_set_loglevel(this->ctx, conf->trace_log_level);
+    this->private = conf;
+    ret = 0;
+out:
+    if (ret == -1) {
+        if (history)
+            GF_FREE(history);
+        if (conf)
+            GF_FREE(conf);
+    }
 
-        return ret;
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        if (!this)
-                return;
+    if (!this)
+        return;
 
-        if (this->history)
-                eh_destroy (this->history);
+    if (this->history)
+        eh_destroy(this->history);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "trace translator unloaded");
-        return;
+    gf_log(this->name, GF_LOG_INFO, "trace translator unloaded");
+    return;
 }
 
 struct xlator_fops fops = {
-        .stat        = trace_stat,
-        .readlink    = trace_readlink,
-        .mknod       = trace_mknod,
-        .mkdir       = trace_mkdir,
-        .unlink      = trace_unlink,
-        .rmdir       = trace_rmdir,
-        .symlink     = trace_symlink,
-        .rename      = trace_rename,
-        .link        = trace_link,
-        .truncate    = trace_truncate,
-        .open        = trace_open,
-        .readv       = trace_readv,
-        .writev      = trace_writev,
-        .statfs      = trace_statfs,
-        .flush       = trace_flush,
-        .fsync       = trace_fsync,
-        .setxattr    = trace_setxattr,
-        .getxattr    = trace_getxattr,
-        .fsetxattr   = trace_fsetxattr,
-        .fgetxattr   = trace_fgetxattr,
-        .removexattr = trace_removexattr,
-        .opendir     = trace_opendir,
-        .readdir     = trace_readdir,
-        .readdirp    = trace_readdirp,
-        .fsyncdir    = trace_fsyncdir,
-        .access      = trace_access,
-        .ftruncate   = trace_ftruncate,
-        .fstat       = trace_fstat,
-        .create      = trace_create,
-        .lk          = trace_lk,
-        .inodelk     = trace_inodelk,
-        .finodelk    = trace_finodelk,
-        .entrylk     = trace_entrylk,
-        .fentrylk    = trace_fentrylk,
-        .lookup      = trace_lookup,
-        .rchecksum   = trace_rchecksum,
-        .xattrop     = trace_xattrop,
-        .fxattrop    = trace_fxattrop,
-        .setattr     = trace_setattr,
-        .fsetattr    = trace_fsetattr,
-        .seek        = trace_seek,
+    .stat = trace_stat,
+    .readlink = trace_readlink,
+    .mknod = trace_mknod,
+    .mkdir = trace_mkdir,
+    .unlink = trace_unlink,
+    .rmdir = trace_rmdir,
+    .symlink = trace_symlink,
+    .rename = trace_rename,
+    .link = trace_link,
+    .truncate = trace_truncate,
+    .open = trace_open,
+    .readv = trace_readv,
+    .writev = trace_writev,
+    .statfs = trace_statfs,
+    .flush = trace_flush,
+    .fsync = trace_fsync,
+    .setxattr = trace_setxattr,
+    .getxattr = trace_getxattr,
+    .fsetxattr = trace_fsetxattr,
+    .fgetxattr = trace_fgetxattr,
+    .removexattr = trace_removexattr,
+    .opendir = trace_opendir,
+    .readdir = trace_readdir,
+    .readdirp = trace_readdirp,
+    .fsyncdir = trace_fsyncdir,
+    .access = trace_access,
+    .ftruncate = trace_ftruncate,
+    .fstat = trace_fstat,
+    .create = trace_create,
+    .lk = trace_lk,
+    .inodelk = trace_inodelk,
+    .finodelk = trace_finodelk,
+    .entrylk = trace_entrylk,
+    .fentrylk = trace_fentrylk,
+    .lookup = trace_lookup,
+    .rchecksum = trace_rchecksum,
+    .xattrop = trace_xattrop,
+    .fxattrop = trace_fxattrop,
+    .setattr = trace_setattr,
+    .fsetattr = trace_fsetattr,
+    .seek = trace_seek,
 };
 
 struct xlator_cbks cbks = {
-        .release     = trace_release,
-        .releasedir  = trace_releasedir,
-        .forget      = trace_forget,
+    .release = trace_release,
+    .releasedir = trace_releasedir,
+    .forget = trace_forget,
 };
 
 struct volume_options options[] = {
-        { .key  = {"include-ops", "include"},
-          .type = GF_OPTION_TYPE_STR,
-          /*.value = { ""} */
-        },
-        { .key  = {"exclude-ops", "exclude"},
-          .type = GF_OPTION_TYPE_STR
-          /*.value = { ""} */
-        },
-        { .key  = {"history-size"},
-          .type = GF_OPTION_TYPE_SIZET,
-          .default_value = "1024",
-        },
-        { .key  = {"log-file"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-        },
-        { .key  = {"log-history"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-        },
-        { .key  = {NULL} },
+    {
+        .key = {"include-ops", "include"},
+        .type = GF_OPTION_TYPE_STR,
+        /*.value = { ""} */
+    },
+    {
+        .key = {"exclude-ops", "exclude"},
+        .type = GF_OPTION_TYPE_STR
+        /*.value = { ""} */
+    },
+    {
+        .key = {"history-size"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .default_value = "1024",
+    },
+    {
+        .key = {"log-file"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+    },
+    {
+        .key = {"log-history"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+    },
+    {.key = {NULL}},
 };
 
-struct xlator_dumpops dumpops = {
-        .history = trace_dump_history
+struct xlator_dumpops dumpops = {.history = trace_dump_history};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "trace",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/debug/trace/src/trace.h b/xlators/debug/trace/src/trace.h
index 3b5f7891d00..b16304799da 100644
--- a/xlators/debug/trace/src/trace.h
+++ b/xlators/debug/trace/src/trace.h
@@ -10,47 +10,46 @@
 
 #include <time.h>
 #include <errno.h>
-#include "glusterfs.h"
-#include "xlator.h"
-#include "common-utils.h"
-#include "event-history.h"
-#include "logging.h"
-#include "circ-buff.h"
-#include "statedump.h"
-#include "options.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/event-history.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/circ-buff.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/options.h>
 
 #define TRACE_DEFAULT_HISTORY_SIZE 1024
 
 typedef struct {
-        /* Since the longest fop name is fremovexattr i.e 12 characters, array size
-         * is kept 24, i.e double of the maximum.
-         */
-        char name[24];
-        int enabled;
+    /* Since the longest fop name is fremovexattr i.e 12 characters, array size
+     * is kept 24, i.e double of the maximum.
+     */
+    char name[24];
+    int enabled;
 } trace_fop_name_t;
 
 trace_fop_name_t trace_fop_names[GF_FOP_MAXVALUE];
 
 typedef struct {
-        gf_boolean_t log_file;
-        gf_boolean_t log_history;
-        size_t       history_size;
-        int trace_log_level;
+    gf_boolean_t log_file;
+    gf_boolean_t log_history;
+    uint64_t history_size;
+    int trace_log_level;
 } trace_conf_t;
 
-#define TRACE_STACK_UNWIND(op, frame, params ...)                       \
-        do {                                                            \
-                frame->local = NULL;                                    \
-                STACK_UNWIND_STRICT (op, frame, params);                \
-        } while (0);
-
-#define LOG_ELEMENT(_conf, _string)                                     \
-        do {                                                            \
-                if (_conf) {                                            \
-                        if ((_conf->log_history) == _gf_true)           \
-                                gf_log_eh ("%s", _string);              \
-                        if ((_conf->log_file) == _gf_true)              \
-                                gf_log (THIS->name, _conf->trace_log_level, \
-                                        "%s", _string);                 \
-                }                                                       \
-        } while (0);
+#define TRACE_STACK_UNWIND(op, frame, params...)                               \
+    do {                                                                       \
+        frame->local = NULL;                                                   \
+        STACK_UNWIND_STRICT(op, frame, params);                                \
+    } while (0);
+
+#define LOG_ELEMENT(_conf, _string)                                            \
+    do {                                                                       \
+        if (_conf) {                                                           \
+            if ((_conf->log_history) == _gf_true)                              \
+                gf_log_eh("%s", _string);                                      \
+            if ((_conf->log_file) == _gf_true)                                 \
+                gf_log(THIS->name, _conf->trace_log_level, "%s", _string);     \
+        }                                                                      \
+    } while (0);
diff --git a/xlators/encryption/Makefile.am b/xlators/encryption/Makefile.am
deleted file mode 100644
index 36efc6698bd..00000000000
--- a/xlators/encryption/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = rot-13 crypt
-
-CLEANFILES = 
diff --git a/xlators/encryption/crypt/Makefile.am b/xlators/encryption/crypt/Makefile.am
deleted file mode 100644
index d471a3f9243..00000000000
--- a/xlators/encryption/crypt/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/encryption/crypt/src/Makefile.am b/xlators/encryption/crypt/src/Makefile.am
deleted file mode 100644
index 05fd3d5096b..00000000000
--- a/xlators/encryption/crypt/src/Makefile.am
+++ /dev/null
@@ -1,26 +0,0 @@
-if ENABLE_CRYPT_XLATOR
-
-xlator_LTLIBRARIES = crypt.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/encryption
-
-crypt_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-
-crypt_la_SOURCES = keys.c data.c metadata.c atom.c crypt.c
-crypt_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-	-lssl -lcrypto
-
-noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES =
-
-else
-
-noinst_DIST = keys.c data.c metadata.c atom.c crypt.c
-noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
-
-endif
diff --git a/xlators/encryption/crypt/src/atom.c b/xlators/encryption/crypt/src/atom.c
deleted file mode 100644
index 6d7b351b370..00000000000
--- a/xlators/encryption/crypt/src/atom.c
+++ /dev/null
@@ -1,957 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include "defaults.h"
-#include "crypt-common.h"
-#include "crypt.h"
-
-/*
- *                              Glossary
- *
- *
- * cblock            (or cipher block). A logical unit in a file.
- *                   cblock size is defined as the number of bits
- *                   in an input (or output) block of the block
- *                   cipher (*). Cipher block size is a property of
- *                   cipher algorithm. E.g. cblock size is 64 bits
- *                   for DES, 128 bits for AES, etc.
- *
- * atomic cipher     A cipher algorithm, which requires some chunks of
- * algorithm         text to be padded at left and(or) right sides before
- *                   cipher transaform.
- *
- *
- * block (atom)      Minimal chunk of file's data, which doesn't require
- *                   padding. We'll consider logical units in a file of
- *                   block size (atom size).
- *
- * cipher algorithm  Atomic cipher algorithm, which requires the last
- * with EOF issue    incomplete cblock in a file to be padded with some
- *                   data (usually zeros).
- *
- *
- * operation, which  reading/writing from offset, which is not aligned to
- * forms a gap at    to atom size
- * the beginning
- *
- *
- * operation, which  reading/writing count bytes starting from offset off,
- * forms a gap at    so that off+count is not aligned to atom_size
- * the end
- *
- * head block        the first atom affected by an operation, which forms
- *                   a gap at the beginning, or(and) at the end.
- *                   Сomment. Head block has at least one gap (either at
- *                   the beginning, or at the end)
- *
- *
- * tail block        the last atom different from head, affected by an
- *                   operation, which forms a gap at the end.
- *                   Сomment: Tail block has exactly one gap (at the end).
- *
- *
- * partial block     head or tail block
- *
- *
- * full block        block without gaps.
- *
- *
- * (*) Recommendation for Block Cipher Modes of Operation
- *     Methods and Techniques
- *     NIST Special Publication 800-38A Edition 2001
- */
-
-/*
- * atom->offset_at()
- */
-static off_t offset_at_head(struct avec_config *conf)
-{
-	return conf->aligned_offset;
-}
-
-static off_t offset_at_hole_head(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_head(get_hole_conf(frame));
-}
-
-static off_t offset_at_data_head(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_head(get_data_conf(frame));
-}
-
-
-static off_t offset_at_tail(struct avec_config *conf,
-				    struct object_cipher_info *object)
-{
-	return conf->aligned_offset +
-		(conf->off_in_head ? get_atom_size(object) : 0) +
-		(conf->nr_full_blocks << get_atom_bits(object));
-}
-
-static off_t offset_at_hole_tail(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_tail(get_hole_conf(frame), object);
-}
-
-
-static off_t offset_at_data_tail(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_tail(get_data_conf(frame), object);
-}
-
-static off_t offset_at_full(struct avec_config *conf,
-				    struct object_cipher_info *object)
-{
-	return conf->aligned_offset +
-		(conf->off_in_head ? get_atom_size(object) : 0);
-}
-
-static off_t offset_at_data_full(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_full(get_data_conf(frame), object);
-}
-
-static off_t offset_at_hole_full(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_full(get_hole_conf(frame), object);
-}
-
-/*
- * atom->io_size_nopad()
- */
-
-static uint32_t io_size_nopad_head(struct avec_config *conf,
-				   struct object_cipher_info *object)
-{
-	uint32_t gap_at_beg;
-	uint32_t gap_at_end;
-
-	check_head_block(conf);
-
-	gap_at_beg = conf->off_in_head;
-
-	if (has_tail_block(conf) || has_full_blocks(conf) || conf->off_in_tail == 0 )
-		gap_at_end = 0;
-	else
-		gap_at_end = get_atom_size(object) - conf->off_in_tail;
-
-	return get_atom_size(object) - (gap_at_beg + gap_at_end);
-}
-
-static uint32_t io_size_nopad_tail(struct avec_config *conf,
-				   struct object_cipher_info *object)
-{
-	check_tail_block(conf);
-	return conf->off_in_tail;
-}
-
-static uint32_t io_size_nopad_full(struct avec_config *conf,
-				   struct object_cipher_info *object)
-{
-	check_full_block(conf);
-	return get_atom_size(object);
-}
-
-static uint32_t io_size_nopad_data_head(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_head(get_data_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_hole_head(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_head(get_hole_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_data_tail(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_tail(get_data_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_hole_tail(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_tail(get_hole_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_data_full(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_full(get_data_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_hole_full(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_full(get_hole_conf(frame), object);
-}
-
-static uint32_t offset_in_head(struct avec_config *conf)
-{
-	check_cursor_head(conf);
-
-	return conf->off_in_head;
-}
-
-static uint32_t offset_in_tail(call_frame_t *frame,
-			       struct object_cipher_info *object)
-{
-	return 0;
-}
-
-static uint32_t offset_in_full(struct avec_config *conf,
-			       struct object_cipher_info *object)
-{
-	check_cursor_full(conf);
-
-	if (has_head_block(conf))
-		return (conf->cursor - 1) << get_atom_bits(object);
-	else
-		return conf->cursor << get_atom_bits(object);
-}
-
-static uint32_t offset_in_data_head(call_frame_t *frame,
-				    struct object_cipher_info *object)
-{
-	return offset_in_head(get_data_conf(frame));
-}
-
-static uint32_t offset_in_hole_head(call_frame_t *frame,
-				    struct object_cipher_info *object)
-{
-	return offset_in_head(get_hole_conf(frame));
-}
-
-static uint32_t offset_in_data_full(call_frame_t *frame,
-				    struct object_cipher_info *object)
-{
-	return offset_in_full(get_data_conf(frame), object);
-}
-
-static uint32_t offset_in_hole_full(call_frame_t *frame,
-				    struct object_cipher_info *object)
-{
-	return offset_in_full(get_hole_conf(frame), object);
-}
-
-/*
- * atom->rmw()
- */
-/*
- * Pre-conditions:
- * @vec contains plain text of the latest
- * version.
- *
- * Uptodate gaps of the @partial block with
- * this plain text, encrypt the whole block
- * and write the result to disk.
- */
-static int32_t rmw_partial_block(call_frame_t *frame,
-				 void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret,
-				 int32_t op_errno,
-				 struct iovec *vec,
-				 int32_t count,
-				 struct iatt *stbuf,
-				 struct iobref *iobref,
-				 struct rmw_atom *atom)
-{
-	size_t was_read = 0;
-	uint64_t file_size;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	struct iovec *partial = atom->get_iovec(frame, 0);
-	struct avec_config *conf = atom->get_config(frame);
-	end_writeback_handler_t end_writeback_partial_block;
-#if DEBUG_CRYPT
-	gf_boolean_t check_last_cblock = _gf_false;
-#endif
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0)
-		goto exit;
-
-	file_size = local->cur_file_size;
-	was_read = op_ret;
-
-	if (atom->locality == HEAD_ATOM && conf->off_in_head) {
-		/*
-		 * head atom with a non-uptodate gap
-		 * at the beginning
-		 *
-		 * fill the gap with plain text of the
-		 * latest version. Convert a part of hole
-		 * (if any) to zeros.
-		 */
-		int32_t i;
-		int32_t copied = 0;
-		int32_t to_gap; /* amount of data needed to uptodate
-				   the gap at the beginning */
-#if 0
-		int32_t hole = 0; /* The part of the hole which
-				   * got in the head block */
-#endif /* 0 */
-		to_gap = conf->off_in_head;
-
-		if (was_read < to_gap) {
-			if (file_size >
-			    offset_at_head(conf) + was_read) {
-				/*
-				 * It is impossible to uptodate
-				 * head block: too few bytes have
-				 * been read from disk, so that
-				 * partial write is impossible.
-				 *
-				 * It could happen because of many
-				 * reasons: IO errors, (meta)data
-				 * corruption in the local file system,
-				 * etc.
-				 */
-				gf_log(this->name, GF_LOG_WARNING,
-				     "Can not uptodate a gap at the beginning");
-				local->op_ret = -1;
-				local->op_errno = EIO;
-				goto exit;
-			}
-#if 0
-			hole = to_gap - was_read;
-#endif /* 0 */
-			to_gap = was_read;
-		}
-		/*
-		 * uptodate the gap at the beginning
-		 */
-		for (i = 0; i < count && copied < to_gap; i++) {
-			int32_t to_copy;
-
-			to_copy = vec[i].iov_len;
-			if (to_copy > to_gap - copied)
-				to_copy = to_gap - copied;
-
-			memcpy(partial->iov_base, vec[i].iov_base, to_copy);
-			copied += to_copy;
-		}
-#if 0
-		/*
-		 * If possible, convert part of the
-		 * hole, which got in the head block
-		 */
-		ret = TRY_LOCK(&local->hole_lock);
-		if (!ret) {
-			if (local->hole_handled)
-				/*
-				 * already converted by
-				 * crypt_writev_cbk()
-				 */
-				UNLOCK(&local->hole_lock);
-			else {
-				/*
-				 * convert the part of the hole
-				 * which got in the head block
-				 * to zeros.
-				 *
-				 * Update the orig_offset to make
-				 * sure writev_cbk() won't care
-				 * about this part of the hole.
-				 *
-				 */
-				memset(partial->iov_base + to_gap, 0, hole);
-
-				conf->orig_offset -= hole;
-				conf->orig_size += hole;
-				UNLOCK(&local->hole_lock);
-			}
-		}
-		else /*
-		      * conversion is being performed
-		      * by crypt_writev_cbk()
-		      */
-			;
-#endif /* 0 */
-	}
-	if (atom->locality == TAIL_ATOM ||
-	    (!has_tail_block(conf) && conf->off_in_tail)) {
-		/*
-		 * tail atom, or head atom with a non-uptodate
-		 * gap at the end.
-		 *
-		 * fill the gap at the end of the block
-		 * with plain text of the latest version.
-		 * Pad the result, (if needed)
-		 */
-		int32_t i;
-		int32_t to_gap;
-		int copied;
-		off_t off_in_tail;
-		int32_t to_copy;
-
-		off_in_tail = conf->off_in_tail;
-		to_gap = conf->gap_in_tail;
-
-		if (to_gap && was_read < off_in_tail + to_gap) {
-			/*
-			 * It is impossible to uptodate
-			 * the gap at the end: too few bytes
-			 * have been read from disk, so that
-			 * partial write is impossible.
-			 *
-			 * It could happen because of many
-			 * reasons: IO errors, (meta)data
-			 * corruption in the local file system,
-			 * etc.
-			 */
-			gf_log(this->name, GF_LOG_WARNING,
-			       "Can not uptodate a gap at the end");
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			goto exit;
-		}
-		/*
-		 * uptodate the gap at the end
-		 */
-		copied = 0;
-		to_copy = to_gap;
-		for(i = count - 1; i >= 0 && to_copy > 0; i--) {
-			uint32_t from_vec, off_in_vec;
-
-			off_in_vec = 0;
-			from_vec = vec[i].iov_len;
-			if (from_vec > to_copy) {
-				off_in_vec = from_vec - to_copy;
-				from_vec = to_copy;
-			}
-			memcpy(partial->iov_base +
-			       off_in_tail + to_gap - copied - from_vec,
-			       vec[i].iov_base + off_in_vec,
-			       from_vec);
-
-			gf_log(this->name, GF_LOG_DEBUG,
-			       "uptodate %d bytes at tail. Offset at target(source): %d(%d)",
-			       (int)from_vec,
-			       (int)off_in_tail + to_gap - copied - from_vec,
-			       (int)off_in_vec);
-
-			copied += from_vec;
-			to_copy -= from_vec;
-		}
-		partial->iov_len = off_in_tail + to_gap;
-
-		if (object_alg_should_pad(object)) {
-			int32_t resid = 0;
-			resid = partial->iov_len & (object_alg_blksize(object) - 1);
-			if (resid) {
-				/*
-				 * append a new EOF padding
-				 */
-				local->eof_padding_size =
-					object_alg_blksize(object) - resid;
-
-				gf_log(this->name, GF_LOG_DEBUG,
-				       "set padding size %d",
-				       local->eof_padding_size);
-
-				memset(partial->iov_base + partial->iov_len,
-				       1,
-				       local->eof_padding_size);
-				partial->iov_len += local->eof_padding_size;
-#if DEBUG_CRYPT
-				gf_log(this->name, GF_LOG_DEBUG,
-				       "pad cblock with %d zeros:",
-				       local->eof_padding_size);
-				dump_cblock(this,
-					    (unsigned char *)partial->iov_base +
-					    partial->iov_len - object_alg_blksize(object));
-				check_last_cblock = _gf_true;
-#endif
-			}
-		}
-	}
-	/*
-	 * encrypt the whole block
-	 */
-	encrypt_aligned_iov(object,
-			    partial,
-			    1,
-			    atom->offset_at(frame, object));
-#if DEBUG_CRYPT
-	if (check_last_cblock == _gf_true) {
-		gf_log(this->name, GF_LOG_DEBUG,
-		       "encrypt last cblock with offset %llu",
-		       (unsigned long long)atom->offset_at(frame, object));
-		dump_cblock(this, (unsigned char *)partial->iov_base +
-			    partial->iov_len - object_alg_blksize(object));
-	}
-#endif
-	set_local_io_params_writev(frame, object, atom,
-				   atom->offset_at(frame, object),
-				   iov_length(partial, 1));
-	/*
-	 * write the whole block to disk
-	 */
-	end_writeback_partial_block = dispatch_end_writeback(local->fop);
-	conf->cursor ++;
-	STACK_WIND(frame,
-		   end_writeback_partial_block,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->writev,
-		   local->fd,
-		   partial,
-		   1,
-		   atom->offset_at(frame, object),
-		   local->flags,
-		   local->iobref_data,
-		   local->xdata);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "submit partial block: %d bytes from %d offset",
-	       (int)iov_length(partial, 1),
-	       (int)atom->offset_at(frame, object));
- exit:
-	return 0;
-}
-
-/*
- * Perform a (read-)modify-write sequence.
- * This should be performed only after approval
- * of upper server-side manager, i.e. the caller
- * needs to make sure this is his turn to rmw.
- */
-void submit_partial(call_frame_t *frame,
-		    xlator_t *this,
-		    fd_t *fd,
-		    atom_locality_type ltype)
-{
-	int32_t ret;
-	dict_t *dict;
-	struct rmw_atom *atom;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	atom = atom_by_types(local->active_setup, ltype);
-	/*
-	 * To perform the "read" component of the read-modify-write
-	 * sequence the crypt translator does stack_wind to itself.
-	 *
-	 * Pass current file size to crypt_readv()
-	 */
-	dict = dict_new();
-	if (!dict) {
-		/*
-		 * FIXME: Handle the error
-		 */
-		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
-		return;
-	}
-	ret = dict_set(dict,
-		       FSIZE_XATTR_PREFIX,
-		       data_from_uint64(local->cur_file_size));
-	if (ret) {
-		/*
-		 * FIXME: Handle the error
-		 */
-		dict_unref(dict);
-		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
-		goto exit;
-	}
-	STACK_WIND(frame,
-		   atom->rmw,
-		   this,
-		   this->fops->readv, /* crypt_readv */
-		   fd,
-		   atom->count_to_uptodate(frame, object), /* count */
-		   atom->offset_at(frame, object), /* offset to read from */
-		   0,
-		   dict);
- exit:
-	dict_unref(dict);
-}
-
-/*
- * submit blocks of FULL_ATOM type
- */
-void submit_full(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-	struct rmw_atom *atom = atom_by_types(local->active_setup, FULL_ATOM);
-	uint32_t count; /* total number of full blocks to submit */
-	uint32_t granularity; /* number of blocks to submit in one iteration */
-
-	uint64_t off_in_file; /* start offset in the file, bytes */
-	uint32_t off_in_atom; /* start offset in the atom, blocks */
-	uint32_t blocks_written = 0; /* blocks written for this submit */
-
-	struct avec_config *conf = atom->get_config(frame);
-	end_writeback_handler_t end_writeback_full_block;
-	/*
-	 * Write full blocks by groups of granularity size.
-	 */
-	end_writeback_full_block = dispatch_end_writeback(local->fop);
-
-	if (is_ordered_mode(frame)) {
-		uint32_t skip = has_head_block(conf) ? 1 : 0;
-		count = 1;
-		granularity = 1;
-		/*
-		 * calculate start offset using cursor value;
-		 * here we should take into accout head block,
-		 * which corresponds to cursor value 0.
-		 */
-		off_in_file = atom->offset_at(frame, object) +
-			((conf->cursor - skip) << get_atom_bits(object));
-		off_in_atom = conf->cursor - skip;
-	}
-	else {
-		/*
-		 * in parallel mode
-		 */
-		count = conf->nr_full_blocks;
-		granularity = MAX_IOVEC;
-		off_in_file = atom->offset_at(frame, object);
-		off_in_atom = 0;
-	}
-	while (count) {
-		uint32_t blocks_to_write = count;
-
-		if (blocks_to_write > granularity)
-			blocks_to_write = granularity;
-		if (conf->type == HOLE_ATOM)
-			/*
-			 * reset iovec before encryption
-			 */
-			memset(atom->get_iovec(frame, 0)->iov_base,
-			       0,
-			       get_atom_size(object));
-		/*
-		 * encrypt the group
-		 */
-		encrypt_aligned_iov(object,
-				    atom->get_iovec(frame,
-						    off_in_atom +
-						    blocks_written),
-				    blocks_to_write,
-				    off_in_file + (blocks_written <<
-						   get_atom_bits(object)));
-
-		set_local_io_params_writev(frame, object, atom,
-		        off_in_file + (blocks_written << get_atom_bits(object)),
-			blocks_to_write <<  get_atom_bits(object));
-
-		conf->cursor += blocks_to_write;
-
-		STACK_WIND(frame,
-			   end_writeback_full_block,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->writev,
-			   local->fd,
-			   atom->get_iovec(frame, off_in_atom + blocks_written),
-			   blocks_to_write,
-			   off_in_file + (blocks_written << get_atom_bits(object)),
-			   local->flags,
-			   local->iobref_data ? local->iobref_data : local->iobref,
-			   local->xdata);
-
-		gf_log("crypt", GF_LOG_DEBUG, "submit %d full blocks from %d offset",
-		       blocks_to_write,
-		       (int)(off_in_file + (blocks_written << get_atom_bits(object))));
-
-		count -= blocks_to_write;
-		blocks_written += blocks_to_write;
-	}
-	return;
-}
-
-static int32_t rmw_data_head(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     struct iovec *vec,
-			     int32_t count,
-			     struct iatt *stbuf,
-			     struct iobref *iobref,
-			     dict_t *xdata)
-{
-	return rmw_partial_block(frame,
-				 cookie,
-				 this,
-				 op_ret,
-				 op_errno,
-				 vec,
-				 count,
-				 stbuf,
-				 iobref,
-				 atom_by_types(DATA_ATOM, HEAD_ATOM));
-}
-
-static int32_t rmw_data_tail(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     struct iovec *vec,
-			     int32_t count,
-			     struct iatt *stbuf,
-			     struct iobref *iobref,
-			     dict_t *xdata)
-{
-	return rmw_partial_block(frame,
-				 cookie,
-				 this,
-				 op_ret,
-				 op_errno,
-				 vec,
-				 count,
-				 stbuf,
-				 iobref,
-				 atom_by_types(DATA_ATOM, TAIL_ATOM));
-}
-
-static int32_t rmw_hole_head(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     struct iovec *vec,
-			     int32_t count,
-			     struct iatt *stbuf,
-			     struct iobref *iobref,
-			     dict_t *xdata)
-{
-	return rmw_partial_block(frame,
-				 cookie,
-				 this,
-				 op_ret,
-				 op_errno,
-				 vec,
-				 count,
-				 stbuf,
-				 iobref,
-				 atom_by_types(HOLE_ATOM, HEAD_ATOM));
-}
-
-static int32_t rmw_hole_tail(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     struct iovec *vec,
-			     int32_t count,
-			     struct iatt *stbuf,
-			     struct iobref *iobref,
-			     dict_t *xdata)
-{
-	return rmw_partial_block(frame,
-				 cookie,
-				 this,
-				 op_ret,
-				 op_errno,
-				 vec,
-				 count,
-				 stbuf,
-				 iobref,
-				 atom_by_types(HOLE_ATOM, TAIL_ATOM));
-}
-
-/*
- * atom->count_to_uptodate()
- */
-static uint32_t count_to_uptodate_head(struct avec_config *conf,
-				       struct object_cipher_info *object)
-{
-	if (conf->acount == 1 && conf->off_in_tail)
-		return get_atom_size(object);
-	else
-		/* there is no need to read the whole head block */
-		return conf->off_in_head;
-}
-
-static uint32_t count_to_uptodate_tail(struct avec_config *conf,
-				       struct object_cipher_info *object)
-{
-	/* we need to read the whole tail block */
-	return get_atom_size(object);
-}
-
-static uint32_t count_to_uptodate_data_head(call_frame_t *frame,
-					    struct object_cipher_info *object)
-{
-	return count_to_uptodate_head(get_data_conf(frame), object);
-}
-
-static uint32_t count_to_uptodate_data_tail(call_frame_t *frame,
-					    struct object_cipher_info *object)
-{
-	return count_to_uptodate_tail(get_data_conf(frame), object);
-}
-
-static uint32_t count_to_uptodate_hole_head(call_frame_t *frame,
-					    struct object_cipher_info *object)
-{
-	return count_to_uptodate_head(get_hole_conf(frame), object);
-}
-
-static uint32_t count_to_uptodate_hole_tail(call_frame_t *frame,
-					    struct object_cipher_info *object)
-{
-	return count_to_uptodate_tail(get_hole_conf(frame), object);
-}
-
-/* atom->get_config() */
-
-static struct avec_config *get_config_data(call_frame_t *frame)
-{
-	return &((crypt_local_t *)frame->local)->data_conf;
-}
-
-static struct avec_config *get_config_hole(call_frame_t *frame)
-{
-	return &((crypt_local_t *)frame->local)->hole_conf;
-}
-
-/*
- * atom->get_iovec()
- */
-static struct iovec *get_iovec_hole_head(call_frame_t *frame,
-					 uint32_t count)
-{
-	struct avec_config *conf = get_hole_conf(frame);
-
-	return conf->avec;
-}
-
-static struct iovec *get_iovec_hole_full(call_frame_t *frame,
-					 uint32_t count)
-{
-	struct avec_config *conf = get_hole_conf(frame);
-
-	return conf->avec + (conf->off_in_head ? 1 : 0);
-}
-
-static struct iovec *get_iovec_hole_tail(call_frame_t *frame,
-						uint32_t count)
-{
-	struct avec_config *conf = get_hole_conf(frame);
-
-	return conf->avec + (conf->blocks_in_pool - 1);
-}
-
-static struct iovec *get_iovec_data_head(call_frame_t *frame,
-						uint32_t count)
-{
-	struct avec_config *conf = get_data_conf(frame);
-
-	return conf->avec;
-}
-
-static struct iovec *get_iovec_data_full(call_frame_t *frame,
-						uint32_t count)
-{
-	struct avec_config *conf = get_data_conf(frame);
-
-	return conf->avec + (conf->off_in_head ? 1 : 0) + count;
-}
-
-static struct iovec *get_iovec_data_tail(call_frame_t *frame,
-						uint32_t count)
-{
-	struct avec_config *conf = get_data_conf(frame);
-
-	return conf->avec +
-		(conf->off_in_head ? 1 : 0) +
-		conf->nr_full_blocks;
-}
-
-static struct rmw_atom atoms[LAST_DATA_TYPE][LAST_LOCALITY_TYPE] = {
-	[DATA_ATOM][HEAD_ATOM] =
-	{ .locality = HEAD_ATOM,
-	  .rmw = rmw_data_head,
-	  .offset_at = offset_at_data_head,
-	  .offset_in = offset_in_data_head,
-	  .get_iovec = get_iovec_data_head,
-	  .io_size_nopad = io_size_nopad_data_head,
-	  .count_to_uptodate = count_to_uptodate_data_head,
-	  .get_config = get_config_data
-	},
-	[DATA_ATOM][TAIL_ATOM] =
-	{ .locality = TAIL_ATOM,
-	  .rmw = rmw_data_tail,
-	  .offset_at = offset_at_data_tail,
-	  .offset_in = offset_in_tail,
-	  .get_iovec = get_iovec_data_tail,
-	  .io_size_nopad = io_size_nopad_data_tail,
-	  .count_to_uptodate = count_to_uptodate_data_tail,
-	  .get_config = get_config_data
-	},
-	[DATA_ATOM][FULL_ATOM] =
-	{ .locality = FULL_ATOM,
-	  .offset_at = offset_at_data_full,
-	  .offset_in = offset_in_data_full,
-	  .get_iovec = get_iovec_data_full,
-	  .io_size_nopad = io_size_nopad_data_full,
-	  .get_config = get_config_data
-	},
-	[HOLE_ATOM][HEAD_ATOM] =
-	{ .locality = HEAD_ATOM,
-	  .rmw = rmw_hole_head,
-	  .offset_at = offset_at_hole_head,
-	  .offset_in = offset_in_hole_head,
-	  .get_iovec = get_iovec_hole_head,
-	  .io_size_nopad = io_size_nopad_hole_head,
-	  .count_to_uptodate = count_to_uptodate_hole_head,
-	  .get_config = get_config_hole
-	},
-	[HOLE_ATOM][TAIL_ATOM] =
-	{ .locality = TAIL_ATOM,
-	  .rmw = rmw_hole_tail,
-	  .offset_at = offset_at_hole_tail,
-	  .offset_in = offset_in_tail,
-	  .get_iovec = get_iovec_hole_tail,
-	  .io_size_nopad = io_size_nopad_hole_tail,
-	  .count_to_uptodate = count_to_uptodate_hole_tail,
-	  .get_config = get_config_hole
-	},
-	[HOLE_ATOM][FULL_ATOM] =
-	{ .locality = FULL_ATOM,
-	  .offset_at = offset_at_hole_full,
-	  .offset_in = offset_in_hole_full,
-	  .get_iovec = get_iovec_hole_full,
-	  .io_size_nopad = io_size_nopad_hole_full,
-	  .get_config = get_config_hole
-	}
-};
-
-struct rmw_atom *atom_by_types(atom_data_type data,
-			       atom_locality_type locality)
-{
-	return &atoms[data][locality];
-}
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/crypt-common.h b/xlators/encryption/crypt/src/crypt-common.h
deleted file mode 100644
index 7c212ad5d25..00000000000
--- a/xlators/encryption/crypt/src/crypt-common.h
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __CRYPT_COMMON_H__
-#define __CRYPT_COMMON_H__
-
-#define INVAL_SUBVERSION_NUMBER (0xff)
-#define CRYPT_INVAL_OP   (GF_FOP_NULL)
-
-#define CRYPTO_FORMAT_PREFIX  "trusted.glusterfs.crypt.att.cfmt"
-#define FSIZE_XATTR_PREFIX    "trusted.glusterfs.crypt.att.size"
-#define SUBREQ_PREFIX         "trusted.glusterfs.crypt.msg.sreq"
-#define FSIZE_MSG_PREFIX      "trusted.glusterfs.crypt.msg.size"
-#define DE_MSG_PREFIX         "trusted.glusterfs.crypt.msg.dent"
-#define REQUEST_ID_PREFIX     "trusted.glusterfs.crypt.msg.rqid"
-#define MSGFLAGS_PREFIX       "trusted.glusterfs.crypt.msg.xfgs"
-
-
-/* messages for crypt_open() */
-#define MSGFLAGS_REQUEST_MTD_RLOCK   1 /* take read lock and don't unlock */
-#define MSGFLAGS_REQUEST_MTD_WLOCK   2 /* take write lock and don't unlock */
-
-#define AES_BLOCK_BITS (4) /* AES_BLOCK_SIZE == 1 << AES_BLOCK_BITS */
-
-#define noop   do {; } while (0)
-#define cassert(cond) ({ switch (-1) { case (cond): case 0: break; } })
-#define __round_mask(x, y) ((__typeof__(x))((y)-1))
-#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
-
-/*
- * Format of file's metadata
- */
-struct crypt_format {
-	uint8_t loader_id;  /* version of metadata loader */
-	uint8_t versioned[0]; /* file's metadata of specific version */
-} __attribute__((packed));
-
-typedef enum {
-	AES_CIPHER_ALG,
-	LAST_CIPHER_ALG
-} cipher_alg_t;
-
-typedef enum {
-	XTS_CIPHER_MODE,
-	LAST_CIPHER_MODE
-} cipher_mode_t;
-
-typedef enum {
-	MTD_LOADER_V1,
-	LAST_MTD_LOADER
-} mtd_loader_id;
-
-static inline void msgflags_set_mtd_rlock(uint32_t *flags)
-{
-	*flags |= MSGFLAGS_REQUEST_MTD_RLOCK;
-}
-
-static inline void msgflags_set_mtd_wlock(uint32_t *flags)
-{
-	*flags |= MSGFLAGS_REQUEST_MTD_WLOCK;
-}
-
-static inline gf_boolean_t msgflags_check_mtd_rlock(uint32_t *flags)
-{
-	return *flags & MSGFLAGS_REQUEST_MTD_RLOCK;
-}
-
-static inline gf_boolean_t msgflags_check_mtd_wlock(uint32_t *flags)
-{
-	return *flags & MSGFLAGS_REQUEST_MTD_WLOCK;
-}
-
-static inline gf_boolean_t msgflags_check_mtd_lock(uint32_t *flags)
-{
-	return  msgflags_check_mtd_rlock(flags) ||
-		msgflags_check_mtd_wlock(flags);
-}
-
-/*
- * returns number of logical blocks occupied
- * (maybe partially) by @count bytes
- * at offset @start.
- */
-static inline off_t logical_blocks_occupied(uint64_t start, off_t count,
-					    int blkbits)
-{
-	return ((start + count - 1) >> blkbits) - (start >> blkbits) + 1;
-}
-
-/*
- * are two bytes (represented by offsets @off1
- * and @off2 respectively) in the same logical
- * block.
- */
-static inline int in_same_lblock(uint64_t off1, uint64_t off2,
-				 int blkbits)
-{
-	return off1 >> blkbits == off2 >> blkbits;
-}
-
-static inline void dump_cblock(xlator_t *this, unsigned char *buf)
-{
-	gf_log(this->name, GF_LOG_DEBUG,
-	       "dump cblock: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x",
-	       (buf)[0],
-	       (buf)[1],
-	       (buf)[2],
-	       (buf)[3],
-	       (buf)[4],
-	       (buf)[5],
-	       (buf)[6],
-	       (buf)[7],
-	       (buf)[8],
-	       (buf)[9],
-	       (buf)[10],
-	       (buf)[11],
-	       (buf)[12],
-	       (buf)[13],
-	       (buf)[14],
-	       (buf)[15]);
-}
-
-#endif /* __CRYPT_COMMON_H__ */
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/crypt-mem-types.h b/xlators/encryption/crypt/src/crypt-mem-types.h
deleted file mode 100644
index 1954c579423..00000000000
--- a/xlators/encryption/crypt/src/crypt-mem-types.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef __CRYPT_MEM_TYPES_H__
-#define __CRYPT_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_crypt_mem_types_ {
-	gf_crypt_mt_priv = gf_common_mt_end + 1,
-	gf_crypt_mt_inode,
-	gf_crypt_mt_data,
-	gf_crypt_mt_mtd,
-	gf_crypt_mt_loc,
-	gf_crypt_mt_iatt,
-	gf_crypt_mt_key,
-	gf_crypt_mt_iovec,
-	gf_crypt_mt_char,
-	gf_crypt_mt_local,
-	gf_crypt_mt_end,
-};
-
-#endif /* __CRYPT_MEM_TYPES_H__ */
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
-
-
-
diff --git a/xlators/encryption/crypt/src/crypt.c b/xlators/encryption/crypt/src/crypt.c
deleted file mode 100644
index 9e5bcf79c30..00000000000
--- a/xlators/encryption/crypt/src/crypt.c
+++ /dev/null
@@ -1,4525 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-#include <ctype.h>
-#include <sys/uio.h>
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-
-#include "crypt-common.h"
-#include "crypt.h"
-
-static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd);
-static int32_t init_inode_info_tail(struct crypt_inode_info *info,
-				    struct master_cipher_info *master);
-static int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
-				       uint64_t from, off_t size);
-static int32_t load_file_size(call_frame_t *frame, void *cookie,
-			      xlator_t *this, int32_t op_ret, int32_t op_errno,
-			      dict_t *dict, dict_t *xdata);
-static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
-			      atom_data_type dtype);
-static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
-			       atom_data_type dtype);
-static void put_one_call_open(call_frame_t *frame);
-static void put_one_call_readv(call_frame_t *frame, xlator_t *this);
-static void put_one_call_writev(call_frame_t *frame, xlator_t *this);
-static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this);
-static void free_avec(struct iovec *avec, char **pool, int blocks_in_pool);
-static void free_avec_data(crypt_local_t *local);
-static void free_avec_hole(crypt_local_t *local);
-
-static crypt_local_t *crypt_alloc_local(call_frame_t *frame, xlator_t *this,
-					glusterfs_fop_t fop)
-{
-	crypt_local_t *local = NULL;
-
-	local = GF_CALLOC (1, sizeof (*local), gf_crypt_mt_local);
-	if (!local) {
-		gf_log(this->name, GF_LOG_ERROR, "out of memory");
-		return NULL;
-	}
-	local->fop = fop;
-	LOCK_INIT(&local->hole_lock);
-	LOCK_INIT(&local->call_lock);
-	LOCK_INIT(&local->rw_count_lock);
-
-	frame->local = local;
-	return local;
-}
-
-struct crypt_inode_info *get_crypt_inode_info(inode_t *inode, xlator_t *this)
-{
-	int ret;
-	uint64_t value = 0;
-	struct crypt_inode_info *info;
-
-	ret = inode_ctx_get(inode, this, &value);
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"Can not get inode info");
-		return NULL;
-	}
-	info = (struct crypt_inode_info *)(long)value;
-	if (info == NULL) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"Can not obtain inode info");
-		return NULL;
-	}
-	return info;
-}
-
-static struct crypt_inode_info *local_get_inode_info(crypt_local_t *local,
-						      xlator_t *this)
-{
-	if (local->info)
-		return local->info;
-	local->info = get_crypt_inode_info(local->fd->inode, this);
-	return local->info;
-}
-
-static struct crypt_inode_info *alloc_inode_info(crypt_local_t *local,
-						 loc_t *loc)
-{
-	struct crypt_inode_info *info;
-
-	info = GF_CALLOC(1, sizeof(*info), gf_crypt_mt_inode);
-	if (!info) {
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-		gf_log ("crypt", GF_LOG_WARNING,
-			"Can not allocate inode info");
-		return NULL;
-	}
-	memset(info, 0, sizeof(*info));
-#if DEBUG_CRYPT
-	info->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
-	if (!info->loc) {
-		gf_log("crypt", GF_LOG_WARNING, "Can not allocate loc");
-		GF_FREE(info);
-		return NULL;
-	}
-	if (loc_copy(info->loc, loc)){
-		GF_FREE(info->loc);
-		GF_FREE(info);
-		return NULL;
-	}
-#endif /* DEBUG_CRYPT */
-
-	local->info = info;
-	return info;
-}
-
-static void free_inode_info(struct crypt_inode_info *info)
-{
-#if DEBUG_CRYPT
-	loc_wipe(info->loc);
-	GF_FREE(info->loc);
-#endif
-	memset(info, 0, sizeof(*info));
-	GF_FREE(info);
-}
-
-int crypt_forget (xlator_t *this, inode_t *inode)
-{
-        uint64_t ctx_addr = 0;
-        if (!inode_ctx_del (inode, this, &ctx_addr))
-                free_inode_info((struct crypt_inode_info *)(long)ctx_addr);
-        return 0;
-}
-
-#if DEBUG_CRYPT
-static void check_read(call_frame_t *frame, xlator_t *this, int32_t read,
-		       struct iovec *vec, int32_t count, struct iatt *stbuf)
-{
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = get_object_cinfo(local->info);
-	struct avec_config *conf = &local->data_conf;
-	uint32_t resid = stbuf->ia_size & (object_alg_blksize(object) - 1);
-
-	if (read <= 0)
-		return;
-	if (read != iov_length(vec, count))
-		gf_log ("crypt", GF_LOG_DEBUG,
-			"op_ret differs from amount of read bytes");
-
-	if (object_alg_should_pad(object) && (read & (object_alg_blksize(object) - 1)))
-		gf_log ("crypt", GF_LOG_DEBUG,
-			"bad amount of read bytes (!= 0 mod(cblock size))");
-
-	if (conf->aligned_offset + read >
-	    stbuf->ia_size + (resid ? object_alg_blksize(object) - resid : 0))
-		gf_log ("crypt", GF_LOG_DEBUG,
-			"bad amount of read bytes (too large))");
-
-}
-
-#define PT_BYTES_TO_DUMP (32)
-static void dump_plain_text(crypt_local_t *local, struct iovec *avec)
-{
-	int32_t to_dump;
-	char str[PT_BYTES_TO_DUMP + 1];
-
-	if (!avec)
-		return;
-	to_dump = avec->iov_len;
-	if (to_dump > PT_BYTES_TO_DUMP)
-		to_dump = PT_BYTES_TO_DUMP;
-	memcpy(str, avec->iov_base, to_dump);
-	memset(str + to_dump, '0', 1);
-	gf_log("crypt", GF_LOG_DEBUG, "Read file: %s", str);
-}
-
-static int32_t data_conf_invariant(struct avec_config *conf)
-{
-	return conf->acount ==
-		!!has_head_block(conf) +
-		!!has_tail_block(conf)+
-		conf->nr_full_blocks;
-}
-
-static int32_t hole_conf_invariant(struct avec_config *conf)
-{
-	return conf->blocks_in_pool ==
-		!!has_head_block(conf) +
-		!!has_tail_block(conf)+
-		!!has_full_blocks(conf);
-}
-
-static void crypt_check_conf(struct avec_config *conf)
-{
-	int32_t ret = 0;
-	const char *msg;
-
-	switch (conf->type) {
-	case DATA_ATOM:
-		msg = "data";
-		ret = data_conf_invariant(conf);
-		break;
-	case HOLE_ATOM:
-		msg = "hole";
-		ret = hole_conf_invariant(conf);
-		break;
-	default:
-		msg = "unknown";
-	}
-	if (!ret)
-		gf_log("crypt", GF_LOG_DEBUG, "bad %s conf", msg);
-}
-
-static void check_buf(call_frame_t *frame, xlator_t *this, struct iatt *buf)
-{
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-	uint64_t local_file_size;
-
-	switch(local->fop) {
-	case GF_FOP_FTRUNCATE:
-		return;
-	case GF_FOP_WRITE:
-		local_file_size = local->new_file_size;
-		break;
-	case GF_FOP_READ:
-		if (parent_is_crypt_xlator(frame, this))
-			return;
-		local_file_size = local->cur_file_size;
-		break;
-	default:
-		gf_log("crypt", GF_LOG_DEBUG, "bad file operation");
-		return;
-	}
-	if (buf->ia_size != round_up(local_file_size,
-					 object_alg_blksize(object)))
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "bad ia_size in buf (%llu), should be %llu",
-		       (unsigned long long)buf->ia_size,
-		       (unsigned long long)round_up(local_file_size,
-						    object_alg_blksize(object)));
-}
-
-#else
-#define check_read(frame, this, op_ret, vec, count, stbuf) noop
-#define dump_plain_text(local, avec) noop
-#define crypt_check_conf(conf) noop
-#define check_buf(frame, this, buf) noop
-#endif /* DEBUG_CRYPT */
-
-/*
- * Pre-conditions:
- * @vec represents a ciphertext of expanded size and
- * aligned offset.
- *
- * Compound a temporal vector @avec with block-aligned
- * components, decrypt and fix it up to represent a chunk
- * of data corresponding to the original size and offset.
- * Pass the result to the next translator.
- */
-int32_t crypt_readv_cbk(call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			struct iovec *vec,
-			int32_t count,
-			struct iatt *stbuf,
-			struct iobref *iobref,
-			dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	struct iovec *avec;
-	uint32_t i;
-	uint32_t to_vec;
-	uint32_t to_user;
-
-	check_buf(frame, this, stbuf);
-	check_read(frame, this, op_ret, vec, count, stbuf);
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	local->iobref = iobref_ref(iobref);
-
-	local->buf = *stbuf;
-	local->buf.ia_size = local->cur_file_size;
-
-	if (op_ret <= 0 || count == 0 || vec[0].iov_len == 0)
-		goto put_one_call;
-
-	if (conf->orig_offset >= local->cur_file_size) {
-		local->op_ret = 0;
-		goto put_one_call;
-	}
-	/*
-	 * correct config params with real file size
-	 * and actual amount of bytes read
-	 */
-	set_config_offsets(frame, this,
-			   conf->orig_offset, op_ret, DATA_ATOM, 0);
-
-	if (conf->orig_offset + conf->orig_size > local->cur_file_size)
-		conf->orig_size = local->cur_file_size - conf->orig_offset;
-	/*
-	 * calculate amount of data to be returned
-	 * to user.
-	 */
-	to_user = op_ret;
-	if (conf->aligned_offset + to_user <= conf->orig_offset) {
-		gf_log(this->name, GF_LOG_WARNING, "Incomplete read");
-		local->op_ret = -1;
-		local->op_errno = EIO;
-		goto put_one_call;
-	}
-	to_user -= (conf->aligned_offset - conf->orig_offset);
-
-	if (to_user > conf->orig_size)
-		to_user = conf->orig_size;
-	local->rw_count = to_user;
-
-	op_errno = set_config_avec_data(this, local,
-					conf, object, vec, count);
-	if (op_errno) {
-		local->op_ret = -1;
-		local->op_errno = op_errno;
-		goto put_one_call;
-	}
-	avec = conf->avec;
-#if DEBUG_CRYPT
-	if (conf->off_in_tail != 0 &&
-	    conf->off_in_tail < object_alg_blksize(object) &&
-	    object_alg_should_pad(object))
-		gf_log(this->name, GF_LOG_DEBUG, "Bad offset in tail %d",
-		       conf->off_in_tail);
-	if (iov_length(vec, count) != 0 &&
-	    in_same_lblock(conf->orig_offset + iov_length(vec, count) - 1,
-			   local->cur_file_size - 1,
-			   object_alg_blkbits(object))) {
-		gf_log(this->name, GF_LOG_DEBUG, "Compound last cblock");
-		dump_cblock(this,
-			    (unsigned char *)(avec[conf->acount - 1].iov_base) +
-			    avec[conf->acount - 1].iov_len - object_alg_blksize(object));
-		dump_cblock(this,
-			    (unsigned char *)(vec[count - 1].iov_base) +
-			    vec[count - 1].iov_len - object_alg_blksize(object));
-	}
-#endif
-	decrypt_aligned_iov(object, avec,
-			    conf->acount, conf->aligned_offset);
-	/*
-	 * pass proper plain data to user
-	 */
-	avec[0].iov_base += (conf->aligned_offset - conf->orig_offset);
-	avec[0].iov_len  -= (conf->aligned_offset - conf->orig_offset);
-
-	to_vec = to_user;
-	for (i = 0; i < conf->acount; i++) {
-		if (avec[i].iov_len > to_vec)
-			avec[i].iov_len = to_vec;
-		to_vec -= avec[i].iov_len;
-	}
- put_one_call:
-	put_one_call_readv(frame, this);
-	return 0;
-}
-
-static int32_t do_readv(call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			dict_t *dict,
-			dict_t *xdata)
-{
-	data_t *data;
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;
-	/*
-	 * extract regular file size
-	 */
-	data = dict_get(dict, FSIZE_XATTR_PREFIX);
-	if (!data) {
-		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
-		op_errno = EIO;
-		goto error;
-	}
-	local->cur_file_size = data_to_uint64(data);
-
-	get_one_call(frame);
-	STACK_WIND(frame,
-		   crypt_readv_cbk,
-		   FIRST_CHILD (this),
-		   FIRST_CHILD (this)->fops->readv,
-		   local->fd,
-		   /*
-		    * FIXME: read amount can be reduced
-		    */
-		   local->data_conf.expanded_size,
-		   local->data_conf.aligned_offset,
-		   local->flags,
-		   local->xdata);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-
-	get_one_call(frame);
-	put_one_call_readv(frame, this);
-	return 0;
-}
-
-static int32_t crypt_readv_finodelk_cbk(call_frame_t *frame,
-					void *cookie,
-					xlator_t *this,
-					int32_t op_ret,
-					int32_t op_errno,
-					dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;
-	/*
-	 * An access has been granted,
-	 * retrieve file size
-	 */
-	STACK_WIND(frame,
-		   do_readv,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fgetxattr,
-		   local->fd,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return  0;
- error:
-	fd_unref(local->fd);
-	if (local->xdata)
-		dict_unref(local->xdata);
-	STACK_UNWIND_STRICT(readv,
-			    frame,
-			    -1,
-			    op_errno,
-			    NULL,
-			    0,
-			    NULL,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-static int32_t readv_trivial_completion(call_frame_t *frame,
-					void *cookie,
-					xlator_t *this,
-					int32_t op_ret,
-					int32_t op_errno,
-					struct iatt *buf,
-					dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "stat failed (%d)", op_errno);
-		goto error;
-	}
-	local->buf = *buf;
-	STACK_WIND(frame,
-		   load_file_size,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->getxattr,
-		   local->loc,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno,
-			    NULL, 0, NULL, NULL, NULL);
-	return 0;
-}
-
-int32_t crypt_readv(call_frame_t *frame,
-		    xlator_t *this,
-		    fd_t *fd,
-		    size_t size,
-		    off_t offset,
-		    uint32_t flags, dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-	struct crypt_inode_info *info;
-	struct gf_flock lock = {0, };
-
-#if DEBUG_CRYPT
-	gf_log("crypt", GF_LOG_DEBUG, "reading %d bytes from offset %llu",
-	       (int)size, (long long)offset);
-	if (parent_is_crypt_xlator(frame, this))
-		gf_log("crypt", GF_LOG_DEBUG, "parent is crypt");
-#endif
-	local = crypt_alloc_local(frame, this, GF_FOP_READ);
-	if (!local) {
-		ret = ENOMEM;
-		goto error;
-	}
-	if (size == 0)
-		goto trivial;
-
-	local->fd = fd_ref(fd);
-	local->flags = flags;
-
-	info = local_get_inode_info(local, this);
-	if (info == NULL) {
-		ret = EINVAL;
-		fd_unref(fd);
-		goto error;
-	}
-	if (!object_alg_atomic(&info->cinfo)) {
-		ret = EINVAL;
-		fd_unref(fd);
-		goto error;
-	}
-	set_config_offsets(frame, this, offset, size,
-			   DATA_ATOM, 0);
-	if (parent_is_crypt_xlator(frame, this)) {
-		data_t *data;
-		/*
-		 * We are called by crypt_writev (or cypt_ftruncate)
-		 * to perform the "read" component of the read-modify-write
-		 * (or read-prune-write) sequence for some atom;
-		 *
-		 * don't ask for access:
-		 * it has already been acquired
-		 *
-		 * Retrieve current file size
-		 */
-		if (!xdata) {
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size hasn't been passed");
-			ret = EIO;
-			goto error;
-		}
-		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
-		if (!data) {
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size not found");
-			ret = EIO;
-			goto error;
-		}
-		local->old_file_size =
-			local->cur_file_size = data_to_uint64(data);
-
-		get_one_call(frame);
-		STACK_WIND(frame,
-			   crypt_readv_cbk,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->readv,
-			   local->fd,
-			   /*
-			    * FIXME: read amount can be reduced
-			    */
-			   local->data_conf.expanded_size,
-			   local->data_conf.aligned_offset,
-			   flags,
-			   NULL);
-		return 0;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_RDLCK;
-        lock.l_whence = SEEK_SET;
-
-	STACK_WIND(frame,
-		   crypt_readv_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- trivial:
-	STACK_WIND(frame,
-		   readv_trivial_completion,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fstat,
-		   fd,
-		   NULL);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(readv,
-			    frame,
-			    -1,
-			    ret,
-			    NULL,
-			    0,
-			    NULL,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-void set_local_io_params_writev(call_frame_t *frame,
-				struct object_cipher_info *object,
-				struct rmw_atom *atom,
-				off_t io_offset,
-				uint32_t io_size)
-{
-	crypt_local_t *local = frame->local;
-
-	local->io_offset = io_offset;
-	local->io_size = io_size;
-
-	local->io_offset_nopad =
-		atom->offset_at(frame, object) + atom->offset_in(frame, object);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "set nopad offset to %llu",
-	       (unsigned long long)local->io_offset_nopad);
-
-	local->io_size_nopad = atom->io_size_nopad(frame, object);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "set nopad size to %llu",
-	       (unsigned long long)local->io_size_nopad);
-
-	local->update_disk_file_size = 0;
-	/*
-	 * NOTE: eof_padding_size is 0 for all full atoms;
-	 * For head and tail atoms it will be set up at rmw_partial block()
-	 */
-	local->new_file_size = local->cur_file_size;
-
-	if (local->io_offset_nopad + local->io_size_nopad > local->cur_file_size) {
-
-		local->new_file_size = local->io_offset_nopad + local->io_size_nopad;
-
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "set new file size to %llu",
-		       (unsigned long long)local->new_file_size);
-
-		local->update_disk_file_size = 1;
-	}
-}
-
-void set_local_io_params_ftruncate(call_frame_t *frame,
-				   struct object_cipher_info *object)
-{
-	uint32_t resid;
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-
-	resid = conf->orig_offset & (object_alg_blksize(object) - 1);
-	if (resid) {
-		local->eof_padding_size =
-			object_alg_blksize(object) - resid;
-		local->new_file_size = conf->aligned_offset;
-		local->update_disk_file_size = 0;
-		/*
-		 * file size will be updated
-		 * in the ->writev() stack,
-		 * when submitting file tail
-		 */
-	} else {
-		local->eof_padding_size = 0;
-		local->new_file_size = conf->orig_offset;
-		local->update_disk_file_size = 1;
-		/*
-		 * file size will be updated
-		 * in this ->ftruncate stack
-		 */
-	}
-}
-
-static void submit_head(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	submit_partial(frame, this, local->fd, HEAD_ATOM);
-}
-
-static void submit_tail(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	submit_partial(frame, this, local->fd, TAIL_ATOM);
-}
-
-static void submit_hole(call_frame_t *frame, xlator_t *this)
-{
-	/*
-	 * hole conversion always means
-	 * appended write and goes in ordered fashion
-	 */
-	do_ordered_submit(frame, this, HOLE_ATOM);
-}
-
-static void submit_data(call_frame_t *frame, xlator_t *this)
-{
-	if (is_ordered_mode(frame)) {
-		do_ordered_submit(frame, this, DATA_ATOM);
-		return;
-	}
-	gf_log("crypt", GF_LOG_WARNING, "Bad submit mode");
-	get_nr_calls(frame, nr_calls_data(frame));
-	do_parallel_submit(frame, this, DATA_ATOM);
-	return;
-}
-
-/*
- * heplers called by writev_cbk, fruncate_cbk in ordered mode
- */
-
-static int32_t should_submit_hole(crypt_local_t *local)
-{
-	struct avec_config *conf = &local->hole_conf;
-
-	return conf->avec != NULL;
-}
-
-static int32_t should_resume_submit_hole(crypt_local_t *local)
-{
-	struct avec_config *conf = &local->hole_conf;
-
-	if (local->fop == GF_FOP_WRITE && has_tail_block(conf))
-		/*
-		 * Don't submit a part of hole, which
-		 * fits into a data block:
- 		 * this part of hole will be converted
-		 * as a gap filled by zeros in data head
-		 * block.
-		 */
-		return conf->cursor < conf->acount - 1;
-	else
-		return conf->cursor < conf->acount;
-}
-
-static int32_t should_resume_submit_data(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-
-	if (is_ordered_mode(frame))
-		return conf->cursor < conf->acount;
-	/*
-	 * parallel writes
-	 */
-	return 0;
-}
-
-static int32_t should_submit_data_after_hole(crypt_local_t *local)
-{
-	return local->data_conf.avec != NULL;
-}
-
-static void update_local_file_params(call_frame_t *frame,
-				     xlator_t *this,
-				     struct iatt *prebuf,
-				     struct iatt *postbuf)
-{
-	crypt_local_t *local = frame->local;
-
-	check_buf(frame, this, postbuf);
-
-	local->prebuf = *prebuf;
-	local->postbuf = *postbuf;
-
-	local->prebuf.ia_size  = local->cur_file_size;
-	local->postbuf.ia_size = local->new_file_size;
-
-	local->cur_file_size = local->new_file_size;
-}
-
-static int32_t end_writeback_writev(call_frame_t *frame,
-				    void *cookie,
-				    xlator_t *this,
-				    int32_t op_ret,
-				    int32_t op_errno,
-				    struct iatt *prebuf,
-				    struct iatt *postbuf,
-				    dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret <= 0) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "writev iteration failed");
-		goto put_one_call;
-	}
-	/*
-	 * op_ret includes paddings (atom's head, atom's tail and EOF)
-	 */
-	if (op_ret < local->io_size) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "Incomplete writev iteration");
-		goto put_one_call;
-	}
-	op_ret -= local->eof_padding_size;
-	local->op_ret = op_ret;
-
-	update_local_file_params(frame, this, prebuf, postbuf);
-
-	if (data_write_in_progress(local)) {
-
-		LOCK(&local->rw_count_lock);
-		local->rw_count += op_ret;
-		UNLOCK(&local->rw_count_lock);
-
-		if (should_resume_submit_data(frame))
-			submit_data(frame, this);
-	}
-	else {
-		/*
-		 * hole conversion is going on;
-		 * don't take into account written zeros
-		 */
-		if (should_resume_submit_hole(local))
-			submit_hole(frame, this);
-
-		else if (should_submit_data_after_hole(local))
-			submit_data(frame, this);
-	}
- put_one_call:
-	put_one_call_writev(frame, this);
-	return 0;
-}
-
-#define crypt_writev_cbk end_writeback_writev
-
-#define HOLE_WRITE_CHUNK_BITS 12
-#define HOLE_WRITE_CHUNK_SIZE (1 << HOLE_WRITE_CHUNK_BITS)
-
-/*
- * Convert hole of size @size at offset @off to
- * zeros and prepare respective iovecs for submit.
- * The hole lock should be held.
- *
- * Pre-conditions:
- * @local->file_size is set and valid.
- */
-int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
-				uint64_t off, off_t size)
-{
-	int32_t ret;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	set_config_offsets(frame, this, off, size, HOLE_ATOM, 1);
-
-	ret = set_config_avec_hole(this, local,
-				   &local->hole_conf, object, local->fop);
-	crypt_check_conf(&local->hole_conf);
-
-	return ret;
-}
-
-/*
- * prepare for submit @count bytes at offset @from
- */
-int32_t prepare_for_submit_data(call_frame_t *frame, xlator_t *this,
-				off_t from, int32_t size, struct iovec *vec,
-				int32_t vec_count, int32_t setup_gap)
-{
-	uint32_t ret;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	set_config_offsets(frame, this, from, size,
-			   DATA_ATOM, setup_gap);
-
-	ret = set_config_avec_data(this, local,
-				   &local->data_conf, object, vec, vec_count);
-	crypt_check_conf(&local->data_conf);
-
-	return ret;
-}
-
-static void free_avec(struct iovec *avec,
-		      char **pool, int blocks_in_pool)
-{
-	if (!avec)
-		return;
-	GF_FREE(pool);
-	GF_FREE(avec);
-}
-
-static void free_avec_data(crypt_local_t *local)
-{
-	return free_avec(local->data_conf.avec,
-			 local->data_conf.pool,
-			 local->data_conf.blocks_in_pool);
-}
-
-static void free_avec_hole(crypt_local_t *local)
-{
-	return free_avec(local->hole_conf.avec,
-			 local->hole_conf.pool,
-			 local->hole_conf.blocks_in_pool);
-}
-
-
-static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
-			       atom_data_type dtype)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf;
-
-	local->active_setup = dtype;
-	conf = conf_by_type(frame, dtype);
-
-	if (has_head_block(conf))
-		submit_head(frame, this);
-
-	if (has_full_blocks(conf))
-		submit_full(frame, this);
-
-	if (has_tail_block(conf))
-		submit_tail(frame, this);
-	return;
-}
-
-static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
-			      atom_data_type dtype)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf;
-
-	local->active_setup = dtype;
-	conf = conf_by_type(frame, dtype);
-
-	if (should_submit_head_block(conf)) {
-		get_one_call_nolock(frame);
-		submit_head(frame, this);
-	}
-	else if (should_submit_full_block(conf)) {
-		get_one_call_nolock(frame);
-		submit_full(frame, this);
-	}
-	else if (should_submit_tail_block(conf)) {
-		get_one_call_nolock(frame);
-		submit_tail(frame, this);
-	}
-	else
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "nothing has been submitted in ordered mode");
-	return;
-}
-
-static int32_t do_writev(call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dict_t *dict,
-			 dict_t *xdata)
-{
-	data_t *data;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-	/*
-	 * extract regular file size
-	 */
-	data = dict_get(dict, FSIZE_XATTR_PREFIX);
-	if (!data) {
-		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
-		op_ret = -1;
-		op_errno = EIO;
-		goto error;
-	}
-	local->old_file_size = local->cur_file_size = data_to_uint64(data);
-
-	set_gap_at_end(frame, object, &local->data_conf, DATA_ATOM);
-
-	if (local->cur_file_size < local->data_conf.orig_offset) {
-		/*
-		 * Set up hole config
-		 */
-		op_errno = prepare_for_submit_hole(frame,
-			       this,
-			       local->cur_file_size,
-			       local->data_conf.orig_offset - local->cur_file_size);
-		if (op_errno) {
-			local->op_ret = -1;
-			local->op_errno = op_errno;
-			goto error;
-		}
-	}
-	if (should_submit_hole(local))
-		submit_hole(frame, this);
-	else
-		submit_data(frame, this);
-	return 0;
- error:
-	get_one_call_nolock(frame);
-	put_one_call_writev(frame, this);
-	return 0;
-}
-
-static int32_t crypt_writev_finodelk_cbk(call_frame_t *frame,
-					 void *cookie,
-					 xlator_t *this,
-					 int32_t op_ret,
-					 int32_t op_errno,
-					 dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0)
-		goto error;
-	/*
-	 * An access has been granted,
-	 * retrieve file size first
-	 */
-	STACK_WIND(frame,
-		   do_writev,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fgetxattr,
-		   local->fd,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return 0;
- error:
-	get_one_call_nolock(frame);
-	put_one_call_writev(frame, this);
-	return 0;
-}
-
-static int32_t writev_trivial_completion(call_frame_t *frame,
-					 void *cookie,
-					 xlator_t *this,
-					 int32_t op_ret,
-					 int32_t op_errno,
-					 struct iatt *buf,
-					 dict_t *dict)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	local->prebuf = *buf;
-	local->postbuf = *buf;
-
-	local->prebuf.ia_size = local->cur_file_size;
-	local->postbuf.ia_size = local->cur_file_size;
-
-	get_one_call(frame);
-	put_one_call_writev(frame, this);
-	return 0;
-}
-
-int crypt_writev(call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 struct iovec *vec,
-		 int32_t count,
-		 off_t offset,
-		 uint32_t flags,
-		 struct iobref *iobref,
-		 dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-	struct crypt_inode_info *info;
-	struct gf_flock lock = {0, };
-#if DEBUG_CRYPT
-	gf_log ("crypt", GF_LOG_DEBUG, "writing %d bytes from offset %llu",
-		(int)iov_length(vec, count), (long long)offset);
-#endif
-	local = crypt_alloc_local(frame, this, GF_FOP_WRITE);
-	if (!local) {
-		ret = ENOMEM;
-		goto error;
-	}
-	local->fd = fd_ref(fd);
-
-	if (iobref)
-		local->iobref = iobref_ref(iobref);
-	/*
-	 * to update real file size on the server
-	 */
-	local->xattr = dict_new();
-	if (!local->xattr) {
-		ret = ENOMEM;
-		goto error;
-	}
-	local->flags = flags;
-
-	info = local_get_inode_info(local, this);
-	if (info == NULL) {
-		ret = EINVAL;
-		goto error;
-	}
-	if (!object_alg_atomic(&info->cinfo)) {
-		ret = EINVAL;
-		goto error;
-	}
-	if (iov_length(vec, count) == 0)
-		goto trivial;
-
-	ret = prepare_for_submit_data(frame, this, offset,
-				      iov_length(vec, count),
-				      vec, count, 0 /* don't setup gup
-						       in tail: we don't
-						       know file size yet */);
-	if (ret)
-		goto error;
-
-	if (parent_is_crypt_xlator(frame, this)) {
-		data_t *data;
-		/*
-		 * we are called by shinking crypt_ftruncate(),
-		 * which doesn't perform hole conversion;
-		 *
-		 * don't ask for access:
-		 * it has already been acquired
-		 */
-
-		/*
-		 * extract file size
-		 */
-		if (!xdata) {
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size hasn't been passed");
-			ret = EIO;
-			goto error;
-		}
-		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
-		if (!data) {
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size not found");
-			ret = EIO;
-			goto error;
-		}
-		local->old_file_size =
-			local->cur_file_size = data_to_uint64(data);
-
-		submit_data(frame, this);
-		return 0;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-	/*
-	 * lock the file and retrieve its size
-	 */
-	lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_WRLCK;
-        lock.l_whence = SEEK_SET;
-
-	STACK_WIND(frame,
-		   crypt_writev_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- trivial:
-	STACK_WIND(frame,
-		   writev_trivial_completion,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fstat,
-		   fd,
-		   NULL);
-	return 0;
- error:
-	if (local && local->fd)
-		fd_unref(fd);
-	if (local && local->iobref)
-		iobref_unref(iobref);
-	if (local && local->xdata)
-		dict_unref(xdata);
-	if (local && local->xattr)
-		dict_unref(local->xattr);
-	if (local && local->info)
-		free_inode_info(local->info);
-
-	STACK_UNWIND_STRICT(writev, frame, -1, ret, NULL, NULL, NULL);
-	return 0;
-}
-
-int32_t prepare_for_prune(call_frame_t *frame, xlator_t *this, uint64_t offset)
-{
-	set_config_offsets(frame, this,
-			   offset,
-			   0, /* count */
-			   DATA_ATOM,
-			   0 /* since we prune, there is no
-				gap in tail to uptodate */);
-	return 0;
-}
-
-/*
- * Finish the read-prune-modify sequence
- *
- * Can be invoked as
- * 1) ->ftruncate_cbk() for cblock-aligned, or trivial prune
- * 2) ->writev_cbk() for non-cblock-aligned prune
- */
-
-static int32_t prune_complete(call_frame_t *frame,
-			      void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret,
-			      int32_t op_errno,
-			      struct iatt *prebuf,
-			      struct iatt *postbuf,
-			      dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	update_local_file_params(frame, this, prebuf, postbuf);
-
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * This is called as ->ftruncate_cbk()
- *
- * Perform the "write" component of the
- * read-prune-write sequence.
- *
- * submuit the rest of the file
- */
-static int32_t prune_submit_file_tail(call_frame_t *frame,
-				      void *cookie,
-				      xlator_t *this,
-				      int32_t op_ret,
-				      int32_t op_errno,
-				      struct iatt *prebuf,
-				      struct iatt *postbuf,
-				      dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-	dict_t *dict;
-
-	if (op_ret < 0)
-		goto put_one_call;
-
-	if (local->xdata) {
-		dict_unref(local->xdata);
-		local->xdata = NULL;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	dict = dict_new();
-	if (!dict) {
-		op_errno = ENOMEM;
-		goto error;
-	}
-
-	update_local_file_params(frame, this, prebuf, postbuf);
-	local->new_file_size = conf->orig_offset;
-
-	/*
-	 * The rest of the file is a partial block and, hence,
-	 * should be written via RMW sequence, so the crypt xlator
-	 * does STACK_WIND to itself.
-	 *
-	 * Pass current file size to crypt_writev()
-	 */
-	op_errno = dict_set(dict,
-			    FSIZE_XATTR_PREFIX,
-			    data_from_uint64(local->cur_file_size));
-	if (op_errno) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "can not set key to update file size");
-		dict_unref(dict);
-		goto error;
-	}
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "passing current file size (%llu) to crypt_writev",
-	       (unsigned long long)local->cur_file_size);
-	/*
-	 * Padding will be filled with
-	 * zeros by rmw_partial_block()
-	 */
-	STACK_WIND(frame,
-		   prune_complete,
-		   this,
-		   this->fops->writev, /* crypt_writev */
-		   local->fd,
-		   &local->vec,
-		   1,
-		   conf->aligned_offset, /* offset to write from */
-		   0,
-		   local->iobref,
-		   dict);
-
-	dict_unref(dict);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
- put_one_call:
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * This is called as a callback of ->writev() invoked in behalf
- * of ftruncate(): it can be
- * 1) ordered writes issued by hole conversion in the case of
- *    expanded truncate, or
- * 2) an rmw partial data block issued by non-cblock-aligned
- * prune.
- */
-int32_t end_writeback_ftruncate(call_frame_t *frame,
-				void *cookie,
-				xlator_t *this,
-				int32_t op_ret,
-				int32_t op_errno,
-				struct iatt *prebuf,
-				struct iatt *postbuf,
-				dict_t *xdata)
-{
-	crypt_local_t  *local = frame->local;
-	/*
-	 * if nothing has been written,
-	 * then it must be an error
-	 */
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0)
-		goto put_one_call;
-
-	update_local_file_params(frame, this, prebuf, postbuf);
-
-	if (data_write_in_progress(local))
-		/* case (2) */
-		goto put_one_call;
-	/* case (1) */
-	if (should_resume_submit_hole(local))
-		submit_hole(frame, this);
-	/*
-	 * case of hole, when we should't resume
-	 */
- put_one_call:
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * Perform prune and write components of the
- * read-prune-write sequence.
- *
- * Called as ->readv_cbk()
- *
- * Pre-conditions:
- * @vec contains the latest atom of the file
- * (plain text)
- */
-static int32_t prune_write(call_frame_t *frame,
-			   void *cookie,
-			   xlator_t *this,
-			   int32_t op_ret,
-			   int32_t op_errno,
-			   struct iovec *vec,
-			   int32_t count,
-			   struct iatt *stbuf,
-			   struct iobref *iobref,
-			   dict_t *xdata)
-{
-	int32_t i;
-	size_t to_copy;
-	size_t copied = 0;
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	if (op_ret == -1)
-		goto put_one_call;
-
-	/*
-	 * At first, uptodate head block
-	 */
-	if (iov_length(vec, count) < conf->off_in_head) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "Failed to uptodate head block for prune");
-		local->op_ret = -1;
-		local->op_errno = EIO;
-		goto put_one_call;
-	}
-	local->vec.iov_len = conf->off_in_head;
-	local->vec.iov_base = GF_CALLOC(1, local->vec.iov_len,
-					gf_crypt_mt_data);
-
-	if (local->vec.iov_base == NULL) {
-	        gf_log(this->name, GF_LOG_WARNING,
-                       "Failed to calloc head block for prune");
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-	        goto put_one_call;
-	}
-	for (i = 0; i < count; i++) {
-		to_copy = vec[i].iov_len;
-		if (to_copy > local->vec.iov_len - copied)
-			to_copy = local->vec.iov_len - copied;
-
-		memcpy((char *)local->vec.iov_base + copied,
-		       vec[i].iov_base,
-		       to_copy);
-		copied += to_copy;
-		if (copied == local->vec.iov_len)
-			break;
-	}
-	/*
-	 * perform prune with aligned offset
-	 * (i.e. at this step we prune a bit
-	 * more then it is needed
-	 */
-	STACK_WIND(frame,
-		   prune_submit_file_tail,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->ftruncate,
-		   local->fd,
-		   conf->aligned_offset,
-		   local->xdata);
-	return 0;
- put_one_call:
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * Perform a read-prune-write sequence
- */
-int32_t read_prune_write(call_frame_t *frame, xlator_t *this)
-{
-	int32_t ret = 0;
-	dict_t *dict = NULL;
-	crypt_local_t  *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	set_local_io_params_ftruncate(frame, object);
-	get_one_call_nolock(frame);
-
-	if ((conf->orig_offset & (object_alg_blksize(object) - 1)) == 0) {
-		/*
-		 * cblock-aligned prune:
-		 * we don't need read and write components,
-		 * just cut file body
-		 */
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "prune without RMW (at offset %llu",
-		       (unsigned long long)conf->orig_offset);
-
-		STACK_WIND(frame,
-			   prune_complete,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->ftruncate,
-			   local->fd,
-			   conf->orig_offset,
-			   local->xdata);
-		return 0;
-	}
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "prune with RMW (at offset %llu",
-	       (unsigned long long)conf->orig_offset);
-	/*
-	 * We are about to perform the "read" component of the
-	 * read-prune-write sequence. It means that we need to
-	 * read encrypted data from disk and decrypt it.
-	 * So, the crypt translator does STACK_WIND to itself.
-	 *
-	 * Pass current file size to crypt_readv()
-
-	 */
-	dict = dict_new();
-	if (!dict) {
-		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
-		ret = ENOMEM;
-		goto exit;
-	}
-	ret = dict_set(dict,
-		       FSIZE_XATTR_PREFIX,
-		       data_from_uint64(local->cur_file_size));
-	if (ret) {
-		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
-		goto exit;
-	}
-	STACK_WIND(frame,
-		   prune_write,
-		   this,
-		   this->fops->readv, /* crypt_readv */
-		   local->fd,
-		   get_atom_size(object), /* bytes to read */
-		   conf->aligned_offset, /* offset to read from */
-		   0,
-		   dict);
- exit:
-	if (dict)
-		dict_unref(dict);
-	return ret;
-}
-
-/*
- * File prune is more complicated than expand.
- * First we need to read the latest atom to not lose info
- * needed for proper update. Also we need to make sure that
- * every component of read-prune-write sequence leaves data
- * consistent
- *
- * Non-cblock aligned prune is performed as read-prune-write
- * sequence:
- *
- * 1) read the latest atom;
- * 2) perform cblock-aligned prune
- * 3) issue a write request for the end-of-file
- */
-int32_t prune_file(call_frame_t *frame, xlator_t *this, uint64_t offset)
-{
-	int32_t ret;
-
-	ret = prepare_for_prune(frame, this, offset);
-	if (ret)
-		return ret;
-	return read_prune_write(frame, this);
-}
-
-int32_t expand_file(call_frame_t *frame, xlator_t *this,
-		    uint64_t offset)
-{
-	int32_t ret;
-	crypt_local_t *local = frame->local;
-
-	ret = prepare_for_submit_hole(frame, this,
-				      local->old_file_size,
-				      offset - local->old_file_size);
-	if (ret)
-		return ret;
-	submit_hole(frame, this);
-	return 0;
-}
-
-static int32_t ftruncate_trivial_completion(call_frame_t *frame,
-					    void *cookie,
-					    xlator_t *this,
-					    int32_t op_ret,
-					    int32_t op_errno,
-					    struct iatt *buf,
-					    dict_t *dict)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	local->prebuf = *buf;
-	local->postbuf = *buf;
-
-	local->prebuf.ia_size = local->cur_file_size;
-	local->postbuf.ia_size = local->cur_file_size;
-
-	get_one_call(frame);
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-static int32_t do_ftruncate(call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    dict_t *dict,
-			    dict_t *xdata)
-{
-	data_t *data;
-	crypt_local_t *local = frame->local;
-
-	if (op_ret)
-		goto error;
-	/*
-	 * extract regular file size
-	 */
-	data = dict_get(dict, FSIZE_XATTR_PREFIX);
-	if (!data) {
-		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
-		op_errno = EIO;
-		goto error;
-	}
-	local->old_file_size = local->cur_file_size = data_to_uint64(data);
-
-	if (local->data_conf.orig_offset == local->cur_file_size) {
-#if DEBUG_CRYPT
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "trivial ftruncate (current file size %llu)",
-		       (unsigned long long)local->cur_file_size);
-#endif
-		goto trivial;
-	}
-	else if (local->data_conf.orig_offset < local->cur_file_size) {
-#if DEBUG_CRYPT
-		gf_log("crypt", GF_LOG_DEBUG, "prune from %llu to %llu",
-		       (unsigned long long)local->cur_file_size,
-		       (unsigned long long)local->data_conf.orig_offset);
-#endif
-		op_errno = prune_file(frame,
-				      this,
-				      local->data_conf.orig_offset);
-	}
-	else {
-#if DEBUG_CRYPT
-		gf_log("crypt", GF_LOG_DEBUG, "expand from %llu to %llu",
-		       (unsigned long long)local->cur_file_size,
-		       (unsigned long long)local->data_conf.orig_offset);
-#endif
-		op_errno = expand_file(frame,
-				       this,
-				       local->data_conf.orig_offset);
-	}
-	if (op_errno)
-		goto error;
-	return 0;
- trivial:
-	STACK_WIND(frame,
-		   ftruncate_trivial_completion,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fstat,
-		   local->fd,
-		   NULL);
-	return 0;
- error:
-	/*
-	 * finish with ftruncate
-	 */
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-
-	get_one_call_nolock(frame);
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-static int32_t crypt_ftruncate_finodelk_cbk(call_frame_t *frame,
-					    void *cookie,
-					    xlator_t *this,
-					    int32_t op_ret,
-					    int32_t op_errno,
-					    dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0)
-		goto error;
-	/*
-	 * An access has been granted,
-	 * retrieve file size first
-	 */
-	STACK_WIND(frame,
-		   do_ftruncate,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fgetxattr,
-		   local->fd,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return 0;
- error:
-	get_one_call_nolock(frame);
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * ftruncate is performed in 2 steps:
- * . receive file size;
- * . expand or prune file.
- */
-static int32_t crypt_ftruncate(call_frame_t *frame,
-			       xlator_t *this,
-			       fd_t *fd,
-			       off_t offset,
-			       dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-	struct crypt_inode_info *info;
-	struct gf_flock lock = {0, };
-
-	local = crypt_alloc_local(frame, this, GF_FOP_FTRUNCATE);
-	if (!local) {
-		ret = ENOMEM;
-		goto error;
-	}
-	local->xattr = dict_new();
-	if (!local->xattr) {
-		ret = ENOMEM;
-		goto error;
-	}
-	local->fd = fd_ref(fd);
-	info = local_get_inode_info(local, this);
-	if (info == NULL) {
-		ret = EINVAL;
-		goto error;
-	}
-	if (!object_alg_atomic(&info->cinfo)) {
-		ret = EINVAL;
-		goto error;
-	}
-	local->data_conf.orig_offset = offset;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_WRLCK;
-        lock.l_whence = SEEK_SET;
-
-	STACK_WIND(frame,
-		   crypt_ftruncate_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- error:
-	if (local && local->fd)
-		fd_unref(fd);
-	if (local && local->xdata)
-		dict_unref(xdata);
-	if (local && local->xattr)
-		dict_unref(local->xattr);
-	if (local && local->info)
-		free_inode_info(local->info);
-
-	STACK_UNWIND_STRICT(ftruncate, frame, -1, ret, NULL, NULL, NULL);
-	return 0;
-}
-
-/* ->flush_cbk() */
-int32_t truncate_end(call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	STACK_UNWIND_STRICT(truncate,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local->xdata);
-	return 0;
-}
-
-/* ftruncate_cbk() */
-int32_t truncate_flush(call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct iatt *prebuf,
-		       struct iatt *postbuf,
-		       dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	fd_t *fd = local->fd;
-	local->prebuf = *prebuf;
-	local->postbuf = *postbuf;
-
-	STACK_WIND(frame,
-		   truncate_end,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->flush,
-		   fd,
-		   NULL);
-	fd_unref(fd);
-	return 0;
-}
-
-/*
- * is called as ->open_cbk()
- */
-static int32_t truncate_begin(call_frame_t *frame,
-			      void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret,
-			      int32_t op_errno,
-			      fd_t *fd,
-			      dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0) {
-		fd_unref(fd);
-		STACK_UNWIND_STRICT(truncate,
-				    frame,
-				    op_ret,
-				    op_errno, NULL, NULL, NULL);
-		return 0;
-	} else {
-	        fd_bind (fd);
-        }
-	/*
-	 * crypt_truncate() is implemented via crypt_ftruncate(),
-	 * so the crypt xlator does STACK_WIND to itself here
-	 */
-	STACK_WIND(frame,
-		   truncate_flush,
-		   this,
-		   this->fops->ftruncate, /* crypt_ftruncate */
-		   fd,
-		   local->offset,
-		   NULL);
-	return 0;
-}
-
-/*
- * crypt_truncate() is implemented via crypt_ftruncate() as a
- * sequence crypt_open() - crypt_ftruncate() - truncate_flush()
- */
-int32_t crypt_truncate(call_frame_t *frame,
-		       xlator_t *this,
-		       loc_t *loc,
-		       off_t offset,
-		       dict_t *xdata)
-{
-	fd_t *fd;
-	crypt_local_t *local;
-
-#if DEBUG_CRYPT
-	gf_log(this->name, GF_LOG_DEBUG,
-	       "truncate file %s at offset %llu",
-	       loc->path, (unsigned long long)offset);
-#endif
-	local = crypt_alloc_local(frame, this, GF_FOP_TRUNCATE);
-	if (!local)
-		goto error;
-
-	fd = fd_create(loc->inode, frame->root->pid);
-	if (!fd) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
-		goto error;
-	}
-	local->fd = fd;
-	local->offset = offset;
-	local->xdata = xdata;
-	STACK_WIND(frame,
-		   truncate_begin,
-		   this,
-		   this->fops->open, /* crypt_open() */
-		   loc,
-		   O_RDWR,
-		   fd,
-		   NULL);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(truncate, frame, -1, EINVAL, NULL, NULL, NULL);
-	return 0;
-}
-
-end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop)
-{
-	switch (fop) {
-	case GF_FOP_WRITE:
-		return end_writeback_writev;
-	case GF_FOP_FTRUNCATE:
-		return end_writeback_ftruncate;
-	default:
-		gf_log("crypt", GF_LOG_WARNING, "Bad wb operation %d", fop);
-		return NULL;
-	}
-}
-
-/*
- * true, if the caller needs metadata string
- */
-static int32_t is_custom_mtd(dict_t *xdata)
-{
-	data_t *data;
-	uint32_t flags;
-
-	if (!xdata)
-		return 0;
-
-	data = dict_get(xdata, MSGFLAGS_PREFIX);
-	if (!data)
-		return 0;
-	if (data->len != sizeof(uint32_t)) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "Bad msgflags size (%d)", data->len);
-		return -1;
-	}
-	flags = *((uint32_t *)data->data);
-	return msgflags_check_mtd_lock(&flags);
-}
-
-static int32_t crypt_open_done(call_frame_t *frame,
-			       void *cookie,
-			       xlator_t *this,
-			       int32_t op_ret,
-			       int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	if (op_ret < 0)
-		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
-		       op_errno);
-	put_one_call_open(frame);
-	return 0;
-}
-
-static void crypt_open_tail(call_frame_t *frame, xlator_t *this)
-{
-	struct gf_flock  lock  = {0, };
-	crypt_local_t *local = frame->local;
-
-        lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
-
-	STACK_WIND(frame,
-		   crypt_open_done,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-}
-
-/*
- * load private inode info at open time
- * called as ->fgetxattr_cbk()
- */
-static int load_mtd_open(call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dict_t *dict,
-			 dict_t *xdata)
-{
-	int32_t ret;
-	gf_boolean_t upload_info;
-	data_t *mtd;
-	uint64_t value = 0;
-	struct crypt_inode_info *info;
-	crypt_local_t *local = frame->local;
-	crypt_private_t *priv = this->private;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (local->fd->inode->ia_type == IA_IFLNK)
-		goto exit;
-	if (op_ret < 0)
-		goto exit;
-	/*
-	 * first, check for cached info
-	 */
-	ret = inode_ctx_get(local->fd->inode, this, &value);
-	if (ret != -1) {
-		info = (struct crypt_inode_info *)(long)value;
-		if (info == NULL) {
-			gf_log(this->name, GF_LOG_WARNING,
-			       "Inode info expected, but not found");
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			goto exit;
-		}
-		/*
-		 * info has been found in the cache
-		 */
-		upload_info = _gf_false;
-	}
-	else {
-		/*
-		 * info hasn't been found in the cache.
-		 */
-		info = alloc_inode_info(local, local->loc);
-		if (!info) {
-			local->op_ret = -1;
-			local->op_errno = ENOMEM;
-			goto exit;
-		}
-		init_inode_info_head(info, local->fd);
-		upload_info = _gf_true;
-	}
-	/*
-	 * extract metadata
-	 */
-	mtd = dict_get(dict, CRYPTO_FORMAT_PREFIX);
-	if (!mtd) {
-		local->op_ret = -1;
-		local->op_errno = ENOENT;
-		gf_log (this->name, GF_LOG_WARNING,
-			"Format string wasn't found");
-		goto exit;
-	}
-	/*
-	 * authenticate metadata against the path
-	 */
-	ret = open_format((unsigned char *)mtd->data,
-			  mtd->len,
-			  local->loc,
-			  info,
-			  get_master_cinfo(priv),
-			  local,
-			  upload_info);
-	if (ret) {
-		local->op_ret = -1;
-		local->op_errno = ret;
-		goto exit;
-	}
-	if (upload_info) {
-		ret = init_inode_info_tail(info, get_master_cinfo(priv));
-		if (ret) {
-			local->op_ret = -1;
-			local->op_errno = ret;
-			goto exit;
-		}
-		ret = inode_ctx_put(local->fd->inode,
-				    this, (uint64_t)(long)info);
-		if (ret == -1) {
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			goto exit;
-		}
-	}
-	if (local->custom_mtd) {
-		/*
-		 * pass the metadata string to the customer
-		 */
-		ret = dict_set_static_bin(local->xdata,
-					  CRYPTO_FORMAT_PREFIX,
-					  mtd->data,
-					  mtd->len);
-		if (ret) {
-			local->op_ret = -1;
-			local->op_errno = ret;
-			goto exit;
-		}
-	}
- exit:
-	if (!local->custom_mtd)
-		crypt_open_tail(frame, this);
-	else
-		put_one_call_open(frame);
-	return 0;
-}
-
-static int32_t crypt_open_finodelk_cbk(call_frame_t *frame,
-				       void *cookie,
-				       xlator_t *this,
-				       int32_t op_ret,
-				       int32_t op_errno,
-				       dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0) {
-		gf_log(this->name, GF_LOG_WARNING, "finodelk (LOCK) failed");
-		goto exit;
-	}
-	STACK_WIND(frame,
-		   load_mtd_open,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fgetxattr,
-		   local->fd,
-		   CRYPTO_FORMAT_PREFIX,
-		   NULL);
-	return 0;
- exit:
-	put_one_call_open(frame);
-	return 0;
-}
-
-/*
- * verify metadata against the specified pathname
- */
-static int32_t crypt_open_cbk(call_frame_t *frame,
-			      void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret,
-			      int32_t op_errno,
-			      fd_t *fd,
-			      dict_t *xdata)
-{
-	struct gf_flock lock = {0, };
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (local->fd->inode->ia_type == IA_IFLNK)
-		goto exit;
-	if (op_ret < 0)
-		goto exit;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-	else if (local->custom_mtd){
-		local->xdata = dict_new();
-		if (!local->xdata) {
-			local->op_ret = -1;
-			local->op_errno = ENOMEM;
-			gf_log ("crypt", GF_LOG_ERROR,
-				"Can not get new dict for mtd string");
-			goto exit;
-		}
-	}
-	lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = local->custom_mtd ? F_WRLCK : F_RDLCK;
-        lock.l_whence = SEEK_SET;
-
-	STACK_WIND(frame,
-		   crypt_open_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- exit:
-	put_one_call_open(frame);
-	return 0;
-}
-
-static int32_t crypt_open(call_frame_t *frame,
-			  xlator_t *this,
-			  loc_t *loc,
-			  int32_t flags,
-			  fd_t *fd,
-			  dict_t *xdata)
-{
-	int32_t ret = ENOMEM;
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, GF_FOP_OPEN);
-	if (!local)
-		goto error;
-	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
-	if (!local->loc) {
-		ret = ENOMEM;
-		goto error;
-	}
-	memset(local->loc, 0, sizeof(*local->loc));
-	ret = loc_copy(local->loc, loc);
-	if (ret) {
-		GF_FREE(local->loc);
-		goto error;
-	}
-	local->fd = fd_ref(fd);
-
-	ret = is_custom_mtd(xdata);
-	if (ret < 0) {
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-		ret = EINVAL;
-		goto error;
-	}
-	local->custom_mtd = ret;
-
-	if ((flags & O_ACCMODE) == O_WRONLY)
-		/*
-		 * we can't open O_WRONLY, because
-		 * we need to do read-modify-write
-		 */
-		flags = (flags & ~O_ACCMODE) | O_RDWR;
-	/*
-	 * Make sure that out translated offsets
-	 * and counts won't be ignored
-	 */
-	flags &= ~O_APPEND;
-	get_one_call_nolock(frame);
-	STACK_WIND(frame,
-		   crypt_open_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->open,
-		   loc,
-		   flags,
-		   fd,
-		   xdata);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(open,
-			    frame,
-			    -1,
-			    ret,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-static int32_t init_inode_info_tail(struct crypt_inode_info *info,
-				    struct master_cipher_info *master)
-{
-	int32_t ret;
-	struct object_cipher_info *object = &info->cinfo;
-
-#if DEBUG_CRYPT
-	gf_log("crypt", GF_LOG_DEBUG, "Init inode info for object %s",
-	       uuid_utoa(info->oid));
-#endif
-	ret = data_cipher_algs[object->o_alg][object->o_mode].set_private(info,
-									master);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Set private info failed");
-		return ret;
-	}
-	return 0;
-}
-
-/*
- * Init inode info at ->create() time
- */
-static void init_inode_info_create(struct crypt_inode_info *info,
-				   struct master_cipher_info *master,
-				   data_t *data)
-{
-	struct object_cipher_info *object;
-
-	info->nr_minor = CRYPT_XLATOR_ID;
-	memcpy(info->oid, data->data, data->len);
-
-	object = &info->cinfo;
-
-	object->o_alg        = master->m_alg;
-	object->o_mode       = master->m_mode;
-	object->o_block_bits = master->m_block_bits;
-	object->o_dkey_size  = master->m_dkey_size;
-}
-
-static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd)
-{
-	memcpy(info->oid, fd->inode->gfid, sizeof(uuid_t));
-}
-
-static int32_t crypt_create_done(call_frame_t *frame,
-				 void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret,
-				 int32_t op_errno, dict_t *xdata)
-{
-	crypt_private_t *priv = this->private;
-	crypt_local_t *local = frame->local;
-	struct crypt_inode_info *info = local->info;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	inode_t *local_inode = local->inode;
-
-	if (op_ret < 0) {
-		free_inode_info(info);
-		goto unwind;
-	}
-	op_errno = init_inode_info_tail(info, get_master_cinfo(priv));
-	if (op_errno) {
-		op_ret = -1;
-		free_inode_info(info);
-		goto unwind;
-	}
-	/*
-	 * FIXME: drop major subversion number
-	 */
-	op_ret = inode_ctx_put(local->fd->inode, this, (uint64_t)(long)info);
-	if (op_ret == -1) {
-		op_errno = EIO;
-		free_inode_info(info);
-		goto unwind;
-	}
- unwind:
-	free_format(local);
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    local_fd,
-			    local_inode,
-			    &local->buf,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local_xdata);
-	fd_unref(local_fd);
-	inode_unref(local_inode);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	return 0;
-}
-
-static int crypt_create_tail(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     dict_t *xdata)
-{
-	struct gf_flock  lock  = {0, };
-	crypt_local_t *local = frame->local;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	inode_t *local_inode = local->inode;
-
-	dict_unref(local->xattr);
-
-	if (op_ret < 0)
-		goto error;
-
-        lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
-
-	STACK_WIND(frame,
-		   crypt_create_done,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- error:
-	free_inode_info(local->info);
-	free_format(local);
-
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    local_fd,
-			    local_inode,
-			    &local->buf,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local_xdata);
-
-	fd_unref(local_fd);
-	inode_unref(local_inode);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	return 0;
-}
-
-static int32_t crypt_create_finodelk_cbk(call_frame_t *frame,
-					 void *cookie,
-					 xlator_t *this,
-					 int32_t op_ret,
-					 int32_t op_errno,
-					 dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	struct crypt_inode_info *info = local->info;
-
-	if (op_ret < 0)
-		goto error;
-
-	STACK_WIND(frame,
-		   crypt_create_tail,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fsetxattr,
-		   local->fd,
-		   local->xattr, /* CRYPTO_FORMAT_PREFIX */
-		   0,
-		   NULL);
-	return 0;
- error:
-	free_inode_info(info);
-	free_format(local);
-	fd_unref(local->fd);
-	dict_unref(local->xattr);
-	if (local->xdata)
-		dict_unref(local->xdata);
-
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    NULL,
-			    NULL,
-			    NULL,
-			    NULL,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-/*
- * Create and store crypt-specific format on disk;
- * Populate cache with private inode info
- */
-static int32_t crypt_create_cbk(call_frame_t *frame,
-				void *cookie,
-				xlator_t *this,
-				int32_t op_ret,
-				int32_t op_errno,
-				fd_t *fd,
-				inode_t *inode,
-				struct iatt *buf,
-				struct iatt *preparent,
-				struct iatt *postparent,
-				dict_t *xdata)
-{
-	struct gf_flock lock = {0, };
-	crypt_local_t *local = frame->local;
-	struct crypt_inode_info *info = local->info;
-
-	if (op_ret < 0)
-		goto error;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-	local->inode = inode_ref(inode);
-	local->buf = *buf;
-	local->prebuf = *preparent;
-	local->postbuf = *postparent;
-
-        lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_WRLCK;
-        lock.l_whence = SEEK_SET;
-
-        STACK_WIND(frame,
-		   crypt_create_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- error:
-	free_inode_info(info);
-	free_format(local);
-	fd_unref(local->fd);
-	dict_unref(local->xattr);
-
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    NULL, NULL, NULL,
-			    NULL, NULL, NULL);
-	return 0;
-}
-
-static int32_t crypt_create(call_frame_t *frame,
-			    xlator_t *this,
-			    loc_t *loc,
-			    int32_t flags,
-			    mode_t mode,
-			    mode_t umask,
-			    fd_t *fd,
-			    dict_t *xdata)
-{
-	int ret;
-	data_t *data;
-	crypt_local_t *local;
-	crypt_private_t *priv;
-	struct master_cipher_info *master;
-	struct crypt_inode_info *info;
-
-	priv = this->private;
-	master = get_master_cinfo(priv);
-
-	if (master_alg_atomic(master)) {
-		/*
-		 * We can't open O_WRONLY, because we
-		 * need to do read-modify-write.
-		 */
-		if ((flags & O_ACCMODE) == O_WRONLY)
-			flags = (flags & ~O_ACCMODE) | O_RDWR;
-		/*
-		 * Make sure that out translated offsets
-		 * and counts won't be ignored
-		 */
-		flags &= ~O_APPEND;
-	}
-	local = crypt_alloc_local(frame, this, GF_FOP_CREATE);
-	if (!local) {
-		ret = ENOMEM;
-		goto error;
-	}
-	data = dict_get(xdata, "gfid-req");
-	if (!data) {
-		ret = EINVAL;
-		gf_log("crypt", GF_LOG_WARNING, "gfid not found");
-		goto error;
-	}
-	if (data->len != sizeof(uuid_t)) {
-		ret = EINVAL;
-		gf_log("crypt", GF_LOG_WARNING,
-		       "bad gfid size (%d), should be %d",
-		       (int)data->len, (int)sizeof(uuid_t));
-		goto error;
-	}
-	info = alloc_inode_info(local, loc);
-	if (!info){
-		ret = ENOMEM;
-		goto error;
-	}
-	/*
-	 * NOTE:
-	 * format has to be created BEFORE
-	 * proceeding to the untrusted server
-	 */
-	ret = alloc_format_create(local);
-	if (ret) {
-		free_inode_info(info);
-		goto error;
-	}
-	init_inode_info_create(info, master, data);
-
-	ret = create_format(local->format,
-			    loc,
-			    info,
-			    master);
-	if (ret) {
-		free_inode_info(info);
-		goto error;
-	}
-	local->xattr = dict_new();
-	if (!local->xattr) {
-		free_inode_info(info);
-		free_format(local);
-		goto error;
-	}
-	ret = dict_set_static_bin(local->xattr,
-				  CRYPTO_FORMAT_PREFIX,
-				  local->format,
-				  new_format_size());
-	if (ret) {
-		dict_unref(local->xattr);
-		free_inode_info(info);
-		free_format(local);
-		goto error;
-	}
-	ret = dict_set(local->xattr, FSIZE_XATTR_PREFIX, data_from_uint64(0));
-	if (ret) {
-		dict_unref(local->xattr);
-		free_inode_info(info);
-		free_format(local);
-		goto error;
-	}
-	local->fd = fd_ref(fd);
-
-	STACK_WIND(frame,
-		   crypt_create_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->create,
-		   loc,
-		   flags,
-		   mode,
-		   umask,
-		   fd,
-		   xdata);
-	return 0;
- error:
-	gf_log("crypt", GF_LOG_WARNING, "can not create file");
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    -1,
-			    ret,
-			    NULL, NULL, NULL,
-			    NULL, NULL, NULL);
-	return 0;
-}
-
-/*
- * FIXME: this should depends on the version of format string
- */
-static int32_t filter_crypt_xattr(dict_t *dict,
-				  char *key, data_t *value, void *data)
-{
-        dict_del(dict, key);
-        return 0;
-}
-
-static int32_t crypt_fsetxattr(call_frame_t *frame,
-			       xlator_t *this,
-			       fd_t *fd,
-			       dict_t *dict,
-			       int32_t flags, dict_t *xdata)
-{
-	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
-			     filter_crypt_xattr, NULL);
-	STACK_WIND(frame,
-		   default_fsetxattr_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fsetxattr,
-		   fd,
-		   dict,
-		   flags,
-		   xdata);
-	return 0;
-}
-
-/*
- * TBD: verify file metadata before wind
- */
-static int32_t crypt_setxattr(call_frame_t *frame,
-			      xlator_t *this,
-			      loc_t *loc,
-			      dict_t *dict,
-			      int32_t flags, dict_t *xdata)
-{
-	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
-			     filter_crypt_xattr, NULL);
-	STACK_WIND(frame,
-		   default_setxattr_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->setxattr,
-		   loc,
-		   dict,
-		   flags,
-		   xdata);
-	return 0;
-}
-
-/*
- * called as flush_cbk()
- */
-static int32_t linkop_end(call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	linkop_unwind_handler_t unwind_fn;
-	unwind_fn = linkop_unwind_dispatch(local->fop);
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0 &&
-	    op_errno == ENOENT &&
-	    local->loc->inode->ia_type == IA_IFLNK) {
-		local->op_ret = 0;
-		local->op_errno = 0;
-	}
-	unwind_fn(frame);
-	return 0;
-}
-
-/*
- * unpin inode on the server
- */
-static int32_t link_flush(call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  inode_t *inode,
-			  struct iatt *buf,
-			  struct iatt *preparent,
-			  struct iatt *postparent, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;
-	if (local->xdata) {
-		dict_unref(local->xdata);
-		local->xdata = NULL;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-	local->inode = inode_ref(inode);
-	local->buf = *buf;
-	local->prebuf = *preparent;
-	local->postbuf = *postparent;
-
-	STACK_WIND(frame,
-		   linkop_end,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->flush,
-		   local->fd,
-		   NULL);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-	link_unwind(frame);
-	return 0;
-}
-
-void link_unwind(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	dict_t *xdata;
-	dict_t *xattr;
-	inode_t *inode;
-
-	if (!local) {
-		STACK_UNWIND_STRICT(link,
-				    frame,
-				    -1,
-				    ENOMEM,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL);
-		return;
-	}
-	xdata = local->xdata;
-	xattr = local->xattr;
-	inode = local->inode;
-
-	if (local->loc){
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	if (local->newloc) {
-		loc_wipe(local->newloc);
-		GF_FREE(local->newloc);
-	}
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->format)
-		GF_FREE(local->format);
-
-	STACK_UNWIND_STRICT(link,
-			    frame,
-			    local->op_ret,
-			    local->op_errno,
-			    inode,
-			    &local->buf,
-			    &local->prebuf,
-			    &local->postbuf,
-			    xdata);
-	if (xdata)
-		dict_unref(xdata);
-	if (xattr)
-		dict_unref(xattr);
-	if (inode)
-		inode_unref(inode);
-}
-
-void link_wind(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-
-	STACK_WIND(frame,
-		   link_flush,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->link,
-		   local->loc,
-		   local->newloc,
-		   local->xdata);
-}
-
-/*
- * unlink()
- */
-static int32_t unlink_flush(call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    struct iatt *preparent,
-			    struct iatt *postparent, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;
-	local->prebuf = *preparent;
-	local->postbuf = *postparent;
-	if (local->xdata) {
-		dict_unref(local->xdata);
-		local->xdata = NULL;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	STACK_WIND(frame,
-		   linkop_end,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->flush,
-		   local->fd,
-		   NULL);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-	unlink_unwind(frame);
-	return 0;
-}
-
-void unlink_unwind(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	dict_t *xdata;
-	dict_t *xattr;
-
-	if (!local) {
-		STACK_UNWIND_STRICT(unlink,
-				    frame,
-				    -1,
-				    ENOMEM,
-				    NULL,
-				    NULL,
-				    NULL);
-		return;
-	}
-	xdata = local->xdata;
-	xattr = local->xattr;
-	if (local->loc){
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->format)
-		GF_FREE(local->format);
-
-	STACK_UNWIND_STRICT(unlink,
-			    frame,
-			    local->op_ret,
-			    local->op_errno,
-			    &local->prebuf,
-			    &local->postbuf,
-			    xdata);
-	if (xdata)
-		dict_unref(xdata);
-	if (xattr)
-		dict_unref(xattr);
-}
-
-void unlink_wind(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-
-	STACK_WIND(frame,
-		   unlink_flush,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->unlink,
-		   local->loc,
-		   local->flags,
-		   local->xdata);
-}
-
-void rename_unwind(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	dict_t *xdata;
-	dict_t *xattr;
-	struct iatt *prenewparent;
-	struct iatt *postnewparent;
-
-	if (!local) {
-		STACK_UNWIND_STRICT(rename,
-				    frame,
-				    -1,
-				    ENOMEM,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL);
-		return;
-	}
-	xdata = local->xdata;
-	xattr = local->xattr;
-	prenewparent = local->prenewparent;
-	postnewparent = local->postnewparent;
-
-	if (local->loc){
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	if (local->newloc){
-		loc_wipe(local->newloc);
-		GF_FREE(local->newloc);
-	}
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->format)
-		GF_FREE(local->format);
-
-	STACK_UNWIND_STRICT(rename,
-			    frame,
-			    local->op_ret,
-			    local->op_errno,
-			    &local->buf,
-			    &local->prebuf,
-			    &local->postbuf,
-			    prenewparent,
-			    postnewparent,
-			    xdata);
-	if (xdata)
-		dict_unref(xdata);
-	if (xattr)
-		dict_unref(xattr);
-	if (prenewparent)
-		GF_FREE(prenewparent);
-	if (postnewparent)
-		GF_FREE(postnewparent);
-}
-
-/*
- * called as flush_cbk()
- */
-static int32_t rename_end(call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	rename_unwind(frame);
-	return 0;
-}
-
-static int32_t rename_flush(call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    struct iatt *buf,
-			    struct iatt *preoldparent,
-			    struct iatt *postoldparent,
-			    struct iatt *prenewparent,
-			    struct iatt *postnewparent,
-			    dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;
-	dict_unref(local->xdata);
-	local->xdata = NULL;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	local->buf = *buf;
-	local->prebuf = *preoldparent;
-	local->postbuf = *postoldparent;
-	if (prenewparent) {
-		local->prenewparent = GF_CALLOC(1, sizeof(*prenewparent),
-						gf_crypt_mt_iatt);
-		if (!local->prenewparent) {
-			op_errno = ENOMEM;
-			goto error;
-		}
-		*local->prenewparent = *prenewparent;
-	}
-	if (postnewparent) {
-		local->postnewparent = GF_CALLOC(1, sizeof(*postnewparent),
-						 gf_crypt_mt_iatt);
-		if (!local->postnewparent) {
-			op_errno = ENOMEM;
-			goto error;
-		}
-		*local->postnewparent = *postnewparent;
-	}
-	STACK_WIND(frame,
-		   rename_end,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->flush,
-		   local->fd,
-		   NULL);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-	rename_unwind(frame);
-	return 0;
-}
-
-void rename_wind(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-
-	STACK_WIND(frame,
-		   rename_flush,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->rename,
-		   local->loc,
-		   local->newloc,
-		   local->xdata);
-}
-
-static int32_t __do_linkop(call_frame_t *frame,
-			   void *cookie,
-			   xlator_t *this,
-			   int32_t op_ret,
-			   int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	linkop_wind_handler_t wind_fn;
-	linkop_unwind_handler_t unwind_fn;
-
-	wind_fn = linkop_wind_dispatch(local->fop);
-	unwind_fn = linkop_unwind_dispatch(local->fop);
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret >= 0)
-		wind_fn(frame, this);
-	else {
-		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
-		       op_errno);
-		unwind_fn(frame);
-	}
-	return 0;
-}
-
-static int32_t do_linkop(call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dict_t *xdata)
-{
-	struct gf_flock  lock  = {0, };
-	crypt_local_t *local = frame->local;
-	linkop_unwind_handler_t unwind_fn;
-
-	unwind_fn = linkop_unwind_dispatch(local->fop);
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if(op_ret < 0)
-		goto error;
-
-	lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
-
-	STACK_WIND(frame,
-		   __do_linkop,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- error:
-	unwind_fn(frame);
-	return 0;
-}
-
-/*
- * Update the metadata string (against the new pathname);
- * submit the result
- */
-static int32_t linkop_begin(call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    fd_t *fd,
-			    dict_t *xdata)
-{
-	gf_boolean_t upload_info;
-	crypt_local_t *local = frame->local;
-	crypt_private_t *priv = this->private;
-	struct crypt_inode_info *info;
-	data_t *old_mtd;
-	uint32_t new_mtd_size;
-	uint64_t value = 0;
-	void (*unwind_fn)(call_frame_t *frame);
-	mtd_op_t mop;
-
-	unwind_fn = linkop_unwind_dispatch(local->fop);
-	mop = linkop_mtdop_dispatch(local->fop);
-
-	if (op_ret < 0) {
-		/*
-		 * verification failed
-		 */
-		goto error;
-        } else {
-                fd_bind (fd);
-        }
-
-	old_mtd = dict_get(xdata, CRYPTO_FORMAT_PREFIX);
-	if (!old_mtd) {
-		op_errno = EIO;
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Metadata string wasn't found");
-		goto error;
-	}
-	new_mtd_size = format_size(mop, old_mtd->len);
-	op_errno = alloc_format(local, new_mtd_size);
-	if (op_errno)
-		goto error;
-	/*
-	 * check for cached info
-	 */
-	op_ret = inode_ctx_get(fd->inode, this, &value);
-	if (op_ret != -1) {
-		info = (struct crypt_inode_info *)(long)value;
-		if (info == NULL) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Inode info was not found");
-			op_errno = EINVAL;
-			goto error;
-		}
-		/*
-		 * info was found in the cache
-		 */
-		local->info = info;
-		upload_info = _gf_false;
-	}
-	else {
-		/*
-		 * info wasn't found in the cache;
-		 */
-		info = alloc_inode_info(local, local->loc);
-		if (!info)
-			goto error;
-		init_inode_info_head(info, fd);
-		local->info = info;
-		upload_info = _gf_true;
-	}
-	op_errno = open_format((unsigned char *)old_mtd->data,
-			       old_mtd->len,
-			       local->loc,
-			       info,
-			       get_master_cinfo(priv),
-			       local,
-			       upload_info);
-	if (op_errno)
-		goto error;
-	if (upload_info == _gf_true) {
-		op_errno = init_inode_info_tail(info,
-						get_master_cinfo(priv));
-		if (op_errno)
-			goto error;
-		op_errno = inode_ctx_put(fd->inode, this,
-					 (uint64_t)(long)(info));
-		if (op_errno == -1) {
-			op_errno = EIO;
-			goto error;
-		}
-	}
-	/*
-	 * update the format string (append/update/cup a MAC)
-	 */
-	op_errno = update_format(local->format,
-				 (unsigned char *)old_mtd->data,
-				 old_mtd->len,
-				 local->mac_idx,
-				 mop,
-				 local->newloc,
-				 info,
-				 get_master_cinfo(priv),
-				 local);
-	if (op_errno)
-		goto error;
-	/*
-	 * store the new format string on the server
-	 */
-	if (new_mtd_size) {
-		op_errno = dict_set_static_bin(local->xattr,
-					       CRYPTO_FORMAT_PREFIX,
-					       local->format,
-					       new_mtd_size);
-		if (op_errno)
-			goto error;
-	}
-	STACK_WIND(frame,
-		   do_linkop,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->setxattr,
-		   local->loc,
-		   local->xattr,
-		   0,
-		   NULL);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-	unwind_fn(frame);
-	return 0;
-}
-
-static int32_t linkop_grab_local(call_frame_t *frame,
-				 xlator_t *this,
-				 loc_t *oldloc,
-				 loc_t *newloc,
-				 int flags, dict_t *xdata,
-				 glusterfs_fop_t op)
-{
-	int32_t ret = ENOMEM;
-	fd_t *fd;
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, op);
-	if (!local)
-		goto error;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	fd = fd_create(oldloc->inode, frame->root->pid);
-	if (!fd) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
-		goto error;
-	}
-	local->fd = fd;
-	local->flags = flags;
-	local->loc = GF_CALLOC(1, sizeof(*oldloc), gf_crypt_mt_loc);
-	if (!local->loc)
-		goto error;
-	memset(local->loc, 0, sizeof(*local->loc));
-	ret = loc_copy(local->loc, oldloc);
-	if (ret) {
-		GF_FREE(local->loc);
-		local->loc = NULL;
-		goto error;
-	}
-	if (newloc) {
-		local->newloc = GF_CALLOC(1, sizeof(*newloc), gf_crypt_mt_loc);
-		if (!local->newloc) {
-			loc_wipe(local->loc);
-			GF_FREE(local->loc);
-			goto error;
-		}
-		memset(local->newloc, 0, sizeof(*local->newloc));
-		ret = loc_copy(local->newloc, newloc);
-		if (ret) {
-			loc_wipe(local->loc);
-			GF_FREE(local->loc);
-			GF_FREE(local->newloc);
-			goto error;
-		}
-	}
-	local->xattr = dict_new();
-	if (!local->xattr) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
-		ret = ENOMEM;
-		goto error;
-	}
-	return 0;
-
-error:
-        if (local) {
-                if (local->xdata)
-                        dict_unref(local->xdata);
-                if (local->fd)
-                        fd_unref(local->fd);
-                local->fd = 0;
-                local->loc = NULL;
-                local->newloc = NULL;
-                local->op_ret = -1;
-                local->op_errno = ret;
-        }
-
-        return ret;
-}
-
-/*
- * read and verify locked metadata against the old pathname (via open);
- * update the metadata string in accordance with the new pathname;
- * submit modified metadata;
- * wind;
- */
-static int32_t linkop(call_frame_t *frame,
-		      xlator_t *this,
-		      loc_t *oldloc,
-		      loc_t *newloc,
-		      int flags,
-		      dict_t *xdata,
-		      glusterfs_fop_t op)
-{
-	int32_t ret;
-	dict_t *dict;
-	crypt_local_t *local;
-	void (*unwind_fn)(call_frame_t *frame);
-        void (*wind_fn)(call_frame_t *frame, xlator_t *this);
-
-        wind_fn = linkop_wind_dispatch(op);
-	unwind_fn = linkop_unwind_dispatch(op);
-
-	ret = linkop_grab_local(frame, this, oldloc, newloc, flags, xdata, op);
-	local = frame->local;
-	if (ret)
-		goto error;
-
-        if (local->fd->inode->ia_type == IA_IFLNK)
-                goto wind;
-
-	dict = dict_new();
-	if (!dict) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
-		ret = ENOMEM;
-		goto error;
-	}
-	/*
-	 * Set a message to crypt_open() that we need
-	 * locked metadata string.
-	 * All link operations (link, unlink, rename)
-	 * need write lock
-	 */
-	msgflags_set_mtd_wlock(&local->msgflags);
-	ret = dict_set_static_bin(dict,
-				  MSGFLAGS_PREFIX,
-				  &local->msgflags,
-				  sizeof(local->msgflags));
-	if (ret) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not set dict");
-		dict_unref(dict);
-		goto error;
-	}
-	/*
-	 * verify metadata against the old pathname
-	 * and retrieve locked metadata string
-	 */
-	STACK_WIND(frame,
-		   linkop_begin,
-		   this,
-		   this->fops->open, /* crypt_open() */
-		   oldloc,
-		   O_RDWR,
-		   local->fd,
-		   dict);
-	dict_unref(dict);
-	return 0;
-
-wind:
-        wind_fn(frame, this);
-        return 0;
-
-error:
-	local->op_ret = -1;
-	local->op_errno = ret;
-	unwind_fn(frame);
-	return 0;
-}
-
-static int32_t crypt_link(call_frame_t *frame, xlator_t *this,
-			  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_LINK);
-}
-
-static int32_t crypt_unlink(call_frame_t *frame, xlator_t *this,
-			    loc_t *loc, int flags, dict_t *xdata)
-{
-	return linkop(frame, this, loc, NULL, flags, xdata, GF_FOP_UNLINK);
-}
-
-static int32_t crypt_rename(call_frame_t *frame, xlator_t *this,
-			    loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_RENAME);
-}
-
-static void put_one_call_open(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	if (put_one_call(local)) {
-		fd_t *fd = local->fd;
-		loc_t *loc = local->loc;
-		dict_t *xdata = local->xdata;
-
-		STACK_UNWIND_STRICT(open,
-				    frame,
-				    local->op_ret,
-				    local->op_errno,
-				    fd,
-				    xdata);
-		fd_unref(fd);
-		if (xdata)
-			dict_unref(xdata);
-		loc_wipe(loc);
-		GF_FREE(loc);
-	}
-}
-
-static int32_t __crypt_readv_done(call_frame_t *frame,
-				  void *cookie,
-				  xlator_t *this,
-				  int32_t op_ret,
-				  int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	/* read deals with data configs only */
-	struct iovec *avec = local->data_conf.avec;
-	char **pool = local->data_conf.pool;
-	int blocks_in_pool = local->data_conf.blocks_in_pool;
-	struct iobref *iobref = local->iobref;
-	struct iobref *iobref_data = local->iobref_data;
-
-	if (op_ret < 0) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "readv unlock failed (%d)", op_errno);
-		if (local->op_ret >= 0) {
-			local->op_ret = op_ret;
-			local->op_errno = op_errno;
-		}
-	}
-	dump_plain_text(local, avec);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "readv: ret_to_user: %d, iovec len: %d, ia_size: %llu",
-	       (int)(local->rw_count > 0 ? local->rw_count : local->op_ret),
-	       (int)(local->rw_count > 0 ? iov_length(avec, local->data_conf.acount) : 0),
-	       (unsigned long long)local->buf.ia_size);
-
-	STACK_UNWIND_STRICT(readv,
-			    frame,
-			    local->rw_count > 0 ? local->rw_count : local->op_ret,
-			    local->op_errno,
-			    avec,
-			    avec ? local->data_conf.acount : 0,
-			    &local->buf,
-			    local->iobref,
-			    local_xdata);
-
-	free_avec(avec, pool, blocks_in_pool);
-	fd_unref(local_fd);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	if (iobref)
-		iobref_unref(iobref);
-	if (iobref_data)
-		iobref_unref(iobref_data);
-	return 0;
-}
-
-static void crypt_readv_done(call_frame_t *frame, xlator_t *this)
-{
-	if (parent_is_crypt_xlator(frame, this))
-		/*
-		 *  don't unlock (it will be done by the parent)
-		 */
-		__crypt_readv_done(frame, NULL, this, 0, 0, NULL);
-	else {
-		crypt_local_t *local = frame->local;
-		struct gf_flock  lock  = {0, };
-
-		lock.l_type   = F_UNLCK;
-		lock.l_whence = SEEK_SET;
-		lock.l_start  = 0;
-		lock.l_len    = 0;
-		lock.l_pid    = 0;
-
-		STACK_WIND(frame,
-			   __crypt_readv_done,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->finodelk,
-			   this->name,
-			   local->fd,
-			   F_SETLKW,
-			   &lock,
-			   NULL);
-	}
-}
-
-static void put_one_call_readv(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	if (put_one_call(local))
-		crypt_readv_done(frame, this);
-}
-
-static int32_t __crypt_writev_done(call_frame_t *frame,
-				   void *cookie,
-				   xlator_t *this,
-				   int32_t op_ret,
-				   int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	int32_t ret_to_user;
-
-	if (local->xattr)
-		dict_unref(local->xattr);
-	/*
-	 * Calculate amout of butes to be returned
-	 * to user. We need to subtract paddings that
-	 * have been written as a part of atom.
-	 */
-	/*
-	 * subtract head padding
-	 */
-	if (local->rw_count == 0)
-		/*
-		 * Nothing has been written, it must be an error
-		 */
-		ret_to_user = local->op_ret;
-	else if (local->rw_count <= local->data_conf.off_in_head) {
-		gf_log("crypt", GF_LOG_WARNING, "Incomplete write");
-		ret_to_user = 0;
-	}
-	else
-		ret_to_user = local->rw_count -
-			local->data_conf.off_in_head;
-	/*
-	 * subtract tail padding
-	 */
-	if (ret_to_user > local->data_conf.orig_size)
-		ret_to_user = local->data_conf.orig_size;
-
-	if (local->iobref)
-		iobref_unref(local->iobref);
-	if (local->iobref_data)
-		iobref_unref(local->iobref_data);
-	free_avec_data(local);
-	free_avec_hole(local);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "writev: ret_to_user: %d", ret_to_user);
-
-	STACK_UNWIND_STRICT(writev,
-			    frame,
-			    ret_to_user,
-			    local->op_errno,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local_xdata);
-	fd_unref(local_fd);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	return 0;
-}
-
-static int32_t crypt_writev_done(call_frame_t *frame,
-				 void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret,
-				 int32_t op_errno,
-				 dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
-
-	if (parent_is_crypt_xlator(frame, this))
-		/*
-		 * don't unlock (it will be done by the parent)
-		 */
-		__crypt_writev_done(frame, NULL, this, 0, 0, NULL);
-	else {
-		struct gf_flock  lock  = {0, };
-
-		lock.l_type   = F_UNLCK;
-		lock.l_whence = SEEK_SET;
-		lock.l_start  = 0;
-		lock.l_len    = 0;
-		lock.l_pid    = 0;
-
-		STACK_WIND(frame,
-			   __crypt_writev_done,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->finodelk,
-			   this->name,
-			   local->fd,
-			   F_SETLKW,
-			   &lock,
-			   NULL);
-	}
-	return 0;
-}
-
-static void put_one_call_writev(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	if (put_one_call(local)) {
-		if (local->update_disk_file_size) {
-			int32_t ret;
-			/*
-			 * update file size, unlock the file and unwind
-			 */
-			ret = dict_set(local->xattr,
-				       FSIZE_XATTR_PREFIX,
-				       data_from_uint64(local->cur_file_size));
-			if (ret) {
-				gf_log("crypt", GF_LOG_WARNING,
-				       "can not set key to update file size");
-				crypt_writev_done(frame, NULL,
-						  this, 0, 0, NULL);
-				return;
-			}
-			gf_log("crypt", GF_LOG_DEBUG,
-			       "Updating disk file size to %llu",
-			       (unsigned long long)local->cur_file_size);
-			STACK_WIND(frame,
-				   crypt_writev_done,
-				   FIRST_CHILD(this),
-				   FIRST_CHILD(this)->fops->fsetxattr,
-				   local->fd,
-				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
-				   0,
-				   NULL);
-		}
-		else
-			crypt_writev_done(frame, NULL, this, 0, 0, NULL);
-	}
-}
-
-static int32_t __crypt_ftruncate_done(call_frame_t *frame,
-				      void *cookie,
-				      xlator_t *this,
-				      int32_t op_ret,
-				      int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	char *iobase = local->vec.iov_base;
-
-	if (op_ret < 0) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "ftruncate unlock failed (%d)", op_errno);
-		if (local->op_ret >= 0) {
-			local->op_ret = op_ret;
-			local->op_errno = op_errno;
-		}
-	}
-	if (local->iobref_data)
-		iobref_unref(local->iobref_data);
-	free_avec_data(local);
-	free_avec_hole(local);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "ftruncate, return to user: presize=%llu, postsize=%llu",
-	       (unsigned long long)local->prebuf.ia_size,
-	       (unsigned long long)local->postbuf.ia_size);
-
-	STACK_UNWIND_STRICT(ftruncate,
-			    frame,
-			    ((local->op_ret < 0) ? -1 : 0),
-			    local->op_errno,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local_xdata);
-	fd_unref(local_fd);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	if (iobase)
-		GF_FREE(iobase);
-	return 0;
-}
-
-static int32_t crypt_ftruncate_done(call_frame_t *frame,
-				    void *cookie,
-				    xlator_t *this,
-				    int32_t op_ret,
-				    int32_t op_errno,
-				    dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	struct gf_flock  lock  = {0, };
-
-	dict_unref(local->xattr);
-	if (op_ret < 0)
-		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
-
-	lock.l_type   = F_UNLCK;
-	lock.l_whence = SEEK_SET;
-	lock.l_start  = 0;
-	lock.l_len    = 0;
-	lock.l_pid    = 0;
-
-	STACK_WIND(frame,
-		   __crypt_ftruncate_done,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
-}
-
-static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	if (put_one_call(local)) {
-		if (local->update_disk_file_size) {
-			int32_t ret;
-			/*
-			 * update file size, unlock the file and unwind
-			 */
-			ret = dict_set(local->xattr,
-				       FSIZE_XATTR_PREFIX,
-				       data_from_uint64(local->cur_file_size));
-			if (ret) {
-				gf_log("crypt", GF_LOG_WARNING,
-				       "can not set key to update file size");
-				crypt_ftruncate_done(frame, NULL,
-						     this, 0, 0, NULL);
-				return;
-			}
-			gf_log("crypt", GF_LOG_DEBUG,
-			       "Updating disk file size to %llu",
-			       (unsigned long long)local->cur_file_size);
-			STACK_WIND(frame,
-				   crypt_ftruncate_done,
-				   FIRST_CHILD(this),
-				   FIRST_CHILD(this)->fops->fsetxattr,
-				   local->fd,
-				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
-				   0,
-				   NULL);
-		}
-		else
-			crypt_ftruncate_done(frame, NULL, this, 0, 0, NULL);
-	}
-}
-
-/*
- * load regular file size for some FOPs
- */
-static int32_t load_file_size(call_frame_t *frame,
-			      void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret,
-			      int32_t op_errno,
-			      dict_t *dict,
-			      dict_t *xdata)
-{
-	data_t *data;
-	crypt_local_t *local = frame->local;
-
-	dict_t *local_xdata = local->xdata;
-	inode_t *local_inode = local->inode;
-
-	if (op_ret < 0)
-		goto unwind;
-	/*
-	 * load regular file size
-	 */
-	data = dict_get(dict, FSIZE_XATTR_PREFIX);
-	if (!data) {
-		if (local->xdata)
-			dict_unref(local->xdata);
-		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
-		op_ret = -1;
-		op_errno = EIO;
-		goto unwind;
-	}
-	local->buf.ia_size = data_to_uint64(data);
-
-	gf_log(this->name, GF_LOG_DEBUG,
-	       "FOP %d: Translate regular file to %llu",
-	       local->fop,
-	       (unsigned long long)local->buf.ia_size);
- unwind:
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->loc) {
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	switch (local->fop) {
-	case GF_FOP_FSTAT:
-		STACK_UNWIND_STRICT(fstat,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? &local->buf : NULL,
-				    local->xdata);
-		break;
-	case GF_FOP_STAT:
-		STACK_UNWIND_STRICT(stat,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? &local->buf : NULL,
-				    local->xdata);
-		break;
-	case GF_FOP_LOOKUP:
-		STACK_UNWIND_STRICT(lookup,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? local->inode : NULL,
-				    op_ret >= 0 ? &local->buf : NULL,
-				    local->xdata,
-				    op_ret >= 0 ? &local->postbuf : NULL);
-		break;
-	case GF_FOP_READ:
-		STACK_UNWIND_STRICT(readv,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    NULL,
-				    0,
-				    op_ret >= 0 ? &local->buf : NULL,
-				    NULL,
-				    NULL);
-		break;
-	default:
-		gf_log(this->name, GF_LOG_WARNING,
-		       "Improper file operation %d", local->fop);
-	}
-	if (local_xdata)
-		dict_unref(local_xdata);
-	if (local_inode)
-		inode_unref(local_inode);
-	return 0;
-}
-
-static int32_t crypt_stat_common_cbk(call_frame_t *frame,
-				     void *cookie,
-				     xlator_t *this,
-				     int32_t op_ret,
-				     int32_t op_errno,
-				     struct iatt *buf, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto unwind;
-	if (!IA_ISREG(buf->ia_type))
-		goto unwind;
-
-	local->buf = *buf;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	switch (local->fop) {
-	case GF_FOP_FSTAT:
-		STACK_WIND(frame,
-			   load_file_size,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->fgetxattr,
-			   local->fd,
-			   FSIZE_XATTR_PREFIX,
-			   NULL);
-		break;
-	case GF_FOP_STAT:
-		STACK_WIND(frame,
-			   load_file_size,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->getxattr,
-			   local->loc,
-			   FSIZE_XATTR_PREFIX,
-			   NULL);
-		break;
-	default:
-		gf_log (this->name, GF_LOG_WARNING,
-			"Improper file operation %d", local->fop);
-	}
-	return 0;
- unwind:
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->loc) {
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	switch (local->fop) {
-	case GF_FOP_FSTAT:
-		STACK_UNWIND_STRICT(fstat,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? buf : NULL,
-				    op_ret >= 0 ? xdata : NULL);
-		break;
-	case GF_FOP_STAT:
-		STACK_UNWIND_STRICT(stat,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? buf : NULL,
-				    op_ret >= 0 ? xdata : NULL);
-		break;
-	default:
-		gf_log (this->name, GF_LOG_WARNING,
-			"Improper file operation %d", local->fop);
-	}
-	return 0;
-}
-
-static int32_t crypt_fstat(call_frame_t *frame,
-			   xlator_t *this,
-			   fd_t *fd, dict_t *xdata)
-{
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, GF_FOP_FSTAT);
-	if (!local)
-		goto error;
-	local->fd = fd_ref(fd);
-	STACK_WIND(frame,
-		   crypt_stat_common_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fstat,
-		   fd,
-		   xdata);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(fstat,
-			    frame,
-			    -1,
-			    ENOMEM,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-static int32_t crypt_stat(call_frame_t *frame,
-			  xlator_t *this,
-			  loc_t *loc, dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, GF_FOP_STAT);
-	if (!local)
-		goto error;
-	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
-	if (!local->loc)
-		goto error;
-	memset(local->loc, 0, sizeof(*local->loc));
-	ret = loc_copy(local->loc, loc);
-	if (ret) {
-		GF_FREE(local->loc);
-		goto error;
-	}
-	STACK_WIND(frame,
-		   crypt_stat_common_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->stat,
-		   loc,
-		   xdata);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(stat,
-			    frame,
-			    -1,
-			    ENOMEM,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-static int32_t crypt_lookup_cbk(call_frame_t *frame,
-				void *cookie,
-				xlator_t *this,
-				int32_t op_ret,
-				int32_t op_errno,
-				inode_t *inode,
-				struct iatt *buf, dict_t *xdata,
-				struct iatt *postparent)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto unwind;
-	if (!IA_ISREG(buf->ia_type))
-		goto unwind;
-
-	local->inode = inode_ref(inode);
-	local->buf = *buf;
-	local->postbuf = *postparent;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-	gf_uuid_copy(local->loc->gfid, buf->ia_gfid);
-
-	STACK_WIND(frame,
-		   load_file_size,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->getxattr,
-		   local->loc,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return 0;
- unwind:
-	loc_wipe(local->loc);
-	GF_FREE(local->loc);
-	STACK_UNWIND_STRICT(lookup,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    inode,
-			    buf,
-			    xdata,
-			    postparent);
-	return 0;
-}
-
-static int32_t crypt_lookup(call_frame_t *frame,
-			    xlator_t *this,
-			    loc_t *loc, dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, GF_FOP_LOOKUP);
-	if (!local)
-		goto error;
-	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
-	if (!local->loc)
-		goto error;
-	memset(local->loc, 0, sizeof(*local->loc));
-	ret = loc_copy(local->loc, loc);
-	if (ret) {
-		GF_FREE(local->loc);
-		goto error;
-	}
-	gf_log(this->name, GF_LOG_DEBUG, "Lookup %s", loc->path);
-	STACK_WIND(frame,
-		   crypt_lookup_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->lookup,
-		   loc,
-		   xdata);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(lookup,
-			    frame,
-			    -1,
-			    ENOMEM,
-			    NULL,
-			    NULL,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-/*
- * for every regular directory entry find its real file size
- * and update stat's buf properly
- */
-static int32_t crypt_readdirp_cbk(call_frame_t *frame,
-				  void *cookie,
-				  xlator_t *this,
-				  int32_t op_ret,
-				  int32_t op_errno,
-				  gf_dirent_t *entries, dict_t *xdata)
-{
-	gf_dirent_t *entry = NULL;
-
-	if (op_ret < 0)
-		goto unwind;
-
-	list_for_each_entry (entry, (&entries->list), list) {
-		data_t *data;
-
-		if (!IA_ISREG(entry->d_stat.ia_type))
-			continue;
-		data = dict_get(entry->dict, FSIZE_XATTR_PREFIX);
-		if (!data){
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size of direntry not found");
-			op_errno = EIO;
-			op_ret = -1;
-			break;
-		}
-		entry->d_stat.ia_size = data_to_uint64(data);
-	}
- unwind:
-	STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
-	return 0;
-}
-
-/*
- * ->readdirp() fills in-core inodes, so we need to set proper
- * file sizes for all directory entries of the parent @fd.
- * Actual updates take place in ->crypt_readdirp_cbk()
- */
-static int32_t crypt_readdirp(call_frame_t *frame, xlator_t *this,
-			      fd_t *fd, size_t size, off_t offset,
-			      dict_t *xdata)
-{
-	int32_t ret = ENOMEM;
-
-	if (!xdata) {
-		xdata = dict_new();
-		if (!xdata)
-			goto error;
-	}
-	else
-		dict_ref(xdata);
-	/*
-	 * make sure that we'll have real file sizes at ->readdirp_cbk()
-	 */
-	ret = dict_set(xdata, FSIZE_XATTR_PREFIX, data_from_uint64(0));
-	if (ret) {
-		dict_unref(xdata);
-		goto error;
-	}
-	STACK_WIND(frame,
-		   crypt_readdirp_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->readdirp,
-		   fd,
-		   size,
-		   offset,
-		   xdata);
-	dict_unref(xdata);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(readdirp, frame, -1, ret, NULL, NULL);
-	return 0;
-}
-
-static int32_t crypt_access(call_frame_t *frame,
-			    xlator_t *this,
-			    loc_t *loc,
-			    int32_t mask, dict_t *xdata)
-{
-	gf_log(this->name, GF_LOG_WARNING,
-	       "NFS mounts of encrypted volumes are unsupported");
-	STACK_UNWIND_STRICT(access, frame, -1, EPERM, NULL);
-	return 0;
-}
-
-int32_t master_set_block_size (xlator_t *this, crypt_private_t *priv,
-			       dict_t *options)
-{
-	uint64_t block_size = 0;
-	struct master_cipher_info *master = get_master_cinfo(priv);
-
-	if (options != NULL)
-		GF_OPTION_RECONF("block-size", block_size, options,
-				 size_uint64, error);
-	else
-		GF_OPTION_INIT("block-size", block_size, size_uint64, error);
-
-	switch (block_size) {
-	case 512:
-		master->m_block_bits = 9;
-		break;
-	case 1024:
-		master->m_block_bits = 10;
-		break;
-	case 2048:
-		master->m_block_bits = 11;
-		break;
-	case 4096:
-		master->m_block_bits = 12;
-		break;
-	default:
-		gf_log("crypt", GF_LOG_ERROR,
-		       "FATAL: unsupported block size %llu",
-		       (unsigned long long)block_size);
-		goto error;
-	}
-	return 0;
- error:
-	return -1;
-}
-
-int32_t master_set_alg(xlator_t *this, crypt_private_t *priv)
-{
-	struct master_cipher_info *master = get_master_cinfo(priv);
-	master->m_alg = AES_CIPHER_ALG;
-	return 0;
-}
-
-int32_t master_set_mode(xlator_t *this, crypt_private_t *priv)
-{
-	struct master_cipher_info *master = get_master_cinfo(priv);
-	master->m_mode = XTS_CIPHER_MODE;
-	return 0;
-}
-
-/*
- * set key size in bits to the master info
- * Pre-conditions: cipher mode in the master info is uptodate.
- */
-static int master_set_data_key_size (xlator_t *this, crypt_private_t *priv,
-				     dict_t *options)
-{
-	int32_t ret;
-	uint64_t key_size = 0;
-	struct master_cipher_info *master = get_master_cinfo(priv);
-
-	if (options != NULL)
-		GF_OPTION_RECONF("data-key-size", key_size, options,
-				 uint64, error);
-	else
-		GF_OPTION_INIT("data-key-size", key_size, uint64, error);
-
-	ret = data_cipher_algs[master->m_alg][master->m_mode].check_key(key_size);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "FATAL: wrong bin key size %llu for alg %d mode %d",
-		       (unsigned long long)key_size,
-		       (int)master->m_alg,
-		       (int)master->m_mode);
-		goto error;
-	}
-	master->m_dkey_size = key_size;
-	return 0;
- error:
-	return -1;
-}
-
-static int is_hex(char *s) {
-	return  ('0' <= *s && *s <= '9') || ('a' <= *s && *s <= 'f');
-}
-
-static int parse_hex_buf(xlator_t *this, char *src, unsigned char *dst,
-			 int hex_size)
-{
-	int i;
-	int hex_byte = 0;
-
-	for (i = 0; i < (hex_size / 2); i++) {
-		if (!is_hex(src + i*2) || !is_hex(src + i*2 + 1)) {
-			gf_log("crypt", GF_LOG_ERROR,
-			       "FATAL: not hex symbol in key");
-			return -1;
-		}
-		if (sscanf(src + i*2, "%2x", &hex_byte) != 1) {
-			gf_log("crypt", GF_LOG_ERROR,
-			       "FATAL: can not parse hex key");
-			return -1;
-		}
-		dst[i] = hex_byte & 0xff;
-	}
-	return 0;
-}
-
-/*
- * Parse options;
- * install master volume key
- */
-int32_t master_set_master_vol_key(xlator_t *this, crypt_private_t *priv)
-{
-	int32_t ret;
-	FILE *file = NULL;
-
-	int32_t key_size;
-	char *opt_key_file_pathname = NULL;
-
-	unsigned char bin_buf[MASTER_VOL_KEY_SIZE];
-	char hex_buf[2 * MASTER_VOL_KEY_SIZE];
-
-	struct master_cipher_info *master = get_master_cinfo(priv);
-	/*
-	 * extract master key passed via option
-	 */
-	GF_OPTION_INIT("master-key", opt_key_file_pathname, path, bad_key);
-
-	if (!opt_key_file_pathname) {
-		gf_log(this->name, GF_LOG_ERROR, "FATAL: missing master key");
-		return -1;
-	}
-	gf_log(this->name, GF_LOG_DEBUG, "handling file key %s",
-	       opt_key_file_pathname);
-
-	file = fopen(opt_key_file_pathname, "r");
-	if (file == NULL) {
-		gf_log(this->name, GF_LOG_ERROR,
-		       "FATAL: can not open file with master key");
-		return -1;
-	}
-	/*
-	 * extract hex key
-	 */
-	key_size = fread(hex_buf, 1, sizeof(hex_buf), file);
-	if (key_size < sizeof(hex_buf)) {
-		gf_log(this->name, GF_LOG_ERROR,
-		       "FATAL: master key is too short");
-		goto bad_key;
-	}
-	ret = parse_hex_buf(this, hex_buf, bin_buf, key_size);
-	if (ret)
-		goto bad_key;
-	memcpy(master->m_key, bin_buf, MASTER_VOL_KEY_SIZE);
-	memset(hex_buf, 0, sizeof(hex_buf));
-	fclose(file);
-
- 	memset(bin_buf, 0, sizeof(bin_buf));
-	return 0;
- bad_key:
-	gf_log(this->name, GF_LOG_ERROR, "FATAL: bad master key");
-	if (file)
-		fclose(file);
-	memset(bin_buf, 0, sizeof(bin_buf));
-	return -1;
-}
-
-/*
- * Derive volume key for object-id authentication
- */
-int32_t master_set_nmtd_vol_key(xlator_t *this, crypt_private_t *priv)
-{
-	return get_nmtd_vol_key(get_master_cinfo(priv));
-}
-
-int32_t crypt_init_xlator(xlator_t *this)
-{
-	int32_t ret;
-	crypt_private_t *priv = this->private;
-
-	ret = master_set_alg(this, priv);
-	if (ret)
-		return ret;
-	ret = master_set_mode(this, priv);
-	if (ret)
-		return ret;
-	ret = master_set_block_size(this, priv, NULL);
-	if (ret)
-		return ret;
-	ret = master_set_data_key_size(this, priv, NULL);
-	if (ret)
-		return ret;
-	ret = master_set_master_vol_key(this, priv);
-	if (ret)
-		return ret;
-	return master_set_nmtd_vol_key(this, priv);
-}
-
-static int32_t crypt_alloc_private(xlator_t *this)
-{
-	this->private = GF_CALLOC(1, sizeof(crypt_private_t), gf_crypt_mt_priv);
-	if (!this->private) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "Can not allocate memory for private data");
-		return ENOMEM;
-	}
-	return 0;
-}
-
-static void crypt_free_private(xlator_t *this)
-{
-	crypt_private_t *priv = this->private;
-	if (priv) {
-		memset(priv, 0, sizeof(*priv));
-		GF_FREE(priv);
-	}
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_crypt_mt_end);
-
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-int32_t reconfigure (xlator_t *this, dict_t *options)
-{
-	int32_t ret = -1;
-	crypt_private_t *priv = NULL;
-
-	GF_VALIDATE_OR_GOTO ("crypt", this, error);
-	GF_VALIDATE_OR_GOTO (this->name, this->private, error);
-	GF_VALIDATE_OR_GOTO (this->name, options, error);
-
-	priv = this->private;
-
-	ret = master_set_block_size(this, priv, options);
-	if (ret) {
-		gf_log("this->name", GF_LOG_ERROR,
-		       "Failed to reconfure block size");
-		goto error;
-	}
-	ret = master_set_data_key_size(this, priv, options);
-	if (ret) {
-		gf_log("this->name", GF_LOG_ERROR,
-		       "Failed to reconfure data key size");
-		goto error;
-	}
-	return 0;
- error:
-	return ret;
-}
-
-int32_t init(xlator_t *this)
-{
-	int32_t ret;
-
-	if (!this->children || this->children->next) {
-		gf_log ("crypt", GF_LOG_ERROR,
-			"FATAL: crypt should have exactly one child");
-		return EINVAL;
-	}
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-	ret = crypt_alloc_private(this);
-	if (ret)
-		return ret;
-	ret = crypt_init_xlator(this);
-	if (ret)
-		goto error;
-	this->local_pool = mem_pool_new(crypt_local_t, 64);
-        if (!this->local_pool) {
-		gf_log(this->name, GF_LOG_ERROR,
-		       "failed to create local_t's memory pool");
-		ret = ENOMEM;
-                goto error;
-        }
-	gf_log ("crypt", GF_LOG_INFO, "crypt xlator loaded");
-	return 0;
- error:
-	crypt_free_private(this);
-	return ret;
-}
-
-void fini (xlator_t *this)
-{
-	crypt_free_private(this);
-}
-
-struct xlator_fops fops = {
-	.readv        = crypt_readv,
-	.writev       = crypt_writev,
-	.truncate     = crypt_truncate,
-	.ftruncate    = crypt_ftruncate,
-	.setxattr     = crypt_setxattr,
-	.fsetxattr    = crypt_fsetxattr,
-	.link         = crypt_link,
-	.unlink       = crypt_unlink,
-	.rename       = crypt_rename,
-	.open         = crypt_open,
-	.create       = crypt_create,
-	.stat         = crypt_stat,
-	.fstat        = crypt_fstat,
-	.lookup       = crypt_lookup,
-	.readdirp     = crypt_readdirp,
-	.access       = crypt_access
-};
-
-struct xlator_cbks cbks = {
-	.forget       = crypt_forget
-};
-
-struct volume_options options[] = {
-	{ .key = {"master-key"},
-	  .type = GF_OPTION_TYPE_PATH,
-	  .description = "Pathname of regular file which contains master volume key"
-	},
-	{ .key = {"data-key-size"},
-	  .type = GF_OPTION_TYPE_SIZET,
-	  .description = "Data key size (bits)",
-	  .min = 256,
-	  .max = 512,
-	  .default_value = "256",
-	},
-	{ .key = {"block-size"},
-	  .type = GF_OPTION_TYPE_SIZET,
-	  .description = "Atom size (bits)",
-	  .min = 512,
-	  .max = 4096,
-	  .default_value = "4096"
-	},
-	{ .key  = {NULL} },
-};
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/crypt.h b/xlators/encryption/crypt/src/crypt.h
deleted file mode 100644
index 6c014535f0a..00000000000
--- a/xlators/encryption/crypt/src/crypt.h
+++ /dev/null
@@ -1,890 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __CRYPT_H__
-#define __CRYPT_H__
-
-#include <openssl/aes.h>
-#include <openssl/evp.h>
-#include <openssl/sha.h>
-#include <openssl/hmac.h>
-#include <openssl/cmac.h>
-#include <openssl/modes.h>
-#include "crypt-mem-types.h"
-#include "compat.h"
-
-#define CRYPT_XLATOR_ID  (0)
-
-#define MAX_IOVEC_BITS (3)
-#define MAX_IOVEC (1 << MAX_IOVEC_BITS)
-#define KEY_FACTOR_BITS  (6)
-
-#define DEBUG_CRYPT (0)
-#define TRIVIAL_TFM (0)
-
-#define CRYPT_MIN_BLOCK_BITS (9)
-#define CRYPT_MAX_BLOCK_BITS (12)
-
-#define MASTER_VOL_KEY_SIZE (32)
-#define NMTD_VOL_KEY_SIZE (16)
-
-#if !defined(GF_LINUX_HOST_OS)
-typedef off_t loff_t;
-#endif
-
-struct crypt_key {
-	uint32_t len;
-	const char *label;
-};
-
-/*
- * Add new key types to the end of this
- * enumeration but before LAST_KEY_TYPE
- */
-typedef enum {
-	MASTER_VOL_KEY,
-	NMTD_VOL_KEY,
-	NMTD_LINK_KEY,
-	EMTD_FILE_KEY,
-	DATA_FILE_KEY_256,
-	DATA_FILE_KEY_512,
-	LAST_KEY_TYPE
-}crypt_key_type;
-
-struct kderive_context {
-	const unsigned char *pkey;/* parent key */
-	uint32_t pkey_len;        /* parent key size, bits */
-	uint32_t ckey_len;        /* child key size, bits */
-	unsigned char *fid;       /* fixed input data, NIST 800-108, 5.1 */
-	uint32_t fid_len;         /* fid len, bytes */
-	unsigned char *out;       /* contains child keying material */
-	uint32_t out_len;         /* out len, bytes */
-};
-
-typedef enum {
-	DATA_ATOM,
-	HOLE_ATOM,
-	LAST_DATA_TYPE
-}atom_data_type;
-
-typedef enum {
-	HEAD_ATOM,
-	TAIL_ATOM,
-	FULL_ATOM,
-	LAST_LOCALITY_TYPE
-}atom_locality_type;
-
-typedef enum {
-	MTD_CREATE,
-	MTD_APPEND,
-	MTD_OVERWRITE,
-	MTD_CUT,
-	MTD_LAST_OP
-} mtd_op_t;
-
-struct xts128_context {
-	void      *key1, *key2;
-	block128_f block1,block2;
-};
-
-struct object_cipher_info {
-	cipher_alg_t  o_alg;
-        cipher_mode_t o_mode;
-	uint32_t      o_block_bits;
-        uint32_t      o_dkey_size; /* raw data key size in bits */
-	union {
-		struct {
-			unsigned char ivec[16];
-			AES_KEY dkey[2];
-			AES_KEY tkey;    /* key used for tweaking */
-			XTS128_CONTEXT xts;
-		} aes_xts;
-	} u;
-};
-
-struct master_cipher_info {
-	/*
-	 * attributes inherited by newly created regular files
-	 */
-	cipher_alg_t  m_alg;
-        cipher_mode_t m_mode;
-	uint32_t      m_block_bits;
-        uint32_t      m_dkey_size; /* raw key size in bits */
-	/*
-	 * master key
-	 */
-	unsigned char m_key[MASTER_VOL_KEY_SIZE];
-	/*
-	 * volume key for oid authentication
-	 */
-	unsigned char m_nmtd_key[NMTD_VOL_KEY_SIZE];
-};
-
-/*
-* This info is not changed during file's life
- */
-struct crypt_inode_info {
-#if DEBUG_CRYPT
-	loc_t *loc; /* pathname that the file has been
-		       opened, or created with */
-#endif
-	uint16_t nr_minor;
-	uuid_t oid;
-	struct object_cipher_info cinfo;
-};
-
-/*
- * this should locate in secure memory
- */
-typedef struct {
-	struct master_cipher_info master;
-} crypt_private_t;
-
-static inline struct master_cipher_info *get_master_cinfo(crypt_private_t *priv)
-{
-	return &priv->master;
-}
-
-static inline struct object_cipher_info *get_object_cinfo(struct crypt_inode_info
-							 *info)
-{
-	return &info->cinfo;
-}
-
-/*
- * this describes layouts and properties
- * of atoms in an aligned vector
- */
-struct avec_config {
-	uint32_t atom_size;
-	atom_data_type type;
-	size_t orig_size;
-	off_t orig_offset;
-	size_t expanded_size;
-	off_t aligned_offset;
-
-	uint32_t off_in_head;
-	uint32_t off_in_tail;
-	uint32_t gap_in_tail;
-	uint32_t nr_full_blocks;
-
-	struct iovec *avec; /* aligned vector */
-	uint32_t acount; /* number of avec components. The same
-			  * as number of occupied logical blocks */
-	char **pool;
-	uint32_t blocks_in_pool;
-	uint32_t cursor; /* makes sense only for ordered writes,
-			  * so there is no races on this counter.
-			  *
-			  * Cursor is per-config object, we don't
-			  * reset cursor for atoms of different
-			  * localities (head, tail, full)
-			  */
-};
-
-
-typedef struct {
-	glusterfs_fop_t fop; /* code of FOP this local info built for */
-	fd_t *fd;
-	inode_t *inode;
-	loc_t *loc;
-	int32_t mac_idx;
-	loc_t *newloc;
-	int32_t flags;
-	int32_t wbflags;
-	struct crypt_inode_info *info;
-	struct iobref *iobref;
-	struct iobref *iobref_data;
-	off_t offset;
-
-	uint64_t old_file_size; /* per FOP, retrieved under lock held */
-	uint64_t cur_file_size; /* per iteration, before issuing IOs */
-	uint64_t new_file_size; /* per iteration, after issuing IOs */
-
-	uint64_t io_offset; /* offset of IOs issued per iteration */
-	uint64_t io_offset_nopad; /* offset of user's data in the atom */
-	uint32_t io_size; /* size of IOs issued per iteration */
-	uint32_t io_size_nopad; /* size of user's data in the IOs */
-	uint32_t eof_padding_size; /* size od EOF padding in the IOs */
-
-	gf_lock_t call_lock; /* protect nr_calls from many cbks */
-	int32_t nr_calls;
-
-	atom_data_type active_setup; /* which setup (hole or date)
-					is currently active */
-	/* data setup */
-	struct avec_config data_conf;
-
-	/* hole setup */
-	int hole_conv_in_proggress;
-	gf_lock_t hole_lock; /* protect hole config from many cbks */
-	int hole_handled;
-	struct avec_config hole_conf;
-	struct iatt buf;
-	struct iatt prebuf;
-	struct iatt postbuf;
-	struct iatt *prenewparent;
-	struct iatt *postnewparent;
-	int32_t op_ret;
-	int32_t op_errno;
-	int32_t rw_count; /* total read or written */
-	gf_lock_t rw_count_lock; /* protect the counter above */
-	unsigned char *format; /* for create, update format string */
-	uint32_t format_size;
-	uint32_t msgflags; /* messages for crypt_open() */
-	dict_t *xdata;
-	dict_t *xattr;
-	struct iovec vec; /* contains last file's atom for
-			     read-prune-write sequence */
-	gf_boolean_t custom_mtd;
-	/*
-	 * the next 3 fields are used by readdir and friends
-	 */
-	gf_dirent_t *de; /* directory entry */
-	char *de_path; /* pathname of directory entry */
-	uint32_t de_prefix_len; /* length of the parent's pathname */
-	gf_dirent_t *entries;
-
-	uint32_t update_disk_file_size:1;
-} crypt_local_t;
-
-/* This represents a (read)modify-write atom */
-struct rmw_atom {
-	atom_locality_type locality;
-	/*
-	 * read-modify-write sequence of the atom
-	 */
-	int32_t (*rmw)(call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct iovec *vec,
-		       int32_t count,
-		       struct iatt *stbuf,
-		       struct iobref *iobref,
-		       dict_t *xdata);
-	/*
-	 * offset of the logical block in a file
-	 */
-	loff_t (*offset_at)(call_frame_t *frame,
-			    struct object_cipher_info *object);
-	/*
-	 * IO offset in an atom
-	 */
-	uint32_t (*offset_in)(call_frame_t *frame,
-			      struct object_cipher_info *object);
-	/*
-	 * number of bytes of plain text of this atom that user
-	 * wants to read/write.
-	 * It can be smaller than atom_size in the case of head
-	 * or tail atoms.
-	 */
-	uint32_t (*io_size_nopad)(call_frame_t *frame,
-				  struct object_cipher_info *object);
-	/*
-	 * which iovec represents the atom
-	 */
-	struct iovec *(*get_iovec)(call_frame_t *frame, uint32_t count);
-	/*
-	 * how many bytes of partial block should be uptodated by
-	 * reading from disk.
-	 * This is used to perform a read component of RMW (read-modify-write).
-	 */
-	uint32_t (*count_to_uptodate)(call_frame_t *frame, struct object_cipher_info *object);
-	struct avec_config *(*get_config)(call_frame_t *frame);
-};
-
-struct data_cipher_alg {
-	gf_boolean_t atomic; /* true means that algorithm requires
-				to pad data before cipher transform */
-	gf_boolean_t should_pad; /* true means that algorithm requires
-				    to pad the end of file with extra-data */
-	uint32_t blkbits; /* blksize = 1 << blkbits */
-	/*
-	 * any preliminary sanity checks goes here
-	 */
-	int32_t (*init)(void);
-	/*
-	 * set alg-mode specific inode info
-	 */
-	int32_t (*set_private)(struct crypt_inode_info *info,
-			       struct master_cipher_info *master);
-	/*
-	 * check alg-mode specific data key
-	 */
-	int32_t (*check_key)(uint32_t key_size);
-	void (*set_iv)(off_t offset, struct object_cipher_info *object);
-	int32_t (*encrypt)(const unsigned char *from, unsigned char *to,
-			   size_t length, off_t offset, const int enc,
-			   struct object_cipher_info *object);
-};
-
-/*
- * version-dependent metadata loader
- */
-struct crypt_mtd_loader {
-	/*
-	 * return core format size
-	 */
-	size_t (*format_size)(mtd_op_t op, size_t old_size);
-	/*
-	 * pack version-specific metadata of an object
-	 * at ->create()
-	 */
-	int32_t (*create_format)(unsigned char *wire,
-				 loc_t *loc,
-				 struct crypt_inode_info *info,
-				 struct master_cipher_info *master);
-	/*
-	 * extract version-specific metadata of an object
-	 * at ->open() time
-	 */
-	int32_t (*open_format)(unsigned char *wire,
-			       int32_t len,
-			       loc_t *loc,
-			       struct crypt_inode_info *info,
-			       struct master_cipher_info *master,
-			       crypt_local_t *local,
-			       gf_boolean_t load_info);
-	int32_t (*update_format)(unsigned char *new,
-				 unsigned char *old,
-				 size_t old_len,
-				 int32_t mac_idx,
-				 mtd_op_t op,
-				 loc_t *loc,
-				 struct crypt_inode_info *info,
-				 struct master_cipher_info *master,
-				 crypt_local_t *local);
-};
-
-typedef int32_t (*end_writeback_handler_t)(call_frame_t *frame,
-					   void *cookie,
-					   xlator_t *this,
-					   int32_t op_ret,
-					   int32_t op_errno,
-					   struct iatt *prebuf,
-					   struct iatt *postbuf,
-					   dict_t *xdata);
-typedef void (*linkop_wind_handler_t)(call_frame_t *frame, xlator_t *this);
-typedef void (*linkop_unwind_handler_t)(call_frame_t *frame);
-
-
-/* Declarations */
-
-/* keys.c */
-extern struct crypt_key crypt_keys[LAST_KEY_TYPE];
-int32_t get_nmtd_vol_key(struct master_cipher_info *master);
-int32_t get_nmtd_link_key(loc_t *loc,
-			  struct master_cipher_info *master,
-			  unsigned char *result);
-int32_t get_emtd_file_key(struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  unsigned char *result);
-int32_t get_data_file_key(struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  uint32_t keysize,
-			  unsigned char *key);
-/* data.c */
-extern struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE];
-void encrypt_aligned_iov(struct object_cipher_info *object,
-			 struct iovec *vec,
-			 int count,
-			 off_t off);
-void decrypt_aligned_iov(struct object_cipher_info *object,
-			 struct iovec *vec,
-			 int count,
-			 off_t off);
-int32_t align_iov_by_atoms(xlator_t *this,
-			   crypt_local_t *local,
-			   struct object_cipher_info *object,
-			   struct iovec *vec /* input vector */,
-			   int32_t count /* number of vec components */,
-			   struct iovec *avec /* aligned vector */,
-			   char **blocks /* pool of blocks */,
-			   uint32_t *blocks_allocated,
-			   struct avec_config *conf);
-int32_t set_config_avec_data(xlator_t *this,
-			     crypt_local_t *local,
-			     struct avec_config *conf,
-			     struct object_cipher_info *object,
-			     struct iovec *vec,
-			     int32_t vec_count);
-int32_t set_config_avec_hole(xlator_t *this,
-			     crypt_local_t *local,
-			     struct avec_config *conf,
-			     struct object_cipher_info *object,
-			     glusterfs_fop_t fop);
-void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
-		    struct avec_config *conf, atom_data_type dtype);
-void set_config_offsets(call_frame_t *frame,
-			xlator_t *this,
-			uint64_t offset,
-			uint64_t count,
-			atom_data_type dtype,
-			int32_t setup_gap_in_tail);
-
-/* metadata.c */
-extern struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER];
-
-int32_t alloc_format(crypt_local_t *local, size_t size);
-int32_t alloc_format_create(crypt_local_t *local);
-void free_format(crypt_local_t *local);
-size_t format_size(mtd_op_t op, size_t old_size);
-size_t new_format_size(void);
-int32_t open_format(unsigned char *str, int32_t len, loc_t *loc,
-		    struct crypt_inode_info *info,
-		    struct master_cipher_info *master, crypt_local_t *local,
-		    gf_boolean_t load_info);
-int32_t update_format(unsigned char *new, unsigned char *old,
-		      size_t old_len, int32_t mac_idx, mtd_op_t op, loc_t *loc,
-		      struct crypt_inode_info *info,
-		      struct master_cipher_info *master,
-		      crypt_local_t *local);
-int32_t create_format(unsigned char *wire,
-		      loc_t *loc,
-		      struct crypt_inode_info *info,
-		      struct master_cipher_info *master);
-
-/* atom.c */
-struct rmw_atom *atom_by_types(atom_data_type data,
-			       atom_locality_type locality);
-void submit_partial(call_frame_t *frame,
-		    xlator_t *this,
-		    fd_t *fd,
-		    atom_locality_type ltype);
-void submit_full(call_frame_t *frame, xlator_t *this);
-
-/* crypt.c */
-
-end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop);
-void set_local_io_params_writev(call_frame_t *frame,
-				struct object_cipher_info *object,
-				struct rmw_atom *atom, off_t io_offset,
-				uint32_t io_size);
-void link_wind(call_frame_t *frame, xlator_t *this);
-void unlink_wind(call_frame_t *frame, xlator_t *this);
-void link_unwind(call_frame_t *frame);
-void unlink_unwind(call_frame_t *frame);
-void rename_wind(call_frame_t *frame, xlator_t *this);
-void rename_unwind(call_frame_t *frame);
-
-/* Inline functions */
-
-static inline int32_t crypt_xlator_id(void)
-{
-	return CRYPT_XLATOR_ID;
-}
-
-static inline mtd_loader_id current_mtd_loader(void)
-{
-	return MTD_LOADER_V1;
-}
-
-static inline uint32_t master_key_size (void)
-{
-	return crypt_keys[MASTER_VOL_KEY].len >> 3;
-}
-
-static inline uint32_t nmtd_vol_key_size (void)
-{
-	return crypt_keys[NMTD_VOL_KEY].len >> 3;
-}
-
-static inline uint32_t alg_mode_blkbits(cipher_alg_t alg,
-					cipher_mode_t mode)
-{
-	return data_cipher_algs[alg][mode].blkbits;
-}
-
-static inline uint32_t alg_mode_blksize(cipher_alg_t alg,
-					cipher_mode_t mode)
-{
-	return 1 << alg_mode_blkbits(alg, mode);
-}
-
-static inline gf_boolean_t alg_mode_atomic(cipher_alg_t alg,
-					   cipher_mode_t mode)
-{
-	return data_cipher_algs[alg][mode].atomic;
-}
-
-static inline gf_boolean_t alg_mode_should_pad(cipher_alg_t alg,
-					       cipher_mode_t mode)
-{
-	return data_cipher_algs[alg][mode].should_pad;
-}
-
-static inline uint32_t master_alg_blksize(struct master_cipher_info *mr)
-{
-	return alg_mode_blksize(mr->m_alg, mr->m_mode);
-}
-
-static inline uint32_t master_alg_blkbits(struct master_cipher_info *mr)
-{
-	return alg_mode_blkbits(mr->m_alg, mr->m_mode);
-}
-
-static inline gf_boolean_t master_alg_atomic(struct master_cipher_info *mr)
-{
-	return alg_mode_atomic(mr->m_alg, mr->m_mode);
-}
-
-static inline gf_boolean_t master_alg_should_pad(struct master_cipher_info *mr)
-{
-	return alg_mode_should_pad(mr->m_alg, mr->m_mode);
-}
-
-static inline uint32_t object_alg_blksize(struct object_cipher_info *ob)
-{
-	return alg_mode_blksize(ob->o_alg, ob->o_mode);
-}
-
-static inline uint32_t object_alg_blkbits(struct object_cipher_info *ob)
-{
-	return alg_mode_blkbits(ob->o_alg, ob->o_mode);
-}
-
-static inline gf_boolean_t object_alg_atomic(struct object_cipher_info *ob)
-{
-	return alg_mode_atomic(ob->o_alg, ob->o_mode);
-}
-
-static inline gf_boolean_t object_alg_should_pad(struct object_cipher_info *ob)
-{
-	return alg_mode_should_pad(ob->o_alg, ob->o_mode);
-}
-
-static inline uint32_t aes_raw_key_size(struct master_cipher_info *master)
-{
-	return master->m_dkey_size >> 3;
-}
-
-static inline struct avec_config *get_hole_conf(call_frame_t *frame)
-{
-	return &(((crypt_local_t *)frame->local)->hole_conf);
-}
-
-static inline struct avec_config *get_data_conf(call_frame_t *frame)
-{
-	return &(((crypt_local_t *)frame->local)->data_conf);
-}
-
-static inline int32_t get_atom_bits (struct object_cipher_info *object)
-{
-	return object->o_block_bits;
-}
-
-static inline int32_t get_atom_size (struct object_cipher_info *object)
-{
-	return 1 << get_atom_bits(object);
-}
-
-static inline int32_t has_head_block(struct avec_config *conf)
-{
-	return conf->off_in_head ||
-		(conf->acount == 1 && conf->off_in_tail);
-}
-
-static inline int32_t has_tail_block(struct avec_config *conf)
-{
-	return conf->off_in_tail && conf->acount > 1;
-}
-
-static inline int32_t has_full_blocks(struct avec_config *conf)
-{
-	return conf->nr_full_blocks;
-}
-
-static inline int32_t should_submit_head_block(struct avec_config *conf)
-{
-	return has_head_block(conf) && (conf->cursor == 0);
-}
-
-static inline int32_t should_submit_tail_block(struct avec_config *conf)
-{
-	return has_tail_block(conf) && (conf->cursor == conf->acount - 1);
-}
-
-static inline int32_t should_submit_full_block(struct avec_config *conf)
-{
-	uint32_t start = has_head_block(conf) ? 1 : 0;
-
-	return has_full_blocks(conf) &&
-		conf->cursor >= start &&
-		conf->cursor < start + conf->nr_full_blocks;
-}
-
-#if DEBUG_CRYPT
-static inline void crypt_check_input_len(size_t len,
-					 struct object_cipher_info *object)
-{
-	if (object_alg_should_pad(object) && (len & (object_alg_blksize(object) - 1)))
-		gf_log ("crypt", GF_LOG_DEBUG, "bad input len: %d", (int)len);
-}
-
-static inline void check_head_block(struct avec_config *conf)
-{
-	if (!has_head_block(conf))
-		gf_log("crypt", GF_LOG_DEBUG, "not a head atom");
-}
-
-static inline void check_tail_block(struct avec_config *conf)
-{
-	if (!has_tail_block(conf))
-		gf_log("crypt", GF_LOG_DEBUG, "not a tail atom");
-}
-
-static inline void check_full_block(struct avec_config *conf)
-{
-	if (!has_full_blocks(conf))
-		gf_log("crypt", GF_LOG_DEBUG, "not a full atom");
-}
-
-static inline void check_cursor_head(struct avec_config *conf)
-{
-	if (!has_head_block(conf))
-		gf_log("crypt",
-		       GF_LOG_DEBUG, "Illegal call of head atom method");
-	else if (conf->cursor != 0)
-		gf_log("crypt",
-		       GF_LOG_DEBUG, "Cursor (%d) is not at head atom",
-		       conf->cursor);
-}
-
-static inline void check_cursor_full(struct avec_config *conf)
-{
-	if (!has_full_blocks(conf))
-		gf_log("crypt",
-		       GF_LOG_DEBUG, "Illegal call of full atom method");
-	if (has_head_block(conf) && (conf->cursor == 0))
-		gf_log("crypt",
-		       GF_LOG_DEBUG, "Cursor is not at full atom");
-}
-
-/*
- * FIXME: use avec->iov_len to check setup
- */
-static inline int data_local_invariant(crypt_local_t *local)
-{
-	return 0;
-}
-
-#else
-#define crypt_check_input_len(len, object) noop
-#define check_head_block(conf) noop
-#define check_tail_block(conf) noop
-#define check_full_block(conf) noop
-#define check_cursor_head(conf) noop
-#define check_cursor_full(conf) noop
-
-#endif /* DEBUG_CRYPT */
-
-static inline struct avec_config *conf_by_type(call_frame_t *frame,
-					       atom_data_type dtype)
-{
-	struct avec_config *conf = NULL;
-
-	switch (dtype) {
-	case HOLE_ATOM:
-		conf = get_hole_conf(frame);
-		break;
-	case DATA_ATOM:
-		conf = get_data_conf(frame);
-		break;
-	default:
-		gf_log("crypt", GF_LOG_DEBUG, "bad atom type");
-	}
-	return conf;
-}
-
-static inline uint32_t nr_calls_head(struct avec_config *conf)
-{
-	return has_head_block(conf) ? 1 : 0;
-}
-
-static inline uint32_t nr_calls_tail(struct avec_config *conf)
-{
-	return has_tail_block(conf) ? 1 : 0;
-}
-
-static inline uint32_t nr_calls_full(struct avec_config *conf)
-{
-	switch(conf->type) {
-	case HOLE_ATOM:
-		return has_full_blocks(conf);
-	case DATA_ATOM:
-		return has_full_blocks(conf) ?
-			logical_blocks_occupied(0,
-						conf->nr_full_blocks,
-						MAX_IOVEC_BITS)	: 0;
-	default:
-		gf_log("crypt", GF_LOG_DEBUG, "bad atom data type");
-		return 0;
-	}
-}
-
-static inline uint32_t nr_calls(struct avec_config *conf)
-{
-	return nr_calls_head(conf) + nr_calls_tail(conf) + nr_calls_full(conf);
-}
-
-static inline uint32_t nr_calls_data(call_frame_t *frame)
-{
-	return nr_calls(get_data_conf(frame));
-}
-
-static inline uint32_t nr_calls_hole(call_frame_t *frame)
-{
-	return nr_calls(get_hole_conf(frame));
-}
-
-static inline void get_one_call_nolock(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-
-	++local->nr_calls;
-
-	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", 1);
-}
-
-static inline void get_one_call(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-
-	LOCK(&local->call_lock);
-	get_one_call_nolock(frame);
-	UNLOCK(&local->call_lock);
-}
-
-static inline void get_nr_calls_nolock(call_frame_t *frame, int32_t nr)
-{
-	crypt_local_t *local = frame->local;
-
-	local->nr_calls += nr;
-
-	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", nr);
-}
-
-static inline void get_nr_calls(call_frame_t *frame, int32_t nr)
-{
-	crypt_local_t *local = frame->local;
-
-	LOCK(&local->call_lock);
-	get_nr_calls_nolock(frame, nr);
-	UNLOCK(&local->call_lock);
-}
-
-static inline int put_one_call(crypt_local_t *local)
-{
-	uint32_t last = 0;
-
-	LOCK(&local->call_lock);
-	if (--local->nr_calls == 0)
-		last = 1;
-
-	//gf_log("crypt", GF_LOG_DEBUG, "put %d calls", 1);
-
-	UNLOCK(&local->call_lock);
-	return last;
-}
-
-static inline int is_appended_write(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = get_data_conf(frame);
-
-	return conf->orig_offset + conf->orig_size > local->old_file_size;
-}
-
-static inline int is_ordered_mode(call_frame_t *frame)
-{
-#if 0
-	crypt_local_t *local = frame->local;
-	return local->fop == GF_FOP_FTRUNCATE ||
-		(local->fop == GF_FOP_WRITE && is_appended_write(frame));
-#endif
-	return 1;
-}
-
-static inline int32_t hole_conv_completed(crypt_local_t *local)
-{
-	struct avec_config *conf = &local->hole_conf;
-	return conf->cursor == conf->acount;
-}
-
-static inline int32_t data_write_in_progress(crypt_local_t *local)
-{
-	return local->active_setup == DATA_ATOM;
-}
-
-static inline int32_t parent_is_crypt_xlator(call_frame_t *frame,
-					     xlator_t *this)
-{
-	return frame->parent->this == this;
-}
-
-static inline linkop_wind_handler_t linkop_wind_dispatch(glusterfs_fop_t fop)
-{
-	switch(fop){
-	case GF_FOP_LINK:
-		return link_wind;
-	case GF_FOP_UNLINK:
-		return unlink_wind;
-	case GF_FOP_RENAME:
-		return rename_wind;
-	default:
-		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
-		return NULL;
-	}
-}
-
-static inline linkop_unwind_handler_t linkop_unwind_dispatch(glusterfs_fop_t fop)
-{
-	switch(fop){
-	case GF_FOP_LINK:
-		return link_unwind;
-	case GF_FOP_UNLINK:
-		return unlink_unwind;
-	case GF_FOP_RENAME:
-		return rename_unwind;
-	default:
-		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
-		return NULL;
-	}
-}
-
-static inline mtd_op_t linkop_mtdop_dispatch(glusterfs_fop_t fop)
-{
-	switch (fop) {
-	case GF_FOP_LINK:
-		return MTD_APPEND;
-	case GF_FOP_UNLINK:
-		return MTD_CUT;
-	case GF_FOP_RENAME:
-		return MTD_OVERWRITE;
-	default:
-		gf_log("crypt", GF_LOG_WARNING, "Bad link operation %d", fop);
-		return MTD_LAST_OP;
-	}
-}
-
-#endif /* __CRYPT_H__ */
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/data.c b/xlators/encryption/crypt/src/data.c
deleted file mode 100644
index 8b467bf2c60..00000000000
--- a/xlators/encryption/crypt/src/data.c
+++ /dev/null
@@ -1,764 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include "defaults.h"
-#include "crypt-common.h"
-#include "crypt.h"
-
-static void set_iv_aes_xts(off_t offset, struct object_cipher_info *object)
-{
-	unsigned char *ivec;
-
-	ivec = object->u.aes_xts.ivec;
-
-	/* convert the tweak into a little-endian byte
-	 * array (IEEE P1619/D16, May 2007, section 5.1)
-	 */
-
-	*((uint64_t *)ivec) = htole64(offset);
-
-	/* ivec is padded with zeroes */
-}
-
-static int32_t aes_set_keys_common(unsigned char *raw_key, uint32_t key_size,
-				   AES_KEY *keys)
-{
-	int32_t ret;
-
-	ret = AES_set_encrypt_key(raw_key,
-				  key_size,
-				  &keys[AES_ENCRYPT]);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Set encrypt key failed");
-		return ret;
-	}
-	ret = AES_set_decrypt_key(raw_key,
-				  key_size,
-				  &keys[AES_DECRYPT]);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Set decrypt key failed");
-		return ret;
-	}
-	return 0;
-}
-
-/*
- * set private cipher info for xts mode
- */
-static int32_t set_private_aes_xts(struct crypt_inode_info *info,
-				   struct master_cipher_info *master)
-{
-	int ret;
-	struct object_cipher_info *object = get_object_cinfo(info);
-	unsigned char *data_key;
-	uint32_t subkey_size;
-
-	/* init tweak value */
-	memset(object->u.aes_xts.ivec, 0, 16);
-
-	data_key = GF_CALLOC(1, object->o_dkey_size, gf_crypt_mt_key);
-	if (!data_key)
-		return ENOMEM;
-
-	/*
-	 * retrieve data keying meterial
-	 */
-	ret = get_data_file_key(info, master, object->o_dkey_size, data_key);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Failed to retrieve data key");
-		GF_FREE(data_key);
-		return ret;
-	}
-	/*
-	 * parse compound xts key
-	 */
-	subkey_size = object->o_dkey_size >> 4; /* (xts-key-size-in-bytes / 2) */
-	/*
-	 * install key for data encryption
-	 */
-	ret = aes_set_keys_common(data_key,
-				  subkey_size << 3, object->u.aes_xts.dkey);
-	if (ret) {
-		GF_FREE(data_key);
-		return ret;
-	}
-	/*
-	 * set up key used to encrypt tweaks
-	 */
-	ret = AES_set_encrypt_key(data_key + subkey_size,
-				  object->o_dkey_size / 2,
-				  &object->u.aes_xts.tkey);
-	if (ret < 0)
-		gf_log("crypt", GF_LOG_ERROR, "Set tweak key failed");
-
-	GF_FREE(data_key);
-	return ret;
-}
-
-static int32_t aes_xts_init(void)
-{
-	cassert(AES_BLOCK_SIZE == (1 << AES_BLOCK_BITS));
-	return 0;
-}
-
-static int32_t check_key_aes_xts(uint32_t keysize)
-{
-	switch(keysize) {
-	case 256:
-	case 512:
-		return 0;
-	default:
-		break;
-	}
-	return -1;
-}
-
-static int32_t encrypt_aes_xts(const unsigned char *from,
-			       unsigned char *to, size_t length,
-			       off_t offset, const int enc,
-			       struct object_cipher_info *object)
-{
-	XTS128_CONTEXT ctx;
-	if (enc) {
-		ctx.key1 = &object->u.aes_xts.dkey[AES_ENCRYPT];
-		ctx.block1 = (block128_f)AES_encrypt;
-	}
-	else {
-		ctx.key1 = &object->u.aes_xts.dkey[AES_DECRYPT];
-		ctx.block1 = (block128_f)AES_decrypt;
-	}
-	ctx.key2 = &object->u.aes_xts.tkey;
-	ctx.block2 = (block128_f)AES_encrypt;
-
-	return CRYPTO_xts128_encrypt(&ctx,
-				     object->u.aes_xts.ivec,
-				     from,
-				     to,
-				     length, enc);
-}
-
-/*
- * Cipher input chunk @from of length @len;
- * @to: result of cipher transform;
- * @off: offset in a file (must be cblock-aligned);
- */
-static void cipher_data(struct object_cipher_info *object,
-			char *from,
-			char *to,
-			off_t off,
-			size_t len,
-			const int enc)
-{
-	crypt_check_input_len(len, object);
-
-#if TRIVIAL_TFM && DEBUG_CRYPT
-	return;
-#endif
-	data_cipher_algs[object->o_alg][object->o_mode].set_iv(off, object);
-	data_cipher_algs[object->o_alg][object->o_mode].encrypt
-		((const unsigned char *)from,
-		 (unsigned char *)to,
-		 len,
-		 off,
-		 enc,
-		 object);
-}
-
-#define MAX_CIPHER_CHUNK (1 << 30)
-
-/*
- * Do cipher (encryption/decryption) transform of a
- * continuous region of memory.
- *
- * @len: a number of bytes to transform;
- * @buf: data to transform;
- * @off: offset in a file, should be block-aligned
- *       for atomic cipher modes and ksize-aligned
- *       for other modes).
- * @dir: direction of transform (encrypt/decrypt).
- */
-static void cipher_region(struct object_cipher_info *object,
-			  char *from,
-			  char *to,
-			  off_t off,
-			  size_t len,
-			  int dir)
-{
-	while (len > 0) {
-		size_t to_cipher;
-
-		to_cipher = len;
-		if (to_cipher > MAX_CIPHER_CHUNK)
-			to_cipher = MAX_CIPHER_CHUNK;
-
-		/* this will reset IV */
-		cipher_data(object,
-			    from,
-			    to,
-			    off,
-			    to_cipher,
-			    dir);
-		from += to_cipher;
-		to   += to_cipher;
-		off  += to_cipher;
-		len  -= to_cipher;
-	}
-}
-
-/*
- * Do cipher transform (encryption/decryption) of
- * plaintext/ciphertext represented by @vec.
- *
- * Pre-conditions: @vec represents a continuous piece
- * of data in a file at offset @off to be ciphered
- * (encrypted/decrypted).
- * @count is the number of vec's components. All the
- * components must be block-aligned, the caller is
- * responsible for this. @dir is "direction" of
- * transform (encrypt/decrypt).
- */
-static void cipher_aligned_iov(struct object_cipher_info *object,
-			       struct iovec *vec,
-			       int count,
-			       off_t off,
-			       int32_t dir)
-{
-	int i;
-	int len = 0;
-
-	for (i = 0; i < count; i++) {
-		cipher_region(object,
-			      vec[i].iov_base,
-			      vec[i].iov_base,
-			      off + len,
-			      vec[i].iov_len,
-			      dir);
-		len += vec[i].iov_len;
-	}
-}
-
-void encrypt_aligned_iov(struct object_cipher_info *object,
-			 struct iovec *vec,
-			 int count,
-			 off_t off)
-{
-	cipher_aligned_iov(object, vec, count, off, 1);
-}
-
-void decrypt_aligned_iov(struct object_cipher_info *object,
-			 struct iovec *vec,
-			 int count,
-			 off_t off)
-{
-	cipher_aligned_iov(object, vec, count, off, 0);
-}
-
-#if DEBUG_CRYPT
-static void compound_stream(struct iovec *vec, int count, char *buf, off_t skip)
-{
-	int i;
-	int off = 0;
-	for (i = 0; i < count; i++) {
-		memcpy(buf + off,
-		       vec[i].iov_base + skip,
-		       vec[i].iov_len - skip);
-
-		off += (vec[i].iov_len - skip);
-		skip = 0;
-	}
-}
-
-static void check_iovecs(struct iovec *vec, int cnt,
-			 struct iovec *avec, int acnt, uint32_t off_in_head)
-{
-	char *s1, *s2;
-	uint32_t size, asize;
-
-	size = iov_length(vec, cnt);
-	asize = iov_length(avec, acnt) - off_in_head;
-	if (size != asize) {
-		gf_log("crypt", GF_LOG_DEBUG, "size %d is not eq asize %d",
-		       size, asize);
-		return;
-	}
-	s1 = GF_CALLOC(1, size, gf_crypt_mt_data);
-	if (!s1) {
-		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
-		return;
-	}
-	s2 = GF_CALLOC(1, asize, gf_crypt_mt_data);
-	if (!s2) {
-		GF_FREE(s1);
-		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
-		return;
-	}
-	compound_stream(vec, cnt, s1, 0);
-	compound_stream(avec, acnt, s2, off_in_head);
-	if (memcmp(s1, s2, size))
-		gf_log("crypt", GF_LOG_DEBUG, "chunks of different data");
-	GF_FREE(s1);
-	GF_FREE(s2);
-}
-
-#else
-#define check_iovecs(vec, count, avec, avecn, off) noop
-#endif /* DEBUG_CRYPT */
-
-static char *data_alloc_block(xlator_t *this, crypt_local_t *local,
-			      int32_t block_size)
-{
-	struct iobuf *iobuf = NULL;
-
-	iobuf = iobuf_get2(this->ctx->iobuf_pool, block_size);
-	if (!iobuf) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "Failed to get iobuf");
-		return NULL;
-	}
-	if (!local->iobref_data) {
-		local->iobref_data = iobref_new();
-		if (!local->iobref_data) {
-			gf_log("crypt", GF_LOG_ERROR,
-			       "Failed to get iobref");
-			iobuf_unref(iobuf);
-			return NULL;
-		}
-	}
-	iobref_add(local->iobref_data, iobuf);
-	return iobuf->ptr;
-}
-
-/*
- * Compound @avec, which represent the same data
- * chunk as @vec, but has aligned components of
- * specified block size. Alloc blocks, if needed.
- * In particular, incomplete head and tail blocks
- * must be allocated.
- * Put number of allocated blocks to @num_blocks.
- *
- * Example:
- *
- * input: data chunk represented by 4 components
- * [AB],[BC],[CD],[DE];
- * output: 5 logical blocks (0, 1, 2, 3, 4).
- *
- *   A     B       C     D               E
- *   *-----*+------*-+---*----+--------+-*
- *   |     ||      | |   |    |        | |
- * *-+-----+*------+-*---+----*--------*-+------*
- * 0        1        2        3        4
- *
- * 0 - incomplete compound (head);
- * 1, 2 - full compound;
- * 3 - full non-compound (the case of reuse);
- * 4 - incomplete non-compound (tail).
- */
-int32_t align_iov_by_atoms(xlator_t *this,
-			   crypt_local_t *local,
-			   struct object_cipher_info *object,
-			   struct iovec *vec /* input vector */,
-			   int32_t count /* number of vec components */,
-			   struct iovec *avec /* aligned vector */,
-			   char **blocks /* pool of blocks */,
-			   uint32_t *blocks_allocated,
-			   struct avec_config *conf)
-{
-	int vecn = 0;      /* number of the current component in vec */
-	int avecn = 0;     /* number of the current component in avec */
-	off_t vec_off = 0; /* offset in the current vec component,
-			    * i.e. the number of bytes have already
-			    * been copied */
-	int32_t block_size = get_atom_size(object);
-	size_t to_process; /* number of vec's bytes to copy and(or) re-use */
-	int32_t off_in_head = conf->off_in_head;
-
-	to_process = iov_length(vec, count);
-
-	while (to_process > 0) {
-		if (off_in_head ||
-		    vec[vecn].iov_len - vec_off < block_size) {
-			/*
-			 * less than block_size:
-			 * the case of incomplete (head or tail),
-			 * or compound block
-			 */
-			size_t copied = 0;
-			/*
-			 * populate the pool with a new block
-			 */
-			blocks[*blocks_allocated] = data_alloc_block(this,
-								    local,
-								    block_size);
-			if (!blocks[*blocks_allocated])
-				return -ENOMEM;
-			memset(blocks[*blocks_allocated], 0, off_in_head);
-			/*
-			 * fill the block with vec components
-			 */
-			do {
-				size_t to_copy;
-
-				to_copy = vec[vecn].iov_len - vec_off;
-				if (to_copy > block_size - off_in_head)
-					to_copy = block_size - off_in_head;
-
-				memcpy(blocks[*blocks_allocated] + off_in_head + copied,
-				       vec[vecn].iov_base + vec_off,
-				       to_copy);
-
-				copied += to_copy;
-				to_process -= to_copy;
-
-				vec_off += to_copy;
-				if (vec_off == vec[vecn].iov_len) {
-					/* finished with this vecn */
-					vec_off = 0;
-					vecn++;
-				}
-			} while (copied < (block_size - off_in_head) && to_process > 0);
-			/*
-			 * update avec
-			 */
-			avec[avecn].iov_len = off_in_head + copied;
-			avec[avecn].iov_base = blocks[*blocks_allocated];
-
-			(*blocks_allocated)++;
-			off_in_head = 0;
-		} else {
-			/*
-			 * the rest of the current vec component
-			 * is not less than block_size, so reuse
-			 * the memory buffer of the component.
-			 */
-			size_t to_reuse;
-			to_reuse = (to_process > block_size ?
-				    block_size :
-				    to_process);
-			avec[avecn].iov_len = to_reuse;
-			avec[avecn].iov_base = vec[vecn].iov_base + vec_off;
-
-			vec_off += to_reuse;
-			if (vec_off == vec[vecn].iov_len) {
-				/* finished with this vecn */
-				vec_off = 0;
-				vecn++;
-			}
-			to_process -= to_reuse;
-		}
-		avecn++;
-	}
-	check_iovecs(vec, count, avec, avecn, conf->off_in_head);
-	return 0;
-}
-
-/*
- * allocate and setup aligned vector for data submission
- * Pre-condition: @conf is set.
- */
-int32_t set_config_avec_data(xlator_t *this,
-			     crypt_local_t *local,
-			     struct avec_config *conf,
-			     struct object_cipher_info *object,
-			     struct iovec *vec,
-			     int32_t vec_count)
-{
-	int32_t ret = ENOMEM;
-	struct iovec *avec;
-	char **pool;
-	uint32_t blocks_in_pool = 0;
-
-	conf->type = DATA_ATOM;
-
-	avec = GF_CALLOC(conf->acount, sizeof(*avec), gf_crypt_mt_iovec);
-	if (!avec)
-		return ret;
-	pool = GF_CALLOC(conf->acount, sizeof(pool), gf_crypt_mt_char);
-	if (!pool) {
-		GF_FREE(avec);
-		return ret;
-	}
-	if (!vec) {
-		/*
-		 * degenerated case: no data
-		 */
-		pool[0] = data_alloc_block(this, local, get_atom_size(object));
-		if (!pool[0])
-			goto free;
-		blocks_in_pool = 1;
-		avec->iov_base = pool[0];
-		avec->iov_len = conf->off_in_tail;
-	}
-	else {
-		ret = align_iov_by_atoms(this, local, object, vec, vec_count,
-					 avec, pool, &blocks_in_pool, conf);
-		if (ret)
-			goto free;
-	}
-	conf->avec = avec;
-	conf->pool = pool;
-	conf->blocks_in_pool = blocks_in_pool;
-	return 0;
- free:
-	GF_FREE(avec);
-	GF_FREE(pool);
-	return ret;
-}
-
-/*
- * allocate and setup aligned vector for hole submission
- */
-int32_t set_config_avec_hole(xlator_t *this,
-			     crypt_local_t *local,
-			     struct avec_config *conf,
-			     struct object_cipher_info *object,
-			     glusterfs_fop_t fop)
-{
-	uint32_t i, idx;
-	struct iovec *avec;
-	char **pool;
-	uint32_t num_blocks;
-	uint32_t blocks_in_pool = 0;
-
-	conf->type = HOLE_ATOM;
-
-	num_blocks = conf->acount -
-		(conf->nr_full_blocks ? conf->nr_full_blocks - 1 : 0);
-
-	switch (fop) {
-	case GF_FOP_WRITE:
-		/*
-		 * hole goes before data
-		 */
-		if (num_blocks == 1 && conf->off_in_tail != 0)
-		/*
-		 * we won't submit a hole which fits into
-		 * a data atom: this part of hole will be
-		 * submitted with data write
-		 */
-			return 0;
-		break;
-	case GF_FOP_FTRUNCATE:
-		/*
-		 * expanding truncate, hole goes after data,
-		 * and will be submited in any case.
- 		 */
-		break;
-	default:
-		gf_log("crypt", GF_LOG_WARNING,
-		       "bad file operation %d", fop);
-		return 0;
-	}
-	avec = GF_CALLOC(num_blocks, sizeof(*avec), gf_crypt_mt_iovec);
-	if (!avec)
-		return ENOMEM;
-	pool = GF_CALLOC(num_blocks, sizeof(pool), gf_crypt_mt_char);
-	if (!pool) {
-		GF_FREE(avec);
-		return ENOMEM;
-	}
-	for (i = 0; i < num_blocks; i++) {
-		pool[i] = data_alloc_block(this, local, get_atom_size(object));
-		if (pool[i] == NULL)
-			goto free;
-		blocks_in_pool++;
-	}
-	if (has_head_block(conf)) {
-		/* set head block */
-		idx = 0;
-		avec[idx].iov_base = pool[idx];
-		avec[idx].iov_len = get_atom_size(object);
-		memset(avec[idx].iov_base + conf->off_in_head,
-		       0,
-		       get_atom_size(object) - conf->off_in_head);
-	}
-	if (has_tail_block(conf)) {
-		/* set tail block */
-		idx = num_blocks - 1;
-		avec[idx].iov_base = pool[idx];
-		avec[idx].iov_len = get_atom_size(object);
-		memset(avec[idx].iov_base, 0, conf->off_in_tail);
-	}
-	if (has_full_blocks(conf)) {
-		/* set full block */
-		idx = conf->off_in_head ? 1 : 0;
-		avec[idx].iov_base = pool[idx];
-		avec[idx].iov_len = get_atom_size(object);
-		/*
-		 * since we re-use the buffer,
-		 * zeroes will be set every time
-		 * before encryption, see submit_full()
-		 */
-	}
-	conf->avec = avec;
-	conf->pool = pool;
-	conf->blocks_in_pool = blocks_in_pool;
-	return 0;
- free:
-	GF_FREE(avec);
-	GF_FREE(pool);
-	return ENOMEM;
-}
-
-/* A helper for setting up config of partial atoms (which
- * participate in read-modify-write sequence).
- *
- * Calculate and setup precise amount of "extra-bytes"
- * that should be uptodated at the end of partial (not
- * necessarily tail!) block.
- *
- * Pre-condition: local->old_file_size is valid!
- * @conf contains setup, which is enough for correct calculation
- * of has_tail_block(), ->get_offset().
- */
-void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
-		    struct avec_config *conf, atom_data_type dtype)
-{
-	uint32_t to_block;
-	crypt_local_t *local = frame->local;
-	uint64_t old_file_size = local->old_file_size;
-	struct rmw_atom *partial = atom_by_types(dtype,
-						 has_tail_block(conf) ?
-						 TAIL_ATOM : HEAD_ATOM);
-
-	if (old_file_size <= partial->offset_at(frame, object))
-		to_block = 0;
-	else {
-		to_block = old_file_size - partial->offset_at(frame, object);
-		if (to_block > get_atom_size(object))
-			to_block = get_atom_size(object);
-	}
-	if (to_block > conf->off_in_tail)
-		conf->gap_in_tail = to_block - conf->off_in_tail;
-	else
-		/*
-		 * nothing to uptodate
-		 */
-		conf->gap_in_tail = 0;
-}
-
-/*
- * fill struct avec_config with offsets layouts
- */
-void set_config_offsets(call_frame_t *frame,
-			xlator_t *this,
-			uint64_t offset,
-			uint64_t count,
-			atom_data_type dtype,
-			int32_t set_gap)
-{
-	crypt_local_t *local;
-	struct object_cipher_info *object;
-	struct avec_config *conf;
-	uint32_t resid;
-
-	uint32_t atom_size;
-	uint32_t atom_bits;
-
-	size_t orig_size;
-	off_t orig_offset;
-	size_t expanded_size;
-	off_t aligned_offset;
-
-	uint32_t off_in_head = 0;
-	uint32_t off_in_tail = 0;
-	uint32_t nr_full_blocks;
-	int32_t size_full_blocks;
-
-	uint32_t acount; /* number of alifned components to write.
-			  * The same as number of occupied logical
-			  * blocks (atoms)
-			  */
-	local = frame->local;
-	object = &local->info->cinfo;
-	conf = (dtype == DATA_ATOM ?
-		get_data_conf(frame) : get_hole_conf(frame));
-
-	orig_offset = offset;
-	orig_size = count;
-
-	atom_size = get_atom_size(object);
-	atom_bits = get_atom_bits(object);
-
-	/*
-	 * Round-down the start,
-	 * round-up the end.
-	 */
-	resid = offset & (uint64_t)(atom_size - 1);
-
-	if (resid)
-		off_in_head = resid;
-	aligned_offset = offset - off_in_head;
-	expanded_size = orig_size + off_in_head;
-
-	/* calculate tail,
-	   expand size forward  */
-	resid = (offset + orig_size) & (uint64_t)(atom_size - 1);
-
-	if (resid) {
-		off_in_tail = resid;
-		expanded_size += (atom_size - off_in_tail);
-	}
-	/*
-	 * calculate number of occupied blocks
-	 */
-	acount = expanded_size >> atom_bits;
-	/*
-	 * calculate number of full blocks
-	 */
-	size_full_blocks = expanded_size;
-	if (off_in_head)
-		size_full_blocks -= atom_size;
-	if (off_in_tail && size_full_blocks > 0)
-		size_full_blocks -= atom_size;
-	nr_full_blocks = size_full_blocks >> atom_bits;
-
-	conf->atom_size = atom_size;
-	conf->orig_size = orig_size;
-	conf->orig_offset = orig_offset;
-	conf->expanded_size = expanded_size;
-	conf->aligned_offset = aligned_offset;
-
-	conf->off_in_head = off_in_head;
-	conf->off_in_tail = off_in_tail;
-	conf->nr_full_blocks = nr_full_blocks;
-	conf->acount = acount;
-	/*
-	 * Finally, calculate precise amount of
-	 * "extra-bytes" that should be uptodated
-	 * at the end.
-	 * Only if RMW is expected.
-	 */
-	if (off_in_tail && set_gap)
-		set_gap_at_end(frame, object, conf, dtype);
-}
-
-struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE] = {
-	[AES_CIPHER_ALG][XTS_CIPHER_MODE] =
-	{ .atomic = _gf_true,
-	  .should_pad = _gf_true,
-	  .blkbits = AES_BLOCK_BITS,
-	  .init = aes_xts_init,
-	  .set_private = set_private_aes_xts,
-	  .check_key = check_key_aes_xts,
-	  .set_iv = set_iv_aes_xts,
-	  .encrypt = encrypt_aes_xts
-	}
-};
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/keys.c b/xlators/encryption/crypt/src/keys.c
deleted file mode 100644
index e9da55960c8..00000000000
--- a/xlators/encryption/crypt/src/keys.c
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include "defaults.h"
-#include "crypt-common.h"
-#include "crypt.h"
-
-/* Key hierarchy
-
-                   +----------------+
-                   | MASTER_VOL_KEY |
-                   +-------+--------+
-                           |
-                           |
-          +----------------+----------------+
-          |                |                |
-          |                |                |
-  +-------+------+ +-------+-------+ +------+--------+
-  | NMTD_VOL_KEY | | EMTD_FILE_KEY | | DATA_FILE_KEY |
-  +-------+------+ +---------------+ +---------------+
-          |
-          |
-  +-------+-------+
-  | NMTD_LINK_KEY |
-  +---------------+
-
- */
-
-#if DEBUG_CRYPT
-static void check_prf_iters(uint32_t num_iters)
-{
-	if (num_iters == 0)
-		gf_log ("crypt", GF_LOG_DEBUG,
-			"bad number of prf iterations : %d", num_iters);
-}
-#else
-#define check_prf_iters(num_iters) noop
-#endif /* DEBUG_CRYPT */
-
-unsigned char crypt_fake_oid[16] =
-	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-
-/*
- * derive key in the counter mode using
- * sha256-based HMAC as PRF, see
- * NIST Special Publication 800-108, 5.1)
- */
-
-#define PRF_OUTPUT_SIZE  SHA256_DIGEST_LENGTH
-
-static int32_t kderive_init(struct kderive_context *ctx,
-			    const unsigned char *pkey, /* parent key */
-			    uint32_t pkey_size,  /* parent key size */
-			    const unsigned char *idctx, /* id-context */
-			    uint32_t idctx_size,
-			    crypt_key_type type  /* type of child key */)
-{
-	unsigned char *pos;
-	uint32_t llen = strlen(crypt_keys[type].label);
-	/*
-	 * Compoud the fixed input data for KDF:
-	 * [i]_2 || Label || 0x00 || Id-Context || [L]_2),
-	 * NIST SP 800-108, 5.1
-	 */
-	ctx->fid_len =
-		sizeof(uint32_t) +
-		llen +
-		1 +
-		idctx_size +
-		sizeof(uint32_t);
-
-	ctx->fid = GF_CALLOC(ctx->fid_len, 1, gf_crypt_mt_key);
-	if (!ctx->fid)
-		return ENOMEM;
-	ctx->out_len = round_up(crypt_keys[type].len >> 3,
-				PRF_OUTPUT_SIZE);
-	ctx->out = GF_CALLOC(ctx->out_len, 1, gf_crypt_mt_key);
-	if (!ctx->out) {
-		GF_FREE(ctx->fid);
-		return ENOMEM;
-	}
-	ctx->pkey = pkey;
-	ctx->pkey_len = pkey_size;
-	ctx->ckey_len = crypt_keys[type].len;
-
-	pos = ctx->fid;
-
-	/* counter will be set up in kderive_rfn() */
-	pos += sizeof(uint32_t);
-
-	memcpy(pos, crypt_keys[type].label, llen);
-	pos += llen;
-
-	/* set up zero octet */
-	*pos = 0;
-	pos += 1;
-
-	memcpy(pos, idctx, idctx_size);
-	pos += idctx_size;
-
-	*((uint32_t *)pos) = htobe32(ctx->ckey_len);
-
-	return 0;
-}
-
-static void kderive_update(struct kderive_context *ctx)
-{
-	uint32_t i;
-#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
-	HMAC_CTX hctx;
-#endif
-        HMAC_CTX *phctx = NULL;
-	unsigned char *pos = ctx->out;
-	uint32_t *p_iter = (uint32_t *)ctx->fid;
-	uint32_t num_iters = ctx->out_len / PRF_OUTPUT_SIZE;
-
-	check_prf_iters(num_iters);
-
-#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
-	HMAC_CTX_init(&hctx);
-        phctx = &hctx;
-#else
-        phctx = HMAC_CTX_new();
-        /* I guess we presume it was successful? */
-#endif
-	for (i = 0; i < num_iters; i++) {
-		/*
-		 * update the iteration number in the fid
-		 */
-		*p_iter = htobe32(i);
-		HMAC_Init_ex(phctx,
-			     ctx->pkey, ctx->pkey_len >> 3,
-			     EVP_sha256(),
-			     NULL);
-		HMAC_Update(phctx, ctx->fid, ctx->fid_len);
-		HMAC_Final(phctx, pos, NULL);
-
-		pos += PRF_OUTPUT_SIZE;
-	}
-#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
-	HMAC_CTX_cleanup(phctx);
-#else
-        HMAC_CTX_free(phctx);
-#endif
-}
-
-static void kderive_final(struct kderive_context *ctx, unsigned char *child)
-{
-	memcpy(child, ctx->out, ctx->ckey_len >> 3);
-	GF_FREE(ctx->fid);
-	GF_FREE(ctx->out);
-	memset(ctx, 0, sizeof(*ctx));
-}
-
-/*
- * derive per-volume key for object ids aithentication
- */
-int32_t get_nmtd_vol_key(struct master_cipher_info *master)
-{
-	int32_t ret;
-	struct kderive_context ctx;
-
-	ret = kderive_init(&ctx,
-			   master->m_key,
-			   master_key_size(),
-			   crypt_fake_oid, sizeof(uuid_t), NMTD_VOL_KEY);
-	if (ret)
-		return ret;
-	kderive_update(&ctx);
-	kderive_final(&ctx, master->m_nmtd_key);
-	return 0;
-}
-
-/*
- * derive per-link key for aithentication of non-encrypted
- * meta-data (nmtd)
- */
-int32_t get_nmtd_link_key(loc_t *loc,
-			  struct master_cipher_info *master,
-			  unsigned char *result)
-{
-	int32_t ret;
-	struct kderive_context ctx;
-
-	ret = kderive_init(&ctx,
-			   master->m_nmtd_key,
-			   nmtd_vol_key_size(),
-			   (const unsigned char *)loc->path,
-			   strlen(loc->path), NMTD_LINK_KEY);
-	if (ret)
-		return ret;
-	kderive_update(&ctx);
-	kderive_final(&ctx, result);
-	return 0;
-}
-
-/*
- * derive per-file key for encryption and authentication
- * of encrypted part of metadata (emtd)
- */
-int32_t get_emtd_file_key(struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  unsigned char *result)
-{
-	int32_t ret;
-	struct kderive_context ctx;
-
-	ret = kderive_init(&ctx,
-			   master->m_key,
-			   master_key_size(),
-			   info->oid, sizeof(uuid_t), EMTD_FILE_KEY);
-	if (ret)
-		return ret;
-	kderive_update(&ctx);
-	kderive_final(&ctx, result);
-	return 0;
-}
-
-static int32_t data_key_type_by_size(uint32_t keysize, crypt_key_type *type)
-{
-	int32_t ret = 0;
-	switch (keysize) {
-	case 256:
-		*type = DATA_FILE_KEY_256;
-		break;
-	case 512:
-		*type = DATA_FILE_KEY_512;
-		break;
-	default:
-		gf_log("crypt", GF_LOG_ERROR, "Unsupported data key size %d",
-		       keysize);
-		ret = ENOTSUP;
-		break;
-	}
-	return ret;
-}
-
-/*
- * derive per-file key for data encryption
- */
-int32_t get_data_file_key(struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  uint32_t keysize,
-			  unsigned char *key)
-{
-	int32_t ret;
-	struct kderive_context ctx;
-	crypt_key_type type;
-
-	ret = data_key_type_by_size(keysize, &type);
-	if (ret)
-		return ret;
-	ret = kderive_init(&ctx,
-			   master->m_key,
-			   master_key_size(),
-			   info->oid, sizeof(uuid_t), type);
-	if (ret)
-		return ret;
-	kderive_update(&ctx);
-	kderive_final(&ctx, key);
-	return 0;
-}
-
-/*
- * NOTE: Don't change existing keys: it will break compatibility;
- */
-struct crypt_key crypt_keys[LAST_KEY_TYPE] = {
-	[MASTER_VOL_KEY] =
-	{ .len = MASTER_VOL_KEY_SIZE << 3,
-	  .label = "volume-master",
-	},
-	[NMTD_VOL_KEY] =
-	{ .len = NMTD_VOL_KEY_SIZE << 3,
-	  .label = "volume-nmtd-key-generation"
-	},
-	[NMTD_LINK_KEY] =
-	{ .len = 128,
-	  .label = "link-nmtd-authentication"
-	},
-	[EMTD_FILE_KEY] =
-	{ .len = 128,
-	  .label = "file-emtd-encryption-and-auth"
-	},
-	[DATA_FILE_KEY_256] =
-	{ .len = 256,
-	  .label = "file-data-encryption-256"
-	},
-	[DATA_FILE_KEY_512] =
-	{ .len = 512,
-	  .label = "file-data-encryption-512"
-	}
-};
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/metadata.c b/xlators/encryption/crypt/src/metadata.c
deleted file mode 100644
index 1364f825a98..00000000000
--- a/xlators/encryption/crypt/src/metadata.c
+++ /dev/null
@@ -1,614 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include "defaults.h"
-#include "crypt-common.h"
-#include "crypt.h"
-#include "metadata.h"
-
-int32_t alloc_format(crypt_local_t *local, size_t size)
-{
-	if (size > 0) {
-		local->format = GF_CALLOC(1, size, gf_crypt_mt_mtd);
-		if (!local->format)
-			return ENOMEM;
-	}
-	local->format_size = size;
-	return 0;
-}
-
-int32_t alloc_format_create(crypt_local_t *local)
-{
-	return alloc_format(local, new_format_size());
-}
-
-void free_format(crypt_local_t *local)
-{
-	GF_FREE(local->format);
-}
-
-/*
- * Check compatibility with extracted metadata
- */
-static int32_t check_file_metadata(struct crypt_inode_info *info)
-{
-	struct object_cipher_info *object = &info->cinfo;
-
-	if (info->nr_minor != CRYPT_XLATOR_ID) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "unsupported minor subversion %d", info->nr_minor);
-		return EINVAL;
-	}
-	if (object->o_alg > LAST_CIPHER_ALG) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "unsupported cipher algorithm %d",
-		       object->o_alg);
-		return EINVAL;
-	}
-	if (object->o_mode > LAST_CIPHER_MODE) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "unsupported cipher mode %d",
-		       object->o_mode);
-		return EINVAL;
-	}
-	if (object->o_block_bits < CRYPT_MIN_BLOCK_BITS ||
-	    object->o_block_bits > CRYPT_MAX_BLOCK_BITS) {
-		gf_log("crypt", GF_LOG_WARNING, "unsupported block bits %d",
-		       object->o_block_bits);
-		return EINVAL;
-	}
-	/* TBD: check data key size */
-	return 0;
-}
-
-static size_t format_size_v1(mtd_op_t op, size_t old_size)
-{
-
-	switch (op) {
-	case MTD_CREATE:
-		return sizeof(struct mtd_format_v1);
-	case MTD_OVERWRITE:
-		return old_size;
-	case MTD_APPEND:
-		return old_size + NMTD_8_MAC_SIZE;
-	case MTD_CUT:
-		if (old_size > sizeof(struct mtd_format_v1))
-			return old_size - NMTD_8_MAC_SIZE;
-		else
-			return 0;
-	default:
-		gf_log("crypt", GF_LOG_WARNING, "Bad mtd operation");
-		return 0;
-	}
-}
-
-/*
- * Calculate size of the updated format string.
- * Returned zero means that we don't need to update the format string.
- */
-size_t format_size(mtd_op_t op, size_t old_size)
-{
-	size_t versioned;
-
-	versioned = mtd_loaders[current_mtd_loader()].format_size(op,
-				 old_size - sizeof(struct crypt_format));
-	if (versioned != 0)
-		return versioned + sizeof(struct crypt_format);
-	return 0;
-}
-
-/*
- * size of the format string of newly created file (nr_links = 1)
- */
-size_t new_format_size(void)
-{
-	return format_size(MTD_CREATE, 0);
-}
-
-/*
- * Calculate per-link MAC by pathname
- */
-static int32_t calc_link_mac_v1(struct mtd_format_v1 *fmt,
-				loc_t *loc,
-				unsigned char *result,
-				struct crypt_inode_info *info,
-				struct master_cipher_info *master)
-{
-	int32_t ret;
-	unsigned char nmtd_link_key[16];
-	CMAC_CTX *cctx;
-	size_t len;
-
-	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Can not get nmtd link key");
-		return -1;
-	}
-	cctx = CMAC_CTX_new();
-	if (!cctx) {
-		gf_log("crypt", GF_LOG_ERROR, "CMAC_CTX_new failed");
-		return -1;
-	}
-	ret = CMAC_Init(cctx, nmtd_link_key, sizeof(nmtd_link_key),
-			EVP_aes_128_cbc(), 0);
-	if (!ret) {
-		gf_log("crypt", GF_LOG_ERROR, "CMAC_Init failed");
-		CMAC_CTX_free(cctx);
-		return -1;
-	}
-	ret = CMAC_Update(cctx, get_NMTD_V1(info), SIZE_OF_NMTD_V1);
-	if (!ret) {
-		gf_log("crypt", GF_LOG_ERROR, "CMAC_Update failed");
-		CMAC_CTX_free(cctx);
-		return -1;
-	}
-	ret = CMAC_Final(cctx, result, &len);
-	CMAC_CTX_free(cctx);
-	if (!ret) {
-		gf_log("crypt", GF_LOG_ERROR, "CMAC_Final failed");
-		return -1;
-	}
-	return 0;
-}
-
-/*
- * Create per-link MAC of index @idx by pathname
- */
-static int32_t create_link_mac_v1(struct mtd_format_v1 *fmt,
-				  uint32_t idx,
-				  loc_t *loc,
-				  struct crypt_inode_info *info,
-				  struct master_cipher_info *master)
-{
-	int32_t ret;
-	unsigned char *mac;
-	unsigned char cmac[16];
-
-	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
-
-	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
-	if (ret)
-		return -1;
-	memcpy(mac, cmac, SIZE_OF_NMTD_V1_MAC);
-	return 0;
-}
-
-static int32_t create_format_v1(unsigned char *wire,
-				loc_t *loc,
-				struct crypt_inode_info *info,
-				struct master_cipher_info *master)
-{
-	int32_t ret;
-	struct mtd_format_v1 *fmt;
-	unsigned char mtd_key[16];
-	AES_KEY EMTD_KEY;
-	unsigned char nmtd_link_key[16];
-	uint32_t ad;
-	GCM128_CONTEXT *gctx;
-
-	fmt = (struct mtd_format_v1 *)wire;
-
-	fmt->minor_id = info->nr_minor;
-	fmt->alg_id = AES_CIPHER_ALG;
-	fmt->dkey_factor = master->m_dkey_size >> KEY_FACTOR_BITS;
-	fmt->block_bits = master->m_block_bits;
-	fmt->mode_id = master->m_mode;
-	/*
-	 * retrieve keys for the parts of metadata
-	 */
-	ret = get_emtd_file_key(info, master, mtd_key);
-	if (ret)
-		return ret;
-	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
-	if (ret)
-		return ret;
-
-	AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
-
-	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
-
-	/* TBD: Check return values */
-
-	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
-
-	ad = htole32(MTD_LOADER_V1);
-	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
-		CRYPTO_gcm128_release(gctx);
-		return ret;
-	}
-	ret = CRYPTO_gcm128_encrypt(gctx,
-				    get_EMTD_V1(fmt),
-				    get_EMTD_V1(fmt),
-				    SIZE_OF_EMTD_V1);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_encrypt failed");
-		CRYPTO_gcm128_release(gctx);
-		return ret;
-	}
-	/*
-	 * set MAC of encrypted part of metadata
-	 */
-	CRYPTO_gcm128_tag(gctx, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC);
-	CRYPTO_gcm128_release(gctx);
-	/*
-	 * set the first MAC of non-encrypted part of metadata
-	 */
-	return create_link_mac_v1(fmt, 0, loc, info, master);
-}
-
-/*
- * Called by fops:
- * ->create();
- * ->link();
- *
- * Pack common and version-specific parts of file's metadata
- * Pre-conditions: @info contains valid object-id.
- */
-int32_t create_format(unsigned char *wire,
-		      loc_t *loc,
-		      struct crypt_inode_info *info,
-		      struct master_cipher_info *master)
-{
-	struct crypt_format *fmt = (struct crypt_format *)wire;
-
-	fmt->loader_id = current_mtd_loader();
-
-	wire += sizeof(struct crypt_format);
-	return mtd_loaders[current_mtd_loader()].create_format(wire, loc,
-							       info, master);
-}
-
-/*
- * Append or overwrite per-link mac of @mac_idx index
- * in accordance with the new pathname
- */
-int32_t appov_link_mac_v1(unsigned char *new,
-			  unsigned char *old,
-			  uint32_t old_size,
-			  int32_t mac_idx,
-			  loc_t *loc,
-			  struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  crypt_local_t *local)
-{
-	memcpy(new, old, old_size);
-	return create_link_mac_v1((struct mtd_format_v1 *)new, mac_idx,
-				  loc, info, master);
-}
-
-/*
- * Cut per-link mac of @mac_idx index
- */
-static int32_t cut_link_mac_v1(unsigned char *new,
-			       unsigned char *old,
-			       uint32_t old_size,
-			       int32_t mac_idx,
-			       loc_t *loc,
-			       struct crypt_inode_info *info,
-			       struct master_cipher_info *master,
-			       crypt_local_t *local)
-{
-	memcpy(new,
-	       old,
-	       sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1));
-
-	memcpy(new + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1),
-	       old + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx,
-	       old_size - (sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx));
-	return 0;
-}
-
-int32_t update_format_v1(unsigned char *new,
-			 unsigned char *old,
-				size_t old_len,
-				int32_t mac_idx, /* of old name */
-				mtd_op_t op,
-				loc_t *loc,
-				struct crypt_inode_info *info,
-				struct master_cipher_info *master,
-				crypt_local_t *local)
-{
-	switch (op) {
-	case MTD_APPEND:
-		mac_idx = 1 + (old_len - sizeof(struct mtd_format_v1))/8;
-	case MTD_OVERWRITE:
-		return appov_link_mac_v1(new, old, old_len, mac_idx,
-					 loc, info, master, local);
-	case MTD_CUT:
-		return cut_link_mac_v1(new, old, old_len, mac_idx,
-				       loc, info, master, local);
-	default:
-		gf_log("crypt", GF_LOG_ERROR, "Bad  mtd operation %d", op);
-		return -1;
-	}
-}
-
-/*
- * Called by fops:
- *
- * ->link()
- * ->unlink()
- * ->rename()
- *
- */
-int32_t update_format(unsigned char *new,
-		      unsigned char *old,
-		      size_t old_len,
-		      int32_t mac_idx,
-		      mtd_op_t op,
-		      loc_t *loc,
-		      struct crypt_inode_info *info,
-		      struct master_cipher_info *master,
-		      crypt_local_t *local)
-{
-	if (!new)
-		return 0;
-	memcpy(new, old, sizeof(struct crypt_format));
-
-	old += sizeof(struct crypt_format);
-	new += sizeof(struct crypt_format);
-	old_len -= sizeof(struct crypt_format);
-
-	return mtd_loaders[current_mtd_loader()].update_format(new, old,
-							       old_len,
-							       mac_idx, op,
-							       loc, info,
-							       master, local);
-}
-
-/*
- * Perform preliminary checks of found metadata
- * Return < 0 on errors;
- * Return number of object-id MACs (>= 1) on success
- */
-int32_t check_format_v1(uint32_t len, unsigned char *wire)
-{
-	uint32_t nr_links;
-
-	if (len < sizeof(struct mtd_format_v1)) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "v1-loader: bad metadata size %d", len);
-		goto error;
-	}
-	len -= sizeof(struct mtd_format_v1);
-	if (len % sizeof(nmtd_8_mac_t)) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "v1-loader: bad metadata format");
-		goto error;
-	}
-	nr_links = 1 + len / sizeof(nmtd_8_mac_t);
-	if (nr_links > _POSIX_LINK_MAX)
-		goto error;
-	return nr_links;
- error:
-	return EIO;
-}
-
-/*
- * Verify per-link MAC specified by index @idx
- *
- * return:
- * -1 on errors;
- *  0 on failed verification;
- *  1 on successful verification
- */
-static int32_t verify_link_mac_v1(struct mtd_format_v1 *fmt,
-				  uint32_t idx /* index of the mac to verify */,
-				  loc_t *loc,
-				  struct crypt_inode_info *info,
-				  struct master_cipher_info *master)
-{
-	int32_t ret;
-	unsigned char *mac;
-	unsigned char cmac[16];
-
-	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
-
-	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
-	if (ret)
-		return -1;
-	if (memcmp(cmac, mac, SIZE_OF_NMTD_V1_MAC))
-		return 0;
-	return 1;
-}
-
-/*
- * Lookup per-link MAC by pathname.
- *
- * return index of the MAC, if it was found;
- * return < 0 on errors, or if the MAC wasn't found
- */
-static int32_t lookup_link_mac_v1(struct mtd_format_v1 *fmt,
-				  uint32_t nr_macs,
-				  loc_t *loc,
-				  struct crypt_inode_info *info,
-				  struct master_cipher_info *master)
-{
-	int32_t ret;
-	uint32_t idx;
-
-	for (idx = 0; idx < nr_macs; idx++) {
-		ret = verify_link_mac_v1(fmt, idx, loc, info, master);
-		if (ret < 0)
-			return ret;
-		if (ret > 0)
-			return idx;
-	}
-	return -ENOENT;
-}
-
-/*
- * Extract version-specific part of metadata
- */
-static int32_t open_format_v1(unsigned char *wire,
-			      int32_t len,
-			      loc_t *loc,
-			      struct crypt_inode_info *info,
-			      struct master_cipher_info *master,
-			      crypt_local_t *local,
-			      gf_boolean_t load_info)
-{
-	int32_t ret;
-	int32_t num_nmtd_macs;
-	struct mtd_format_v1 *fmt;
-	unsigned char mtd_key[16];
-	AES_KEY EMTD_KEY;
-	GCM128_CONTEXT *gctx;
-	uint32_t ad;
-	emtd_8_mac_t gmac;
-	struct object_cipher_info *object;
-
-	num_nmtd_macs = check_format_v1(len, wire);
-	if (num_nmtd_macs <= 0)
-		return EIO;
-
-	ret = lookup_link_mac_v1((struct mtd_format_v1 *)wire,
-				 num_nmtd_macs, loc, info, master);
-	if (ret < 0) {
-		gf_log("crypt", GF_LOG_ERROR, "NMTD verification failed");
-		return EINVAL;
-	}
-
-	local->mac_idx = ret;
-	if (load_info == _gf_false)
-		/* the case of partial open */
-		return 0;
-
-	fmt = GF_CALLOC(1, len, gf_crypt_mt_mtd);
-	if (!fmt)
-		return ENOMEM;
-	memcpy(fmt, wire, len);
-
-	object = &info->cinfo;
-
-	ret = get_emtd_file_key(info, master, mtd_key);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Can not retrieve metadata key");
-		goto out;
-	}
-	/*
-	 * decrypt encrypted meta-data
-	 */
-	ret = AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
-	if (ret < 0) {
-		gf_log("crypt", GF_LOG_ERROR, "Can not set encrypt key");
-		ret = EIO;
-		goto out;
-	}
-	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
-	if (!gctx) {
-		gf_log("crypt", GF_LOG_ERROR, "Can not alloc gcm context");
-		ret = ENOMEM;
-		goto out;
-	}
-	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
-
-	ad = htole32(MTD_LOADER_V1);
-	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
-		CRYPTO_gcm128_release(gctx);
-		ret = EIO;
-		goto out;
-	}
-	ret = CRYPTO_gcm128_decrypt(gctx,
-				    get_EMTD_V1(fmt),
-				    get_EMTD_V1(fmt),
-				    SIZE_OF_EMTD_V1);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_decrypt failed");
-		CRYPTO_gcm128_release(gctx);
-		ret = EIO;
-		goto out;
-	}
-	/*
-	 * verify metadata
-	 */
-	CRYPTO_gcm128_tag(gctx, gmac, sizeof(gmac));
-	CRYPTO_gcm128_release(gctx);
-	if (memcmp(gmac, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC)) {
-		gf_log("crypt", GF_LOG_ERROR, "EMTD verification failed");
-		ret = EINVAL;
-		goto out;
-	}
-	/*
-	 * load verified metadata to the private part of inode
-	 */
-	info->nr_minor = fmt->minor_id;
-
-	object->o_alg = fmt->alg_id;
-	object->o_dkey_size = fmt->dkey_factor << KEY_FACTOR_BITS;
-	object->o_block_bits = fmt->block_bits;
-	object->o_mode = fmt->mode_id;
-
-	ret = check_file_metadata(info);
- out:
-	GF_FREE(fmt);
-	return ret;
-}
-
-/*
- * perform metadata authentication against @loc->path;
- * extract crypt-specific attribtes and populate @info
- * with them (optional)
- */
-int32_t open_format(unsigned char *str,
-		    int32_t len,
-		    loc_t *loc,
-		    struct crypt_inode_info *info,
-		    struct master_cipher_info *master,
-		    crypt_local_t *local,
-		    gf_boolean_t load_info)
-{
-	struct crypt_format *fmt;
-	if (len < sizeof(*fmt)) {
-		gf_log("crypt", GF_LOG_ERROR, "Bad core format");
-		return EIO;
-	}
-	fmt = (struct crypt_format *)str;
-
-	if (fmt->loader_id >= LAST_MTD_LOADER) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "Unsupported loader id %d", fmt->loader_id);
-		return EINVAL;
-	}
-	str += sizeof(*fmt);
-	len -= sizeof(*fmt);
-
-	return mtd_loaders[fmt->loader_id].open_format(str,
-						       len,
-						       loc,
-						       info,
-						       master,
-						       local,
-						       load_info);
-}
-
-struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER] = {
-	[MTD_LOADER_V1] =
-	{.format_size = format_size_v1,
-	 .create_format = create_format_v1,
-	 .open_format = open_format_v1,
-	 .update_format = update_format_v1
-	}
-};
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/metadata.h b/xlators/encryption/crypt/src/metadata.h
deleted file mode 100644
index b67ae25b58c..00000000000
--- a/xlators/encryption/crypt/src/metadata.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __METADATA_H__
-#define __METADATA_H__
-
-#define NMTD_8_MAC_SIZE   (8)
-#define EMTD_8_MAC_SIZE   (8)
-
-typedef uint8_t nmtd_8_mac_t[NMTD_8_MAC_SIZE];
-typedef uint8_t emtd_8_mac_t[EMTD_8_MAC_SIZE] ;
-
-/*
- * Version "v1" of file's metadata.
- * Metadata of this version has 4 components:
- *
- * 1) EMTD (Encrypted part of MeTaData);
- * 2) NMTD (Non-encrypted part of MeTaData);
- * 3) EMTD_MAC; (EMTD Message Authentication Code);
- * 4) Array of per-link NMTD MACs (for every (hard)link it includes
- *    exactly one MAC)
- */
-struct mtd_format_v1 {
-	/* EMTD, encrypted part of meta-data */
-	uint8_t            alg_id;  /* cipher algorithm id (only AES for now) */
-        uint8_t            mode_id; /* cipher mode id; (only XTS for now) */
-	uint8_t            block_bits; /* encoded block size */
-	uint8_t            minor_id; /* client translator id */
-	uint8_t            dkey_factor; /* encoded size of the data key */
-	/* MACs */
-	emtd_8_mac_t       gmac; /* MAC of the encrypted meta-data, 8 bytes */
-	nmtd_8_mac_t       omac; /* per-link MACs of the non-encrypted
-				  * meta-data: at least one such MAC is always
-				  * present */
-} __attribute__((packed));
-
-/*
- * NMTD, the non-encrypted part of metadata of version "v1"
- * is file's gfid, which is generated on trusted machines.
- */
-#define SIZE_OF_NMTD_V1 (sizeof(uuid_t))
-#define SIZE_OF_EMTD_V1 (offsetof(struct mtd_format_v1, gmac) -		\
-			 offsetof(struct mtd_format_v1, alg_id))
-#define SIZE_OF_NMTD_V1_MAC (NMTD_8_MAC_SIZE)
-#define SIZE_OF_EMTD_V1_MAC (EMTD_8_MAC_SIZE)
-
-static inline unsigned char *get_EMTD_V1(struct mtd_format_v1 *format)
-{
-        return &format->alg_id;
-}
-
-static inline unsigned char *get_NMTD_V1(struct crypt_inode_info *info)
-{
-        return info->oid;
-}
-
-static inline unsigned char *get_EMTD_V1_MAC(struct mtd_format_v1 *format)
-{
-        return format->gmac;
-}
-
-static inline unsigned char *get_NMTD_V1_MAC(struct mtd_format_v1 *format)
-{
-        return format->omac;
-}
-
-#endif /* __METADATA_H__ */
diff --git a/xlators/encryption/rot-13/Makefile.am b/xlators/encryption/rot-13/Makefile.am
deleted file mode 100644
index d471a3f9243..00000000000
--- a/xlators/encryption/rot-13/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/encryption/rot-13/src/rot-13.c b/xlators/encryption/rot-13/src/rot-13.c
deleted file mode 100644
index 6ec1b47c87b..00000000000
--- a/xlators/encryption/rot-13/src/rot-13.c
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
-   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include <ctype.h>
-#include <sys/uio.h>
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-
-#include "rot-13.h"
-
-/*
- * This is a rot13 ``encryption'' xlator. It rot13's data when
- * writing to disk and rot13's it back when reading it.
- * This xlator is meant as an example, NOT FOR PRODUCTION
- * USE ;) (hence no error-checking)
- */
-
-void
-rot13 (char *buf, int len)
-{
-	int i;
-	for (i = 0; i < len; i++) {
-		if (buf[i] >= 'a' && buf[i] <= 'z')
-			buf[i] = 'a' + ((buf[i] - 'a' + 13) % 26);
-		else if (buf[i] >= 'A' && buf[i] <= 'Z')
-			buf[i] = 'A' + ((buf[i] - 'A' + 13) % 26);
-	}
-}
-
-void
-rot13_iovec (struct iovec *vector, int count)
-{
-	int i;
-	for (i = 0; i < count; i++) {
-		rot13 (vector[i].iov_base, vector[i].iov_len);
-	}
-}
-
-int32_t
-rot13_readv_cbk (call_frame_t *frame,
-                 void *cookie,
-                 xlator_t *this,
-                 int32_t op_ret,
-                 int32_t op_errno,
-                 struct iovec *vector,
-                 int32_t count,
-		 struct iatt *stbuf,
-                 struct iobref *iobref, dict_t *xdata)
-{
-	rot_13_private_t *priv = (rot_13_private_t *)this->private;
-
-	if (priv->decrypt_read)
-		rot13_iovec (vector, count);
-
-	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref, xdata);
-	return 0;
-}
-
-int32_t
-rot13_readv (call_frame_t *frame,
-             xlator_t *this,
-             fd_t *fd,
-             size_t size,
-             off_t offset, uint32_t flags, dict_t *xdata)
-{
-	STACK_WIND (frame,
-		    rot13_readv_cbk,
-		    FIRST_CHILD (this),
-		    FIRST_CHILD (this)->fops->readv,
-		    fd, size, offset, flags, xdata);
-	return 0;
-}
-
-int32_t
-rot13_writev_cbk (call_frame_t *frame,
-                  void *cookie,
-                  xlator_t *this,
-                  int32_t op_ret,
-                  int32_t op_errno,
-                  struct iatt *prebuf,
-		  struct iatt *postbuf, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-	return 0;
-}
-
-int32_t
-rot13_writev (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd,
-              struct iovec *vector,
-              int32_t count,
-              off_t offset, uint32_t flags,
-              struct iobref *iobref, dict_t *xdata)
-{
-	rot_13_private_t *priv = (rot_13_private_t *)this->private;
-	if (priv->encrypt_write)
-		rot13_iovec (vector, count);
-
-	STACK_WIND (frame,
-		    rot13_writev_cbk,
-		    FIRST_CHILD (this),
-		    FIRST_CHILD (this)->fops->writev,
-		    fd, vector, count, offset, flags,
-                    iobref, xdata);
-	return 0;
-}
-
-int32_t
-init (xlator_t *this)
-{
-	data_t *data = NULL;
-	rot_13_private_t *priv = NULL;
-
-	if (!this->children || this->children->next) {
-		gf_log ("rot13", GF_LOG_ERROR,
-			"FATAL: rot13 should have exactly one child");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-	priv = GF_CALLOC (sizeof (rot_13_private_t), 1, 0);
-        if (!priv)
-                return -1;
-
-	priv->decrypt_read = 1;
-	priv->encrypt_write = 1;
-
-	data = dict_get (this->options, "encrypt-write");
-	if (data) {
-		if (gf_string2boolean (data->data, &priv->encrypt_write) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"encrypt-write takes only boolean options");
-			GF_FREE (priv);
-			return -1;
-		}
-	}
-
-	data = dict_get (this->options, "decrypt-read");
-	if (data) {
-		if (gf_string2boolean (data->data, &priv->decrypt_read) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"decrypt-read takes only boolean options");
-			GF_FREE (priv);
-			return -1;
-		}
-	}
-
-	this->private = priv;
-	gf_log ("rot13", GF_LOG_DEBUG, "rot13 xlator loaded");
-	return 0;
-}
-
-void
-fini (xlator_t *this)
-{
-	rot_13_private_t *priv = this->private;
-
-        if (!priv)
-                return;
-        this->private = NULL;
-	GF_FREE (priv);
-
-	return;
-}
-
-struct xlator_fops fops = {
-	.readv        = rot13_readv,
-	.writev       = rot13_writev
-};
-
-struct xlator_cbks cbks;
-
-struct volume_options options[] = {
-	{ .key  = {"encrypt-write"},
-	  .type = GF_OPTION_TYPE_BOOL
-	},
-	{ .key  = {"decrypt-read"},
-	  .type = GF_OPTION_TYPE_BOOL
-	},
-	{ .key  = {NULL} },
-};
diff --git a/xlators/experimental/Makefile.am b/xlators/experimental/Makefile.am
deleted file mode 100644
index a530845c4c0..00000000000
--- a/xlators/experimental/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = jbr-client jbr-server fdl dht2 posix2
-
-CLEANFILES =
diff --git a/xlators/experimental/README.md b/xlators/experimental/README.md
deleted file mode 100644
index b00f24e114b..00000000000
--- a/xlators/experimental/README.md
+++ /dev/null
@@ -1,107 +0,0 @@
-# Purpose of this directory
-
-This directory is created to host experimental gluster translators. A new
-translator that is *experimental* in nature, would need to create its own
-subdirectory under this directory, to host/publish its work.
-
-Example:
-  The first commit should include the following changes
-    1. xlators/experimental/Makefile.am
-      NOTE: Add foobar to the list of SUBDIRS here
-    2. xlators/experimental/foobar
-    3. xlators/experimental/foobar/Makefle.am
-      NOTE: Can be empty initially in the first commit
-    4. configure.ac
-      NOTE: Include your experimental Makefile under AC_CONFIG_FILES
-    5. xlators/experimental/foobar/README.md
-      NOTE: The readme should cover details as required for the translator to be
-      accepted as experimental, primarily including a link to the specification
-      under the gluster-specs repository [1]. Later the readme should suffice
-      as an entry point for developers and users alike, who wish to experiment
-      with the xlator under development
-    6. xlators/experimental/foobar/TODO.md
-      NOTE: This is a list of TODO's identified during the development process
-      that needs addressing over time. These include exceptions granted during
-      the review process, for things not addressed when commits are merged into
-      the repository
-
-# Why is it provided
-
-Quite often translator development that happens out of tree, does not get
-enough eyeballs early in its development phase, has not undergone CI
-(regression/continuous integration testing), and at times is not well integrated
-with the rest of gluster stack.
-
-Also, when such out of tree translators are submitted for acceptance, it is a
-bulk commit that makes review difficult and inefficient. Such submissions also
-have to be merged forward, and depending on the time spent in developing the
-translator the master branch could have moved far ahead, making this a painful
-activity.
-
-Experimental is born out of such needs, to provide xlator developers,
-  - Early access to CI
-  - Ability to adapt to ongoing changes in other parts of gluster
-  - More eye balls on the code and design aspects of the translator
-  - TBD: What else?
-
-and for maintainers,
-  - Ability to look at smaller change sets in the review process
-  - Ability to verify/check implementation against the specification provided
-
-# General rules
-
-1. If a new translator is added under here it should, at the very least, pass
-compilation.
-
-2. All translators under the experimental directory are shipped as a part of
-gluster-experimental RPMs.
-TBD: Spec file and other artifacts for the gluster-experimental RPM needs to be
-fleshed out.
-
-3. Experimental translators can leverage the CI framework as needed. Tests need
-to be hosted under xlators/experimental/tests initially, and later moved to the
-appropriate tests/ directory as the xlator matures. It is encouraged to provide
-tests for each commit or series of commits, so that code and tests can be
-inspected together.
-
-4. If any experimental translator breaks CI, it is quarantined till demonstrable
-proof towards the contrary is provided. This is applicable as tests are moved
-out of experimental tests directory to the CI framework directory, as otherwise
-experimental tests are not a part of regular CI regression runs.
-
-5. An experimental translator need not function at all, as a result commits can
-be merged pretty much at will as long as other rules as stated are not violated.
-
-6. Experimental submissions will be assigned a existing maintainer, to aid
-merging commits and ensure aspects of gluster code submissions are respected.
-When an experimental xlator is proposed and the first commit posted
-a mail to gluster-devel@gluster.org requesting attention, will assign the
-maintainer buddy for the submission.
-NOTE: As we scale, this may change.
-
-6. More?
-
-# Getting out of the experimental jail
-
-So you now think your xlator is ready to leave experimental and become part of
-mainline!
-- TBD: guidelines pending.
-
-# FAQs
-
-1. How do I submit/commit experimental framework changes outside of my
-experimental xlator?
-  - Provide such framework changes as a separate commit
-  - Conditionally ensure these are built or activated only when the experimental
-  feature is activated, so as to prevent normal gluster workflow to function as
-  before
-  - TBD: guidelines and/or examples pending.
-
-2. Ask your question either on gluster-devel@gluster.org or as a change request
-to this file in gluster gerrit [2] for an answer that will be assimilated into
-this readme.
-
-# Links
-[1] http://review.gluster.org/#/q/project:glusterfs-specs
-
-[2] http://review.gluster.org/#/q/project:glusterfs
diff --git a/xlators/experimental/dht2/Makefile.am b/xlators/experimental/dht2/Makefile.am
deleted file mode 100644
index 9d910a66056..00000000000
--- a/xlators/experimental/dht2/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = dht2-client dht2-server
-
-CLEANFILES =
diff --git a/xlators/experimental/dht2/README.md b/xlators/experimental/dht2/README.md
deleted file mode 100644
index 8f249a83673..00000000000
--- a/xlators/experimental/dht2/README.md
+++ /dev/null
@@ -1,47 +0,0 @@
-# DHT2 Experimental README
-
-DHT2 is the new distribution scheme being developed for Gluster, that
-aims to remove the subdirectory spread across all DHT subvolumes.
-
-As a result of this work, the Gluster backend file layouts and on disk
-representation of directories and files are modified, thus making DHT2
-volumes incompatible to existing DHT based Gluster deployments.
-
-This document presents interested users with relevant data to play around
-with DHT2 volumes and provide feedback towards the same.
-
-REMOVEME: Design details currently under review here,
-	- http://review.gluster.org/#/c/13395/
-
-TODO: Add more information as relevant code is pulled in
-
-# Directory strucutre elaborated
-
-## dht2-server
-This directory contains code for the server side DHT2 xlator. This xlator is
-intended to run on the brick graph, and is responsible for FOP synchronization,
-redirection, transactions, and journal replays.
-
-NOTE: The server side code also handles changes to volume/cluster map and
-also any rebalance activities.
-
-## dht2-client
-This directory contains code for the client side DHT2 xlator. This xlator is
-intended to run on the client/access protocol/mount graph, and is responsible
-for FOP routing to the right DHT2 subvolume. It uses a volume/cluster wide map
-of the routing (layout), to achieve the same.
-
-## dht2-common
-This directory contains code that is used in common across other parts of DHT2.
-For example, FOP routing store/consult abstractions that are common across the
-client and server side of DHT2.
-
-## Issue: How to build dht2-common?
-  1. Build a shared object
-    - We cannot ship this as a part of both the client xlator RPM
-  2. Build an archive
-    - Symbol clashes? when both the client and server xlators are loaded as a
-    part of the same graph
-  3. Compile with other parts of the code that needs it 
-    - Not a very different from (2) above
-    - This is what is chosen at present, and maybe would be revised later
diff --git a/xlators/experimental/dht2/TODO.md b/xlators/experimental/dht2/TODO.md
deleted file mode 100644
index 1e2c53c5b36..00000000000
--- a/xlators/experimental/dht2/TODO.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# DHT2 TODO list
-
-<Items will be added as code is pulled into the repository>
diff --git a/xlators/experimental/dht2/dht2-client/src/Makefile.am b/xlators/experimental/dht2/dht2-client/src/Makefile.am
deleted file mode 100644
index a16b9df2f76..00000000000
--- a/xlators/experimental/dht2/dht2-client/src/Makefile.am
+++ /dev/null
@@ -1,21 +0,0 @@
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
-xlator_LTLIBRARIES = dht2c.la
-
-dht2c_sources = dht2-client-main.c
-
-dht2common_sources = $(top_srcdir)/xlators/experimental/dht2/dht2-common/src/dht2-common-map.c
-
-dht2c_la_SOURCES = $(dht2c_sources) $(dht2common_sources)
-dht2c_la_LDFLAGS = -module -avoid-version
-dht2c_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-AM_CPPFLAGS = $(GF_CPPFLAGS)
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/experimental/dht2/dht2-common/src/
-AM_CPPFLAGS += -I$(top_srcdir)/libglusterfs/src
-AM_CPPFLAGS += -I$(top_srcdir)/rpc/xdr/src
-AM_CPPFLAGS += -I$(top_builddir)/rpc/xdr/src
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/lib/src
-
-CLEANFILES =
diff --git a/xlators/experimental/dht2/dht2-client/src/dht2-client-main.c b/xlators/experimental/dht2/dht2-client/src/dht2-client-main.c
deleted file mode 100644
index bd1d446e2b5..00000000000
--- a/xlators/experimental/dht2/dht2-client/src/dht2-client-main.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/* File: dht2-client-main.c
- * This file contains the xlator loading functions, FOP entry points
- * and options.
- * The entire functionality including comments is TODO.
- */
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "statedump.h"
-
-int32_t
-dht2_client_init (xlator_t *this)
-{
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Missing children in volume graph, this (%s) is"
-                        " not a leaf translator", this->name);
-                return -1;
-        }
-
-        return 0;
-}
-
-void
-dht2_client_fini (xlator_t *this)
-{
-        return;
-}
-
-class_methods_t class_methods = {
-        .init           = dht2_client_init,
-        .fini           = dht2_client_fini,
-};
-
-struct xlator_fops fops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-/*
-struct xlator_dumpops dumpops = {
-};
-*/
-
-struct volume_options options[] = {
-        { .key  = {NULL} },
-};
diff --git a/xlators/experimental/dht2/dht2-common/src/dht2-common-map.c b/xlators/experimental/dht2/dht2-common/src/dht2-common-map.c
deleted file mode 100644
index d959483b8a4..00000000000
--- a/xlators/experimental/dht2/dht2-common/src/dht2-common-map.c
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/* File: dht2-common-map.c
- * This file contains helper routines to store, consult, the volume map
- * for subvolume to GFID relations.
- * The entire functionality including comments is TODO.
- */
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "statedump.h"
diff --git a/xlators/experimental/dht2/dht2-server/src/Makefile.am b/xlators/experimental/dht2/dht2-server/src/Makefile.am
deleted file mode 100644
index 12d66d126f0..00000000000
--- a/xlators/experimental/dht2/dht2-server/src/Makefile.am
+++ /dev/null
@@ -1,23 +0,0 @@
-if WITH_SERVER
-xlator_LTLIBRARIES = dht2s.la
-endif
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
-
-dht2s_sources = dht2-server-main.c
-
-dht2common_sources = $(top_srcdir)/xlators/experimental/dht2/dht2-common/src/dht2-common-map.c
-
-dht2s_la_SOURCES = $(dht2s_sources) $(dht2common_sources)
-dht2s_la_LDFLAGS = -module -avoid-version
-dht2s_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-AM_CPPFLAGS = $(GF_CPPFLAGS)
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/experimental/dht2/dht2-common/src/
-AM_CPPFLAGS += -I$(top_srcdir)/libglusterfs/src
-AM_CPPFLAGS += -I$(top_srcdir)/rpc/xdr/src
-AM_CPPFLAGS += -I$(top_builddir)/rpc/xdr/src
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/lib/src
-
-CLEANFILES =
diff --git a/xlators/experimental/dht2/dht2-server/src/dht2-server-main.c b/xlators/experimental/dht2/dht2-server/src/dht2-server-main.c
deleted file mode 100644
index 1f232cc3430..00000000000
--- a/xlators/experimental/dht2/dht2-server/src/dht2-server-main.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/* File: dht2-server-main.c
- * This file contains the xlator loading functions, FOP entry points
- * and options.
- * The entire functionality including comments is TODO.
- */
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "statedump.h"
-
-int32_t
-dht2_server_init (xlator_t *this)
-{
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Missing children in volume graph, this (%s) is"
-                        " not a leaf translator", this->name);
-                return -1;
-        }
-
-        return 0;
-}
-
-void
-dht2_server_fini (xlator_t *this)
-{
-        return;
-}
-
-class_methods_t class_methods = {
-        .init           = dht2_server_init,
-        .fini           = dht2_server_fini,
-};
-
-struct xlator_fops fops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-/*
-struct xlator_dumpops dumpops = {
-};
-*/
-
-struct volume_options options[] = {
-        { .key  = {NULL} },
-};
diff --git a/xlators/experimental/fdl/src/Makefile.am b/xlators/experimental/fdl/src/Makefile.am
deleted file mode 100644
index da80ce28317..00000000000
--- a/xlators/experimental/fdl/src/Makefile.am
+++ /dev/null
@@ -1,48 +0,0 @@
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
-if WITH_SERVER
-xlator_LTLIBRARIES = fdl.la
-endif
-
-noinst_HEADERS = fdl.h
-
-nodist_fdl_la_SOURCES = fdl.c
-fdl_la_LDFLAGS = -module -avoid-version
-fdl_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-if WITH_SERVER
-sbin_PROGRAMS = gf_logdump gf_recon
-endif
-gf_logdump_SOURCES = logdump.c
-nodist_gf_logdump_SOURCES = libfdl.c
-gf_logdump_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-	$(top_builddir)/api/src/libgfapi.la $(GFAPI_LIBS) $(UUID_LIBS)
-
-# Eventually recon(ciliation) code will move elsewhere, but for now it's
-# easier to have it next to the similar logdump code.
-gf_recon_SOURCES = recon.c
-nodist_gf_recon_SOURCES = librecon.c
-gf_recon_LDADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-	$(top_builddir)/api/src/libgfapi.la $(GFAPI_LIBS) $(UUID_LIBS)
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
-	-I$(top_srcdir)/api/src -fPIC \
-	-D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) \
-	-DDATADIR=\"$(localstatedir)\"
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-noinst_PYTHON = gen_fdl.py gen_dumper.py gen_recon.py
-EXTRA_DIST = fdl-tmpl.c dump-tmpl.c recon-tmpl.c
-
-CLEANFILES = $(nodist_fdl_la_SOURCES) $(nodist_gf_logdump_SOURCES) \
-	$(nodist_gf_recon_SOURCES)
-
-fdl.c: fdl-tmpl.c gen_fdl.py
-	$(PYTHON) $(srcdir)/gen_fdl.py $(srcdir)/fdl-tmpl.c > $@
-
-libfdl.c: dump-tmpl.c gen_dumper.py
-	$(PYTHON) $(srcdir)/gen_dumper.py $(srcdir)/dump-tmpl.c > $@
-
-librecon.c: recon-tmpl.c gen_recon.py
-	$(PYTHON) $(srcdir)/gen_recon.py $(srcdir)/recon-tmpl.c > $@
diff --git a/xlators/experimental/fdl/src/dump-tmpl.c b/xlators/experimental/fdl/src/dump-tmpl.c
deleted file mode 100644
index 32b0fef6af3..00000000000
--- a/xlators/experimental/fdl/src/dump-tmpl.c
+++ /dev/null
@@ -1,187 +0,0 @@
-#pragma fragment PROLOG
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#include <ctype.h>
-#endif
-
-#include "glfs.h"
-#include "iatt.h"
-#include "xlator.h"
-#include "fdl.h"
-
-/*
- * Returns 0 if the string is ASCII printable *
- * and -1 if it's not ASCII printable         *
- */
-int str_isprint (char *s)
-{
-        int ret = -1;
-
-        if (!s)
-                goto out;
-
-        while (s[0] != '\0') {
-                if (!isprint(s[0]))
-                        goto out;
-                else
-                        s++;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-#pragma fragment DICT
-        {
-                int key_len, data_len;
-                char *key_ptr;
-                char *key_val;
-                printf ("@ARGNAME@ = dict {\n");
-                for (;;) {
-                        key_len = *((int *)new_meta);
-                        new_meta += sizeof(int);
-                        if (!key_len) {
-                                break;
-                        }
-                        key_ptr = new_meta;
-                        new_meta += key_len;
-                        data_len = *((int *)new_meta);
-                        key_val = new_meta + sizeof(int);
-                        new_meta += sizeof(int) + data_len;
-                        if (str_isprint(key_val))
-                                printf (" %s = <%d bytes>\n",
-                                        key_ptr, data_len);
-                        else
-                                printf (" %s = %s <%d bytes>\n",
-                                        key_ptr, key_val, data_len);
-                }
-                printf ("}\n");
-        }
-
-#pragma fragment DOUBLE
-        printf ("@ARGNAME@ = @FORMAT@\n", *((uint64_t *)new_meta),
-                *((uint64_t *)new_meta));
-        new_meta += sizeof(uint64_t);
-
-#pragma fragment GFID
-        printf ("@ARGNAME@ = <gfid %s>\n", uuid_utoa(*((uuid_t *)new_meta)));
-        new_meta += 16;
-
-#pragma fragment INTEGER
-        printf ("@ARGNAME@ = @FORMAT@\n", *((uint32_t *)new_meta),
-                *((uint32_t *)new_meta));
-        new_meta += sizeof(uint32_t);
-
-#pragma fragment LOC
-        printf ("@ARGNAME@ = loc {\n");
-        printf ("  gfid = %s\n", uuid_utoa(*((uuid_t *)new_meta)));
-        new_meta += 16;
-        printf ("  pargfid = %s\n", uuid_utoa(*((uuid_t *)new_meta)));
-        new_meta += 16;
-        if (*(new_meta++)) {
-                printf ("  name = %s\n", new_meta);
-                new_meta += (strlen(new_meta) + 1);
-        }
-        printf ("}\n");
-
-#pragma fragment STRING
-        if (*(new_meta++)) {
-                printf ("@ARGNAME@ = %s\n", new_meta);
-                new_meta += (strlen(new_meta) + 1);
-        }
-
-#pragma fragment VECTOR
-        {
-                size_t len = *((size_t *)new_meta);
-                new_meta += sizeof(len);
-                printf ("@ARGNAME@ = <%zu bytes>\n", len);
-                new_data += len;
-        }
-
-#pragma fragment IATT
-        {
-                ia_prot_t *myprot = ((ia_prot_t *)new_meta);
-                printf ("@ARGNAME@ = iatt {\n");
-                printf ("  ia_prot = %c%c%c",
-                        myprot->suid ? 'S' : '-',
-                        myprot->sgid ? 'S' : '-',
-                        myprot->sticky ? 'T' : '-');
-                printf ("%c%c%c",
-                        myprot->owner.read ? 'r' : '-',
-                        myprot->owner.write ? 'w' : '-',
-                        myprot->owner.exec ? 'x' : '-');
-                printf ("%c%c%c",
-                        myprot->group.read ? 'r' : '-',
-                        myprot->group.write ? 'w' : '-',
-                        myprot->group.exec ? 'x' : '-');
-                printf ("%c%c%c\n",
-                        myprot->other.read ? 'r' : '-',
-                        myprot->other.write ? 'w' : '-',
-                        myprot->other.exec ? 'x' : '-');
-                new_meta += sizeof(ia_prot_t);
-                uint32_t *myints = (uint32_t *)new_meta;
-                printf ("  ia_uid = %u\n", myints[0]);
-                printf ("  ia_gid = %u\n", myints[1]);
-                printf ("  ia_atime = %u.%09u\n", myints[2], myints[3]);
-                printf ("  ia_mtime = %u.%09u\n", myints[4], myints[5]);
-                new_meta += sizeof(*myints) * 6;
-        }
-
-#pragma fragment FOP
-void
-fdl_dump_@NAME@ (char **old_meta, char **old_data)
-{
-        char    *new_meta	= *old_meta;
-        char	*new_data	= *old_data;
-
-        /* TBD: word size/endianness */
-@FUNCTION_BODY@
-
-        *old_meta = new_meta;
-        *old_data = new_data;
-}
-
-#pragma fragment CASE
-        case GF_FOP_@UPNAME@:
-                printf ("=== GF_FOP_@UPNAME@\n");
-                fdl_dump_@NAME@ (&new_meta, &new_data);
-                break;
-
-#pragma fragment EPILOG
-int
-fdl_dump (char **old_meta, char **old_data)
-{
-        char            *new_meta       = *old_meta;
-        char            *new_data       = *old_data;
-        static glfs_t   *fs             = NULL;
-        int             recognized      = 1;
-        event_header_t  *eh;
-
-        /*
-         * We don't really call anything else in GFAPI, but this is the most
-         * convenient way to satisfy all of the spurious dependencies on how it
-         * or glusterfsd initialize (e.g. setting up THIS).
-         */
-        if (!fs) {
-                fs = glfs_new ("dummy");
-        }
-
-        eh = (event_header_t *)new_meta;
-        new_meta += sizeof (*eh);
-
-        /* TBD: check event_type instead of assuming NEW_REQUEST */
-
-        switch (eh->fop_type) {
-@SWITCH_BODY@
-
-        default:
-                printf ("unknown fop %u\n", eh->fop_type);
-                recognized = 0;
-        }
-
-        *old_meta = new_meta;
-        *old_data = new_data;
-        return recognized;
-}
diff --git a/xlators/experimental/fdl/src/fdl-tmpl.c b/xlators/experimental/fdl/src/fdl-tmpl.c
deleted file mode 100644
index 145dad7964a..00000000000
--- a/xlators/experimental/fdl/src/fdl-tmpl.c
+++ /dev/null
@@ -1,536 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <fcntl.h>
-#include <unistd.h>
-#include <sys/mman.h>
-#include "call-stub.h"
-#include "iatt.h"
-#include "defaults.h"
-#include "syscall.h"
-#include "xlator.h"
-#include "fdl.h"
-
-/* TBD: make tunable */
-#define META_FILE_SIZE  (1 << 20)
-#define DATA_FILE_SIZE  (1 << 24)
-
-enum gf_fdl {
-        gf_fdl_mt_fdl_private_t = gf_common_mt_end + 1,
-        gf_fdl_mt_end
-};
-
-typedef struct {
-        char            *type;
-        off_t           size;
-        char            *path;
-        int             fd;
-        void *          ptr;
-        off_t           max_offset;
-} log_obj_t;
-
-typedef struct {
-        struct list_head        reqs;
-        pthread_mutex_t         req_lock;
-        pthread_cond_t          req_cond;
-        char                    *log_dir;
-        pthread_t               worker;
-        gf_boolean_t            should_stop;
-        gf_boolean_t            change_term;
-        log_obj_t               meta_log;
-        log_obj_t               data_log;
-        int                     term;
-        int                     first_term;
-} fdl_private_t;
-
-int32_t
-fdl_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata);
-
-void
-fdl_enqueue (xlator_t *this, call_stub_t *stub)
-{
-        fdl_private_t   *priv   = this->private;
-
-        pthread_mutex_lock (&priv->req_lock);
-        list_add_tail (&stub->list, &priv->reqs);
-        pthread_mutex_unlock (&priv->req_lock);
-
-        pthread_cond_signal (&priv->req_cond);
-}
-
-#pragma generate
-
-char *
-fdl_open_term_log (xlator_t *this, log_obj_t *obj, int term)
-{
-        fdl_private_t   *priv   = this->private;
-        int             ret;
-        char *          ptr     = NULL;
-
-        /*
-         * Use .jnl instead of .log so that we don't get test info (mistakenly)
-         * appended to our journal files.
-         */
-        if (this->ctx->cmd_args.log_ident) {
-                ret = gf_asprintf (&obj->path, "%s/%s-%s-%d.jnl",
-                                   priv->log_dir, this->ctx->cmd_args.log_ident,
-                                   obj->type, term);
-        }
-        else {
-                ret = gf_asprintf (&obj->path, "%s/fubar-%s-%d.jnl",
-                                   priv->log_dir, obj->type, term);
-        }
-        if ((ret <= 0) || !obj->path) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to construct log-file path");
-                goto err;
-        }
-
-        gf_log (this->name, GF_LOG_INFO, "opening %s (size %ld)",
-                obj->path, obj->size);
-
-        obj->fd = open (obj->path, O_RDWR|O_CREAT|O_TRUNC, 0666);
-        if (obj->fd < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to open log file (%s)", strerror(errno));
-                goto err;
-        }
-
-#if !defined(GF_BSD_HOST_OS)
-        /*
-         * NetBSD can just go die in a fire.  Even though it claims to support
-         * fallocate/posix_fallocate they don't actually *do* anything so the
-         * file size remains zero.  Then mmap succeeds anyway, but any access
-         * to the mmap'ed region will segfault.  It would be acceptable for
-         * fallocate to do what it says, for mmap to fail, or for access to
-         * extend the file.  NetBSD managed to hit the trifecta of Getting
-         * Everything Wrong, and debugging in that environment to get this far
-         * has already been painful enough (systems I worked on in 1990 were
-         * better that way).  We'll fall through to the lseek/write method, and
-         * performance will be worse, and TOO BAD.
-         */
-        if (sys_fallocate(obj->fd,0,0,obj->size) < 0)
-#endif
-        {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to fallocate space for log file");
-                /* Have to do this the ugly page-faulty way. */
-                (void) sys_lseek (obj->fd, obj->size-1, SEEK_SET);
-                (void) sys_write (obj->fd, "", 1);
-        }
-
-        ptr = mmap (NULL, obj->size, PROT_WRITE, MAP_SHARED, obj->fd, 0);
-        if (ptr == MAP_FAILED) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to mmap log (%s)",
-                        strerror(errno));
-                goto err;
-        }
-
-        obj->ptr = ptr;
-        obj->max_offset = 0;
-        return ptr;
-
-err:
-        if (obj->fd >= 0) {
-                sys_close (obj->fd);
-                obj->fd = (-1);
-        }
-        if (obj->path) {
-                GF_FREE (obj->path);
-                obj->path = NULL;
-        }
-        return ptr;
-}
-
-void
-fdl_close_term_log (xlator_t *this, log_obj_t *obj)
-{
-        fdl_private_t   *priv           = this->private;
-
-        if (obj->ptr) {
-                (void) munmap (obj->ptr, obj->size);
-                obj->ptr = NULL;
-        }
-
-        if (obj->fd >= 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "truncating term %d %s journal to %ld",
-                        priv->term, obj->type, obj->max_offset);
-                if (sys_ftruncate(obj->fd,obj->max_offset) < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to truncate journal (%s)",
-                                strerror(errno));
-                }
-                sys_close (obj->fd);
-                obj->fd = (-1);
-        }
-
-        if (obj->path) {
-                GF_FREE (obj->path);
-                obj->path = NULL;
-        }
-}
-
-gf_boolean_t
-fdl_change_term (xlator_t *this, char **meta_ptr, char **data_ptr)
-{
-        fdl_private_t   *priv           = this->private;
-
-        fdl_close_term_log (this, &priv->meta_log);
-        fdl_close_term_log (this, &priv->data_log);
-
-        ++(priv->term);
-
-        *meta_ptr = fdl_open_term_log (this, &priv->meta_log, priv->term);
-        if (!*meta_ptr) {
-                return _gf_false;
-        }
-
-        *data_ptr = fdl_open_term_log (this, &priv->data_log, priv->term);
-        if (!*data_ptr) {
-                return _gf_false;
-        }
-
-        return _gf_true;
-}
-
-void *
-fdl_worker (void *arg)
-{
-        xlator_t        *this           = arg;
-        fdl_private_t   *priv           = this->private;
-        call_stub_t     *stub;
-        char *          meta_ptr        = NULL;
-        off_t           *meta_offset    = &priv->meta_log.max_offset;
-        char *          data_ptr        = NULL;
-        off_t           *data_offset    = &priv->data_log.max_offset;
-        unsigned long   base_as_ul;
-        void *          msync_ptr;
-        size_t          msync_len;
-        gf_boolean_t    recycle;
-        void            *err_label      = &&err_unlocked;
-
-        priv->meta_log.type = "meta";
-        priv->meta_log.size = META_FILE_SIZE;
-        priv->meta_log.path = NULL;
-        priv->meta_log.fd = (-1);
-        priv->meta_log.ptr = NULL;
-
-        priv->data_log.type = "data";
-        priv->data_log.size = DATA_FILE_SIZE;
-        priv->data_log.path = NULL;
-        priv->data_log.fd = (-1);
-        priv->data_log.ptr = NULL;
-
-        /* TBD: initial term should come from persistent storage (e.g. etcd) */
-        priv->first_term = ++(priv->term);
-        meta_ptr = fdl_open_term_log (this, &priv->meta_log, priv->term);
-        if (!meta_ptr) {
-                goto *err_label;
-        }
-        data_ptr = fdl_open_term_log (this, &priv->data_log, priv->term);
-        if (!data_ptr) {
-                fdl_close_term_log (this, &priv->meta_log);
-                goto *err_label;
-        }
-
-        for (;;) {
-                pthread_mutex_lock (&priv->req_lock);
-                err_label = &&err_locked;
-                while (list_empty(&priv->reqs)) {
-                        pthread_cond_wait (&priv->req_cond, &priv->req_lock);
-                        if (priv->should_stop) {
-                                goto *err_label;
-                        }
-                        if (priv->change_term) {
-                                if (!fdl_change_term(this, &meta_ptr,
-                                                           &data_ptr)) {
-                                        goto *err_label;
-                                }
-                                priv->change_term = _gf_false;
-                                continue;
-                        }
-                }
-                stub = list_entry (priv->reqs.next, call_stub_t, list);
-                list_del_init (&stub->list);
-                pthread_mutex_unlock (&priv->req_lock);
-                err_label = &&err_unlocked;
-                /*
-                 * TBD: batch requests
-                 *
-                 * What we should do here is gather up *all* of the requests
-                 * that have accumulated since we were last at this point,
-                 * blast them all out in one big writev, and then dispatch them
-                 * all before coming back for more.  That maximizes throughput,
-                 * at some cost to latency (due to queuing effects at the log
-                 * stage).  Note that we're likely to be above io-threads, so
-                 * the dispatch itself will be parallelized (at further cost to
-                 * latency).  For now, we just do the simplest thing and handle
-                 * one request all the way through before fetching the next.
-                 *
-                 * So, why mmap/msync instead of writev/fdatasync?  Because it's
-                 * faster.  Much faster.  So much faster that I half-suspect
-                 * cheating, but it's more convenient for now than having to
-                 * ensure that everything's page-aligned for O_DIRECT (the only
-                 * alternative that still might avoid ridiculous levels of
-                 * local-FS overhead).
-                 *
-                 * TBD: check that msync really does get our data to disk.
-                 */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "logging %u+%u bytes for op %d",
-                        stub->jnl_meta_len, stub->jnl_data_len, stub->fop);
-                recycle = _gf_false;
-                if ((*meta_offset + stub->jnl_meta_len) > priv->meta_log.size) {
-                        recycle = _gf_true;
-                }
-                if ((*data_offset + stub->jnl_data_len) > priv->data_log.size) {
-                        recycle = _gf_true;
-                }
-                if (recycle && !fdl_change_term(this,&meta_ptr,&data_ptr)) {
-                        goto *err_label;
-                }
-                meta_ptr = priv->meta_log.ptr;
-                data_ptr = priv->data_log.ptr;
-                gf_log (this->name, GF_LOG_DEBUG, "serializing to %p/%p",
-                        meta_ptr + *meta_offset, data_ptr + *data_offset);
-                stub->serialize (stub, meta_ptr + *meta_offset,
-                                       data_ptr + *data_offset);
-                if (stub->jnl_meta_len > 0) {
-                        base_as_ul = (unsigned long) (meta_ptr + *meta_offset);
-                        msync_ptr = (void *) (base_as_ul & ~0x0fff);
-                        msync_len = (size_t) (base_as_ul &  0x0fff);
-                        if (msync (msync_ptr, msync_len+stub->jnl_meta_len,
-                                              MS_SYNC) < 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to log request meta (%s)",
-                                        strerror(errno));
-                        }
-                        *meta_offset += stub->jnl_meta_len;
-                }
-                if (stub->jnl_data_len > 0) {
-                        base_as_ul = (unsigned long) (data_ptr + *data_offset);
-                        msync_ptr = (void *) (base_as_ul & ~0x0fff);
-                        msync_len = (size_t) (base_as_ul &  0x0fff);
-                        if (msync (msync_ptr, msync_len+stub->jnl_data_len,
-                                              MS_SYNC) < 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to log request data (%s)",
-                                        strerror(errno));
-                        }
-                        *data_offset += stub->jnl_data_len;
-                }
-                call_resume (stub);
-        }
-
-err_locked:
-        pthread_mutex_unlock (&priv->req_lock);
-err_unlocked:
-        fdl_close_term_log (this, &priv->meta_log);
-        fdl_close_term_log (this, &priv->data_log);
-        return NULL;
-}
-
-int32_t
-fdl_ipc_continue (call_frame_t *frame, xlator_t *this,
-                  int32_t op, dict_t *xdata)
-{
-        /*
-         * Nothing to be done here. Just Unwind. *
-         */
-        STACK_UNWIND_STRICT (ipc, frame, 0, 0, xdata);
-
-        return 0;
-}
-
-int32_t
-fdl_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
-{
-        call_stub_t     *stub;
-        fdl_private_t   *priv   = this->private;
-        dict_t          *tdict;
-        int32_t         gt_err  = EIO;
-
-        switch (op) {
-
-        case FDL_IPC_CHANGE_TERM:
-                gf_log (this->name, GF_LOG_INFO, "got CHANGE_TERM op");
-                priv->change_term = _gf_true;
-                pthread_cond_signal (&priv->req_cond);
-                STACK_UNWIND_STRICT (ipc, frame, 0, 0, NULL);
-                break;
-
-        case FDL_IPC_GET_TERMS:
-                gf_log (this->name, GF_LOG_INFO, "got GET_TERMS op");
-                tdict = dict_new ();
-                if (!tdict) {
-                        gt_err = ENOMEM;
-                        goto gt_done;
-                }
-                if (dict_set_int32(tdict,"first",priv->first_term) != 0) {
-                        goto gt_done;
-                }
-                if (dict_set_int32(tdict,"last",priv->term) != 0) {
-                        goto gt_done;
-                }
-                gt_err = 0;
-        gt_done:
-                if (gt_err) {
-                        STACK_UNWIND_STRICT (ipc, frame, -1, gt_err, NULL);
-                } else {
-                        STACK_UNWIND_STRICT (ipc, frame, 0, 0, tdict);
-                }
-                if (tdict) {
-                        dict_unref (tdict);
-                }
-                break;
-
-        case FDL_IPC_JBR_SERVER_ROLLBACK:
-                /*
-                 * In case of a rollback from jbr-server, dump  *
-                 * the term and index number in the journal,    *
-                 * which will later be used to rollback the fop *
-                 */
-                stub = fop_ipc_stub (frame, fdl_ipc_continue,
-                                     op, xdata);
-                fdl_len_ipc (stub);
-                stub->serialize = fdl_serialize_ipc;
-                fdl_enqueue (this, stub);
-
-                break;
-
-        default:
-                STACK_WIND_TAIL (frame,
-                                 FIRST_CHILD(this),
-                                 FIRST_CHILD(this)->fops->ipc,
-                                 op, xdata);
-        }
-
-        return 0;
-}
-
-int
-fdl_init (xlator_t *this)
-{
-        fdl_private_t   *priv   = NULL;
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_fdl_mt_fdl_private_t);
-        if (!priv) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to allocate fdl_private");
-                goto err;
-        }
-
-        INIT_LIST_HEAD (&priv->reqs);
-        if (pthread_mutex_init (&priv->req_lock, NULL) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to initialize req_lock");
-                goto err;
-        }
-        if (pthread_cond_init (&priv->req_cond, NULL) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to initialize req_cond");
-                goto err;
-        }
-
-        GF_OPTION_INIT ("log-path", priv->log_dir, path, err);
-
-        this->private = priv;
-        /*
-         * The rest of the fop table is automatically generated, so this is a
-         * bit cleaner than messing with the generation to add a hand-written
-         * exception.
-         */
-
-        if (gf_thread_create (&priv->worker, NULL, fdl_worker, this,
-                              "fdlwrker") != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to start fdl_worker");
-                goto err;
-        }
-
-        return 0;
-
-err:
-        if (priv) {
-                GF_FREE(priv);
-        }
-        return -1;
-}
-
-void
-fdl_fini (xlator_t *this)
-{
-        fdl_private_t   *priv   = this->private;
-
-        if (priv) {
-                priv->should_stop = _gf_true;
-                pthread_cond_signal (&priv->req_cond);
-                pthread_join (priv->worker, NULL);
-                GF_FREE(priv);
-        }
-}
-
-int
-fdl_reconfigure (xlator_t *this, dict_t *options)
-{
-        fdl_private_t   *priv   = this->private;
-
-	GF_OPTION_RECONF ("log_dir", priv->log_dir, options, path, out);
-        /* TBD: react if it changed */
-
-out:
-        return 0;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("fdl", this, out);
-
-        ret = xlator_mem_acct_init (this, gf_fdl_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
-out:
-        return ret;
-}
-
-class_methods_t class_methods = {
-        .init           = fdl_init,
-        .fini           = fdl_fini,
-        .reconfigure    = fdl_reconfigure,
-        .notify         = default_notify,
-};
-
-struct volume_options options[] = {
-        { .key = {"log-path"},
-          .type = GF_OPTION_TYPE_PATH,
-          .default_value = DEFAULT_LOG_FILE_DIRECTORY,
-          .description = "Directory for FDL files."
-        },
-        { .key  = {NULL} },
-};
-
-struct xlator_cbks cbks = {
-        .release        = default_release,
-        .releasedir     = default_releasedir,
-        .forget         = default_forget,
-};
diff --git a/xlators/experimental/fdl/src/fdl.h b/xlators/experimental/fdl/src/fdl.h
deleted file mode 100644
index 32e38c93f2d..00000000000
--- a/xlators/experimental/fdl/src/fdl.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
-   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef _FDL_H_
-#define _FDL_H_
-
-#define NEW_REQUEST     (uint8_t)'N'
-
-typedef struct {
-        uint8_t         event_type;     /* e.g. NEW_REQUEST */
-        uint8_t         fop_type;       /* e.g. GF_FOP_SETATTR */
-        uint16_t        request_id;
-        uint32_t        ext_length;
-} event_header_t;
-
-enum {
-        FDL_IPC_BASE = 0xfeedbee5,       /* ... and they make honey */
-        FDL_IPC_CHANGE_TERM,
-        FDL_IPC_GET_TERMS,
-        FDL_IPC_JBR_SERVER_ROLLBACK
-};
-
-#endif /* _FDL_H_ */
diff --git a/xlators/experimental/fdl/src/gen_dumper.py b/xlators/experimental/fdl/src/gen_dumper.py
deleted file mode 100755
index 363ba6ce63f..00000000000
--- a/xlators/experimental/fdl/src/gen_dumper.py
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/usr/bin/python2
-
-from __future__ import print_function
-import os
-import re
-import sys
-
-curdir = os.path.dirname (sys.argv[0])
-gendir = os.path.join (curdir, '../../../../libglusterfs/src')
-sys.path.append (gendir)
-from generator import ops, fop_subs, cbk_subs, generate
-
-# See the big header comment at the start of gen_fdl.py to see how the stages
-# fit together.  The big difference here is that *all* of the C code is in the
-# template file as labelled fragments, instead of as Python strings.  That
-# makes it much easier to edit in one place, with proper syntax highlighting
-# and indentation.
-#
-#   Stage 1 uses type-specific fragments to generate FUNCTION_BODY, instead of
-#   LEN_*_TEMPLATE and SERLZ_*_TEMPLATE to generate LEN_CODE and SER_CODE.
-#
-#   Stage 2 uses the FOP and CASE fragments instead of RECON_TEMPLATE and
-#   FOP_TEMPLATE.  The expanded FOP code (including FUNCTION_BODY substitution
-#   in the middle of each function) is emitted immediately; the expanded CASE
-#   code is saved for the next stage.
-#
-#   Stage 3 uses the PROLOG and EPILOG fragments, with the expanded CASE code
-#   in the middle of EPILOG, to generate the whole output file.
-#
-# Another way of looking at it is to consider how the fragments appear in
-# the final output:
-#
-#   PROLOG
-#   FOP (expanded for CREATE)
-#       FOP before FUNCTION_BODY
-#       LOC, INTEGER, GFID, etc. (one per arg, by type)
-#       FOP after FUNCTION_BODY
-#   FOP (expanded for WRITEV)
-#       FOP before FUNCTION_BODY
-#       GFID, VECTOR, etc. (on per arg, by type)
-#       FOP after FUNCTION_BODY
-#   (more FOPs)
-#   EPILOG
-#       EPILOG before CASE
-#       CASE statements (one per fop)
-#       EPILOG after CASE
-
-typemap = {
-	'dict_t *':				( "DICT",		""),
-	'fd_t *':				( "GFID",		""),
-	'dev_t':				( "DOUBLE",		"%ld (0x%lx)"),
-	'gf_xattrop_flags_t':	( "INTEGER",	"%d (0x%x)"),
-	'int32_t':				( "INTEGER",	"%d (0x%x)"),
-	'mode_t':				( "INTEGER",	"%d (0x%x)"),
-	'off_t':				( "DOUBLE",		"%ld (0x%lx)"),
-	'size_t':				( "DOUBLE",		"%ld (0x%lx)"),
-	'uint32_t':				( "INTEGER",	"%d (0x%x)"),
-	'loc_t *':				( "LOC",		""),
-	'const char *':			( "STRING",		""),
-	'struct iovec *':		( "VECTOR",		""),
-	'struct iatt *':		( "IATT",		""),
-}
-
-def get_special_subs (args):
-	code = ""
-	for arg in args:
-		if (arg[0] != 'fop-arg') or (len(arg) < 4):
-			continue
-		recon_type, recon_fmt = typemap[arg[2]]
-		code += fragments[recon_type].replace("@ARGNAME@",arg[3])		\
-									 .replace("@FORMAT@",recon_fmt)
-	return code
-
-def gen_functions ():
-	code = ""
-	for name, value in ops.iteritems():
-		if "journal" not in [ x[0] for x in value ]:
-			continue
-		fop_subs[name]["@FUNCTION_BODY@"] = get_special_subs(value)
-		# Print the FOP fragment with @FUNCTION_BODY@ in the middle.
-		code += generate(fragments["FOP"],name,fop_subs)
-	return code
-
-def gen_cases ():
-	code = ""
-	for name, value in ops.iteritems():
-		if "journal" not in [ x[0] for x in value ]:
-			continue
-		# Add the CASE fragment for this fop.
-		code += generate(fragments["CASE"],name,fop_subs)
-	return code
-
-def load_fragments (path="recon-tmpl.c"):
-	pragma_re = re.compile('pragma fragment (.*)')
-	cur_symbol = None
-	cur_value = ""
-	result = {}
-	for line in open(path,"r").readlines():
-		m = pragma_re.search(line)
-		if m:
-			if cur_symbol:
-				result[cur_symbol] = cur_value
-			cur_symbol = m.group(1)
-			cur_value = ""
-		else:
-			cur_value += line
-	if cur_symbol:
-		result[cur_symbol] = cur_value
-	return result
-
-if __name__ == "__main__":
-	fragments = load_fragments(sys.argv[1])
-	print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
-	print(fragments["PROLOG"])
-	print(gen_functions())
-	print(fragments["EPILOG"].replace("@SWITCH_BODY@",gen_cases()))
-	print("/* END GENERATED CODE */")
diff --git a/xlators/experimental/fdl/src/gen_fdl.py b/xlators/experimental/fdl/src/gen_fdl.py
deleted file mode 100755
index d2e7dd5dfb2..00000000000
--- a/xlators/experimental/fdl/src/gen_fdl.py
+++ /dev/null
@@ -1,354 +0,0 @@
-#!/usr/bin/python2
-
-from __future__ import print_function
-import os
-import sys
-
-curdir = os.path.dirname (sys.argv[0])
-gendir = os.path.join (curdir, '../../../../libglusterfs/src')
-sys.path.append (gendir)
-from generator import ops, fop_subs, cbk_subs, generate
-
-# Generation occurs in three stages.  In this case, it actually makes more
-# sense to discuss them in the *opposite* order of that in which they
-# actually happen.
-#
-#   Stage 3 is to insert all of the generated code into a file, replacing the
-#   "#pragma generate" that's already there.  The file can thus contain all
-#   sorts of stuff that's not specific to one fop, either before or after the
-#   generated code as appropriate.
-#
-#   Stage 2 is to generate all of the code *for a particular fop*, using a
-#   string-valued template plus a table of substitution values.  Most of these
-#   are built in to the generator itself.  However, we also add a couple that
-#   are specific to this particular translator - LEN_CODE and SER_CODE.  These
-#   are per-fop functions to get the length or the contents (respectively) of
-#   what we'll put in the log.  As with stage 3 allowing per-file boilerplate
-#   before and after generated code, this allows per-fop boilerplate before and
-#   after generated code.
-#
-#   Stage 1, therefore, is to create the LEN_CODE and SER_CODE substitutions for
-#   each fop, and put them in the same table where e.g. NAME and SHORT_ARGS
-#   already are.  We do this by looking at the fop-description table in the
-#   generator module, then doing out own template substitution to plug each
-#   specific argument name into another string-valued template.
-#
-# So, what does this leave us with in terms of variables and files?
-#
-#   For stage 1, we have a series of LEN_*_TEMPLATE and SERLZ_*_TEMPLATE
-#   strings, which are used to generate the length and serialization code for
-#   each argument type.
-#
-#   For stage 2, we have a bunch of *_TEMPLATE strings (no LEN_ or SERLZ_
-#   prefix), which are used (along with the output from stage 1) to generate
-#   whole functions.
-#
-#   For stage 3, we have a whole separate file (fdl_tmpl.c) into which we insert
-#   the collection of all functions defined in stage 2.
-
-
-LEN_TEMPLATE = """
-void
-fdl_len_@NAME@ (call_stub_t *stub)
-{
-        uint32_t    meta_len    = sizeof (event_header_t);
-		uint32_t	data_len	= 0;
-
-        /* TBD: global stuff, e.g. uid/gid */
-@LEN_CODE@
-
-		/* TBD: pad extension length */
-		stub->jnl_meta_len = meta_len;
-		stub->jnl_data_len = data_len;
-}
-"""
-
-SER_TEMPLATE = """
-void
-fdl_serialize_@NAME@ (call_stub_t *stub, char *meta_buf, char *data_buf)
-{
-		event_header_t	*eh;
-		unsigned long	offset = 0;
-
-        /* TBD: word size/endianness */
-		eh = (event_header_t *)meta_buf;
-		eh->event_type = NEW_REQUEST;
-		eh->fop_type = GF_FOP_@UPNAME@;
-		eh->request_id = 0;	// TBD
-		meta_buf += sizeof (*eh);
-@SER_CODE@
-		/* TBD: pad extension length */
-		eh->ext_length = offset;
-}
-"""
-
-CBK_TEMPLATE = """
-int32_t
-fdl_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno,
-                @LONG_ARGS@)
-{
-        STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno,
-                             @SHORT_ARGS@);
-        return 0;
-}
-"""
-
-CONTINUE_TEMPLATE = """
-int32_t
-fdl_@NAME@_continue (call_frame_t *frame, xlator_t *this,
-                     @LONG_ARGS@)
-{
-        STACK_WIND (frame, fdl_@NAME@_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
-                    @SHORT_ARGS@);
-        return 0;
-}
-
-"""
-
-FOP_TEMPLATE = """
-int32_t
-fdl_@NAME@ (call_frame_t *frame, xlator_t *this,
-            @LONG_ARGS@)
-{
-        call_stub_t     *stub;
-
-        stub = fop_@NAME@_stub (frame, default_@NAME@,
-                                @SHORT_ARGS@);
-		fdl_len_@NAME@ (stub);
-        stub->serialize = fdl_serialize_@NAME@;
-        fdl_enqueue (this, stub);
-
-        return 0;
-}
-"""
-
-LEN_DICT_TEMPLATE = """
-		if (@SRC@) {
-			data_pair_t *memb;
-			for (memb = @SRC@->members_list; memb; memb = memb->next) {
-				meta_len += sizeof(int);
-				meta_len += strlen(memb->key) + 1;
-				meta_len += sizeof(int);
-				meta_len += memb->value->len;
-			}
-		}
-		meta_len += sizeof(int);
-"""
-
-LEN_GFID_TEMPLATE = """
-        meta_len += 16;
-"""
-
-LEN_INTEGER_TEMPLATE = """
-        meta_len += sizeof (@SRC@);
-"""
-
-# 16 for gfid, 16 for pargfid, 1 for flag, 0/1 for terminating NUL
-LEN_LOC_TEMPLATE = """
-        if (@SRC@.name) {
-                meta_len += (strlen (@SRC@.name) + 34);
-        } else {
-                meta_len += 33;
-        }
-"""
-
-LEN_STRING_TEMPLATE = """
-        if (@SRC@) {
-                meta_len += (strlen (@SRC@) + 1);
-        } else {
-                meta_len += 1;
-        }
-"""
-
-LEN_VECTOR_TEMPLATE = """
-        meta_len += sizeof(size_t);
-        data_len += iov_length (@VEC@, @CNT@);
-"""
-
-LEN_IATT_TEMPLATE = """
-		meta_len += sizeof(@SRC@.ia_prot);
-		meta_len += sizeof(@SRC@.ia_uid);
-		meta_len += sizeof(@SRC@.ia_gid);
-		meta_len += sizeof(@SRC@.ia_atime);
-		meta_len += sizeof(@SRC@.ia_atime_nsec);
-		meta_len += sizeof(@SRC@.ia_mtime);
-		meta_len += sizeof(@SRC@.ia_mtime_nsec);
-"""
-
-SERLZ_DICT_TEMPLATE = """
-        if (@SRC@) {
-			data_pair_t *memb;
-			for (memb = @SRC@->members_list; memb; memb = memb->next) {
-				*((int *)(meta_buf+offset)) = strlen(memb->key) + 1;
-				offset += sizeof(int);
-				strcpy (meta_buf+offset, memb->key);
-				offset += strlen(memb->key) + 1;
-				*((int *)(meta_buf+offset)) = memb->value->len;
-				offset += sizeof(int);
-				memcpy (meta_buf+offset, memb->value->data, memb->value->len);
-				offset += memb->value->len;
-			}
-        }
-		*((int *)(meta_buf+offset)) = 0;
-		offset += sizeof(int);
-"""
-
-SERLZ_GFID_TEMPLATE = """
-        memcpy (meta_buf+offset, @SRC@->inode->gfid, 16);
-        offset += 16;
-"""
-
-SERLZ_INTEGER_TEMPLATE = """
-        memcpy (meta_buf+offset, &@SRC@, sizeof(@SRC@));
-        offset += sizeof(@SRC@);
-"""
-
-SERLZ_LOC_TEMPLATE = """
-        memcpy (meta_buf+offset, @SRC@.gfid, 16);
-        offset += 16;
-        memcpy (meta_buf+offset, @SRC@.pargfid, 16);
-        offset += 16;
-        if (@SRC@.name) {
-                *(meta_buf+offset) = 1;
-				++offset;
-                strcpy (meta_buf+offset, @SRC@.name);
-                offset += (strlen (@SRC@.name) + 1);
-        } else {
-                *(meta_buf+offset) = 0;
-				++offset;
-        }
-"""
-
-SERLZ_STRING_TEMPLATE = """
-        if (@SRC@) {
-                *(meta_buf+offset) = 1;
-				++offset;
-                strcpy (meta_buf+offset, @SRC@);
-                offset += strlen(@SRC@);
-        } else {
-                *(meta_buf+offset) = 0;
-				++offset;
-        }
-"""
-
-SERLZ_VECTOR_TEMPLATE = """
-        *((size_t *)(meta_buf+offset)) = iov_length (@VEC@, @CNT@);
-        offset += sizeof(size_t);
-        int32_t i;
-        for (i = 0; i < @CNT@; ++i) {
-                memcpy (data_buf, @VEC@[i].iov_base, @VEC@[i].iov_len);
-                data_buf += @VEC@[i].iov_len;
-        }
-"""
-
-# We don't need to save all of the fields - only those affected by chown,
-# chgrp, chmod, and utime.
-SERLZ_IATT_TEMPLATE = """
-		*((ia_prot_t *)(meta_buf+offset)) = @SRC@.ia_prot;
-		offset += sizeof(@SRC@.ia_prot);
-		*((uint32_t *)(meta_buf+offset)) = @SRC@.ia_uid;
-		offset += sizeof(@SRC@.ia_uid);
-		*((uint32_t *)(meta_buf+offset)) = @SRC@.ia_gid;
-		offset += sizeof(@SRC@.ia_gid);
-		*((uint32_t *)(meta_buf+offset)) = @SRC@.ia_atime;
-		offset += sizeof(@SRC@.ia_atime);
-		*((uint32_t *)(meta_buf+offset)) = @SRC@.ia_atime_nsec;
-		offset += sizeof(@SRC@.ia_atime_nsec);
-		*((uint32_t *)(meta_buf+offset)) = @SRC@.ia_mtime;
-		offset += sizeof(@SRC@.ia_mtime);
-		*((uint32_t *)(meta_buf+offset)) = @SRC@.ia_mtime_nsec;
-		offset += sizeof(@SRC@.ia_mtime_nsec);
-"""
-
-typemap = {
-	'dict_t *':				( LEN_DICT_TEMPLATE,	SERLZ_DICT_TEMPLATE),
-	'fd_t *':				( LEN_GFID_TEMPLATE,	SERLZ_GFID_TEMPLATE),
-	'dev_t':				( LEN_INTEGER_TEMPLATE,	SERLZ_INTEGER_TEMPLATE),
-	'gf_xattrop_flags_t':	( LEN_INTEGER_TEMPLATE,	SERLZ_INTEGER_TEMPLATE),
-	'int32_t':				( LEN_INTEGER_TEMPLATE,	SERLZ_INTEGER_TEMPLATE),
-	'mode_t':				( LEN_INTEGER_TEMPLATE, SERLZ_INTEGER_TEMPLATE),
-	'off_t':				( LEN_INTEGER_TEMPLATE,	SERLZ_INTEGER_TEMPLATE),
-	'size_t':				( LEN_INTEGER_TEMPLATE,	SERLZ_INTEGER_TEMPLATE),
-	'uint32_t':				( LEN_INTEGER_TEMPLATE,	SERLZ_INTEGER_TEMPLATE),
-	'loc_t *':				( LEN_LOC_TEMPLATE,		SERLZ_LOC_TEMPLATE),
-	'const char *':			( LEN_STRING_TEMPLATE,	SERLZ_STRING_TEMPLATE),
-	'struct iatt *':		( LEN_IATT_TEMPLATE,	SERLZ_IATT_TEMPLATE),
-}
-
-def get_special_subs (args):
-	len_code = ""
-	ser_code = ""
-	for arg in args:
-		if (arg[0] != 'fop-arg') or (len(arg) < 4):
-			continue
-		# Let this throw an exception if we get an unknown field name.  The
-		# broken build will remind whoever messed with the stub code that a
-		# corresponding update is needed here.
-		if arg[3] == "vector":
-			# Make it as obvious as possible that this is a special case.
-			len_code += LEN_VECTOR_TEMPLATE \
-				.replace("@VEC@","stub->args.vector") \
-				.replace("@CNT@","stub->args.count")
-			ser_code += SERLZ_VECTOR_TEMPLATE \
-				.replace("@VEC@","stub->args.vector") \
-				.replace("@CNT@","stub->args.count")
-		else:
-			len_tmpl, ser_tmpl = typemap[arg[2]]
-			src = "stub->args.%s" % arg[3]
-			len_code += len_tmpl.replace("@SRC@",src)
-			ser_code += ser_tmpl.replace("@SRC@",src)
-	return len_code, ser_code
-
-# Mention those fops in the selective_generate table, for which
-# only a few common functions will be generated, and mention those
-# functions. Rest of the functions can be customized
-selective_generate = {
-		"ipc":			"len,serialize",
-	}
-
-def gen_fdl ():
-	entrypoints = []
-	for name, value in ops.iteritems():
-		if "journal" not in [ x[0] for x in value ]:
-			continue
-
-		# generate all functions for all the fops
-		# except for the ones in selective_generate for which
-		# generate only the functions mentioned in the
-		# selective_generate table
-		gen_funcs = "len,serialize,callback,continue,fop"
-		if name in selective_generate:
-			gen_funcs = selective_generate[name].split(",")
-
-		len_code, ser_code = get_special_subs(value)
-		fop_subs[name]["@LEN_CODE@"] = len_code[:-1]
-		fop_subs[name]["@SER_CODE@"] = ser_code[:-1]
-		if 'len' in gen_funcs:
-			print(generate(LEN_TEMPLATE,name,fop_subs))
-		if 'serialize' in gen_funcs:
-			print(generate(SER_TEMPLATE,name,fop_subs))
-		if name == 'writev':
-			print("#define DESTAGE_ASYNC")
-		if 'callback' in gen_funcs:
-			print(generate(CBK_TEMPLATE,name,cbk_subs))
-		if 'continue' in gen_funcs:
-			print(generate(CONTINUE_TEMPLATE,name,fop_subs))
-		if 'fop' in gen_funcs:
-			print(generate(FOP_TEMPLATE,name,fop_subs))
-		if name == 'writev':
-			print("#undef DESTAGE_ASYNC")
-		entrypoints.append(name)
-	print("struct xlator_fops fops = {")
-	for ep in entrypoints:
-		print("\t.%s = fdl_%s," % (ep, ep))
-	print("};")
-
-for l in open(sys.argv[1],'r').readlines():
-	if l.find('#pragma generate') != -1:
-		print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
-		gen_fdl()
-		print("/* END GENERATED CODE */")
-	else:
-		print(l[:-1])
diff --git a/xlators/experimental/fdl/src/gen_recon.py b/xlators/experimental/fdl/src/gen_recon.py
deleted file mode 100755
index 75323ea3dd9..00000000000
--- a/xlators/experimental/fdl/src/gen_recon.py
+++ /dev/null
@@ -1,218 +0,0 @@
-#!/usr/bin/python2
-
-from __future__ import print_function
-import os
-import re
-import string
-import sys
-
-curdir = os.path.dirname (sys.argv[0])
-gendir = os.path.join (curdir, '../../../../libglusterfs/src')
-sys.path.append (gendir)
-from generator import ops, fop_subs, cbk_subs, generate
-
-# See the big header comment at the start of gen_fdl.py to see how the stages
-# fit together.  The big difference here is that *all* of the C code is in the
-# template file as labelled fragments, instead of as Python strings.  That
-# makes it much easier to edit in one place, with proper syntax highlighting
-# and indentation.
-#
-#   Stage 1 uses type-specific fragments to generate FUNCTION_BODY, instead of
-#   LEN_*_TEMPLATE and SERLZ_*_TEMPLATE to generate LEN_CODE and SER_CODE.
-#
-#   Stage 2 uses the FOP and CASE fragments instead of RECON_TEMPLATE and
-#   FOP_TEMPLATE.  The expanded FOP code (including FUNCTION_BODY substitution
-#   in the middle of each function) is emitted immediately; the expanded CASE
-#   code is saved for the next stage.
-#
-#   Stage 3 uses the PROLOG and EPILOG fragments, with the expanded CASE code
-#   in the middle of EPILOG, to generate the whole output file.
-#
-# Another way of looking at it is to consider how the fragments appear in
-# the final output:
-#
-#   PROLOG
-#   FOP (expanded for CREATE)
-#       FOP before FUNCTION_BODY
-#       LOC, INTEGER, GFID, etc. (one per arg, by type)
-#       FOP after FUNCTION_BODY
-#   FOP (expanded for WRITEV)
-#       FOP before FUNCTION_BODY
-#       GFID, VECTOR, etc. (one per arg, by type)
-#       FOP after FUNCTION_BODY
-#   (more FOPs)
-#   EPILOG
-#       EPILOG before CASE
-#       CASE statements (one per fop)
-#       EPILOG after CASE
-
-typemap = {
-	'dict_t *':				"DICT",
-	'fd_t *':				"FD",
-	'dev_t':				"DOUBLE",
-	'gf_xattrop_flags_t':	"INTEGER",
-	'int32_t':				"INTEGER",
-	'mode_t':				"INTEGER",
-	'off_t':				"DOUBLE",
-	'size_t':				"DOUBLE",
-	'uint32_t':				"INTEGER",
-	'loc_t *':				"LOC",
-	'const char *':			"STRING",
-	'struct iovec *':		"VECTOR",
-	'struct iatt *':		"IATT",
-	'struct iobref *':		"IOBREF",
-}
-
-def get_special_subs (name, args, fop_type):
-	code = ""
-	cleanups = ""
-	links = ""
-	s_args = []
-	for arg in args:
-		if arg[0] == 'extra':
-			code += "\t%s %s;\n\n" % (arg[2], arg[1])
-			s_args.append(arg[3])
-			continue
-		if arg[0] == 'link':
-			links += fragments["LINK"].replace("@INODE_ARG@",arg[1])	\
-									  .replace("@IATT_ARG@",arg[2])
-			continue
-		if arg[0] != 'fop-arg':
-			continue
-		if (name, arg[1]) == ('writev', 'count'):
-			# Special case: just skip this.  We can't mark it as 'nosync'
-			# because of the way the translator and dumper generators look for
-			# that after 'stub-name' which we don't define.  Instead of adding a
-			# bunch of generic infrastructure for this one case, just pound it
-			# here.
-			continue
-		recon_type = typemap[arg[2]]
-		# print "/* %s.%s => %s (%s)*/" % (name, arg[1], recon_type, fop_type)
-		if (name == "create") and (arg[1] == "fd"):
-			# Special case: fd for create is new, not looked up.
-			# print "/* change to NEW_FD */"
-			recon_type = "NEW_FD"
-		elif (recon_type == "LOC") and (fop_type == "entry-op"):
-			# Need to treat this differently for inode vs. entry ops.
-			# Special case: link source is treated like inode-op.
-			if (name != "link") or (arg[1] != "oldloc"):
-				# print "/* change to PARENT_LOC */"
-				recon_type = "PARENT_LOC"
-		code += fragments[recon_type].replace("@ARGNAME@",arg[1])		\
-									 .replace("@ARGTYPE@",arg[2])
-		cleanup_key = recon_type + "_CLEANUP"
-		if fragments.has_key(cleanup_key):
-			new_frag = fragments[cleanup_key].replace("@ARGNAME@",arg[1])
-			# Make sure these get added in *reverse* order.  Otherwise, a
-			# failure for an earlier argument might goto a label that falls
-			# through to the cleanup code for a variable associated with a
-			# later argument, but that variable might not even have been
-			# *declared* (let alone initialized) yet.  Consider the following
-			# case.
-			#
-			#         process argument A (on failure goto cleanup_A)
-			#         set error label to cleanup_A
-			#
-			#         declare pointer variable for argument B
-			#         process argument B (on failure goto cleanup_B)
-			#
-			#     cleanup_A:
-			#         /* whatever */
-			#     cleanup_B:
-			#         free pointer variable <= "USED BUT NOT SET" error here
-			#
-			# By adding these in reverse order, we ensure that cleanup_B is
-			# actually *before* cleanup_A, and nothing will try to do the free
-			# until we've actually attempted processing of B.
-			cleanups = new_frag + cleanups
-		if 'nosync' in arg[4:]:
-			code += "\t(void)%s;\n" % arg[1];
-			continue
-		if arg[2] in ("loc_t *", "struct iatt *"):
-			# These are passed as pointers to the syncop, but they're actual
-			# structures in the generated code.
-			s_args.append("&"+arg[1]);
-		else:
-			s_args.append(arg[1])
-	# We have to handle a couple of special cases here, because some n00b
-	# defined the syncops with a different argument order than the fops they're
-	# based on.
-	if name == 'writev':
-		# Swap 'flags' and 'iobref'.  Also, we need to add the iov count, which
-		# is not stored in or read from the journal.  There are other ways to
-		# do that, but this is the only place we need anything similar and we
-		# already have to treat it as a special case so this is simplest.
-               s_args_str = 'fd, &vector, 1, off, iobref, flags, &preop, &postop, xdata'
-	elif name == 'symlink':
-		# Swap 'linkpath' and 'loc'.
-		s_args_str = '&loc, linkpath, &iatt, xdata'
-        elif name == 'xattrop':
-                s_args_str = '&loc, flags, dict, xdata, NULL'
-        elif name == 'fxattrop':
-                s_args_str = 'fd, flags, dict, xdata, NULL'
-	else:
-		s_args_str = string.join (s_args, ", ")
-	return code, links, s_args_str, cleanups
-
-# TBD: probably need to generate type-specific cleanup code as well - e.g.
-# fd_unref for an fd_t, loc_wipe for a loc_t, and so on.  All of these
-# generated CLEANUP fragments will go at the end of the function, with goto
-# labels.  Meanwhile, the error-checking part of each type-specific fragment
-# (e.g. LOC or FD) will need to update the indirect label that we jump to when
-# an error is detected.  This will probably get messy.
-def gen_functions ():
-	code = ""
-	for name, value in ops.iteritems():
-		fop_type = [ x[1] for x in value if x[0] == "journal" ]
-		if not fop_type:
-			continue
-		body, links, syncop_args, cleanups = get_special_subs (name, value,
-															   fop_type[0])
-		fop_subs[name]["@FUNCTION_BODY@"] = body
-		fop_subs[name]["@LINKS@"] = links
-		fop_subs[name]["@SYNCOP_ARGS@"] = syncop_args
-		fop_subs[name]["@CLEANUPS@"] = cleanups
-		if name == "writev":
-			# Take advantage of the fact that, *during reconciliation*, the
-			# vector is always a single element.  In normal I/O it's not.
-			fop_subs[name]["@SUCCESS_VALUE@"] = "vector.iov_len"
-		else:
-			fop_subs[name]["@SUCCESS_VALUE@"] = "GFAPI_SUCCESS"
-		# Print the FOP fragment with @FUNCTION_BODY@ in the middle.
-		code += generate(fragments["FOP"],name,fop_subs)
-	return code
-
-def gen_cases ():
-	code = ""
-	for name, value in ops.iteritems():
-		if "journal" not in [ x[0] for x in value ]:
-			continue
-		# Add the CASE fragment for this fop.
-		code += generate(fragments["CASE"],name,fop_subs)
-	return code
-
-def load_fragments (path="recon-tmpl.c"):
-	pragma_re = re.compile('pragma fragment (.*)')
-	cur_symbol = None
-	cur_value = ""
-	result = {}
-	for line in open(path,"r").readlines():
-		m = pragma_re.search(line)
-		if m:
-			if cur_symbol:
-				result[cur_symbol] = cur_value
-			cur_symbol = m.group(1)
-			cur_value = ""
-		else:
-			cur_value += line
-	if cur_symbol:
-		result[cur_symbol] = cur_value
-	return result
-
-if __name__ == "__main__":
-	fragments = load_fragments(sys.argv[1])
-	print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
-	print(fragments["PROLOG"])
-	print(gen_functions())
-	print(fragments["EPILOG"].replace("@SWITCH_BODY@",gen_cases()))
-	print("/* END GENERATED CODE */")
diff --git a/xlators/experimental/fdl/src/logdump.c b/xlators/experimental/fdl/src/logdump.c
deleted file mode 100644
index 7c979c32a04..00000000000
--- a/xlators/experimental/fdl/src/logdump.c
+++ /dev/null
@@ -1,50 +0,0 @@
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <sys/mman.h>
-
-extern int fdl_dump (char **, char **);
-
-int
-main (int argc, char **argv)
-{
-        int     meta_fd         = (-1);
-        char    *meta_buf       = NULL;
-        int     data_fd         = (-1);
-        char    *data_buf       = NULL;
-
-        meta_fd = open (argv[1], O_RDONLY);
-        if (meta_fd < 0) {
-                perror ("open");
-                return EXIT_FAILURE;
-        }
-
-        /* TBD: get proper length */
-        meta_buf = mmap (NULL, 1048576, PROT_READ, MAP_PRIVATE, meta_fd, 0);
-        if (meta_buf == MAP_FAILED) {
-                perror ("mmap");
-                return EXIT_FAILURE;
-        }
-
-        data_fd = open (argv[2], O_RDONLY);
-        if (data_fd < 0) {
-                perror ("open");
-                return EXIT_FAILURE;
-        }
-
-        /* TBD: get proper length */
-        data_buf = mmap (NULL, 1048576, PROT_READ, MAP_PRIVATE, data_fd, 0);
-        if (data_buf == MAP_FAILED) {
-                perror ("mmap");
-                return EXIT_FAILURE;
-        }
-
-        for (;;) {
-                if (!fdl_dump(&meta_buf,&data_buf)) {
-                        break;
-                }
-        }
-
-        return EXIT_SUCCESS;
-}
diff --git a/xlators/experimental/fdl/src/recon-tmpl.c b/xlators/experimental/fdl/src/recon-tmpl.c
deleted file mode 100644
index 228860401ae..00000000000
--- a/xlators/experimental/fdl/src/recon-tmpl.c
+++ /dev/null
@@ -1,304 +0,0 @@
-#pragma fragment PROLOG
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "iatt.h"
-#include "syncop.h"
-#include "xlator.h"
-#include "glfs-internal.h"
-
-#include "fdl.h"
-
-#define GFAPI_SUCCESS 0
-
-inode_t *
-recon_get_inode (glfs_t *fs, uuid_t gfid)
-{
-        inode_t         *inode;
-        loc_t           loc     = {NULL,};
-        struct iatt     iatt;
-        int             ret;
-        inode_t         *newinode;
-
-        inode = inode_find (fs->active_subvol->itable, gfid);
-        if (inode) {
-                printf ("=== FOUND %s IN TABLE\n", uuid_utoa(gfid));
-                return inode;
-        }
-
-        loc.inode = inode_new (fs->active_subvol->itable);
-        if (!loc.inode) {
-                return NULL;
-        }
-        gf_uuid_copy (loc.inode->gfid, gfid);
-        gf_uuid_copy (loc.gfid, gfid);
-
-        printf ("=== DOING LOOKUP FOR %s\n", uuid_utoa(gfid));
-
-        ret = syncop_lookup (fs->active_subvol, &loc, &iatt,
-                             NULL, NULL, NULL);
-        if (ret != GFAPI_SUCCESS) {
-                fprintf (stderr, "syncop_lookup failed (%d)\n", ret);
-                return NULL;
-        }
-
-        newinode = inode_link (loc.inode, NULL, NULL, &iatt);
-        if (newinode) {
-                inode_lookup (newinode);
-        }
-
-        return newinode;
-}
-
-#pragma fragment DICT
-        dict_t  *@ARGNAME@;
-
-        @ARGNAME@ = dict_new();
-        if (!@ARGNAME@) {
-                goto *err_label;
-        }
-        err_label = &&cleanup_@ARGNAME@;
-
-        {
-                int     key_len, data_len;
-                char    *key_ptr;
-                int     garbage;
-                for (;;) {
-                        key_len = *((int *)new_meta);
-                        new_meta += sizeof(int);
-                        if (!key_len) {
-                                break;
-                        }
-                        key_ptr = new_meta;
-                        new_meta += key_len;
-                        data_len = *((int *)new_meta);
-                        new_meta += sizeof(int);
-                        garbage = dict_set_static_bin (@ARGNAME@, key_ptr,
-                                                       new_meta, data_len);
-                        /* TBD: check error from dict_set_static_bin */
-                        (void)garbage;
-                        new_meta += data_len;
-                }
-        }
-
-#pragma fragment DICT_CLEANUP
-cleanup_@ARGNAME@:
-        dict_unref (@ARGNAME@);
-
-#pragma fragment DOUBLE
-        @ARGTYPE@       @ARGNAME@       = *((@ARGTYPE@ *)new_meta);
-        new_meta += sizeof(uint64_t);
-
-#pragma fragment FD
-        inode_t *@ARGNAME@_ino;
-        fd_t    *@ARGNAME@;
-
-        @ARGNAME@_ino = recon_get_inode (fs, *((uuid_t *)new_meta));
-        new_meta += 16;
-        if (!@ARGNAME@_ino) {
-                goto *err_label;
-        }
-        err_label = &&cleanup_@ARGNAME@_ino;
-
-        @ARGNAME@ = fd_anonymous (@ARGNAME@_ino);
-        if (!@ARGNAME@) {
-                goto *err_label;
-        }
-        err_label = &&cleanup_@ARGNAME@;
-
-#pragma fragment FD_CLEANUP
-cleanup_@ARGNAME@:
-        fd_unref (@ARGNAME@);
-cleanup_@ARGNAME@_ino:
-        inode_unref (@ARGNAME@_ino);
-
-#pragma fragment NEW_FD
-        /*
-         * This pseudo-type is only used for create, and in that case we know
-         * we'll be using loc.inode, so it's not worth generalizing to take an
-         * extra argument.
-         */
-        fd_t    *@ARGNAME@      = fd_anonymous (loc.inode);
-
-        if (!fd) {
-                goto *err_label;
-        }
-        err_label = &&cleanup_@ARGNAME@;
-        new_meta += 16;
-
-#pragma fragment NEW_FD_CLEANUP
-cleanup_@ARGNAME@:
-        fd_unref (@ARGNAME@);
-
-#pragma fragment INTEGER
-        @ARGTYPE@       @ARGNAME@       = *((@ARGTYPE@ *)new_meta);
-
-        new_meta += sizeof(@ARGTYPE@);
-
-#pragma fragment LOC
-        loc_t           @ARGNAME@       = { NULL, };
-
-        @ARGNAME@.inode = recon_get_inode (fs, *((uuid_t *)new_meta));
-        if (!@ARGNAME@.inode) {
-                goto *err_label;
-        }
-        err_label = &&cleanup_@ARGNAME@;
-        gf_uuid_copy (@ARGNAME@.gfid, @ARGNAME@.inode->gfid);
-        new_meta += 16;
-        new_meta += 16; /* skip over pargfid */
-        if (*(new_meta++)) {
-                @ARGNAME@.name = new_meta;
-                new_meta += strlen(new_meta) + 1;
-        }
-
-#pragma fragment LOC_CLEANUP
-cleanup_@ARGNAME@:
-        loc_wipe (&@ARGNAME@);
-
-#pragma fragment PARENT_LOC
-        loc_t           @ARGNAME@       = { NULL, };
-
-        new_meta += 16; /* skip over gfid */
-        @ARGNAME@.parent = recon_get_inode (fs, *((uuid_t *)new_meta));
-        if (!@ARGNAME@.parent) {
-                goto *err_label;
-        }
-        err_label = &&cleanup_@ARGNAME@;
-        gf_uuid_copy (@ARGNAME@.pargfid, @ARGNAME@.parent->gfid);
-        new_meta += 16;
-        if (!*(new_meta++)) {
-                goto *err_label;
-        }
-        @ARGNAME@.name = new_meta;
-        new_meta += strlen(new_meta) + 1;
-
-        @ARGNAME@.inode = inode_new (fs->active_subvol->itable);
-        if (!@ARGNAME@.inode) {
-                goto *err_label;
-        }
-
-#pragma fragment PARENT_LOC_CLEANUP
-cleanup_@ARGNAME@:
-        loc_wipe (&@ARGNAME@);
-
-#pragma fragment STRING
-        char    *@ARGNAME@;
-        if (*(new_meta++)) {
-                @ARGNAME@ = new_meta;
-                new_meta += (strlen(new_meta) + 1);
-        }
-        else {
-                goto *err_label;
-        }
-
-#pragma fragment VECTOR
-        struct iovec    @ARGNAME@;
-
-        @ARGNAME@.iov_len = *((size_t *)new_meta);
-        new_meta += sizeof(@ARGNAME@.iov_len);
-        @ARGNAME@.iov_base = new_data;
-        new_data += @ARGNAME@.iov_len;
-
-#pragma fragment IATT
-        struct iatt     @ARGNAME@;
-        {
-                @ARGNAME@.ia_prot = *((ia_prot_t *)new_meta);
-                new_meta += sizeof(ia_prot_t);
-                uint32_t *myints = (uint32_t *)new_meta;
-                @ARGNAME@.ia_uid = myints[0];
-                @ARGNAME@.ia_gid = myints[1];
-                @ARGNAME@.ia_atime = myints[2];
-                @ARGNAME@.ia_atime_nsec = myints[3];
-                @ARGNAME@.ia_mtime = myints[4];
-                @ARGNAME@.ia_mtime_nsec = myints[5];
-                new_meta += sizeof(*myints) * 6;
-        }
-
-#pragma fragment IOBREF
-        struct iobref   *@ARGNAME@;
-
-        @ARGNAME@ = iobref_new();
-        if (!@ARGNAME@) {
-                goto *err_label;
-        }
-        err_label = &&cleanup_@ARGNAME@;
-
-#pragma fragment IOBREF_CLEANUP
-cleanup_@ARGNAME@:
-        iobref_unref (@ARGNAME@);
-
-#pragma fragment LINK
-        /* TBD: check error */
-        inode_t *new_inode = inode_link (@INODE_ARG@, NULL, NULL, @IATT_ARG@);
-        if (new_inode) {
-                inode_lookup (new_inode);
-        }
-
-#pragma fragment FOP
-int
-fdl_replay_@NAME@ (glfs_t *fs, char **old_meta, char **old_data)
-{
-        char    *new_meta	= *old_meta;
-        char	*new_data	= *old_data;
-        int     ret;
-        int     status          = 0xbad;
-        void    *err_label      = &&done;
-
-@FUNCTION_BODY@
-
-        ret = syncop_@NAME@ (fs->active_subvol, @SYNCOP_ARGS@, NULL);
-        if (ret != @SUCCESS_VALUE@) {
-                fprintf (stderr, "syncop_@NAME@ returned %d", ret);
-                goto *err_label;
-        }
-
-@LINKS@
-
-        status = 0;
-
-@CLEANUPS@
-
-done:
-        *old_meta = new_meta;
-        *old_data = new_data;
-        return status;
-}
-
-#pragma fragment CASE
-        case GF_FOP_@UPNAME@:
-                printf ("=== GF_FOP_@UPNAME@\n");
-                if (fdl_replay_@NAME@ (fs, &new_meta, &new_data) != 0) {
-                        goto done;
-                }
-                recognized = 1;
-                break;
-
-#pragma fragment EPILOG
-int
-recon_execute (glfs_t *fs, char **old_meta, char **old_data)
-{
-        char            *new_meta       = *old_meta;
-        char            *new_data       = *old_data;
-        int             recognized      = 0;
-        event_header_t  *eh;
-
-        eh = (event_header_t *)new_meta;
-        new_meta += sizeof (*eh);
-
-        /* TBD: check event_type instead of assuming NEW_REQUEST */
-
-        switch (eh->fop_type) {
-@SWITCH_BODY@
-
-        default:
-                printf ("unknown fop %u\n", eh->fop_type);
-        }
-
-done:
-        *old_meta = new_meta;
-        *old_data = new_data;
-        return recognized;
-}
diff --git a/xlators/experimental/fdl/src/recon.c b/xlators/experimental/fdl/src/recon.c
deleted file mode 100644
index 14168a011e0..00000000000
--- a/xlators/experimental/fdl/src/recon.c
+++ /dev/null
@@ -1,89 +0,0 @@
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <sys/mman.h>
-
-#include "glusterfs.h"
-#include "fd.h"
-#include "syncop.h"
-#include "glfs-internal.h"
-
-#define GFAPI_SUCCESS 0
-
-extern int recon_execute (glfs_t *, char **, char **);
-
-int
-main (int argc, char **argv)
-{
-        glfs_t  *fs;
-        int     ret;
-        int     meta_fd         = (-1);
-        char    *meta_buf       = NULL;
-        int     data_fd         = (-1);
-        char    *data_buf       = NULL;
-
-        fs = glfs_new ("whocares");
-        if (!fs) {
-                fprintf (stderr, "glfs_new failed\n");
-                return EXIT_FAILURE;
-        }
-
-        if (getenv("RECON_DEBUG")) {
-                ret = glfs_set_logging (fs, "/dev/stderr", 7);
-        }
-        else {
-                ret = glfs_set_logging (fs, "/dev/null", 0);
-        }
-
-        if (ret != GFAPI_SUCCESS) {
-                fprintf (stderr, "glfs_set_logging failed (%d)\n", errno);
-                return EXIT_FAILURE;
-        }
-
-        ret = glfs_set_volfile (fs, argv[1]);
-        if (ret != GFAPI_SUCCESS) {
-                fprintf (stderr, "glfs_set_volfile failed (%d)\n", errno);
-                return EXIT_FAILURE;
-        }
-
-        ret = glfs_init (fs);
-        if (ret != GFAPI_SUCCESS) {
-                fprintf (stderr, "glfs_init failed (%d)\n", errno);
-                return EXIT_FAILURE;
-        }
-
-        meta_fd = open (argv[2], O_RDONLY);
-        if (meta_fd < 0) {
-                perror ("open");
-                return EXIT_FAILURE;
-        }
-
-        /* TBD: get proper length */
-        meta_buf = mmap (NULL, 1048576, PROT_READ, MAP_PRIVATE, meta_fd, 0);
-        if (meta_buf == MAP_FAILED) {
-                perror ("mmap");
-                return EXIT_FAILURE;
-        }
-
-        data_fd = open (argv[3], O_RDONLY);
-        if (data_fd < 0) {
-                perror ("open");
-                return EXIT_FAILURE;
-        }
-
-        /* TBD: get proper length */
-        data_buf = mmap (NULL, 1048576, PROT_READ, MAP_PRIVATE, data_fd, 0);
-        if (data_buf == MAP_FAILED) {
-                perror ("mmap");
-                return EXIT_FAILURE;
-        }
-
-        for (;;) {
-                if (!recon_execute(fs,&meta_buf,&data_buf)) {
-                        break;
-                }
-        }
-
-        return EXIT_SUCCESS;
-}
diff --git a/xlators/experimental/jbr-client/src/Makefile.am b/xlators/experimental/jbr-client/src/Makefile.am
deleted file mode 100644
index a894e69c8d7..00000000000
--- a/xlators/experimental/jbr-client/src/Makefile.am
+++ /dev/null
@@ -1,34 +0,0 @@
-if WITH_SERVER
-xlator_LTLIBRARIES = jbrc.la
-endif
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
-
-nodist_jbrc_la_SOURCES = jbrc-cg.c
-CLEANFILES = $(nodist_jbrc_la_SOURCES)
-
-jbrc_la_LDFLAGS = -module -avoid-version
-jbrc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = $(top_srcdir)/xlators/lib/src/libxlator.h \
-	$(top_srcdir)/glusterfsd/src/glusterfsd.h \
-	jbrc.h jbr-messages.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/xlators/lib/src \
-	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
-	-I$(top_srcdir)/rpc/rpc-lib/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-JBRC_PREFIX	= $(top_srcdir)/xlators/experimental/jbr-client/src
-JBRC_GEN_FOPS	= $(JBRC_PREFIX)/gen-fops.py
-JBRC_TEMPLATES	= $(JBRC_PREFIX)/fop-template.c
-JBRC_WRAPPER	= $(JBRC_PREFIX)/jbrc.c
-noinst_PYTHON	= $(JBRC_GEN_FOPS)
-EXTRA_DIST	= $(JBRC_TEMPLATES) $(JBRC_WRAPPER)
-
-jbrc-cg.c: $(JBRC_GEN_FOPS) $(JBRC_TEMPLATES) $(JBRC_WRAPPER)
-	$(PYTHON) $(JBRC_GEN_FOPS) $(JBRC_TEMPLATES) $(JBRC_WRAPPER) > $@
-
-uninstall-local:
-	rm -f $(DESTDIR)$(xlatordir)/jbr.so
diff --git a/xlators/experimental/jbr-client/src/fop-template.c b/xlators/experimental/jbr-client/src/fop-template.c
deleted file mode 100644
index 7719f511f01..00000000000
--- a/xlators/experimental/jbr-client/src/fop-template.c
+++ /dev/null
@@ -1,113 +0,0 @@
-/* template-name fop */
-int32_t
-jbrc_@NAME@ (call_frame_t *frame, xlator_t *this,
-             @LONG_ARGS@)
-{
-        jbrc_local_t    *local          = NULL;
-        xlator_t        *target_xl      = ACTIVE_CHILD(this);
-
-        local = mem_get(this->local_pool);
-        if (!local) {
-                goto err;
-        }
-
-        local->stub = fop_@NAME@_stub (frame, jbrc_@NAME@_continue,
-                                       @SHORT_ARGS@);
-        if (!local->stub) {
-                goto err;
-        }
-        local->curr_xl = target_xl;
-        local->scars = 0;
-
-        frame->local = local;
-        STACK_WIND_COOKIE (frame, jbrc_@NAME@_cbk, target_xl,
-                    target_xl, target_xl->fops->@NAME@,
-                    @SHORT_ARGS@);
-        return 0;
-
-err:
-        if (local) {
-                mem_put(local);
-        }
-        STACK_UNWIND_STRICT (@NAME@, frame, -1, ENOMEM,
-                             @ERROR_ARGS@);
-        return 0;
-}
-
-/* template-name cbk */
-int32_t
-jbrc_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 @LONG_ARGS@)
-{
-        jbrc_local_t    *local          = frame->local;
-        xlator_t        *last_xl        = cookie;
-        xlator_t        *next_xl;
-        jbrc_private_t  *priv           = this->private;
-        struct timespec spec;
-
-        if (op_ret != (-1)) {
-                if (local->scars) {
-                        gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_RETRY_MSG,
-                                HILITE("retried %p OK"), frame->local);
-                }
-                priv->active = last_xl;
-                goto unwind;
-        }
-        if ((op_errno != EREMOTE) && (op_errno != ENOTCONN)) {
-                goto unwind;
-        }
-
-        /* TBD: get leader ID from xdata? */
-        next_xl = next_xlator(this, last_xl);
-        /*
-         * We can't just give up after we've tried all bricks, because it's
-         * quite likely that a new leader election just hasn't finished yet.
-         * We also shouldn't retry endlessly, and especially not at a high
-         * rate, but that's good enough while we work on other things.
-         *
-         * TBD: implement slow/finite retry via a worker thread
-         */
-        if (!next_xl || (local->scars >= SCAR_LIMIT)) {
-                gf_msg (this->name, GF_LOG_DEBUG, 0, J_MSG_RETRY_MSG,
-                        HILITE("ran out of retries for %p"), frame->local);
-                goto unwind;
-        }
-
-        local->curr_xl = next_xl;
-        local->scars += 1;
-        spec.tv_sec = 1;
-        spec.tv_nsec = 0;
-        /*
-         * WARNING
-         *
-         * Just calling gf_timer_call_after like this leaves open the
-         * possibility that writes will get reordered, if a first write is
-         * rescheduled and then a second comes along to find an updated
-         * priv->active before the first actually executes.  We might need to
-         * implement a stricter (and more complicated) queuing mechanism to
-         * ensure absolute consistency in this case.
-         */
-        if (gf_timer_call_after(this->ctx, spec, jbrc_retry_cb, local)) {
-                return 0;
-        }
-
-unwind:
-        call_stub_destroy(local->stub);
-        STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno,
-                                     @SHORT_ARGS@);
-        return 0;
-}
-
-/* template-name cont-func */
-int32_t
-jbrc_@NAME@_continue (call_frame_t *frame, xlator_t *this,
-                      @LONG_ARGS@)
-{
-        jbrc_local_t    *local  = frame->local;
-
-        STACK_WIND_COOKIE (frame, jbrc_@NAME@_cbk, local->curr_xl,
-                           local->curr_xl, local->curr_xl->fops->@NAME@,
-                           @SHORT_ARGS@);
-        return 0;
-}
diff --git a/xlators/experimental/jbr-client/src/gen-fops.py b/xlators/experimental/jbr-client/src/gen-fops.py
deleted file mode 100755
index 9893e0c5968..00000000000
--- a/xlators/experimental/jbr-client/src/gen-fops.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/python2
-
-from __future__ import print_function
-import os
-import re
-import string
-import sys
-
-curdir = os.path.dirname(sys.argv[0])
-gendir = os.path.join(curdir,'../../../../libglusterfs/src')
-sys.path.append(gendir)
-from generator import ops, fop_subs, cbk_subs, generate
-
-# We really want the callback argument list, even when we're generating fop
-# code, so we propagate here.
-# TBD: this should probably be right in generate.py
-for k, v in cbk_subs.iteritems():
-	fop_subs[k]['@ERROR_ARGS@'] = v['@ERROR_ARGS@']
-
-# Stolen from old codegen.py
-def load_templates (path):
-	templates = {}
-	tmpl_re = re.compile("/\* template-name (.*) \*/")
-	templates = {}
-	t_name = None
-	for line in open(path,"r").readlines():
-		if not line:
-			break
-		m = tmpl_re.match(line)
-		if m:
-			if t_name:
-				templates[t_name] = string.join(t_contents,'')
-			t_name = m.group(1).strip()
-			t_contents = []
-		elif t_name:
-			t_contents.append(line)
-	if t_name:
-		templates[t_name] = string.join(t_contents,'')
-	return templates
-
-# Stolen from gen_fdl.py
-def gen_client (templates):
-	for name, value in ops.iteritems():
-		if name == 'getspec':
-			# It's not real if it doesn't have a stub function.
-			continue
-		print(generate(templates['cbk'],name,cbk_subs))
-		print(generate(templates['cont-func'],name,fop_subs))
-		print(generate(templates['fop'],name,fop_subs))
-
-tmpl = load_templates(sys.argv[1])
-for l in open(sys.argv[2],'r').readlines():
-	if l.find('#pragma generate') != -1:
-		print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
-		gen_client(tmpl)
-		print("/* END GENERATED CODE */")
-	else:
-		print(l[:-1])
diff --git a/xlators/experimental/jbr-client/src/jbr-messages.h b/xlators/experimental/jbr-client/src/jbr-messages.h
deleted file mode 100644
index 9f210184d81..00000000000
--- a/xlators/experimental/jbr-client/src/jbr-messages.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _JBR_MESSAGES_H_
-#define _JBR_MESSAGES_H_
-
-#include "glfs-message-id.h"
-
-/* To add new message IDs, append new identifiers at the end of the list.
- *
- * Never remove a message ID. If it's not used anymore, you can rename it or
- * leave it as it is, but not delete it. This is to prevent reutilization of
- * IDs by other messages.
- *
- * The component name must match one of the entries defined in
- * glfs-message-id.h.
- */
-
-GLFS_MSGID(JBR,
-        J_MSG_INIT_FAIL,
-        J_MSG_RETRY_MSG,
-        J_MSG_MEM_ERR,
-        J_MSG_DICT_FLR,
-        J_MSG_GENERIC,
-        J_MSG_INVALID,
-        J_MSG_NO_DATA,
-        J_MSG_SYS_CALL_FAILURE,
-        J_MSG_QUORUM_NOT_MET,
-        J_MSG_LOCK_FAILURE
-);
-
-#endif /* _JBR_MESSAGES_H_ */
diff --git a/xlators/experimental/jbr-client/src/jbrc.c b/xlators/experimental/jbr-client/src/jbrc.c
deleted file mode 100644
index 9bb9346c5c0..00000000000
--- a/xlators/experimental/jbr-client/src/jbrc.c
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "call-stub.h"
-#include "defaults.h"
-#include "timer.h"
-#include "xlator.h"
-#include "jbr-messages.h"
-#include "jbrc.h"
-#include "statedump.h"
-
-#define SCAR_LIMIT      20
-#define HILITE(x)       ("[1;33m"x"[0m")
-
-/*
- * The fops are actually generated by gen-fops.py; the rest was mostly copied
- * from defaults.c (commit cd253754 on 27 August 2013).
- */
-
-enum gf_dht_mem_types_ {
-        gf_mt_jbrc_private_t = gf_common_mt_end + 1,
-        gf_mt_jbrc_end
-};
-
-char            *JBRC_XATTR     = "user.jbr.active";
-
-static inline
-xlator_t *
-ACTIVE_CHILD (xlator_t *parent)
-{
-        jbrc_private_t  *priv   = parent->private;
-
-        return priv ? priv->active : FIRST_CHILD(parent);
-}
-
-xlator_t *
-next_xlator (xlator_t *this, xlator_t *prev)
-{
-        xlator_list_t   *trav;
-
-        for (trav = this->children; trav; trav = trav->next) {
-                if (trav->xlator == prev) {
-                        return trav->next ? trav->next->xlator
-                                          : this->children->xlator;
-                }
-        }
-
-        return NULL;
-}
-
-void
-jbrc_retry_cb (void *cb_arg)
-{
-        jbrc_local_t    *local  = cb_arg;
-
-        gf_msg (__func__, GF_LOG_INFO, 0, J_MSG_RETRY_MSG,
-                HILITE("retrying %p"), local);
-        call_resume_wind(local->stub);
-}
-
-#pragma generate
-
-int32_t
-jbrc_forget (xlator_t *this, inode_t *inode)
-{
-        gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, J_MSG_INIT_FAIL,
-                          "xlator does not implement forget_cbk");
-        return 0;
-}
-
-
-int32_t
-jbrc_releasedir (xlator_t *this, fd_t *fd)
-{
-        gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, J_MSG_INIT_FAIL,
-                          "xlator does not implement releasedir_cbk");
-        return 0;
-}
-
-int32_t
-jbrc_release (xlator_t *this, fd_t *fd)
-{
-        gf_msg_callingfn (this->name, GF_LOG_WARNING, 0, J_MSG_INIT_FAIL,
-                          "xlator does not implement release_cbk");
-        return 0;
-}
-
-struct xlator_fops fops = {
-        .lookup         = jbrc_lookup,
-        .stat           = jbrc_stat,
-        .fstat          = jbrc_fstat,
-        .truncate       = jbrc_truncate,
-        .ftruncate      = jbrc_ftruncate,
-        .access         = jbrc_access,
-        .readlink       = jbrc_readlink,
-        .mknod          = jbrc_mknod,
-        .mkdir          = jbrc_mkdir,
-        .unlink         = jbrc_unlink,
-        .rmdir          = jbrc_rmdir,
-        .symlink        = jbrc_symlink,
-        .rename         = jbrc_rename,
-        .link           = jbrc_link,
-        .create         = jbrc_create,
-        .open           = jbrc_open,
-        .readv          = jbrc_readv,
-        .writev         = jbrc_writev,
-        .flush          = jbrc_flush,
-        .fsync          = jbrc_fsync,
-        .opendir        = jbrc_opendir,
-        .readdir        = jbrc_readdir,
-        .readdirp       = jbrc_readdirp,
-        .fsyncdir       = jbrc_fsyncdir,
-        .statfs         = jbrc_statfs,
-        .setxattr       = jbrc_setxattr,
-        .getxattr       = jbrc_getxattr,
-        .fsetxattr      = jbrc_fsetxattr,
-        .fgetxattr      = jbrc_fgetxattr,
-        .removexattr    = jbrc_removexattr,
-        .fremovexattr   = jbrc_fremovexattr,
-        .lk             = jbrc_lk,
-        .inodelk        = jbrc_inodelk,
-        .finodelk       = jbrc_finodelk,
-        .entrylk        = jbrc_entrylk,
-        .fentrylk       = jbrc_fentrylk,
-        .rchecksum      = jbrc_rchecksum,
-        .xattrop        = jbrc_xattrop,
-        .fxattrop       = jbrc_fxattrop,
-        .setattr        = jbrc_setattr,
-        .fsetattr       = jbrc_fsetattr,
-	.fallocate	= jbrc_fallocate,
-	.discard        = jbrc_discard,
-};
-
-struct xlator_cbks cbks = {
-};
-
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("jbrc", this, out);
-
-        ret = xlator_mem_acct_init (this, gf_mt_jbrc_end + 1);
-
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, J_MSG_MEM_ERR,
-                        "Memory accounting init failed");
-                return ret;
-        }
-out:
-        return ret;
-}
-
-
-int32_t
-jbrc_init (xlator_t *this)
-{
-        jbrc_private_t  *priv   = NULL;
-        xlator_list_t   *trav   = NULL;
-
-        this->local_pool = mem_pool_new (jbrc_local_t, 128);
-        if (!this->local_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, J_MSG_MEM_ERR,
-                        "failed to create jbrc_local_t pool");
-                goto err;
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_mt_jbrc_private_t);
-        if (!priv) {
-                goto err;
-        }
-
-        for (trav = this->children; trav; trav = trav->next) {
-                ++(priv->n_children);
-        }
-
-        priv->active = FIRST_CHILD(this);
-        this->private = priv;
-        return 0;
-
-err:
-        if (priv) {
-                GF_FREE(priv);
-        }
-        return -1;
-}
-
-void
-jbrc_fini (xlator_t *this)
-{
-        GF_FREE(this->private);
-}
-
-int
-jbrc_get_child_index (xlator_t *this, xlator_t *kid)
-{
-        xlator_list_t   *trav;
-        int             retval = -1;
-
-        for (trav = this->children; trav; trav = trav->next) {
-                ++retval;
-                if (trav->xlator == kid) {
-                        return retval;
-                }
-        }
-
-        return -1;
-}
-
-uint8_t
-jbrc_count_up_kids (jbrc_private_t *priv)
-{
-        uint8_t         retval  = 0;
-        uint8_t         i;
-
-        for (i = 0; i < priv->n_children; ++i) {
-                if (priv->kid_state & (1 << i)) {
-                        ++retval;
-                }
-        }
-
-        return retval;
-}
-
-int32_t
-jbrc_notify (xlator_t *this, int32_t event, void *data, ...)
-{
-        int32_t           ret        = 0;
-        int32_t           index      = 0;
-        jbrc_private_t   *priv       = NULL;
-
-        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        switch (event) {
-        case GF_EVENT_CHILD_UP:
-                index = jbrc_get_child_index(this, data);
-                if (index >= 0) {
-                        priv->kid_state |= (1 << index);
-                        priv->up_children = jbrc_count_up_kids(priv);
-                        gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_GENERIC,
-                                "got CHILD_UP for %s, now %u kids",
-                                ((xlator_t *)data)->name,
-                                priv->up_children);
-                }
-                ret = default_notify (this, event, data);
-                break;
-        case GF_EVENT_CHILD_DOWN:
-                index = jbrc_get_child_index(this, data);
-                if (index >= 0) {
-                        priv->kid_state &= ~(1 << index);
-                        priv->up_children = jbrc_count_up_kids(priv);
-                        gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_GENERIC,
-                                "got CHILD_DOWN for %s, now %u kids",
-                                ((xlator_t *)data)->name,
-                                priv->up_children);
-                }
-                break;
-        default:
-                ret = default_notify (this, event, data);
-        }
-
-out:
-        return ret;
-}
-
-int
-jbrc_priv_dump (xlator_t *this)
-{
-        jbrc_private_t     *priv = NULL;
-        char                key_prefix[GF_DUMP_MAX_BUF_LEN];
-        xlator_list_t      *trav = NULL;
-        int32_t             i    = -1;
-
-        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s",
-                 this->type, this->name);
-        gf_proc_dump_add_section(key_prefix);
-
-        gf_proc_dump_write("up_children", "%u", priv->up_children);
-
-        for (trav = this->children, i = 0; trav; trav = trav->next, i++) {
-                snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "child_%d", i);
-                gf_proc_dump_write(key_prefix, "%s", trav->xlator->name);
-        }
-
-out:
-        return 0;
-}
-
-struct xlator_dumpops dumpops = {
-        .priv       = jbrc_priv_dump,
-};
-
-class_methods_t class_methods = {
-        .init           = jbrc_init,
-        .fini           = jbrc_fini,
-        .notify         = jbrc_notify,
-};
-
-struct volume_options options[] = {
-	{ .key = {NULL} },
-};
diff --git a/xlators/experimental/jbr-client/src/jbrc.h b/xlators/experimental/jbr-client/src/jbrc.h
deleted file mode 100644
index c83259ca1bd..00000000000
--- a/xlators/experimental/jbr-client/src/jbrc.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _JBRC_H_
-#define _JBRC_H_
-
-typedef struct {
-        xlator_t        *active;
-        uint8_t          up_children;
-        uint8_t          n_children;
-        uint32_t         kid_state;
-} jbrc_private_t;
-
-typedef struct {
-        call_stub_t     *stub;
-        xlator_t        *curr_xl;
-        uint16_t        scars;
-} jbrc_local_t;
-
-#endif /* _JBRC_H_ */
diff --git a/xlators/experimental/jbr-server/Makefile.am b/xlators/experimental/jbr-server/Makefile.am
deleted file mode 100644
index a985f42a877..00000000000
--- a/xlators/experimental/jbr-server/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES =
diff --git a/xlators/experimental/jbr-server/src/Makefile.am b/xlators/experimental/jbr-server/src/Makefile.am
deleted file mode 100644
index fe1342dbaff..00000000000
--- a/xlators/experimental/jbr-server/src/Makefile.am
+++ /dev/null
@@ -1,39 +0,0 @@
-if WITH_SERVER
-xlator_LTLIBRARIES = jbr.la
-endif
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
-
-nodist_jbr_la_SOURCES = jbr-cg.c
-CLEANFILES = $(nodist_jbr_la_SOURCES)
-
-jbr_la_LDFLAGS = -module -avoid-version
-jbr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-		$(top_builddir)/api/src/libgfapi.la
-
-noinst_HEADERS = jbr-internal.h \
-	$(top_srcdir)/xlators/lib/src/libxlator.h \
-	$(top_srcdir)/xlators/experimental/fdl/src/fdl.h \
-	$(top_srcdir)/glusterfsd/src/glusterfsd.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
-	-I$(top_srcdir)/xlators/lib/src -I$(top_srcdir)/rpc/rpc-lib/src \
-	-I$(top_srcdir)/xlators/experimental/fdl/src/ \
-	-DSBIN_DIR=\"$(sbindir)\" -I$(top_srcdir)/api/src \
-	-DJBR_SCRIPT_PREFIX=\"$(jbrdir)\" \
-	-I$(top_srcdir)/xlators/experimental/jbr-client/src/
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-JBR_PREFIX	= $(top_srcdir)/xlators/experimental/jbr-server/src
-JBR_GEN_FOPS	= $(JBR_PREFIX)/gen-fops.py
-JBR_TEMPLATES	= $(JBR_PREFIX)/all-templates.c
-JBR_WRAPPER	= $(JBR_PREFIX)/jbr.c
-noinst_PYTHON	= $(JBR_GEN_FOPS)
-EXTRA_DIST	= $(JBR_TEMPLATES) $(JBR_WRAPPER)
-
-jbr-cg.c: $(JBR_GEN_FOPS) $(JBR_TEMPLATES) $(JBR_WRAPPER)
-	$(PYTHON) $(JBR_GEN_FOPS) $(JBR_TEMPLATES) $(JBR_WRAPPER) > $@
-
-uninstall-local:
-	rm -f $(DESTDIR)$(xlatordir)/jbr.so
diff --git a/xlators/experimental/jbr-server/src/all-templates.c b/xlators/experimental/jbr-server/src/all-templates.c
deleted file mode 100644
index 9720442e63f..00000000000
--- a/xlators/experimental/jbr-server/src/all-templates.c
+++ /dev/null
@@ -1,542 +0,0 @@
-/*
- * You can put anything here - it doesn't even have to be a comment - and it
- * will be ignored until we reach the first template-name comment.
- */
-
-
-/* template-name read-fop */
-int32_t
-jbr_@NAME@ (call_frame_t *frame, xlator_t *this,
-            @LONG_ARGS@)
-{
-        jbr_private_t   *priv     = NULL;
-        gf_boolean_t     in_recon = _gf_false;
-        int32_t          op_errno = 0;
-        int32_t          recon_term, recon_index;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, err);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, err);
-        GF_VALIDATE_OR_GOTO (this->name, frame, err);
-
-        op_errno = EREMOTE;
-
-        /* allow reads during reconciliation       *
-         * TBD: allow "dirty" reads on non-leaders *
-         */
-        if (xdata &&
-            (dict_get_int32(xdata, RECON_TERM_XATTR, &recon_term) == 0) &&
-            (dict_get_int32(xdata, RECON_INDEX_XATTR, &recon_index) == 0)) {
-                in_recon = _gf_true;
-        }
-
-        if ((!priv->leader) && (in_recon == _gf_false)) {
-                goto err;
-        }
-
-        STACK_WIND (frame, default_@NAME@_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
-                    @SHORT_ARGS@);
-        return 0;
-
-err:
-        STACK_UNWIND_STRICT (@NAME@, frame, -1, op_errno,
-                             @ERROR_ARGS@);
-        return 0;
-}
-
-/* template-name read-perform_local_op */
-/* No "perform_local_op" function needed for @NAME@ */
-
-/* template-name read-dispatch */
-/* No "dispatch" function needed for @NAME@ */
-
-/* template-name read-call_dispatch */
-/* No "call_dispatch" function needed for @NAME@ */
-
-/* template-name read-fan-in */
-/* No "fan-in" function needed for @NAME@ */
-
-/* template-name read-continue */
-/* No "continue" function needed for @NAME@ */
-
-/* template-name read-complete */
-/* No "complete" function needed for @NAME@ */
-
-/* template-name write-fop */
-int32_t
-jbr_@NAME@ (call_frame_t *frame, xlator_t *this,
-            @LONG_ARGS@)
-{
-        jbr_local_t     *local         = NULL;
-        jbr_private_t   *priv          = NULL;
-        int32_t          ret           = -1;
-        int              op_errno      = ENOMEM;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, err);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, err);
-        GF_VALIDATE_OR_GOTO (this->name, frame, err);
-
-#if defined(JBR_CG_NEED_FD)
-        ret = jbr_leader_checks_and_init (frame, this, &op_errno, xdata, fd);
-#else
-        ret = jbr_leader_checks_and_init (frame, this, &op_errno, xdata, NULL);
-#endif
-        if (ret)
-                goto err;
-
-        local = frame->local;
-
-        /*
-         * If we let it through despite not being the leader, then we just want
-         * to pass it on down without all of the additional xattrs, queuing, and
-         * so on.  However, jbr_*_complete does depend on the initialization
-         * immediately above this.
-         */
-        if (!priv->leader) {
-                STACK_WIND (frame, jbr_@NAME@_complete,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
-                            @SHORT_ARGS@);
-                return 0;
-        }
-
-        ret = jbr_initialize_xdata_set_attrs (this, &xdata);
-        if (ret)
-                goto err;
-
-        local->xdata = dict_ref(xdata);
-        local->stub = fop_@NAME@_stub (frame, jbr_@NAME@_continue,
-                                       @SHORT_ARGS@);
-        if (!local->stub) {
-                goto err;
-        }
-
-        /*
-         * Can be used to just call_dispatch or be customised per fop to *
-         * perform ops specific to that particular fop.                  *
-         */
-        ret = jbr_@NAME@_perform_local_op (frame, this, &op_errno,
-                                           @SHORT_ARGS@);
-        if (ret)
-                goto err;
-
-        return ret;
-err:
-        if (local) {
-                if (local->stub) {
-                        call_stub_destroy(local->stub);
-                }
-                if (local->qstub) {
-                        call_stub_destroy(local->qstub);
-                }
-                if (local->fd) {
-                        fd_unref(local->fd);
-                }
-                mem_put(local);
-        }
-        STACK_UNWIND_STRICT (@NAME@, frame, -1, op_errno,
-                             @ERROR_ARGS@);
-        return 0;
-}
-
-/* template-name write-perform_local_op */
-int32_t
-jbr_@NAME@_perform_local_op (call_frame_t *frame, xlator_t *this, int *op_errno,
-                             @LONG_ARGS@)
-{
-        int32_t          ret    = -1;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-        ret = jbr_@NAME@_call_dispatch (frame, this, op_errno,
-                                        @SHORT_ARGS@);
-
-out:
-        return ret;
-}
-
-/* template-name write-call_dispatch */
-int32_t
-jbr_@NAME@_call_dispatch (call_frame_t *frame, xlator_t *this, int *op_errno,
-                          @LONG_ARGS@)
-{
-        jbr_local_t     *local  = NULL;
-        jbr_private_t   *priv   = NULL;
-        int32_t          ret    = -1;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-
-#if defined(JBR_CG_QUEUE)
-        jbr_inode_ctx_t  *ictx  = jbr_get_inode_ctx(this, fd->inode);
-        if (!ictx) {
-                *op_errno = EIO;
-                goto out;
-        }
-
-        LOCK(&ictx->lock);
-                if (ictx->active) {
-                        gf_msg_debug (this->name, 0,
-                                      "queuing request due to conflict");
-                        /*
-                         * TBD: enqueue only for real conflict
-                         *
-                         * Currently we just act like all writes are in
-                         * conflict with one another.  What we should really do
-                         * is check the active/pending queues and defer only if
-                         * there's a conflict there.
-                         *
-                         * It's important to check the pending queue because we
-                         * might have an active request X which conflicts with
-                         * a pending request Y, and this request Z might
-                         * conflict with Y but not X.  If we checked only the
-                         * active queue then Z could jump ahead of Y, which
-                         * would be incorrect.
-                         */
-                        local->qstub = fop_@NAME@_stub (frame,
-                                                        jbr_@NAME@_dispatch,
-                                                        @SHORT_ARGS@);
-                        if (!local->qstub) {
-                                UNLOCK(&ictx->lock);
-                                goto out;
-                        }
-                        list_add_tail(&local->qlinks, &ictx->pqueue);
-                        ++(ictx->pending);
-                        UNLOCK(&ictx->lock);
-                        ret = 0;
-                        goto out;
-                } else {
-                        list_add_tail(&local->qlinks, &ictx->aqueue);
-                        ++(ictx->active);
-                }
-        UNLOCK(&ictx->lock);
-#endif
-        ret = jbr_@NAME@_dispatch (frame, this, @SHORT_ARGS@);
-
-out:
-        return ret;
-}
-
-/* template-name write-dispatch */
-int32_t
-jbr_@NAME@_dispatch (call_frame_t *frame, xlator_t *this,
-                     @LONG_ARGS@)
-{
-        jbr_local_t     *local  = NULL;
-        jbr_private_t   *priv   = NULL;
-        int32_t          ret    = -1;
-        xlator_list_t   *trav;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-
-        /*
-         * TBD: unblock pending request(s) if we fail after this point but
-         * before we get to jbr_@NAME@_complete (where that code currently
-         * resides).
-         */
-
-        local->call_count = priv->n_children - 1;
-        for (trav = this->children->next; trav; trav = trav->next) {
-                STACK_WIND (frame, jbr_@NAME@_fan_in,
-                            trav->xlator, trav->xlator->fops->@NAME@,
-                            @SHORT_ARGS@);
-        }
-
-        /* TBD: variable Issue count */
-        ret = 0;
-out:
-        return ret;
-}
-
-/* template-name write-fan-in */
-int32_t
-jbr_@NAME@_fan_in (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   @LONG_ARGS@)
-{
-        jbr_local_t   *local  = NULL;
-        int32_t        ret    = -1;
-        uint8_t        call_count;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-
-        gf_msg_trace (this->name, 0, "op_ret = %d, op_errno = %d\n",
-                      op_ret, op_errno);
-
-        LOCK(&frame->lock);
-        call_count = --(local->call_count);
-        if (op_ret != -1) {
-                /* Increment the number of successful acks *
-                 * received for the operation.             *
-                 */
-                (local->successful_acks)++;
-                local->successful_op_ret = op_ret;
-        }
-        gf_msg_debug (this->name, 0, "succ_acks = %d, op_ret = %d, op_errno = %d\n",
-                      op_ret, op_errno, local->successful_acks);
-        UNLOCK(&frame->lock);
-
-        /* TBD: variable Completion count */
-        if (call_count == 0) {
-                call_resume(local->stub);
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-/* template-name write-continue */
-int32_t
-jbr_@NAME@_continue (call_frame_t *frame, xlator_t *this,
-                     @LONG_ARGS@)
-{
-        int32_t          ret       = -1;
-        gf_boolean_t     result    = _gf_false;
-        jbr_local_t     *local     = NULL;
-        jbr_local_t     *new_local = NULL;
-        jbr_private_t   *priv      = NULL;
-        int32_t          op_errno  = 0;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        priv = this->private;
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-
-        /* Perform quorum check to see if the leader needs     *
-         * to perform the operation. If the operation will not *
-         * meet quorum irrespective of the leader's result     *
-         * there is no point in the leader performing the fop  *
-         */
-        result = fop_quorum_check (this, (double)priv->n_children,
-                                   (double)local->successful_acks + 1);
-        if (result == _gf_false) {
-                gf_msg (this->name, GF_LOG_ERROR, EROFS,
-                        J_MSG_QUORUM_NOT_MET, "Didn't receive enough acks "
-                        "to meet quorum. Failing the operation without trying "
-                        "it on the leader.");
-
-#if defined(JBR_CG_QUEUE)
-                /*
-                 * In case of a fop failure, before unwinding need to *
-                 * remove it from queue                               *
-                 */
-                ret = jbr_remove_from_queue (frame, this);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                J_MSG_GENERIC, "Failed to remove from queue.");
-                }
-#endif
-
-                /*
-                 * In this case, the quorum is not met on the followers  *
-                 * So the operation will not be performed on the leader  *
-                 * and a rollback will be sent via GF_FOP_IPC to all the *
-                 * followers, where this particular fop's term and index *
-                 * numbers will be journaled, and later used to rollback *
-                 */
-                call_frame_t    *new_frame;
-
-                new_frame = copy_frame (frame);
-
-                if (new_frame) {
-                        new_local = mem_get0(this->local_pool);
-                        if (new_local) {
-                                INIT_LIST_HEAD(&new_local->qlinks);
-                                ret = dict_set_int32 (local->xdata,
-                                                      "rollback-fop",
-                                                      GF_FOP_@UPNAME@);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                J_MSG_DICT_FLR,
-                                                "failed to set rollback-fop");
-                                } else {
-                                        new_local->xdata = dict_ref(local->xdata);
-                                        new_frame->local = new_local;
-                                        jbr_ipc_call_dispatch (new_frame,
-                                                               this, &op_errno,
-                                                               FDL_IPC_JBR_SERVER_ROLLBACK,
-                                                               new_local->xdata);
-                                }
-                        } else {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Could not create local for new_frame");
-                        }
-                } else {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Could not send rollback ipc");
-                }
-
-                STACK_UNWIND_STRICT (@NAME@, frame, -1, EROFS,
-                                     @ERROR_ARGS@);
-        } else {
-                STACK_WIND (frame, jbr_@NAME@_complete,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
-                            @SHORT_ARGS@);
-        }
-
-out:
-        return 0;
-}
-
-/* template-name write-complete */
-int32_t
-jbr_@NAME@_complete (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     @LONG_ARGS@)
-{
-        int32_t          ret       = -1;
-        gf_boolean_t     result    = _gf_false;
-        jbr_private_t   *priv      = NULL;
-        jbr_local_t     *local     = NULL;
-        jbr_local_t     *new_local = NULL;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, err);
-        GF_VALIDATE_OR_GOTO (this->name, frame, err);
-        priv = this->private;
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, priv, err);
-        GF_VALIDATE_OR_GOTO (this->name, local, err);
-
-        /* If the fop failed on the leader, then reduce one succesful ack
-         * before calculating the fop quorum
-         */
-        LOCK(&frame->lock);
-        if (op_ret == -1)
-                (local->successful_acks)--;
-        UNLOCK(&frame->lock);
-
-#if defined(JBR_CG_QUEUE)
-        ret = jbr_remove_from_queue (frame, this);
-        if (ret)
-                goto err;
-#endif
-
-#if defined(JBR_CG_FSYNC)
-        jbr_mark_fd_dirty(this, local);
-#endif
-
-#if defined(JBR_CG_NEED_FD)
-        fd_unref(local->fd);
-#endif
-
-        /* After the leader completes the fop, a quorum check is      *
-         * performed, taking into account the outcome of the fop      *
-         * on the leader. Irrespective of the fop being successful    *
-         * or failing on the leader, the result of the quorum will    *
-         * determine if the overall fop is successful or not. For     *
-         * example, a fop might have succeeded on every node except   *
-         * the leader, in which case as quorum is being met, the fop  *
-         * will be treated as a successful fop, even though it failed *
-         * on the leader. On follower nodes, no quorum check should   *
-         * be done, and the result is returned to the leader as is.   *
-         */
-        if (priv->leader) {
-                result = fop_quorum_check (this, (double)priv->n_children,
-                                           (double)local->successful_acks + 1);
-                if (result == _gf_false) {
-                        op_ret = -1;
-                        op_errno = EROFS;
-                        gf_msg (this->name, GF_LOG_ERROR, EROFS,
-                                J_MSG_QUORUM_NOT_MET, "Quorum is not met. "
-                                "The operation has failed.");
-                        /*
-                         * In this case, the quorum is not met after the      *
-                         * operation is performed on the leader. Hence a      *
-                         * rollback will be sent via GF_FOP_IPC to the leader *
-                         * where this particular fop's term and index numbers *
-                         * will be journaled, and later used to rollback.     *
-                         * The same will be done on all the followers         *
-                         */
-                        call_frame_t    *new_frame;
-
-                        new_frame = copy_frame (frame);
-                        if (new_frame) {
-                                new_local = mem_get0(this->local_pool);
-                                if (new_local) {
-                                        INIT_LIST_HEAD(&new_local->qlinks);
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                J_MSG_DICT_FLR, "op = %d",
-                                                new_frame->op);
-                                        ret = dict_set_int32 (local->xdata,
-                                                              "rollback-fop",
-                                                              GF_FOP_@UPNAME@);
-                                        if (ret) {
-                                                gf_msg (this->name,
-                                                        GF_LOG_ERROR, 0,
-                                                        J_MSG_DICT_FLR,
-                                                        "failed to set "
-                                                        "rollback-fop");
-                                        } else {
-                                                new_local->xdata = dict_ref (local->xdata);
-                                                new_frame->local = new_local;
-                                                /*
-                                                 * Calling STACK_WIND instead *
-                                                 * of jbr_ipc as it will not  *
-                                                 * unwind to the previous     *
-                                                 * translators like it will   *
-                                                 * in case of jbr_ipc.        *
-                                                 */
-                                                STACK_WIND (new_frame,
-                                                            jbr_ipc_complete,
-                                                            FIRST_CHILD(this),
-                                                            FIRST_CHILD(this)->fops->ipc,
-                                                            FDL_IPC_JBR_SERVER_ROLLBACK,
-                                                            new_local->xdata);
-                                        }
-                                } else {
-                                        gf_log (this->name, GF_LOG_WARNING,
-                                                "Could not create local "
-                                                "for new_frame");
-                                }
-                        } else {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Could not send rollback ipc");
-                        }
-                } else {
-#if defined(JBR_CG_NEED_FD)
-                        op_ret = local->successful_op_ret;
-#else
-                        op_ret = 0;
-#endif
-                        op_errno = 0;
-                        gf_msg_debug (this->name, 0,
-                                      "Quorum has met. The operation has succeeded.");
-                }
-        }
-
-        /*
-         * Unrefing the reference taken in jbr_@NAME@ () *
-         */
-        dict_unref (local->xdata);
-
-        STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno,
-                             @SHORT_ARGS@);
-
-
-        return 0;
-
-err:
-        STACK_UNWIND_STRICT (@NAME@, frame, -1, 0,
-                             @SHORT_ARGS@);
-
-        return 0;
-}
diff --git a/xlators/experimental/jbr-server/src/gen-fops.py b/xlators/experimental/jbr-server/src/gen-fops.py
deleted file mode 100755
index e728f473372..00000000000
--- a/xlators/experimental/jbr-server/src/gen-fops.py
+++ /dev/null
@@ -1,181 +0,0 @@
-#!/usr/bin/python2
-
-# This script generates the boilerplate versions of most fops and cbks in the
-# server.  This allows the details of leadership-status checking, sequencing
-# between leader and followers (including fan-out), and basic error checking
-# to be centralized one place, with per-operation code kept to a minimum.
-
-from __future__ import print_function
-import os
-import re
-import string
-import sys
-
-curdir = os.path.dirname(sys.argv[0])
-gendir = os.path.join(curdir,'../../../../libglusterfs/src')
-sys.path.append(gendir)
-from generator import ops, fop_subs, cbk_subs, generate
-
-# We really want the callback argument list, even when we're generating fop
-# code, so we propagate here.
-# TBD: this should probably be right in generate.py
-for k, v in cbk_subs.iteritems():
-	fop_subs[k]['@ERROR_ARGS@'] = v['@ERROR_ARGS@']
-
-# Stolen from old codegen.py
-def load_templates (path):
-	templates = {}
-	tmpl_re = re.compile("/\* template-name (.*) \*/")
-	templates = {}
-	t_name = None
-	for line in open(path,"r").readlines():
-		if not line:
-			break
-		m = tmpl_re.match(line)
-		if m:
-			if t_name:
-				templates[t_name] = string.join(t_contents,'')
-			t_name = m.group(1).strip()
-			t_contents = []
-		elif t_name:
-			t_contents.append(line)
-	if t_name:
-		templates[t_name] = string.join(t_contents,'')
-	return templates
-
-# We need two types of templates.  The first, for pure read operations, just
-# needs to do a simple am-i-leader check (augmented to allow dirty reads).
-# The second, for pure writes, needs to do fan-out to followers between those
-# initial checks and local execution.  There are other operations that don't
-# fit neatly into either category - e.g. lock ops or fsync - so we'll just have
-# to handle those manually.  The table thus includes entries only for those we
-# can categorize.  The special cases, plus any new operations we've never even
-# heard of, aren't in there.
-#
-# Various keywords can be used to define/undefine preprocessor symbols used
-# in the templates, on a per-function basis.  For example, if the keyword here
-# is "fsync" (lowercase word or abbreviation) that will cause JBR_CG_FSYNC
-# (prefix plus uppercase version) to be defined above all of the generated code
-# for that fop.
-
-fop_table = {
-	"access":		"read",
-	"create":		"write",
-	"discard":		"write",
-#	"entrylk":		"read",
-	"fallocate":	"write",
-#	"fentrylk":		"read",
-	"fgetxattr":	"read",
-#	"finodelk":		"read",
-#	"flush":		"read",
-	"fremovexattr":	"write",
-	"fsetattr":		"write",
-	"fsetxattr":	"write",
-	"fstat":		"read",
-#	"fsync":		"read",
-#	"fsyncdir":		"read",
-	"ftruncate":	"write",
-	"fxattrop":		"write",
-	"getxattr":		"read",
-#	"inodelk":		"read",
-	"link":			"write",
-	"lk":			"write,queue",
-#	"lookup":		"read",
-	"mkdir":		"write",
-	"mknod":		"write",
-	"open":			"write",
-	"opendir":		"read",
-	"rchecksum":	"read",
-	"readdir":		"read",
-	"readdirp":		"read",
-	"readlink":		"read",
-	"readv":		"read",
-	"removexattr":	"write",
-	"rename":		"write",
-	"rmdir":		"write",
-	"setattr":		"write",
-	"setxattr":		"write",
-	"stat":			"read",
-	"statfs":		"read",
-	"symlink":		"write",
-	"truncate":		"write",
-	"unlink":		"write",
-	"writev":		"write,fsync,queue",
-	"xattrop":		"write",
-	"ipc":			"write",
-}
-
-# Mention those fops in the selective_generate table, for which
-# only a few common functions will be generated, and mention those
-# functions. Rest of the functions can be customized
-selective_generate = {
-	"lk":			"fop,dispatch,call_dispatch",
-	"ipc":			"dispatch,call_dispatch",
-}
-
-# Stolen from gen_fdl.py
-def gen_server (templates):
-	fops_done = []
-	for name in fop_table.keys():
-		info = fop_table[name].split(",")
-		kind = info[0]
-		flags = info[1:]
-
-		# generate all functions for the fops in fop_table
-		# except for the ones in selective_generate for which
-		# generate only the functions mentioned in the
-		# selective_generate table
-		gen_funcs = "fop,complete,continue,fan-in,dispatch, \
-			call_dispatch,perform_local_op"
-		if name in selective_generate:
-			gen_funcs = selective_generate[name].split(",")
-
-		if ("fsync" in flags) or ("queue" in flags):
-			flags.append("need_fd")
-		for fname in flags:
-			print("#define JBR_CG_%s" % fname.upper())
-
-		if 'complete' in gen_funcs:
-			print(generate(templates[kind+"-complete"],
-					name,cbk_subs))
-
-		if 'continue' in gen_funcs:
-			print(generate(templates[kind+"-continue"],
-					name,fop_subs))
-
-		if 'fan-in' in gen_funcs:
-			print(generate(templates[kind+"-fan-in"],
-					name,cbk_subs))
-
-		if 'dispatch' in gen_funcs:
-			print(generate(templates[kind+"-dispatch"],
-					name,fop_subs))
-
-		if 'call_dispatch' in gen_funcs:
-			print(generate(templates[kind+"-call_dispatch"],
-					name,fop_subs))
-
-		if 'perform_local_op' in gen_funcs:
-			print(generate(templates[kind+"-perform_local_op"],
-					name, fop_subs))
-
-		if 'fop' in gen_funcs:
-			print(generate(templates[kind+"-fop"],name,fop_subs))
-
-		for fname in flags:
-			print("#undef JBR_CG_%s" % fname.upper())
-		fops_done.append(name)
-	# Just for fun, emit the fops table too.
-	print("struct xlator_fops fops = {")
-	for x in fops_done:
-		print(("	.%s = jbr_%s,"%(x,x)))
-	print("};")
-
-tmpl = load_templates(sys.argv[1])
-for l in open(sys.argv[2],'r').readlines():
-	if l.find('#pragma generate') != -1:
-		print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
-		gen_server(tmpl)
-		print("/* END GENERATED CODE */")
-	else:
-		print(l[:-1])
diff --git a/xlators/experimental/jbr-server/src/jbr-internal.h b/xlators/experimental/jbr-server/src/jbr-internal.h
deleted file mode 100644
index 46a29910d1f..00000000000
--- a/xlators/experimental/jbr-server/src/jbr-internal.h
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
-   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#include <sys/stat.h>
-#include <sys/types.h>
-
-#define LEADER_XATTR            "user.jbr.leader"
-#define SECOND_CHILD(xl)        (xl->children->next->xlator)
-#define RECONCILER_PATH         JBR_SCRIPT_PREFIX"/reconciler.py"
-#define CHANGELOG_ENTRY_SIZE    128
-
-enum {
-        gf_mt_jbr_private_t = gf_common_mt_end + 1,
-        gf_mt_jbr_fd_ctx_t,
-        gf_mt_jbr_inode_ctx_t,
-        gf_mt_jbr_dirty_t,
-        gf_mt_jbr_end
-};
-
-typedef enum jbr_recon_notify_ev_id_t {
-        JBR_RECON_SET_LEADER = 1,
-        JBR_RECON_ADD_CHILD = 2
-} jbr_recon_notify_ev_id_t;
-
-typedef struct _jbr_recon_notify_ev_s {
-        jbr_recon_notify_ev_id_t id;
-        uint32_t index; /* in case of add */
-        struct list_head list;
-} jbr_recon_notify_ev_t;
-
-typedef struct {
-        /*
-         * This is a hack to allow a non-leader to accept requests while the
-         * leader is down, and it only works for n=2.  The way it works is that
-         * "config_leader" indicates the state from our options (via init or
-         * reconfigure) but "leader" is what the fop code actually looks at.  If
-         * config_leader is true, then leader will *always* be true as well,
-         * giving that brick precedence.  If config_leader is false, then
-         * leader will only be true if there is no connection to the other
-         * brick (tracked in jbr_notify).
-         *
-         * TBD: implement real leader election
-         */
-        gf_boolean_t            config_leader;
-        gf_boolean_t            leader;
-        uint8_t                 up_children;
-        uint8_t                 n_children;
-        char                    *vol_file;
-        uint32_t                current_term;
-        uint32_t                kid_state;
-        gf_lock_t               dirty_lock;
-        struct list_head        dirty_fds;
-	uint32_t                index;
-	gf_lock_t               index_lock;
-        double                  quorum_pct;
-        int                     term_fd;
-        long                    term_total;
-        long                    term_read;
-        /*
-         * This is a super-duper hack, but it will do for now.  The reason it's
-         * a hack is that we pass this to dict_set_static_bin, so we don't have
-         * to mess around with allocating and freeing it on every single IPC
-         * request, but it's totally not thread-safe.  On the other hand, there
-         * should only be one reconciliation thread running and calling these
-         * functions at a time, so maybe that doesn't matter.
-         *
-         * TBD: re-evaluate how to manage this
-         */
-        char                    term_buf[CHANGELOG_ENTRY_SIZE];
-        gf_boolean_t            child_up; /* To maintain the state of *
-                                           * the translator */
-} jbr_private_t;
-
-typedef struct {
-        call_stub_t             *stub;
-        call_stub_t             *qstub;
-        uint32_t                call_count;
-        uint32_t                successful_acks;
-        uint32_t                successful_op_ret;
-        fd_t                    *fd;
-        struct list_head        qlinks;
-        dict_t                  *xdata;
-} jbr_local_t;
-
-/*
- * This should match whatever changelog returns on the pre-op for us to pass
- * when we're ready for our post-op.
- */
-typedef uint32_t log_id_t;
-
-typedef struct {
-        struct list_head        links;
-        log_id_t                id;
-} jbr_dirty_list_t;
-
-typedef struct {
-        fd_t                    *fd;
-        struct list_head        dirty_list;
-        struct list_head        fd_list;
-} jbr_fd_ctx_t;
-
-typedef struct {
-        gf_lock_t               lock;
-        uint32_t                active;
-        struct list_head        aqueue;
-        uint32_t                pending;
-        struct list_head        pqueue;
-} jbr_inode_ctx_t;
-
-void jbr_start_reconciler (xlator_t *this);
diff --git a/xlators/experimental/jbr-server/src/jbr.c b/xlators/experimental/jbr-server/src/jbr.c
deleted file mode 100644
index 68ec10a7e54..00000000000
--- a/xlators/experimental/jbr-server/src/jbr.c
+++ /dev/null
@@ -1,1749 +0,0 @@
-/*
-   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <fnmatch.h>
-#include "call-stub.h"
-#include "defaults.h"
-#include "xlator.h"
-#include "glfs.h"
-#include "glfs-internal.h"
-#include "run.h"
-#include "common-utils.h"
-#include "syncop.h"
-#include "syscall.h"
-#include "compat-errno.h"
-#include "fdl.h"
-
-#include "jbr-internal.h"
-#include "jbr-messages.h"
-
-#define JBR_FLUSH_INTERVAL      5
-
-enum {
-        /* echo "cluster/jbr-server" | md5sum | cut -c 1-8 */
-        JBR_SERVER_IPC_BASE = 0x0e2d66a5,
-        JBR_SERVER_TERM_RANGE,
-        JBR_SERVER_OPEN_TERM,
-        JBR_SERVER_NEXT_ENTRY
-};
-
-/*
- * Need to declare jbr_lk_call_dispatch as jbr_lk_continue and *
- * jbr_lk_perform_local_op call it, before code is generated.  *
- */
-int32_t
-jbr_lk_call_dispatch (call_frame_t *frame, xlator_t *this, int *op_errno,
-                      fd_t *fd, int32_t cmd, struct gf_flock *lock,
-                      dict_t *xdata);
-
-int32_t
-jbr_lk_dispatch (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, int32_t cmd, struct gf_flock *lock,
-                 dict_t *xdata);
-
-int32_t
-jbr_ipc_call_dispatch (call_frame_t *frame, xlator_t *this, int *op_errno,
-                       int32_t op, dict_t *xdata);
-
-int32_t
-jbr_ipc_complete (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  dict_t *xdata);
-
-/* Used to check the quorum of acks received after the fop
- * confirming the status of the fop on all the brick processes
- * for this particular subvolume
- */
-gf_boolean_t
-fop_quorum_check (xlator_t *this, double n_children,
-                  double current_state)
-{
-        jbr_private_t   *priv           = NULL;
-        gf_boolean_t     result         = _gf_false;
-        double           required       = 0;
-        double           current        = 0;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        required = n_children * priv->quorum_pct;
-
-        /*
-         * Before performing the fop on the leader, we need to check,
-         * if there is any merit in performing the fop on the leader.
-         * In a case, where even a successful write on the leader, will
-         * not meet quorum, there is no point in trying the fop on the
-         * leader.
-         * When this function is called after the leader has tried
-         * performing the fop, this check will calculate quorum taking into
-         * account the status of the fop on the leader. If the leader's
-         * op_ret was -1, the complete function would account that by
-         * decrementing successful_acks by 1
-         */
-
-        current = current_state * 100.0;
-
-        if (current < required) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_QUORUM_NOT_MET,
-                        "Quorum not met. quorum_pct = %f "
-                        "Current State = %f, Required State = %f",
-                        priv->quorum_pct, current,
-                        required);
-        } else
-                result = _gf_true;
-
-out:
-        return result;
-}
-
-jbr_inode_ctx_t *
-jbr_get_inode_ctx (xlator_t *this, inode_t *inode)
-{
-        uint64_t                ctx_int         = 0LL;
-        jbr_inode_ctx_t         *ctx_ptr;
-
-        if (__inode_ctx_get(inode, this, &ctx_int) == 0) {
-                ctx_ptr = (jbr_inode_ctx_t *)(long)ctx_int;
-        } else {
-                ctx_ptr = GF_CALLOC (1, sizeof(*ctx_ptr),
-                                     gf_mt_jbr_inode_ctx_t);
-                if (ctx_ptr) {
-                        ctx_int = (uint64_t)(long)ctx_ptr;
-                        if (__inode_ctx_set(inode, this, &ctx_int) == 0) {
-                                LOCK_INIT(&ctx_ptr->lock);
-                                INIT_LIST_HEAD(&ctx_ptr->aqueue);
-                                INIT_LIST_HEAD(&ctx_ptr->pqueue);
-                        } else {
-                                GF_FREE(ctx_ptr);
-                                ctx_ptr = NULL;
-                        }
-                }
-
-        }
-
-        return ctx_ptr;
-}
-
-jbr_fd_ctx_t *
-jbr_get_fd_ctx (xlator_t *this, fd_t *fd)
-{
-        uint64_t                ctx_int         = 0LL;
-        jbr_fd_ctx_t            *ctx_ptr;
-
-        if (__fd_ctx_get(fd, this, &ctx_int) == 0) {
-                ctx_ptr = (jbr_fd_ctx_t *)(long)ctx_int;
-        } else {
-                ctx_ptr = GF_CALLOC (1, sizeof(*ctx_ptr), gf_mt_jbr_fd_ctx_t);
-                if (ctx_ptr) {
-                        if (__fd_ctx_set(fd, this, (uint64_t)ctx_ptr) == 0) {
-                                INIT_LIST_HEAD(&ctx_ptr->dirty_list);
-                                INIT_LIST_HEAD(&ctx_ptr->fd_list);
-                        } else {
-                                GF_FREE(ctx_ptr);
-                                ctx_ptr = NULL;
-                        }
-                }
-
-        }
-
-        return ctx_ptr;
-}
-
-void
-jbr_mark_fd_dirty (xlator_t *this, jbr_local_t *local)
-{
-        fd_t                    *fd             = local->fd;
-        jbr_fd_ctx_t            *ctx_ptr;
-        jbr_dirty_list_t        *dirty;
-        jbr_private_t           *priv           = this->private;
-
-        /*
-         * TBD: don't do any of this for O_SYNC/O_DIRECT writes.
-         * Unfortunately, that optimization requires that we distinguish
-         * between writev and other "write" calls, saving the original flags
-         * and checking them in the callback.  Too much work for too little
-         * gain right now.
-         */
-
-        LOCK(&fd->lock);
-                ctx_ptr = jbr_get_fd_ctx(this, fd);
-                dirty = GF_CALLOC(1, sizeof(*dirty), gf_mt_jbr_dirty_t);
-                if (ctx_ptr && dirty) {
-                        gf_msg_trace (this->name, 0,
-                                      "marking fd %p as dirty (%p)", fd, dirty);
-                        /* TBD: fill dirty->id from what changelog gave us */
-                        list_add_tail(&dirty->links, &ctx_ptr->dirty_list);
-                        if (list_empty(&ctx_ptr->fd_list)) {
-                                /* Add a ref so _release doesn't get called. */
-                                ctx_ptr->fd = fd_ref(fd);
-                                LOCK(&priv->dirty_lock);
-                                        list_add_tail (&ctx_ptr->fd_list,
-                                                       &priv->dirty_fds);
-                                UNLOCK(&priv->dirty_lock);
-                        }
-                } else {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                J_MSG_MEM_ERR, "could not mark %p dirty", fd);
-                        if (ctx_ptr) {
-                                GF_FREE(ctx_ptr);
-                        }
-                        if (dirty) {
-                                GF_FREE(dirty);
-                        }
-                }
-        UNLOCK(&fd->lock);
-}
-
-#define JBR_TERM_XATTR          "trusted.jbr.term"
-#define JBR_INDEX_XATTR         "trusted.jbr.index"
-#define JBR_REP_COUNT_XATTR     "trusted.jbr.rep-count"
-#define RECON_TERM_XATTR        "trusted.jbr.recon-term"
-#define RECON_INDEX_XATTR       "trusted.jbr.recon-index"
-
-int32_t
-jbr_leader_checks_and_init (call_frame_t *frame, xlator_t *this, int *op_errno,
-                            dict_t *xdata, fd_t *fd)
-{
-        jbr_local_t     *local         = NULL;
-        jbr_private_t   *priv          = NULL;
-        int32_t          ret           = -1;
-        gf_boolean_t     result        = _gf_false;
-        int              from_leader   = _gf_false;
-        int              from_recon    = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-
-        /*
-         * Our first goal here is to avoid "split brain surprise" for users who
-         * specify exactly 50% with two- or three-way replication.  That means
-         * either a more-than check against half the total replicas or an
-         * at-least check against half of our peers (one less).  Of the two,
-         * only an at-least check supports the intuitive use of 100% to mean
-         * all replicas must be present, because "more than 100%" will never
-         * succeed regardless of which count we use.  This leaves us with a
-         * slightly non-traditional definition of quorum ("at least X% of peers
-         * not including ourselves") but one that's useful enough to be worth
-         * it.
-         *
-         * Note that n_children and up_children *do* include the local
-         * subvolume, so we need to subtract one in each case.
-         */
-        if (priv->leader) {
-                result = fop_quorum_check (this, (double)(priv->n_children - 1),
-                                   (double)(priv->up_children - 1));
-
-                if (result == _gf_false) {
-                        /* Emulate the AFR client-side-quorum behavior. */
-                        gf_msg (this->name, GF_LOG_ERROR, EROFS,
-                                J_MSG_QUORUM_NOT_MET, "Sufficient number of "
-                                "subvolumes are not up to meet quorum.");
-                        *op_errno = EROFS;
-                        goto out;
-                }
-        } else {
-                if (xdata) {
-                        from_leader = !!dict_get(xdata, JBR_TERM_XATTR);
-                        from_recon = !!dict_get(xdata, RECON_TERM_XATTR)
-                                  && !!dict_get(xdata, RECON_INDEX_XATTR);
-                } else {
-                        from_leader = from_recon = _gf_false;
-                }
-
-                /* follower/recon path        *
-                 * just send it to local node *
-                 */
-                if (!from_leader && !from_recon) {
-                        *op_errno = EREMOTE;
-                        goto out;
-                }
-        }
-
-        local = mem_get0(this->local_pool);
-        if (!local) {
-                goto out;
-        }
-
-        if (fd)
-                local->fd = fd_ref(fd);
-        else
-                local->fd = NULL;
-
-        INIT_LIST_HEAD(&local->qlinks);
-        local->successful_acks = 0;
-        frame->local = local;
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-jbr_initialize_xdata_set_attrs (xlator_t *this, dict_t **xdata)
-{
-        jbr_private_t   *priv          = NULL;
-        int32_t          ret           = -1;
-        uint32_t         ti            = 0;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, xdata, out);
-
-        if (!*xdata) {
-                *xdata = dict_new();
-                if (!*xdata) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                J_MSG_MEM_ERR, "failed to allocate xdata");
-                        goto out;
-                }
-        }
-
-        if (dict_set_int32(*xdata, JBR_TERM_XATTR, priv->current_term) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_DICT_FLR, "failed to set jbr-term");
-                goto out;
-        }
-
-        LOCK(&priv->index_lock);
-        ti = ++(priv->index);
-        UNLOCK(&priv->index_lock);
-        if (dict_set_int32(*xdata, JBR_INDEX_XATTR, ti) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_DICT_FLR, "failed to set index");
-                goto out;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-jbr_remove_from_queue (call_frame_t *frame, xlator_t *this)
-{
-        int32_t          ret       = -1;
-        jbr_inode_ctx_t *ictx      = NULL;
-        jbr_local_t     *local     = NULL;
-        jbr_local_t     *next      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-
-        if (local->qlinks.next != &local->qlinks) {
-                list_del(&local->qlinks);
-                ictx = jbr_get_inode_ctx(this, local->fd->inode);
-                if (ictx) {
-                        LOCK(&ictx->lock);
-                                if (ictx->pending) {
-                                        /*
-                                         * TBD: dequeue *all* non-conflicting
-                                         * reqs
-                                         *
-                                         * With the stub implementation there
-                                         * can only be one request active at a
-                                         * time (zero here) so it's not an
-                                         * issue.  In a real implementation
-                                         * there might still be other active
-                                         * requests to check against, and
-                                         * multiple pending requests that could
-                                         * continue.
-                                         */
-                                        gf_msg_debug (this->name, 0,
-                                                     "unblocking next request");
-                                        --(ictx->pending);
-                                        next = list_entry (ictx->pqueue.next,
-                                                           jbr_local_t, qlinks);
-                                        list_del(&next->qlinks);
-                                        list_add_tail(&next->qlinks,
-                                                      &ictx->aqueue);
-                                        call_resume(next->qstub);
-                                } else {
-                                        --(ictx->active);
-                                }
-                        UNLOCK(&ictx->lock);
-                }
-        }
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-int32_t
-jbr_lk_complete (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct gf_flock *flock, dict_t *xdata)
-{
-        int32_t          ret       = -1;
-        jbr_private_t   *priv      = NULL;
-        jbr_local_t     *local     = NULL;
-        gf_boolean_t     result    = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, err);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, err);
-        GF_VALIDATE_OR_GOTO (this->name, frame, err);
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, err);
-        GF_VALIDATE_OR_GOTO (this->name, flock, err);
-        GF_VALIDATE_OR_GOTO (this->name, xdata, err);
-
-        /*
-         * Remove from queue for unlock operation only   *
-         * For lock operation, it will be done in fan-in *
-         */
-        if (flock->l_type == F_UNLCK) {
-                ret = jbr_remove_from_queue (frame, this);
-                if (ret)
-                        goto err;
-        }
-
-        /*
-         * On a follower, unwind with the op_ret and op_errno. On a *
-         * leader, if the fop is a locking fop, and its a failure,  *
-         * send fail, else call stub which will dispatch the fop to *
-         * the followers.                                           *
-         *                                                          *
-         * If the fop is a unlocking fop, check quorum. If quorum   *
-         * is met, then send success. Else Rollback on leader,      *
-         * followed by followers, and then send -ve ack to client.  *
-         */
-        if (priv->leader) {
-
-                /* Increase the successful acks if it's a success. */
-                LOCK(&frame->lock);
-                if (op_ret != -1)
-                        (local->successful_acks)++;
-                UNLOCK(&frame->lock);
-
-                if (flock->l_type == F_UNLCK) {
-                        result = fop_quorum_check (this,
-                                            (double)priv->n_children,
-                                            (double)local->successful_acks);
-                        if (result == _gf_false) {
-                                op_ret = -1;
-                                op_errno = EROFS;
-                                gf_msg (this->name, GF_LOG_ERROR, EROFS,
-                                        J_MSG_QUORUM_NOT_MET,
-                                        "Quorum is not met. "
-                                        "The operation has failed.");
-
-                                /* TODO: PERFORM UNLOCK ROLLBACK ON LEADER *
-                                 * FOLLOWED BY FOLLOWERS. */
-                        } else {
-                                op_ret = 0;
-                                op_errno = 0;
-                        }
-
-                        fd_unref(local->fd);
-                        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno,
-                                             flock, xdata);
-                } else {
-                        if (op_ret == -1) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        J_MSG_LOCK_FAILURE,
-                                        "The lock operation failed on "
-                                        "the leader.");
-
-                                fd_unref(local->fd);
-                                STACK_UNWIND_STRICT (lk, frame, op_ret,
-                                                     op_errno, flock, xdata);
-                        } else {
-                                if (!local->stub) {
-                                        goto err;
-                                }
-
-                                call_resume(local->stub);
-                        }
-                }
-        } else {
-                fd_unref(local->fd);
-                STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno,
-                                     flock, xdata);
-        }
-
-        return 0;
-
-err:
-        if (local) {
-                if (local->stub) {
-                        call_stub_destroy(local->stub);
-                }
-                if (local->qstub) {
-                        call_stub_destroy(local->qstub);
-                }
-                if (local->fd) {
-                        fd_unref(local->fd);
-                }
-                mem_put(local);
-        }
-        STACK_UNWIND_STRICT (lk, frame, -1, op_errno,
-                             flock, xdata);
-        return 0;
-}
-
-int32_t
-jbr_lk_fan_in (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
-               dict_t *xdata)
-{
-        uint8_t          call_count = -1;
-        int32_t          ret        = -1;
-        gf_boolean_t     result     = _gf_false;
-        jbr_local_t     *local      = NULL;
-        jbr_private_t   *priv       = NULL;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        priv = this->private;
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-
-        gf_msg_trace (this->name, 0, "op_ret = %d, op_errno = %d\n",
-                      op_ret, op_errno);
-
-        LOCK(&frame->lock);
-        call_count = --(local->call_count);
-        if (op_ret != -1) {
-                /* Increment the number of successful acks *
-                 * received for the operation.             *
-                 */
-                (local->successful_acks)++;
-                local->successful_op_ret = op_ret;
-        }
-        gf_msg_debug (this->name, 0, "succ_acks = %d, op_ret = %d, op_errno = %d\n",
-                      op_ret, op_errno, local->successful_acks);
-        UNLOCK(&frame->lock);
-
-        if (call_count == 0) {
-                /*
-                 * If the fop is a locking fop, then check quorum. If quorum *
-                 * is met, send successful ack to the client. If quorum is   *
-                 * not met, then rollback locking on followers, followed by  *
-                 * rollback of locking on leader, and then sending -ve ack   *
-                 * to the client.                                            *
-                 *                                                           *
-                 * If the fop is a unlocking fop, then call stub.            *
-                 */
-                if (flock->l_type == F_UNLCK) {
-                        call_resume(local->stub);
-                } else {
-                        /*
-                         * Remove from queue for locking fops, for unlocking *
-                         * fops, it is taken care of in jbr_lk_complete      *
-                         */
-                        ret = jbr_remove_from_queue (frame, this);
-                        if (ret)
-                                goto out;
-
-                        fd_unref(local->fd);
-
-                        result = fop_quorum_check (this,
-                                          (double)priv->n_children,
-                                          (double)local->successful_acks);
-                        if (result == _gf_false) {
-                                gf_msg (this->name, GF_LOG_ERROR, EROFS,
-                                        J_MSG_QUORUM_NOT_MET,
-                                        "Didn't receive enough acks to meet "
-                                        "quorum. Failing the locking "
-                                        "operation and initiating rollback on "
-                                        "followers and the leader "
-                                        "respectively.");
-
-                                /* TODO: PERFORM ROLLBACK OF LOCKING ON
-                                 * FOLLOWERS, FOLLOWED BY ROLLBACK ON
-                                 * LEADER.
-                                 */
-
-                                STACK_UNWIND_STRICT (lk, frame, -1, EROFS,
-                                                     flock, xdata);
-                        } else {
-                                STACK_UNWIND_STRICT (lk, frame, 0, 0,
-                                                     flock, xdata);
-                        }
-                }
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-/*
- * Called from leader for locking fop, being writen as a separate *
- * function so as to support queues.                              *
- */
-int32_t
-jbr_perform_lk_on_leader (call_frame_t *frame, xlator_t *this,
-                         fd_t *fd, int32_t cmd, struct gf_flock *flock,
-                         dict_t *xdata)
-{
-        int32_t          ret    = -1;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, flock, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        STACK_WIND (frame, jbr_lk_complete,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, flock, xdata);
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-jbr_lk_perform_local_op (call_frame_t *frame, xlator_t *this, int *op_errno,
-                         fd_t *fd, int32_t cmd, struct gf_flock *flock,
-                         dict_t *xdata)
-{
-        int32_t          ret    = -1;
-        jbr_local_t     *local  = NULL;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errno, out);
-        GF_VALIDATE_OR_GOTO (this->name, flock, out);
-
-        /*
-         * Check if the fop is a locking fop or unlocking fop, and
-         * handle it accordingly. If it is a locking fop, take the
-         * lock on leader first, and then send it to the followers.
-         * If it is a unlocking fop, unlock the followers first,
-         * and then on meeting quorum perform the unlock on the leader.
-         */
-        if (flock->l_type == F_UNLCK) {
-                ret = jbr_lk_call_dispatch (frame, this, op_errno,
-                                            fd, cmd, flock, xdata);
-                if (ret)
-                        goto out;
-        } else {
-                jbr_inode_ctx_t  *ictx  = jbr_get_inode_ctx(this, fd->inode);
-
-                if (!ictx) {
-                        *op_errno = EIO;
-                        goto out;
-                }
-
-                LOCK(&ictx->lock);
-                        if (ictx->active) {
-                                gf_msg_debug (this->name, 0,
-                                              "queuing request due to conflict");
-
-                                local->qstub = fop_lk_stub (frame,
-                                                       jbr_perform_lk_on_leader,
-                                                       fd, cmd, flock, xdata);
-                                if (!local->qstub) {
-                                        UNLOCK(&ictx->lock);
-                                        goto out;
-                                }
-                                list_add_tail(&local->qlinks, &ictx->pqueue);
-                                ++(ictx->pending);
-                                UNLOCK(&ictx->lock);
-                                ret = 0;
-                                goto out;
-                        } else {
-                                list_add_tail(&local->qlinks, &ictx->aqueue);
-                                ++(ictx->active);
-                        }
-                UNLOCK(&ictx->lock);
-                ret = jbr_perform_lk_on_leader (frame, this, fd, cmd,
-                                                flock, xdata);
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-jbr_lk_continue (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
-{
-        int32_t          ret      = -1;
-        jbr_local_t     *local    = NULL;
-        jbr_private_t   *priv     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        priv = this->private;
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-        GF_VALIDATE_OR_GOTO (this->name, flock, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, xdata, out);
-
-        /*
-         * If it's a locking fop, then call dispatch to followers  *
-         * If it's a unlock fop, then perform the unlock operation *
-         */
-        if (flock->l_type == F_UNLCK) {
-                STACK_WIND (frame, jbr_lk_complete,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->lk,
-                            fd, cmd, flock, xdata);
-        } else {
-                /*
-                 * Directly call jbr_lk_dispatch instead of appending *
-                 * in queue, which is done at jbr_lk_perform_local_op *
-                 * for locking fops                                   *
-                 */
-                ret = jbr_lk_dispatch (frame, this, fd, cmd,
-                                       flock, xdata);
-                if (ret) {
-                        STACK_UNWIND_STRICT (lk, frame, -1, 0,
-                                             flock, xdata);
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-uint8_t
-jbr_count_up_kids (jbr_private_t *priv)
-{
-        uint8_t         retval  = 0;
-        uint8_t         i;
-
-        for (i = 0; i < priv->n_children; ++i) {
-                if (priv->kid_state & (1 << i)) {
-                        ++retval;
-                }
-        }
-
-        return retval;
-}
-
-/*
- * The fsync machinery looks a lot like that for any write call, but there are
- * some important differences that are easy to miss.  First, we don't care
- * about the xdata that shows whether the call came from a leader or
- * reconciliation process.  If we're the leader we fan out; if we're not we
- * don't.  Second, we don't wait for followers before we issue the local call.
- * The code generation system could be updated to handle this, and still might
- * if we need to implement other "almost identical" paths (e.g. for open), but
- * a copy is more readable as long as it's just one.
- */
-
-int32_t
-jbr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf, dict_t *xdata)
-{
-        jbr_local_t     *local  = frame->local;
-        gf_boolean_t    unwind;
-
-        LOCK(&frame->lock);
-                unwind = !--(local->call_count);
-        UNLOCK(&frame->lock);
-
-        if (unwind) {
-                STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf,
-                                     postbuf, xdata);
-        }
-        return 0;
-}
-
-int32_t
-jbr_fsync_local_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
-{
-        jbr_dirty_list_t        *dirty;
-        jbr_dirty_list_t        *dtmp;
-        jbr_local_t             *local  = frame->local;
-
-        list_for_each_entry_safe (dirty, dtmp, &local->qlinks, links) {
-                gf_msg_trace (this->name, 0,
-                              "sending post-op on %p (%p)", local->fd, dirty);
-                GF_FREE(dirty);
-        }
-
-        return jbr_fsync_cbk (frame, cookie, this, op_ret, op_errno,
-                              prebuf, postbuf, xdata);
-}
-
-int32_t
-jbr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-           dict_t *xdata)
-{
-        jbr_private_t   *priv   = this->private;
-        jbr_local_t     *local;
-        uint64_t        ctx_int         = 0LL;
-        jbr_fd_ctx_t    *ctx_ptr;
-        xlator_list_t   *trav;
-
-        local = mem_get0(this->local_pool);
-        if (!local) {
-                STACK_UNWIND_STRICT(fsync, frame, -1, ENOMEM,
-                                    NULL, NULL, xdata);
-                return 0;
-        }
-        INIT_LIST_HEAD(&local->qlinks);
-        frame->local = local;
-
-        /* Move the dirty list from the fd to the fsync request. */
-        LOCK(&fd->lock);
-                if (__fd_ctx_get(fd, this, &ctx_int) == 0) {
-                        ctx_ptr = (jbr_fd_ctx_t *)(long)ctx_int;
-                        list_splice_init (&ctx_ptr->dirty_list,
-                                          &local->qlinks);
-                }
-        UNLOCK(&fd->lock);
-
-        /* Issue the local call. */
-        local->call_count = priv->leader ? priv->n_children : 1;
-        STACK_WIND (frame, jbr_fsync_local_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
-                    fd, flags, xdata);
-
-        /* Issue remote calls if we're the leader. */
-        if (priv->leader) {
-                for (trav = this->children->next; trav; trav = trav->next) {
-                        STACK_WIND (frame, jbr_fsync_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->fsync,
-                                    fd, flags, xdata);
-                }
-        }
-
-        return 0;
-}
-
-int32_t
-jbr_getxattr_special (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      const char *name, dict_t *xdata)
-{
-        dict_t          *result;
-        jbr_private_t   *priv   = this->private;
-
-        if (!priv->leader) {
-                STACK_UNWIND_STRICT (getxattr, frame, -1, EREMOTE, NULL, NULL);
-                return 0;
-        }
-
-        if (!name || (strcmp(name, JBR_REP_COUNT_XATTR) != 0)) {
-                STACK_WIND_TAIL (frame,
-                                 FIRST_CHILD(this),
-                                 FIRST_CHILD(this)->fops->getxattr,
-                                 loc, name, xdata);
-                return 0;
-        }
-
-        result = dict_new();
-        if (!result) {
-                goto dn_failed;
-        }
-
-        priv->up_children = jbr_count_up_kids(this->private);
-        if (dict_set_uint32(result, JBR_REP_COUNT_XATTR,
-                            priv->up_children) != 0) {
-                goto dsu_failed;
-        }
-
-        STACK_UNWIND_STRICT (getxattr, frame, 0, 0, result, NULL);
-        dict_unref(result);
-        return 0;
-
-dsu_failed:
-        dict_unref(result);
-dn_failed:
-        STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
-        return 0;
-}
-
-void
-jbr_flush_fd (xlator_t *this, jbr_fd_ctx_t *fd_ctx)
-{
-        jbr_dirty_list_t        *dirty;
-        jbr_dirty_list_t        *dtmp;
-
-        list_for_each_entry_safe (dirty, dtmp, &fd_ctx->dirty_list, links) {
-                gf_msg_trace (this->name, 0,
-                              "sending post-op on %p (%p)", fd_ctx->fd, dirty);
-                GF_FREE(dirty);
-        }
-
-        INIT_LIST_HEAD(&fd_ctx->dirty_list);
-}
-
-void *
-jbr_flush_thread (void *ctx)
-{
-        xlator_t                *this   = ctx;
-        jbr_private_t           *priv   = this->private;
-        struct list_head        dirty_fds;
-        jbr_fd_ctx_t            *fd_ctx;
-        jbr_fd_ctx_t            *fd_tmp;
-        int                     ret;
-
-        for (;;) {
-                /*
-                 * We have to be very careful to avoid lock inversions here, so
-                 * we can't just hold priv->dirty_lock while we take and
-                 * release locks for each fd.  Instead, we only hold dirty_lock
-                 * at the beginning of each iteration, as we (effectively) make
-                 * a copy of the current list head and then clear the original.
-                 * This leads to four scenarios for adding the first entry to
-                 * an fd and potentially putting it on the global list.
-                 *
-                 * (1) While we're asleep.  No lock contention, it just gets
-                 *     added and will be processed on the next iteration.
-                 *
-                 * (2) After we've made a local copy, but before we've started
-                 *     processing that fd.  The new entry will be added to the
-                 *     fd (under its lock), and we'll process it on the current
-                 *     iteration.
-                 *
-                 * (3) While we're processing the fd.  They'll block on the fd
-                 *     lock, then see that the list is empty and put it on the
-                 *     global list.  We'll process it here on the next
-                 *     iteration.
-                 *
-                 * (4) While we're working, but after we've processed that fd.
-                 *     Same as (1) as far as that fd is concerned.
-                 */
-                INIT_LIST_HEAD(&dirty_fds);
-                LOCK(&priv->dirty_lock);
-                list_splice_init(&priv->dirty_fds, &dirty_fds);
-                UNLOCK(&priv->dirty_lock);
-
-                list_for_each_entry_safe (fd_ctx, fd_tmp, &dirty_fds, fd_list) {
-                        ret = syncop_fsync(FIRST_CHILD(this), fd_ctx->fd, 0,
-                                           NULL, NULL, NULL, NULL);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        J_MSG_SYS_CALL_FAILURE,
-                                        "failed to fsync %p (%d)",
-                                        fd_ctx->fd, -ret);
-                        }
-
-                        LOCK(&fd_ctx->fd->lock);
-                                jbr_flush_fd(this, fd_ctx);
-                                list_del_init(&fd_ctx->fd_list);
-                        UNLOCK(&fd_ctx->fd->lock);
-                        fd_unref(fd_ctx->fd);
-                }
-
-                sleep(JBR_FLUSH_INTERVAL);
-        }
-
-        return NULL;
-}
-
-
-int32_t
-jbr_get_changelog_dir (xlator_t *this, char **cl_dir_p)
-{
-        xlator_t        *cl_xl;
-
-        /* Find our changelog translator. */
-        cl_xl = this;
-        while (cl_xl) {
-                if (strcmp(cl_xl->type, "features/changelog") == 0) {
-                        break;
-                }
-                cl_xl = cl_xl->children->xlator;
-        }
-        if (!cl_xl) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_INIT_FAIL,
-                        "failed to find changelog translator");
-                return ENOENT;
-        }
-
-        /* Find the actual changelog directory. */
-        if (dict_get_str(cl_xl->options, "changelog-dir", cl_dir_p) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_INIT_FAIL,
-                        "failed to find changelog-dir for %s", cl_xl->name);
-                return ENODATA;
-        }
-
-        return 0;
-}
-
-
-void
-jbr_get_terms (call_frame_t *frame, xlator_t *this)
-{
-        int32_t        op_errno      = 0;
-        char          *cl_dir        = NULL;
-        int32_t        term_first    = -1;
-        int32_t        term_contig   = -1;
-        int32_t        term_last     = -1;
-        int            term_num      = 0;
-        char          *probe_str     = NULL;
-        dict_t        *my_xdata      = NULL;
-        DIR           *fp            = NULL;
-        struct dirent *entry         = NULL;
-        struct dirent  scratch[2]    = {{0,},};
-
-        op_errno = jbr_get_changelog_dir(this, &cl_dir);
-        if (op_errno) {
-                goto err;       /* Error was already logged. */
-        }
-        op_errno = ENODATA;     /* Most common error after this. */
-
-        fp = sys_opendir (cl_dir);
-        if (!fp) {
-                op_errno = errno;
-                goto err;
-        }
-
-        /* Find first and last terms. */
-        for (;;) {
-                errno = 0;
-                entry = sys_readdir (fp, scratch);
-                if (!entry || errno != 0) {
-                        if (errno != 0) {
-                                op_errno = errno;
-                                goto err;
-                        }
-                        break;
-                }
-
-                if (fnmatch("TERM.*", entry->d_name, FNM_PATHNAME) != 0) {
-                        continue;
-                }
-                /* +5 points to the character after the period */
-                term_num = atoi(entry->d_name+5);
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        J_MSG_GENERIC,
-                        "%s => %d", entry->d_name, term_num);
-                if (term_num < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                J_MSG_INVALID,
-                                "invalid term file name %s", entry->d_name);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-                if ((term_first < 0) || (term_first > term_num)) {
-                        term_first = term_num;
-                }
-                if ((term_last < 0) || (term_last < term_num)) {
-                        term_last = term_num;
-                }
-        }
-        if ((term_first < 0) || (term_last < 0)) {
-                /* TBD: are we *sure* there should always be at least one? */
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_NO_DATA, "no terms found");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        (void) sys_closedir (fp);
-        fp = NULL;
-
-        /*
-         * Find term_contig, which is the earliest term for which there are
-         * no gaps between it and term_last.
-         */
-        for (term_contig = term_last; term_contig > 0; --term_contig) {
-                if (gf_asprintf(&probe_str, "%s/TERM.%d",
-                                cl_dir, term_contig-1) <= 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                J_MSG_MEM_ERR,
-                                "failed to format term %d", term_contig-1);
-                        goto err;
-                }
-                if (sys_access(probe_str, F_OK) != 0) {
-                        GF_FREE(probe_str);
-                        break;
-                }
-                GF_FREE(probe_str);
-        }
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                J_MSG_GENERIC,
-                "found terms %d-%d (%d)",
-                term_first, term_last, term_contig);
-
-        /* Return what we've found */
-        my_xdata = dict_new();
-        if (!my_xdata) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_MEM_ERR,
-                        "failed to allocate reply dictionary");
-                goto err;
-        }
-        if (dict_set_int32(my_xdata, "term-first", term_first) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_DICT_FLR,
-                        "failed to set term-first");
-                goto err;
-        }
-        if (dict_set_int32(my_xdata, "term-contig", term_contig) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_DICT_FLR,
-                        "failed to set term-contig");
-                goto err;
-        }
-        if (dict_set_int32(my_xdata, "term-last", term_last) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_DICT_FLR,
-                        "failed to set term-last");
-                goto err;
-        }
-
-        /* Finally! */
-        STACK_UNWIND_STRICT (ipc, frame, 0, 0, my_xdata);
-        dict_unref(my_xdata);
-        return;
-
-err:
-        if (fp) {
-                (void) sys_closedir (fp);
-        }
-        if (my_xdata) {
-                dict_unref(my_xdata);
-        }
-        STACK_UNWIND_STRICT (ipc, frame, -1, op_errno, NULL);
-}
-
-
-long
-get_entry_count (xlator_t *this, int fd)
-{
-        struct stat     buf;
-        long            min;            /* last entry not known to be empty */
-        long            max;            /* first entry known to be empty */
-        long            curr;
-        char            entry[CHANGELOG_ENTRY_SIZE];
-
-        if (sys_fstat (fd, &buf) < 0) {
-                return -1;
-        }
-
-        min = 0;
-        max = buf.st_size / CHANGELOG_ENTRY_SIZE;
-
-        while ((min+1) < max) {
-                curr = (min + max) / 2;
-                if (sys_lseek(fd, curr*CHANGELOG_ENTRY_SIZE, SEEK_SET) < 0) {
-                        return -1;
-                }
-                if (sys_read(fd, entry, sizeof(entry)) != sizeof(entry)) {
-                        return -1;
-                }
-                if ((entry[0] == '_') && (entry[1] == 'P')) {
-                        min = curr;
-                } else {
-                        max = curr;
-                }
-        }
-
-        if (sys_lseek(fd, 0, SEEK_SET) < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        J_MSG_SYS_CALL_FAILURE,
-                        "failed to reset offset");
-        }
-        return max;
-}
-
-
-void
-jbr_open_term (call_frame_t *frame, xlator_t *this, dict_t *xdata)
-{
-        int32_t         op_errno;
-        char            *cl_dir;
-        char            *term;
-        char            *path = NULL;
-        jbr_private_t   *priv           = this->private;
-
-        op_errno = jbr_get_changelog_dir(this, &cl_dir);
-        if (op_errno) {
-                goto err;
-        }
-
-        if (dict_get_str(xdata, "term", &term) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_NO_DATA, "missing term");
-                op_errno = ENODATA;
-                goto err;
-        }
-
-        if (gf_asprintf(&path, "%s/TERM.%s", cl_dir, term) < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_MEM_ERR, "failed to construct path");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        if (priv->term_fd >= 0) {
-                sys_close (priv->term_fd);
-        }
-        priv->term_fd = open(path, O_RDONLY);
-        if (priv->term_fd < 0) {
-                op_errno = errno;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_SYS_CALL_FAILURE,
-                        "failed to open term file");
-                goto err;
-        }
-
-        priv->term_total = get_entry_count(this, priv->term_fd);
-        if (priv->term_total < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_NO_DATA, "failed to get entry count");
-                sys_close (priv->term_fd);
-                priv->term_fd = -1;
-                op_errno = EIO;
-                goto err;
-        }
-        priv->term_read = 0;
-
-        /* Success! */
-        STACK_UNWIND_STRICT (ipc, frame, 0, 0, NULL);
-        GF_FREE (path);
-        return;
-
-err:
-        STACK_UNWIND_STRICT (ipc, frame, -1, op_errno, NULL);
-        GF_FREE (path);
-}
-
-
-void
-jbr_next_entry (call_frame_t *frame, xlator_t *this)
-{
-        int32_t         op_errno        = ENOMEM;
-        jbr_private_t   *priv           = this->private;
-        ssize_t          nbytes;
-        dict_t          *my_xdata;
-
-        if (priv->term_fd < 0) {
-                op_errno = EBADFD;
-                goto err;
-        }
-
-        if (priv->term_read >= priv->term_total) {
-                op_errno = ENODATA;
-                goto err;
-        }
-
-        nbytes = sys_read (priv->term_fd, priv->term_buf, CHANGELOG_ENTRY_SIZE);
-        if (nbytes < CHANGELOG_ENTRY_SIZE) {
-                if (nbytes < 0) {
-                        op_errno = errno;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                J_MSG_SYS_CALL_FAILURE,
-                                "error reading next entry: %s",
-                                strerror(errno));
-                } else {
-                        op_errno = EIO;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                J_MSG_SYS_CALL_FAILURE,
-                                "got %zd/%d bytes for next entry",
-                                nbytes, CHANGELOG_ENTRY_SIZE);
-                }
-                goto err;
-        }
-        ++(priv->term_read);
-
-        my_xdata = dict_new();
-        if (!my_xdata) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_MEM_ERR, "failed to allocate reply xdata");
-                goto err;
-        }
-
-        if (dict_set_static_bin(my_xdata, "data",
-                                priv->term_buf, CHANGELOG_ENTRY_SIZE) != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        J_MSG_DICT_FLR, "failed to assign reply xdata");
-                goto err;
-        }
-
-        STACK_UNWIND_STRICT (ipc, frame, 0, 0, my_xdata);
-        dict_unref(my_xdata);
-        return;
-
-err:
-        STACK_UNWIND_STRICT (ipc, frame, -1, op_errno, NULL);
-}
-
-int32_t
-jbr_ipc_fan_in (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        jbr_local_t   *local  = NULL;
-        int32_t        ret    = -1;
-        uint8_t        call_count;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-
-        gf_msg_trace (this->name, 0, "op_ret = %d, op_errno = %d\n",
-                      op_ret, op_errno);
-
-        LOCK(&frame->lock);
-        call_count = --(local->call_count);
-        UNLOCK(&frame->lock);
-
-        if (call_count == 0) {
-#if defined(JBR_CG_QUEUE)
-                ret = jbr_remove_from_queue (frame, this);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                J_MSG_GENERIC, "Failed to remove from queue.");
-                }
-#endif
-                /*
-                 * Unrefing the reference taken in continue() or complete() *
-                 */
-                dict_unref (local->xdata);
-                STACK_DESTROY (frame->root);
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int32_t
-jbr_ipc_complete (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  dict_t *xdata)
-{
-        jbr_local_t     *local     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-
-        jbr_ipc_call_dispatch (frame,
-                               this, &op_errno,
-                               FDL_IPC_JBR_SERVER_ROLLBACK,
-                               local->xdata);
-out:
-        return 0;
-}
-
-int32_t
-jbr_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
-{
-        switch (op) {
-        case JBR_SERVER_TERM_RANGE:
-                jbr_get_terms(frame, this);
-                break;
-        case JBR_SERVER_OPEN_TERM:
-                jbr_open_term(frame, this, xdata);
-                break;
-        case JBR_SERVER_NEXT_ENTRY:
-                jbr_next_entry(frame, this);
-                break;
-        case FDL_IPC_JBR_SERVER_ROLLBACK:
-                /*
-                 * Just send the fop down to fdl. Need not *
-                 * dispatch it to other bricks in the sub- *
-                 * volume, as it will be done where the op *
-                 * has failed.                             *
-                 */
-        default:
-                STACK_WIND_TAIL (frame,
-                                 FIRST_CHILD(this),
-                                 FIRST_CHILD(this)->fops->ipc,
-                                 op, xdata);
-        }
-
-        return 0;
-}
-
-#pragma generate
-
-int32_t
-jbr_forget (xlator_t *this, inode_t *inode)
-{
-        uint64_t        ctx     = 0LL;
-
-        if ((inode_ctx_del(inode, this, &ctx) == 0) && ctx) {
-                GF_FREE((void *)(long)ctx);
-        }
-
-        return 0;
-}
-
-int32_t
-jbr_release (xlator_t *this, fd_t *fd)
-{
-        uint64_t        ctx     = 0LL;
-
-        if ((fd_ctx_del(fd, this, &ctx) == 0) && ctx) {
-                GF_FREE((void *)(long)ctx);
-        }
-
-        return 0;
-}
-
-struct xlator_cbks cbks = {
-        .forget  = jbr_forget,
-        .release = jbr_release,
-};
-
-int
-jbr_reconfigure (xlator_t *this, dict_t *options)
-{
-        jbr_private_t   *priv   = this->private;
-
-        GF_OPTION_RECONF ("leader",
-                          priv->config_leader, options, bool, err);
-        GF_OPTION_RECONF ("quorum-percent",
-                          priv->quorum_pct, options, percent, err);
-        gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_GENERIC,
-                "reconfigure called, config_leader = %d, quorum_pct = %.1f\n",
-                priv->leader, priv->quorum_pct);
-
-        priv->leader = priv->config_leader;
-
-        return 0;
-
-err:
-        return -1;
-}
-
-int
-jbr_get_child_index (xlator_t *this, xlator_t *kid)
-{
-        xlator_list_t   *trav;
-        int             retval = -1;
-
-        for (trav = this->children; trav; trav = trav->next) {
-                ++retval;
-                if (trav->xlator == kid) {
-                        return retval;
-                }
-        }
-
-        return -1;
-}
-
-/*
- * Child notify handling is unreasonably FUBAR.  Sometimes we'll get a
- * CHILD_DOWN for a protocol/client child before we ever got a CHILD_UP for it.
- * Other times we won't.  Because it's effectively random (probably racy), we
- * can't just maintain a count.  We actually have to keep track of the state
- * for each child separately, to filter out the bogus CHILD_DOWN events, and
- * then generate counts on demand.
- */
-int
-jbr_notify (xlator_t *this, int event, void *data, ...)
-{
-        jbr_private_t   *priv         = this->private;
-        int             index         = -1;
-        int             ret           = -1;
-        gf_boolean_t    result        = _gf_false;
-        gf_boolean_t    relevant      = _gf_false;
-
-        switch (event) {
-        case GF_EVENT_CHILD_UP:
-                index = jbr_get_child_index(this, data);
-                if (index >= 0) {
-                        /* Check if the child was previously down
-                         * and it's not a false CHILD_UP
-                         */
-                        if (!(priv->kid_state & (1 << index))) {
-                                relevant = _gf_true;
-                        }
-
-                        priv->kid_state |= (1 << index);
-                        priv->up_children = jbr_count_up_kids(priv);
-                        gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_GENERIC,
-                                "got CHILD_UP for %s, now %u kids",
-                                ((xlator_t *)data)->name,
-                                priv->up_children);
-                        if (!priv->config_leader && (priv->up_children > 1)) {
-                                priv->leader = _gf_false;
-                        }
-
-                        /* If it's not relevant, or we have already *
-                         * sent CHILD_UP just break */
-                        if (!relevant || priv->child_up)
-                                break;
-
-                        /* If it's not a leader, just send the notify up */
-                        if (!priv->leader) {
-                                ret = default_notify(this, event, data);
-                                if (!ret)
-                                        priv->child_up = _gf_true;
-                                break;
-                        }
-
-                        result = fop_quorum_check (this,
-                                                (double)(priv->n_children - 1),
-                                               (double)(priv->up_children - 1));
-                        if (result == _gf_false) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        J_MSG_GENERIC, "Not enough children "
-                                        "are up to meet quorum. Waiting to "
-                                        "send CHILD_UP from leader");
-                        } else {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        J_MSG_GENERIC, "Enough children are up "
-                                        "to meet quorum. Sending CHILD_UP "
-                                        "from leader");
-                                ret = default_notify(this, event, data);
-                                if (!ret)
-                                        priv->child_up = _gf_true;
-                        }
-                }
-                break;
-        case GF_EVENT_CHILD_DOWN:
-                index = jbr_get_child_index(this, data);
-                if (index >= 0) {
-                        /* Check if the child was previously up
-                         * and it's not a false CHILD_DOWN
-                         */
-                        if (priv->kid_state & (1 << index)) {
-                                relevant = _gf_true;
-                        }
-                        priv->kid_state &= ~(1 << index);
-                        priv->up_children = jbr_count_up_kids(priv);
-                        gf_msg (this->name, GF_LOG_INFO, 0, J_MSG_GENERIC,
-                                "got CHILD_DOWN for %s, now %u kids",
-                                ((xlator_t *)data)->name,
-                                priv->up_children);
-                        if (!priv->config_leader && (priv->up_children < 2)
-                            && relevant) {
-                                priv->leader = _gf_true;
-                        }
-
-                        /* If it's not relevant, or we have already *
-                         * sent CHILD_DOWN just break */
-                        if (!relevant || !priv->child_up)
-                                break;
-
-                        /* If it's not a leader, just break coz we shouldn't  *
-                         * propagate the failure from the failure till it     *
-                         * itself goes down                                   *
-                         */
-                        if (!priv->leader) {
-                                break;
-                        }
-
-                        result = fop_quorum_check (this,
-                                           (double)(priv->n_children - 1),
-                                           (double)(priv->up_children - 1));
-                        if (result == _gf_false) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        J_MSG_GENERIC, "Enough children are "
-                                        "to down to fail quorum. "
-                                        "Sending CHILD_DOWN from leader");
-                                ret = default_notify(this, event, data);
-                                if (!ret)
-                                        priv->child_up = _gf_false;
-                        } else {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        J_MSG_GENERIC, "Not enough children "
-                                        "are down to fail quorum. Waiting to "
-                                        "send CHILD_DOWN from leader");
-                        }
-                }
-                break;
-        default:
-                ret = default_notify(this, event, data);
-        }
-
-        return ret;
-}
-
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("jbr", this, out);
-
-        ret = xlator_mem_acct_init (this, gf_mt_jbr_end + 1);
-
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, J_MSG_MEM_ERR,
-                        "Memory accounting init" "failed");
-                return ret;
-        }
-out:
-        return ret;
-}
-
-
-void
-jbr_deallocate_priv (jbr_private_t *priv)
-{
-        if (!priv) {
-                return;
-        }
-
-        GF_FREE(priv);
-}
-
-
-int32_t
-jbr_init (xlator_t *this)
-{
-        xlator_list_t   *remote;
-        xlator_list_t   *local;
-        jbr_private_t   *priv           = NULL;
-        xlator_list_t   *trav;
-        pthread_t       kid;
-        extern xlator_t global_xlator;
-        glusterfs_ctx_t *oldctx         = global_xlator.ctx;
-
-        /*
-         * Any fop that gets special treatment has to be patched in here,
-         * because the compiled-in table is produced by the code generator and
-         * only contains generated functions.  Note that we have to go through
-         * this->fops because of some dynamic-linking strangeness; modifying
-         * the static table doesn't work.
-         */
-        this->fops->getxattr = jbr_getxattr_special;
-        this->fops->fsync = jbr_fsync;
-
-        local = this->children;
-        if (!local) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, J_MSG_NO_DATA,
-                        "no local subvolume");
-                goto err;
-        }
-
-        remote = local->next;
-        if (!remote) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, J_MSG_NO_DATA,
-                        "no remote subvolumes");
-                goto err;
-        }
-
-        this->local_pool = mem_pool_new (jbr_local_t, 128);
-        if (!this->local_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, J_MSG_MEM_ERR,
-                        "failed to create jbr_local_t pool");
-                goto err;
-        }
-
-        priv = GF_CALLOC (1, sizeof(*priv), gf_mt_jbr_private_t);
-        if (!priv) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, J_MSG_MEM_ERR,
-                        "could not allocate priv");
-                goto err;
-        }
-
-        for (trav = this->children; trav; trav = trav->next) {
-                ++(priv->n_children);
-        }
-
-        LOCK_INIT(&priv->dirty_lock);
-	LOCK_INIT(&priv->index_lock);
-        INIT_LIST_HEAD(&priv->dirty_fds);
-        priv->term_fd = -1;
-
-        this->private = priv;
-
-        GF_OPTION_INIT ("leader", priv->config_leader, bool, err);
-        GF_OPTION_INIT ("quorum-percent", priv->quorum_pct, percent, err);
-
-        priv->leader = priv->config_leader;
-        priv->child_up = _gf_false;
-
-        if (gf_thread_create (&kid, NULL, jbr_flush_thread, this,
-                              "jbrflush") != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, J_MSG_SYS_CALL_FAILURE,
-                        "could not start flush thread");
-                /* TBD: treat this as a fatal error? */
-        }
-
-        /*
-         * Calling glfs_new changes old->ctx, even if THIS still points
-         * to global_xlator.  That causes problems later in the main
-         * thread, when gf_log_dump_graph tries to use the FILE after
-         * we've mucked with it and gets a segfault in __fprintf_chk.
-         * We can avoid all that by undoing the damage before we
-         * continue.
-         */
-        global_xlator.ctx = oldctx;
-
-	return 0;
-
-err:
-        jbr_deallocate_priv(priv);
-        return -1;
-}
-
-
-void
-jbr_fini (xlator_t *this)
-{
-        jbr_deallocate_priv(this->private);
-}
-
-class_methods_t class_methods = {
-        .init           = jbr_init,
-        .fini           = jbr_fini,
-        .reconfigure    = jbr_reconfigure,
-        .notify         = jbr_notify,
-};
-
-struct volume_options options[] = {
-        { .key = {"leader"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "false",
-          .description = "Start in the leader role.  This is only for "
-                         "bootstrapping the code, and should go away when we "
-                         "have real leader election."
-        },
-        { .key = {"vol-name"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "volume name"
-        },
-        { .key = {"my-name"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "brick name in form of host:/path"
-        },
-        { .key = {"etcd-servers"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "list of comma separated etc servers"
-        },
-        { .key = {"subvol-uuid"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "UUID for this JBR (sub)volume"
-        },
-        { .key = {"quorum-percent"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .default_value = "50.0",
-          .description = "percentage of rep_count-1 that must be up"
-        },
-	{ .key = {NULL} },
-};
diff --git a/xlators/experimental/posix2/Makefile.am b/xlators/experimental/posix2/Makefile.am
deleted file mode 100644
index 74e5ab0f5bc..00000000000
--- a/xlators/experimental/posix2/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = common mds ds
-
-CLEANFILES =
diff --git a/xlators/experimental/posix2/README.md b/xlators/experimental/posix2/README.md
deleted file mode 100644
index 955a98d061e..00000000000
--- a/xlators/experimental/posix2/README.md
+++ /dev/null
@@ -1,7 +0,0 @@
-# POSIX2 Experimental README
-
-POSIX2 is an implementation of modified storage translator to cater to DHT2
-on disk needs.
-
-For further understanding, refer to xlators/experimental/dht2/README.md for
-details regarding POSIX2
diff --git a/xlators/experimental/posix2/TODO.md b/xlators/experimental/posix2/TODO.md
deleted file mode 100644
index 20cd1e89c1d..00000000000
--- a/xlators/experimental/posix2/TODO.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# POSIX2 TODO List
-
-<Items will be added as code is pulled into the repository>
-\ No newline at end of file
diff --git a/xlators/experimental/posix2/common/Makefile.am b/xlators/experimental/posix2/common/Makefile.am
deleted file mode 100644
index a985f42a877..00000000000
--- a/xlators/experimental/posix2/common/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES =
diff --git a/xlators/experimental/posix2/common/src/Makefile.am b/xlators/experimental/posix2/common/src/Makefile.am
deleted file mode 100644
index 07533d2bf37..00000000000
--- a/xlators/experimental/posix2/common/src/Makefile.am
+++ /dev/null
@@ -1,16 +0,0 @@
-lib_LTLIBRARIES = libposix2common.la
-
-posix2_common_sources = posix2-common.c
-
-libposix2common_la_SOURCES = $(posix2_common_sources)
-libposix2common_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-libposix2common_la_CFLAGS = -Wall $(GF_CFLAGS)
-
-libposix2common_la_CPPFLAGS = $(GF_CPPFLAGS)
-libposix2common_la_CPPFLAGS += -I$(top_srcdir)/libglusterfs/src
-libposix2common_la_CPPFLAGS += -I$(top_srcdir)/rpc/xdr/src
-libposix2common_la_CPPFLAGS += -I$(top_builddir)/rpc/xdr/src
-
-
-CLEANFILES =
diff --git a/xlators/experimental/posix2/common/src/posix2-common.c b/xlators/experimental/posix2/common/src/posix2-common.c
deleted file mode 100644
index 14b51d538b2..00000000000
--- a/xlators/experimental/posix2/common/src/posix2-common.c
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/* File: posix2-common.c
- * This file contains common routines across ds and mds posix xlators
- * The entire functionality including comments is TODO.
- */
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "statedump.h"
diff --git a/xlators/experimental/posix2/ds/Makefile.am b/xlators/experimental/posix2/ds/Makefile.am
deleted file mode 100644
index a985f42a877..00000000000
--- a/xlators/experimental/posix2/ds/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES =
diff --git a/xlators/experimental/posix2/ds/src/Makefile.am b/xlators/experimental/posix2/ds/src/Makefile.am
deleted file mode 100644
index d77ef8cb540..00000000000
--- a/xlators/experimental/posix2/ds/src/Makefile.am
+++ /dev/null
@@ -1,22 +0,0 @@
-if WITH_SERVER
-xlator_LTLIBRARIES = posix2-ds.la
-endif
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
-
-posix2_ds_sources = posix2-ds-main.c
-
-posix2_ds_la_SOURCES = $(posix2_ds_sources)
-posix2_ds_la_LDFLAGS = -module -avoid-version
-posix2_ds_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-posix2_ds_la_LIBADD += $(top_builddir)/xlators/experimental/posix2/common/src/libposix2common.la
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-AM_CPPFLAGS = $(GF_CPPFLAGS)
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/storage/posix2/common/src
-AM_CPPFLAGS += -I$(top_srcdir)/libglusterfs/src
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/lib/src
-AM_CPPFLAGS += -I$(top_srcdir)/rpc/xdr/src
-AM_CPPFLAGS += -I$(top_builddir)/rpc/xdr/src
-
-CLEANFILES =
diff --git a/xlators/experimental/posix2/ds/src/posix2-ds-main.c b/xlators/experimental/posix2/ds/src/posix2-ds-main.c
deleted file mode 100644
index 675c4d7c9da..00000000000
--- a/xlators/experimental/posix2/ds/src/posix2-ds-main.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/* File: posix2-ds-main.c
- * This file contains the xlator loading functions, FOP entry points
- * and options.
- * The entire functionality including comments is TODO.
- */
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "statedump.h"
-
-int32_t
-posix2_ds_init (xlator_t *this)
-{
-        if (this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "This (%s) is a leaf xlator, but found children",
-                        this->name);
-                return -1;
-        }
-
-        return 0;
-}
-
-void
-posix2_ds_fini (xlator_t *this)
-{
-        return;
-}
-
-class_methods_t class_methods = {
-        .init           = posix2_ds_init,
-        .fini           = posix2_ds_fini,
-};
-
-struct xlator_fops fops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-/*
-struct xlator_dumpops dumpops = {
-};
-*/
-
-struct volume_options options[] = {
-        { .key  = {NULL} },
-};
diff --git a/xlators/experimental/posix2/mds/Makefile.am b/xlators/experimental/posix2/mds/Makefile.am
deleted file mode 100644
index a985f42a877..00000000000
--- a/xlators/experimental/posix2/mds/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES =
diff --git a/xlators/experimental/posix2/mds/src/Makefile.am b/xlators/experimental/posix2/mds/src/Makefile.am
deleted file mode 100644
index c6411f46114..00000000000
--- a/xlators/experimental/posix2/mds/src/Makefile.am
+++ /dev/null
@@ -1,22 +0,0 @@
-if WITH_SERVER
-xlator_LTLIBRARIES = posix2-mds.la
-endif
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/experimental
-
-posix2_mds_sources = posix2-mds-main.c
-
-posix2_mds_la_SOURCES = $(posix2_mds_sources)
-posix2_mds_la_LDFLAGS = -module -avoid-version
-posix2_mds_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-posix2_mds_la_LIBADD += $(top_builddir)/xlators/experimental/posix2/common/src/libposix2common.la
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-AM_CPPFLAGS = $(GF_CPPFLAGS)
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/storage/posix2/common/src
-AM_CPPFLAGS += -I$(top_srcdir)/libglusterfs/src
-AM_CPPFLAGS += -I$(top_srcdir)/xlators/lib/src
-AM_CPPFLAGS += -I$(top_srcdir)/rpc/xdr/src
-AM_CPPFLAGS += -I$(top_builddir)/rpc/xdr/src
-
-CLEANFILES =
diff --git a/xlators/experimental/posix2/mds/src/posix2-mds-main.c b/xlators/experimental/posix2/mds/src/posix2-mds-main.c
deleted file mode 100644
index 71ff4e0089c..00000000000
--- a/xlators/experimental/posix2/mds/src/posix2-mds-main.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
-  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/* File: posix2-mds-main.c
- * This file contains the xlator loading functions, FOP entry points
- * and options.
- * The entire functionality including comments is TODO.
- */
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "statedump.h"
-
-int32_t
-posix2_mds_init (xlator_t *this)
-{
-        if (this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "This (%s) is a leaf xlator, but found children",
-                        this->name);
-                return -1;
-        }
-
-        return 0;
-}
-
-void
-posix2_mds_fini (xlator_t *this)
-{
-        return;
-}
-
-class_methods_t class_methods = {
-        .init           = posix2_mds_init,
-        .fini           = posix2_mds_fini,
-};
-
-struct xlator_fops fops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-/*
-struct xlator_dumpops dumpops = {
-};
-*/
-
-struct volume_options options[] = {
-        { .key  = {NULL} },
-};
diff --git a/xlators/features/Makefile.am b/xlators/features/Makefile.am
index 81fbe856e78..c57897f11ea 100644
--- a/xlators/features/Makefile.am
+++ b/xlators/features/Makefile.am
@@ -1,6 +1,14 @@
-SUBDIRS = locks quota read-only quiesce marker index barrier \
-          arbiter compress changelog changetimerecorder \
-          gfid-access $(GLUPY_SUBDIR) upcall snapview-client snapview-server \
-          trash shard bit-rot leases selinux sdfs namespace cloudsync thin-arbiter \
-          utime
+if BUILD_CLOUDSYNC
+  CLOUDSYNC_DIR = cloudsync
+endif
+
+if BUILD_METADISP
+  METADISP_DIR = metadisp
+endif
+
+SUBDIRS = locks quota read-only quiesce marker index barrier arbiter upcall \
+	compress changelog gfid-access snapview-client snapview-server trash \
+	shard bit-rot leases selinux sdfs namespace $(CLOUDSYNC_DIR) thin-arbiter \
+	utime $(METADISP_DIR)
+
 CLEANFILES =
diff --git a/xlators/features/arbiter/src/arbiter-mem-types.h b/xlators/features/arbiter/src/arbiter-mem-types.h
index ccf864cdef0..05d18374c46 100644
--- a/xlators/features/arbiter/src/arbiter-mem-types.h
+++ b/xlators/features/arbiter/src/arbiter-mem-types.h
@@ -9,10 +9,10 @@
 
 #ifndef __ARBITER_MEM_TYPES_H__
 #define __ARBITER_MEM_TYPES_H__
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 typedef enum gf_arbiter_mem_types_ {
-        gf_arbiter_mt_inode_ctx_t =  gf_common_mt_end + 1,
-        gf_arbiter_mt_end
+    gf_arbiter_mt_inode_ctx_t = gf_common_mt_end + 1,
+    gf_arbiter_mt_end
 } gf_arbiter_mem_types_t;
 #endif
diff --git a/xlators/features/arbiter/src/arbiter.c b/xlators/features/arbiter/src/arbiter.c
index cf0841653b7..83a97e3354b 100644
--- a/xlators/features/arbiter/src/arbiter.c
+++ b/xlators/features/arbiter/src/arbiter.c
@@ -10,362 +10,371 @@
 
 #include "arbiter.h"
 #include "arbiter-mem-types.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
 
 static arbiter_inode_ctx_t *
-__arbiter_inode_ctx_get (inode_t *inode, xlator_t *this)
+__arbiter_inode_ctx_get(inode_t *inode, xlator_t *this)
 {
-
-        arbiter_inode_ctx_t *ctx = NULL;
-        int ret = 0;
-        uint64_t ctx_addr = 0;
-
-        ret = __inode_ctx_get (inode, this, &ctx_addr);
-        if (ret == 0) {
-                ctx = (arbiter_inode_ctx_t *) (long) ctx_addr;
-                goto out;
-        }
-
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_arbiter_mt_inode_ctx_t);
-        if (!ctx)
-                goto out;
-
-        ret = __inode_ctx_put (inode, this, (uint64_t)ctx);
-        if (ret) {
-                GF_FREE (ctx);
-                ctx = NULL;
-                gf_log_callingfn (this->name, GF_LOG_ERROR, "failed to "
-                                  "set the inode ctx (%s)",
-                                  uuid_utoa (inode->gfid));
-        }
+    arbiter_inode_ctx_t *ctx = NULL;
+    int ret = 0;
+    uint64_t ctx_addr = 0;
+
+    ret = __inode_ctx_get(inode, this, &ctx_addr);
+    if (ret == 0) {
+        ctx = (arbiter_inode_ctx_t *)(long)ctx_addr;
+        goto out;
+    }
+
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_arbiter_mt_inode_ctx_t);
+    if (!ctx)
+        goto out;
+
+    ret = __inode_ctx_put(inode, this, (uint64_t)(uintptr_t)ctx);
+    if (ret) {
+        GF_FREE(ctx);
+        ctx = NULL;
+        gf_log_callingfn(this->name, GF_LOG_ERROR,
+                         "failed to "
+                         "set the inode ctx (%s)",
+                         uuid_utoa(inode->gfid));
+    }
 out:
-        return ctx;
+    return ctx;
 }
 
 static arbiter_inode_ctx_t *
-arbiter_inode_ctx_get (inode_t *inode, xlator_t *this)
+arbiter_inode_ctx_get(inode_t *inode, xlator_t *this)
 {
-        arbiter_inode_ctx_t *ctx = NULL;
-
-        LOCK(&inode->lock);
-        {
-                ctx = __arbiter_inode_ctx_get (inode, this);
-        }
-        UNLOCK(&inode->lock);
-        return ctx;
+    arbiter_inode_ctx_t *ctx = NULL;
+
+    LOCK(&inode->lock);
+    {
+        ctx = __arbiter_inode_ctx_get(inode, this);
+    }
+    UNLOCK(&inode->lock);
+    return ctx;
 }
 
 int32_t
-arbiter_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+arbiter_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        arbiter_inode_ctx_t *ctx = NULL;
-
-        if (op_ret != 0)
-                goto unwind;
-        ctx = arbiter_inode_ctx_get (inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        memcpy (&ctx->iattbuf, buf, sizeof (ctx->iattbuf));
+    arbiter_inode_ctx_t *ctx = NULL;
+
+    if (op_ret != 0)
+        goto unwind;
+    ctx = arbiter_inode_ctx_get(inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    memcpy(&ctx->iattbuf, buf, sizeof(ctx->iattbuf));
 
 unwind:
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xdata, postparent);
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    return 0;
 }
 
 int32_t
-arbiter_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+arbiter_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        STACK_WIND (frame, arbiter_lookup_cbk, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->lookup, loc, xdata);
-        return 0;
+    STACK_WIND(frame, arbiter_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    return 0;
 }
 
 int32_t
-arbiter_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-                  dict_t *xdata)
+arbiter_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int32_t              op_ret   = 0;
-        int32_t              op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (loc->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = &ctx->iattbuf;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(loc->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
 unwind:
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, buf, buf, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
 int32_t
-arbiter_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                   dict_t *xdata)
+arbiter_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
 
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int32_t              op_ret   = 0;
-        int32_t              op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = &ctx->iattbuf;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
 unwind:
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, buf, buf,
-                             NULL);
-        return 0;
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
-dict_t*
-arbiter_fill_writev_xdata (fd_t *fd, dict_t *xdata, xlator_t *this)
+dict_t *
+arbiter_fill_writev_xdata(fd_t *fd, dict_t *xdata, xlator_t *this)
 {
-        dict_t  *rsp_xdata = NULL;
-        int32_t ret = 0;
-        int is_append = 1;
-
-        if (!fd || !fd->inode || gf_uuid_is_null (fd->inode->gfid)) {
-                goto out;
-        }
-
-        if (!xdata)
-                goto out;
-
-        rsp_xdata = dict_new();
-        if (!rsp_xdata)
-                goto out;
-
-        if (dict_get (xdata, GLUSTERFS_OPEN_FD_COUNT)) {
-                ret = dict_set_uint32 (rsp_xdata, GLUSTERFS_OPEN_FD_COUNT,
-                                       fd->inode->fd_count);
-                if (ret < 0) {
-                        gf_msg_debug (this->name, 0, "Failed to set dict value"
-                                      " for GLUSTERFS_OPEN_FD_COUNT");
-                }
+    dict_t *rsp_xdata = NULL;
+    int32_t ret = 0;
+    int is_append = 1;
+
+    if (!fd || !fd->inode || gf_uuid_is_null(fd->inode->gfid)) {
+        goto out;
+    }
+
+    if (!xdata)
+        goto out;
+
+    rsp_xdata = dict_new();
+    if (!rsp_xdata)
+        goto out;
+
+    if (dict_get(xdata, GLUSTERFS_OPEN_FD_COUNT)) {
+        ret = dict_set_uint32(rsp_xdata, GLUSTERFS_OPEN_FD_COUNT,
+                              fd->inode->fd_count);
+        if (ret < 0) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set dict value"
+                         " for GLUSTERFS_OPEN_FD_COUNT");
         }
-        if (dict_get (xdata, GLUSTERFS_WRITE_IS_APPEND)) {
-                ret = dict_set_uint32 (rsp_xdata, GLUSTERFS_WRITE_IS_APPEND,
-                                       is_append);
-                if (ret < 0) {
-                        gf_msg_debug (this->name, 0, "Failed to set dict value"
-                                      " for GLUSTERFS_WRITE_IS_APPEND");
-                }
+    }
+    if (dict_get(xdata, GLUSTERFS_WRITE_IS_APPEND)) {
+        ret = dict_set_uint32(rsp_xdata, GLUSTERFS_WRITE_IS_APPEND, is_append);
+        if (ret < 0) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set dict value"
+                         " for GLUSTERFS_WRITE_IS_APPEND");
         }
+    }
 out:
-        return rsp_xdata;
+    return rsp_xdata;
 }
 
 int32_t
-arbiter_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iovec *vector, int32_t count, off_t off, uint32_t flags,
-                struct iobref *iobref, dict_t *xdata)
+arbiter_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+               struct iobref *iobref, dict_t *xdata)
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        dict_t           *rsp_xdata   = NULL;
-        int                  op_ret   = 0;
-        int                  op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = &ctx->iattbuf;
-        op_ret = iov_length (vector, count);
-        rsp_xdata = arbiter_fill_writev_xdata (fd, xdata, this);
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    dict_t *rsp_xdata = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
+    op_ret = iov_length(vector, count);
+    rsp_xdata = arbiter_fill_writev_xdata(fd, xdata, this);
 unwind:
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, buf, buf,
-                             rsp_xdata);
-        if (rsp_xdata)
-                dict_unref (rsp_xdata);
-        return 0;
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, buf, buf, rsp_xdata);
+    if (rsp_xdata)
+        dict_unref(rsp_xdata);
+    return 0;
 }
 
 int32_t
-arbiter_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
+arbiter_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
                   int32_t keep_size, off_t offset, size_t len, dict_t *xdata)
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int                  op_ret   = 0;
-        int                  op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = &ctx->iattbuf;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
 unwind:
-        STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, buf, buf, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
 int32_t
-arbiter_discard (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                off_t offset, size_t len, dict_t *xdata)
+arbiter_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                size_t len, dict_t *xdata)
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int                  op_ret   = 0;
-        int                  op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = &ctx->iattbuf;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
 unwind:
-        STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, buf, buf, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
 int32_t
-arbiter_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  off_t offset, off_t len, dict_t *xdata)
+arbiter_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 off_t len, dict_t *xdata)
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int                  op_ret   = 0;
-        int                  op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = &ctx->iattbuf;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
 unwind:
-        STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, buf, buf, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
 static int32_t
-arbiter_readv (call_frame_t *frame,  xlator_t *this, fd_t *fd, size_t size,
-               off_t offset, uint32_t flags, dict_t *xdata)
+arbiter_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, uint32_t flags, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (readv, frame, -1, ENOSYS, NULL, 0, NULL, NULL,
-                             NULL);
-        return 0;
+    STACK_UNWIND_STRICT(readv, frame, -1, ENOSYS, NULL, 0, NULL, NULL, NULL);
+    return 0;
 }
 
 static int32_t
-arbiter_seek (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-              gf_seek_what_t what, dict_t *xdata)
+arbiter_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             gf_seek_what_t what, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (seek, frame, -1, ENOSYS, 0, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(seek, frame, -1, ENOSYS, 0, xdata);
+    return 0;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int ret = -1;
-
-        ret = xlator_mem_acct_init (this, gf_arbiter_mt_end + 1);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting "
-                        "initialization failed.");
-        return ret;
+    int ret = -1;
+
+    ret = xlator_mem_acct_init(this, gf_arbiter_mt_end + 1);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting "
+               "initialization failed.");
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-
-        return 0;
+    return 0;
 }
 
 int
-arbiter_forget (xlator_t *this, inode_t *inode)
+arbiter_forget(xlator_t *this, inode_t *inode)
 {
-        arbiter_inode_ctx_t *ctx = NULL;
-        uint64_t ctx_addr = 0;
-
-        inode_ctx_del (inode, this, &ctx_addr);
-        if (!ctx_addr)
-                return 0;
-        ctx = (arbiter_inode_ctx_t *) (long) ctx_addr;
-        GF_FREE (ctx);
+    arbiter_inode_ctx_t *ctx = NULL;
+    uint64_t ctx_addr = 0;
+
+    inode_ctx_del(inode, this, &ctx_addr);
+    if (!ctx_addr)
         return 0;
+    ctx = (arbiter_inode_ctx_t *)(long)ctx_addr;
+    GF_FREE(ctx);
+    return 0;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "'arbiter' not configured with exactly one child");
+        return -1;
+    }
 
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "'arbiter' not configured with exactly one child");
-                return -1;
-        }
-
-        if (!this->parents)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "dangling volume. check volfile ");
+    if (!this->parents)
+        gf_log(this->name, GF_LOG_ERROR, "dangling volume. check volfile ");
 
-        return 0;
+    return 0;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        return;
+    return;
 }
 
 struct xlator_fops fops = {
-        .lookup = arbiter_lookup,
-
-        /* Return success for these inode write FOPS without winding it down to
-         * posix; this is needed for AFR write transaction logic to work.*/
-        .truncate = arbiter_truncate,
-        .writev = arbiter_writev,
-        .ftruncate = arbiter_ftruncate,
-        .fallocate = arbiter_fallocate,
-        .discard = arbiter_discard,
-        .zerofill = arbiter_zerofill,
-
-        /* AFR is not expected to wind these inode read FOPS initiated by the
-         * application to the arbiter brick. But in case a bug causes them
-         * to be called, we return ENOSYS. */
-        .readv = arbiter_readv,
-        .seek = arbiter_seek,
-
-        /* The following inode read FOPS initiated by the application are not
-         * wound by AFR either but internal logic like  shd, glfsheal and
-         * client side healing in AFR will send them for selfheal/ inode refresh
-         * operations etc.,so we need to wind them down to posix:
-         *
-         * (f)stat, readdir(p), readlink, (f)getxattr.*/
-
-        /* All other FOPs not listed here are safe to be wound down to posix.*/
+    .lookup = arbiter_lookup,
+
+    /* Return success for these inode write FOPS without winding it down to
+     * posix; this is needed for AFR write transaction logic to work.*/
+    .truncate = arbiter_truncate,
+    .writev = arbiter_writev,
+    .ftruncate = arbiter_ftruncate,
+    .fallocate = arbiter_fallocate,
+    .discard = arbiter_discard,
+    .zerofill = arbiter_zerofill,
+
+    /* AFR is not expected to wind these inode read FOPS initiated by the
+     * application to the arbiter brick. But in case a bug causes them
+     * to be called, we return ENOSYS. */
+    .readv = arbiter_readv,
+    .seek = arbiter_seek,
+
+    /* The following inode read FOPS initiated by the application are not
+     * wound by AFR either but internal logic like  shd, glfsheal and
+     * client side healing in AFR will send them for selfheal/ inode refresh
+     * operations etc.,so we need to wind them down to posix:
+     *
+     * (f)stat, readdir(p), readlink, (f)getxattr.*/
+
+    /* All other FOPs not listed here are safe to be wound down to posix.*/
 };
 
 struct xlator_cbks cbks = {
-        .forget = arbiter_forget,
+    .forget = arbiter_forget,
 };
 
 struct volume_options options[] = {
-        { .key  = {NULL} },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "arbiter",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/arbiter/src/arbiter.h b/xlators/features/arbiter/src/arbiter.h
index f52b45fad20..546db7b751a 100644
--- a/xlators/features/arbiter/src/arbiter.h
+++ b/xlators/features/arbiter/src/arbiter.h
@@ -11,11 +11,11 @@
 #ifndef _ARBITER_H
 #define _ARBITER_H
 
-#include "locking.h"
-#include "common-utils.h"
+#include <glusterfs/locking.h>
+#include <glusterfs/common-utils.h>
 
 typedef struct arbiter_inode_ctx_ {
-        struct iatt iattbuf;
+    struct iatt iattbuf;
 } arbiter_inode_ctx_t;
 
 #endif /* _ARBITER_H */
diff --git a/xlators/features/barrier/src/barrier-mem-types.h b/xlators/features/barrier/src/barrier-mem-types.h
index 36647a66966..71ed7898d9c 100644
--- a/xlators/features/barrier/src/barrier-mem-types.h
+++ b/xlators/features/barrier/src/barrier-mem-types.h
@@ -11,10 +11,10 @@
 #ifndef __BARRIER_MEM_TYPES_H__
 #define __BARRIER_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_barrier_mem_types_ {
-        gf_barrier_mt_priv_t = gf_common_mt_end + 1,
-        gf_barrier_mt_end
+    gf_barrier_mt_priv_t = gf_common_mt_end + 1,
+    gf_barrier_mt_end
 };
 #endif
diff --git a/xlators/features/barrier/src/barrier.c b/xlators/features/barrier/src/barrier.c
index 4401e3459a9..852bbacb99d 100644
--- a/xlators/features/barrier/src/barrier.c
+++ b/xlators/features/barrier/src/barrier.c
@@ -9,795 +9,801 @@
 */
 
 #include "barrier.h"
-#include "defaults.h"
-#include "call-stub.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/call-stub.h>
 
-#include "statedump.h"
+#include <glusterfs/statedump.h>
 
 void
-barrier_local_set_gfid (call_frame_t *frame, uuid_t gfid, xlator_t *this)
-{
-        if (gfid) {
-                uuid_t *id = GF_MALLOC (sizeof (uuid_t), gf_common_mt_uuid_t);
-                if (!id) {
-                        gf_log (this->name, GF_LOG_WARNING, "Could not set gfid"
-                                ". gfid will not be dumped in statedump file.");
-                        return;
-                }
-                gf_uuid_copy (*id, gfid);
-                frame->local = id;
+barrier_local_set_gfid(call_frame_t *frame, uuid_t gfid, xlator_t *this)
+{
+    if (gfid) {
+        uuid_t *id = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!id) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "Could not set gfid"
+                   ". gfid will not be dumped in statedump file.");
+            return;
         }
+        gf_uuid_copy(*id, gfid);
+        frame->local = id;
+    }
 }
 
 void
-barrier_local_free_gfid (call_frame_t *frame)
+barrier_local_free_gfid(call_frame_t *frame)
 {
-        if (frame->local) {
-                GF_FREE (frame->local);
-                frame->local = NULL;
-        }
+    if (frame->local) {
+        GF_FREE(frame->local);
+        frame->local = NULL;
+    }
 }
 
 int32_t
-barrier_truncate_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
+barrier_truncate_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            struct iatt *prebuf, struct iatt *postbuf,
+                            dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
+}
+
+int32_t
+barrier_ftruncate_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
                              int32_t op_ret, int32_t op_errno,
                              struct iatt *prebuf, struct iatt *postbuf,
                              dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        return 0;
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int32_t
-barrier_ftruncate_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno,
-                              struct iatt *prebuf, struct iatt *postbuf,
-                              dict_t *xdata)
-{
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
+barrier_unlink_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno,
+                          struct iatt *preparent, struct iatt *postparent,
+                          dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
 int32_t
-barrier_unlink_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *preparent, struct iatt *postparent,
-                           dict_t *xdata)
-{
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-        return 0;
+barrier_rmdir_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno,
+                         struct iatt *preparent, struct iatt *postparent,
+                         dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
 int32_t
-barrier_rmdir_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iatt *preparent, struct iatt *postparent,
+barrier_rename_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                          struct iatt *preoldparent, struct iatt *postoldparent,
+                          struct iatt *prenewparent, struct iatt *postnewparent,
                           dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-        return 0;
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(rename, frame, op_ret, op_errno, buf, preoldparent,
+                        postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
 }
 
 int32_t
-barrier_rename_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                           struct iatt *preoldparent, struct iatt *postoldparent,
-                           struct iatt *prenewparent, struct iatt *postnewparent,
-                           dict_t *xdata)
-{
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent, xdata);
-        return 0;
+barrier_writev_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                          struct iatt *postbuf, dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int32_t
-barrier_writev_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *prebuf, struct iatt *postbuf,
-                           dict_t *xdata)
-{
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        return 0;
+barrier_fsync_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
 }
 
 int32_t
-barrier_fsync_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                          struct iatt *postbuf, dict_t *xdata)
+barrier_removexattr_cbk_resume(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        return 0;
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(removexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-barrier_removexattr_cbk_resume (call_frame_t *frame, void *cookie,
+barrier_fremovexattr_cbk_resume(call_frame_t *frame, void *cookie,
                                 xlator_t *this, int32_t op_ret,
                                 int32_t op_errno, dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(fremovexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-barrier_fremovexattr_cbk_resume (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *xdata)
+barrier_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    BARRIER_FOP_CBK(writev, out, frame, this, op_ret, op_errno, prebuf, postbuf,
+                    xdata);
+out:
+    return 0;
 }
 
 int32_t
-barrier_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf,
-                    dict_t *xdata)
+barrier_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (writev, out, frame, this, op_ret, op_errno,
-                         prebuf, postbuf, xdata);
+    BARRIER_FOP_CBK(fremovexattr, out, frame, this, op_ret, op_errno, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+barrier_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (fremovexattr, out, frame, this, op_ret, op_errno,
-                         xdata);
+    BARRIER_FOP_CBK(removexattr, out, frame, this, op_ret, op_errno, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+barrier_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (removexattr, out, frame, this, op_ret, op_errno,
-                         xdata);
+    BARRIER_FOP_CBK(truncate, out, frame, this, op_ret, op_errno, prebuf,
+                    postbuf, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+barrier_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                       struct iatt *postbuf, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (truncate, out, frame, this, op_ret, op_errno, prebuf,
-                         postbuf, xdata);
+    BARRIER_FOP_CBK(ftruncate, out, frame, this, op_ret, op_errno, prebuf,
+                    postbuf, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
-{
-        BARRIER_FOP_CBK (ftruncate, out, frame, this, op_ret, op_errno, prebuf,
-                         postbuf, xdata);
+barrier_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
+{
+    BARRIER_FOP_CBK(rename, out, frame, this, op_ret, op_errno, buf,
+                    preoldparent, postoldparent, prenewparent, postnewparent,
+                    xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
-{
-        BARRIER_FOP_CBK (rename, out, frame, this, op_ret, op_errno, buf,
-                         preoldparent, postoldparent, prenewparent,
-                         postnewparent, xdata);
+barrier_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+    BARRIER_FOP_CBK(rmdir, out, frame, this, op_ret, op_errno, preparent,
+                    postparent, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+barrier_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
                    struct iatt *postparent, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (rmdir, out, frame, this, op_ret, op_errno, preparent,
-                         postparent, xdata);
+    BARRIER_FOP_CBK(unlink, out, frame, this, op_ret, op_errno, preparent,
+                    postparent, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+barrier_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (unlink, out, frame, this, op_ret, op_errno, preparent,
-                         postparent, xdata);
+    BARRIER_FOP_CBK(fsync, out, frame, this, op_ret, op_errno, prebuf, postbuf,
+                    xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+barrier_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+               struct iobref *iobref, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (fsync, out, frame, this, op_ret, op_errno,
-                         prebuf, postbuf, xdata);
-out:
-        return 0;
-}
-
-int32_t
-barrier_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iovec *vector, int32_t count, off_t off, uint32_t flags,
-                struct iobref *iobref, dict_t *xdata)
-{
-        if (!((flags | fd->flags) & (O_SYNC | O_DSYNC))) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                                 FIRST_CHILD(this)->fops->writev,
-                                 fd, vector, count, off, flags, iobref, xdata);
-
-                return 0;
-        }
+    if (!((flags | fd->flags) & (O_SYNC | O_DSYNC))) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
+                        flags, iobref, xdata);
 
-        barrier_local_set_gfid (frame, fd->inode->gfid, this);
-        STACK_WIND (frame, barrier_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
-                    off, flags, iobref, xdata);
         return 0;
+    }
+
+    barrier_local_set_gfid(frame, fd->inode->gfid, this);
+    STACK_WIND(frame, barrier_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, off, flags,
+               iobref, xdata);
+    return 0;
 }
 
 int32_t
-barrier_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      const char *name, dict_t *xdata)
+barrier_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     const char *name, dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, fd->inode->gfid, this);
-        STACK_WIND (frame, barrier_fremovexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, fd->inode->gfid, this);
+    STACK_WIND(frame, barrier_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
 int32_t
-barrier_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     const char *name, dict_t *xdata)
+barrier_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, loc->inode->gfid, this);
-        STACK_WIND (frame, barrier_removexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, loc->inode->gfid, this);
+    STACK_WIND(frame, barrier_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 }
 
 int32_t
-barrier_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  off_t offset, dict_t *xdata)
+barrier_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, loc->inode->gfid, this);
-        STACK_WIND (frame, barrier_truncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, loc->inode->gfid, this);
+    STACK_WIND(frame, barrier_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
-
 int32_t
-barrier_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                loc_t *newloc, dict_t *xdata)
+barrier_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, oldloc->inode->gfid, this);
-        STACK_WIND (frame, barrier_rename_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->rename,
-                    oldloc, newloc, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, oldloc->inode->gfid, this);
+    STACK_WIND(frame, barrier_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 }
 
 int
-barrier_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-               dict_t *xdata)
+barrier_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, loc->inode->gfid, this);
-        STACK_WIND (frame, barrier_rmdir_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->rmdir,
-                    loc, flags, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, loc->inode->gfid, this);
+    STACK_WIND(frame, barrier_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+    return 0;
 }
 
 int32_t
-barrier_unlink (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int xflag, dict_t *xdata)
+barrier_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+               dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, loc->inode->gfid, this);
-        STACK_WIND (frame, barrier_unlink_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->unlink,
-                    loc, xflag, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, loc->inode->gfid, this);
+    STACK_WIND(frame, barrier_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
 }
 
 int32_t
-barrier_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   off_t offset, dict_t *xdata)
+barrier_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, fd->inode->gfid, this);
-        STACK_WIND (frame, barrier_ftruncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ftruncate,
-                    fd, offset, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, fd->inode->gfid, this);
+    STACK_WIND(frame, barrier_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
 int32_t
-barrier_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               int32_t flags, dict_t *xdata)
+barrier_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+              dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, fd->inode->gfid, this);
-        STACK_WIND (frame, barrier_fsync_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsync,
-                    fd, flags, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, fd->inode->gfid, this);
+    STACK_WIND(frame, barrier_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
 }
 
 call_stub_t *
-__barrier_dequeue (xlator_t *this, struct list_head *queue)
+__barrier_dequeue(xlator_t *this, struct list_head *queue)
 {
-        call_stub_t    *stub            = NULL;
-        barrier_priv_t *priv            = NULL;
+    call_stub_t *stub = NULL;
+    barrier_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (list_empty (queue))
-                goto out;
+    if (list_empty(queue))
+        goto out;
 
-        stub = list_entry (queue->next, call_stub_t, list);
-        list_del_init (&stub->list);
+    stub = list_entry(queue->next, call_stub_t, list);
+    list_del_init(&stub->list);
 
 out:
-        return stub;
+    return stub;
 }
 
 void
-barrier_dequeue_all (xlator_t *this, struct list_head *queue)
+barrier_dequeue_all(xlator_t *this, struct list_head *queue)
 {
-        call_stub_t            *stub    = NULL;
+    call_stub_t *stub = NULL;
 
-        gf_log (this->name, GF_LOG_INFO, "Dequeuing all the barriered fops");
+    gf_log(this->name, GF_LOG_INFO, "Dequeuing all the barriered fops");
 
-        /* TODO: Start the below task in a new thread */
-        while ((stub = __barrier_dequeue (this, queue)))
-                call_resume (stub);
+    /* TODO: Start the below task in a new thread */
+    while ((stub = __barrier_dequeue(this, queue)))
+        call_resume(stub);
 
-        gf_log (this->name, GF_LOG_INFO, "Dequeuing the barriered fops is "
-                                         "finished");
-        return;
+    gf_log(this->name, GF_LOG_INFO,
+           "Dequeuing the barriered fops is "
+           "finished");
+    return;
 }
 
 void
-barrier_timeout (void *data)
+barrier_timeout(void *data)
 {
-        xlator_t               *this    = NULL;
-        barrier_priv_t         *priv    = NULL;
-        struct list_head        queue   = {0,};
+    xlator_t *this = NULL;
+    barrier_priv_t *priv = NULL;
+    struct list_head queue = {
+        0,
+    };
 
-        this = data;
-        THIS = this;
-        priv = this->private;
+    this = data;
+    THIS = this;
+    priv = this->private;
 
-        INIT_LIST_HEAD (&queue);
+    INIT_LIST_HEAD(&queue);
 
-        gf_log (this->name, GF_LOG_CRITICAL, "Disabling barrier because of "
-                                             "the barrier timeout.");
+    gf_log(this->name, GF_LOG_CRITICAL,
+           "Disabling barrier because of "
+           "the barrier timeout.");
 
-        LOCK (&priv->lock);
-        {
-                __barrier_disable (this, &queue);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        __barrier_disable(this, &queue);
+    }
+    UNLOCK(&priv->lock);
 
-        barrier_dequeue_all (this, &queue);
+    barrier_dequeue_all(this, &queue);
 
-        return;
+    return;
 }
 
 void
-__barrier_enqueue (xlator_t *this, call_stub_t *stub)
+__barrier_enqueue(xlator_t *this, call_stub_t *stub)
 {
-        barrier_priv_t *priv    = NULL;
+    barrier_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        list_add_tail (&stub->list, &priv->queue);
-        priv->queue_size++;
+    list_add_tail(&stub->list, &priv->queue);
+    priv->queue_size++;
 
-        return;
+    return;
 }
 
 void
-__barrier_disable (xlator_t *this, struct list_head *queue)
+__barrier_disable(xlator_t *this, struct list_head *queue)
 {
-        GF_UNUSED int   ret     = 0;
-        barrier_priv_t *priv    = NULL;
+    GF_UNUSED int ret = 0;
+    barrier_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (priv->timer) {
-                ret = gf_timer_call_cancel (this->ctx, priv->timer);
-                priv->timer = NULL;
-        }
+    if (priv->timer) {
+        ret = gf_timer_call_cancel(this->ctx, priv->timer);
+        priv->timer = NULL;
+    }
 
-        list_splice_init (&priv->queue, queue);
-        priv->queue_size = 0;
-        priv->barrier_enabled = _gf_false;
+    list_splice_init(&priv->queue, queue);
+    priv->queue_size = 0;
+    priv->barrier_enabled = _gf_false;
 }
 
 int
-__barrier_enable (xlator_t *this, barrier_priv_t *priv)
+__barrier_enable(xlator_t *this, barrier_priv_t *priv)
 {
-        int             ret     = -1;
+    int ret = -1;
 
-        priv->timer = gf_timer_call_after (this->ctx, priv->timeout,
-                                           barrier_timeout, (void *) this);
-        if (!priv->timer) {
-                gf_log (this->name, GF_LOG_CRITICAL, "Couldn't add barrier "
-                                                     "timeout event.");
-                goto out;
-        }
+    priv->timer = gf_timer_call_after(this->ctx, priv->timeout, barrier_timeout,
+                                      (void *)this);
+    if (!priv->timer) {
+        gf_log(this->name, GF_LOG_CRITICAL,
+               "Couldn't add barrier "
+               "timeout event.");
+        goto out;
+    }
 
-        priv->barrier_enabled = _gf_true;
-        ret = 0;
+    priv->barrier_enabled = _gf_true;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        barrier_priv_t  *priv                   = NULL;
-        dict_t          *dict                   = NULL;
-        gf_boolean_t     past                   = _gf_false;
-        int              ret                    = -1;
-        int              barrier_enabled        = _gf_false;
-        struct list_head queue                  = {0,};
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        INIT_LIST_HEAD (&queue);
-
-        switch (event) {
-        case GF_EVENT_TRANSLATOR_OP:
-        {
-                dict = data;
-                barrier_enabled = dict_get_str_boolean (dict, "barrier", -1);
-
-                if (barrier_enabled == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not fetch "
-                                " barrier key from the dictionary.");
-                        goto out;
-                }
-
-                LOCK (&priv->lock);
-                {
-                        past = priv->barrier_enabled;
-
-                        switch (past) {
-                        case _gf_false:
-                                if (barrier_enabled) {
-                                        ret = __barrier_enable (this,priv);
-                                        if (ret)
-                                                goto unlock;
-                                } else {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Already disabled.");
-                                        goto unlock;
-                                }
-                                break;
-
-                        case _gf_true:
-                                if (!barrier_enabled) {
-                                        __barrier_disable(this, &queue);
-                                } else {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Already enabled");
-                                        goto unlock;
-                                }
-                                break;
-                        }
+notify(xlator_t *this, int event, void *data, ...)
+{
+    barrier_priv_t *priv = this->private;
+    dict_t *dict = NULL;
+    int ret = -1;
+    int barrier_enabled = _gf_false;
+    struct list_head queue = {
+        0,
+    };
+
+    GF_ASSERT(priv);
+    INIT_LIST_HEAD(&queue);
+
+    switch (event) {
+        case GF_EVENT_TRANSLATOR_OP: {
+            dict = data;
+            barrier_enabled = dict_get_str_boolean(dict, "barrier", -1);
+
+            if (barrier_enabled == -1) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "Could not fetch "
+                       " barrier key from the dictionary.");
+                goto out;
+            }
+
+            LOCK(&priv->lock);
+            {
+                if (!priv->barrier_enabled) {
+                    if (barrier_enabled) {
+                        ret = __barrier_enable(this, priv);
+                    } else {
+                        UNLOCK(&priv->lock);
+                        gf_log(this->name, GF_LOG_ERROR, "Already disabled.");
+                        goto post_unlock;
+                    }
+                } else {
+                    if (!barrier_enabled) {
+                        __barrier_disable(this, &queue);
                         ret = 0;
+                    } else {
+                        UNLOCK(&priv->lock);
+                        gf_log(this->name, GF_LOG_ERROR, "Already enabled");
+                        goto post_unlock;
+                    }
                 }
-unlock:
-                UNLOCK (&priv->lock);
+            }
+            UNLOCK(&priv->lock);
+        post_unlock:
+            if (!list_empty(&queue))
+                barrier_dequeue_all(this, &queue);
 
-                if (!list_empty (&queue))
-                        barrier_dequeue_all (this, &queue);
-
-                break;
-        }
-        default:
-        {
-                default_notify (this, event, data);
-                ret = 0;
-                goto out;
+            break;
         }
+        default: {
+            default_notify(this, event, data);
+            ret = 0;
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        barrier_priv_t  *priv                   = NULL;
-        gf_boolean_t     past                   = _gf_false;
-        int              ret                    = -1;
-        gf_boolean_t     barrier_enabled        = _gf_false;
-        uint32_t         timeout                = {0,};
-        struct list_head queue                  = {0,};
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_OPTION_RECONF ("barrier", barrier_enabled, options, bool, out);
-        GF_OPTION_RECONF ("barrier-timeout", timeout, options, time, out);
-
-        INIT_LIST_HEAD (&queue);
-
-        LOCK (&priv->lock);
-        {
-                past = priv->barrier_enabled;
-
-                switch (past) {
-                case _gf_false:
-                        if (barrier_enabled) {
-                                ret = __barrier_enable (this, priv);
-                                if (ret) {
-                                        goto unlock;
-                                }
-                        }
-                        break;
-
-                case _gf_true:
-                        if (!barrier_enabled) {
-                                __barrier_disable (this, &queue);
-
-                        }
-                        break;
+reconfigure(xlator_t *this, dict_t *options)
+{
+    barrier_priv_t *priv = NULL;
+    int ret = -1;
+    gf_boolean_t barrier_enabled = _gf_false;
+    uint32_t timeout = {
+        0,
+    };
+    struct list_head queue = {
+        0,
+    };
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_OPTION_RECONF("barrier", barrier_enabled, options, bool, out);
+    GF_OPTION_RECONF("barrier-timeout", timeout, options, time, out);
+
+    INIT_LIST_HEAD(&queue);
+
+    LOCK(&priv->lock);
+    {
+        if (!priv->barrier_enabled) {
+            if (barrier_enabled) {
+                ret = __barrier_enable(this, priv);
+                if (ret) {
+                    goto unlock;
                 }
-                priv->timeout.tv_sec = timeout;
-                ret = 0;
+            }
+        } else {
+            if (!barrier_enabled) {
+                __barrier_disable(this, &queue);
+            }
         }
+        priv->timeout.tv_sec = timeout;
+        ret = 0;
+    }
 unlock:
-        UNLOCK (&priv->lock);
+    UNLOCK(&priv->lock);
 
-        if (!list_empty (&queue))
-                barrier_dequeue_all (this, &queue);
+    if (!list_empty(&queue))
+        barrier_dequeue_all(this, &queue);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_barrier_mt_end + 1);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting "
-                        "initialization failed.");
+    ret = xlator_mem_acct_init(this, gf_barrier_mt_end + 1);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting "
+               "initialization failed.");
 
-        return ret;
+    return ret;
 }
 
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int                     ret     = -1;
-        barrier_priv_t         *priv    = NULL;
-        uint32_t                timeout = {0,};
+    int ret = -1;
+    barrier_priv_t *priv = NULL;
+    uint32_t timeout = {
+        0,
+    };
 
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "'barrier' not configured with exactly one child");
-                goto out;
-        }
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "'barrier' not configured with exactly one child");
+        goto out;
+    }
 
-        if (!this->parents)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
+    if (!this->parents)
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
 
-        priv = GF_CALLOC (1, sizeof (*priv), gf_barrier_mt_priv_t);
-        if (!priv)
-                goto out;
+    priv = GF_CALLOC(1, sizeof(*priv), gf_barrier_mt_priv_t);
+    if (!priv)
+        goto out;
 
-        LOCK_INIT (&priv->lock);
+    LOCK_INIT(&priv->lock);
 
-        GF_OPTION_INIT ("barrier", priv->barrier_enabled, bool, out);
-        GF_OPTION_INIT ("barrier-timeout", timeout, time, out);
-        priv->timeout.tv_sec = timeout;
+    GF_OPTION_INIT("barrier", priv->barrier_enabled, bool, out);
+    GF_OPTION_INIT("barrier-timeout", timeout, time, out);
+    priv->timeout.tv_sec = timeout;
 
-        INIT_LIST_HEAD (&priv->queue);
+    INIT_LIST_HEAD(&priv->queue);
 
-        if (priv->barrier_enabled) {
-                ret = __barrier_enable (this, priv);
-                if (ret == -1)
-                        goto out;
-        }
+    if (priv->barrier_enabled) {
+        ret = __barrier_enable(this, priv);
+        if (ret == -1)
+            goto out;
+    }
 
-        this->private = priv;
-        ret = 0;
+    this->private = priv;
+    ret = 0;
 out:
-        return ret;
+    if (ret && priv)
+        GF_FREE(priv);
+
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        barrier_priv_t         *priv    = NULL;
-        struct list_head        queue   = {0,};
+    barrier_priv_t *priv = NULL;
+    struct list_head queue = {
+        0,
+    };
 
-        priv = this->private;
-        if (!priv)
-                goto out;
+    priv = this->private;
+    if (!priv)
+        goto out;
 
-        INIT_LIST_HEAD (&queue);
+    INIT_LIST_HEAD(&queue);
 
-        gf_log (this->name, GF_LOG_INFO, "Disabling barriering and dequeuing "
-                                         "all the queued fops");
-        LOCK (&priv->lock);
-        {
-                __barrier_disable (this, &queue);
-        }
-        UNLOCK (&priv->lock);
+    gf_log(this->name, GF_LOG_INFO,
+           "Disabling barriering and dequeuing "
+           "all the queued fops");
+    LOCK(&priv->lock);
+    {
+        __barrier_disable(this, &queue);
+    }
+    UNLOCK(&priv->lock);
 
-        if (!list_empty (&queue))
-                barrier_dequeue_all (this, &queue);
+    if (!list_empty(&queue))
+        barrier_dequeue_all(this, &queue);
 
-        this->private = NULL;
+    this->private = NULL;
 
-        LOCK_DESTROY (&priv->lock);
-        GF_FREE (priv);
+    LOCK_DESTROY(&priv->lock);
+    GF_FREE(priv);
 out:
-        return;
+    return;
 }
 
 static void
-barrier_dump_stub (call_stub_t *stub, char *prefix)
+barrier_dump_stub(call_stub_t *stub, char *prefix)
 {
-        char key[GF_DUMP_MAX_BUF_LEN] = {0,};
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
 
-        gf_proc_dump_build_key (key, prefix, "fop");
-        gf_proc_dump_write (key, "%s", gf_fop_list[stub->fop]);
+    gf_proc_dump_build_key(key, prefix, "fop");
+    gf_proc_dump_write(key, "%s", gf_fop_list[stub->fop]);
 
-        if (stub->frame->local) {
-                gf_proc_dump_build_key (key, prefix, "gfid");
-                gf_proc_dump_write (key, "%s",
-                                    uuid_utoa (*(uuid_t*)(stub->frame->local)));
-        }
-        if (stub->args.loc.path) {
-                gf_proc_dump_build_key (key, prefix, "path");
-                gf_proc_dump_write (key, "%s", stub->args.loc.path);
-        }
-        if (stub->args.loc.name) {
-                gf_proc_dump_build_key (key, prefix, "name");
-                gf_proc_dump_write (key, "%s", stub->args.loc.name);
-        }
+    if (stub->frame->local) {
+        gf_proc_dump_build_key(key, prefix, "gfid");
+        gf_proc_dump_write(key, "%s",
+                           uuid_utoa(*(uuid_t *)(stub->frame->local)));
+    }
+    if (stub->args.loc.path) {
+        gf_proc_dump_build_key(key, prefix, "path");
+        gf_proc_dump_write(key, "%s", stub->args.loc.path);
+    }
+    if (stub->args.loc.name) {
+        gf_proc_dump_build_key(key, prefix, "name");
+        gf_proc_dump_write(key, "%s", stub->args.loc.name);
+    }
 
-        return;
+    return;
 }
 
 static void
-__barrier_dump_queue (barrier_priv_t *priv)
+__barrier_dump_queue(barrier_priv_t *priv)
 {
-        call_stub_t *stub = NULL;
-        char key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        int i = 0;
+    call_stub_t *stub = NULL;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    int i = 0;
 
-        GF_VALIDATE_OR_GOTO ("barrier", priv, out);
+    GF_VALIDATE_OR_GOTO("barrier", priv, out);
 
-        list_for_each_entry (stub, &priv->queue, list) {
-                snprintf (key, sizeof (key), "stub.%d", i++);
-                gf_proc_dump_add_section (key);
-                barrier_dump_stub(stub, key);
-        }
+    list_for_each_entry(stub, &priv->queue, list)
+    {
+        snprintf(key, sizeof(key), "stub.%d", i++);
+        gf_proc_dump_add_section("%s", key);
+        barrier_dump_stub(stub, key);
+    }
 
 out:
-        return;
+    return;
 }
 
 int
-barrier_dump_priv (xlator_t *this)
-{
-        int ret = -1;
-        char key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        barrier_priv_t *priv = NULL;
-
-        GF_VALIDATE_OR_GOTO ("barrier", this, out);
-
-        priv = this->private;
-        if (!priv)
-                return 0;
-
-        gf_proc_dump_build_key (key, "xlator.features.barrier", "priv");
-        gf_proc_dump_add_section (key);
-
-        LOCK (&priv->lock);
-        {
-                gf_proc_dump_build_key (key, "barrier", "enabled");
-                gf_proc_dump_write (key, "%d", priv->barrier_enabled);
-                gf_proc_dump_build_key (key, "barrier", "timeout");
-                gf_proc_dump_write (key, "%"PRId64, priv->timeout.tv_sec);
-                if (priv->barrier_enabled) {
-                        gf_proc_dump_build_key (key, "barrier", "queue_size");
-                        gf_proc_dump_write (key, "%d", priv->queue_size);
-                        __barrier_dump_queue (priv);
-                }
+barrier_dump_priv(xlator_t *this)
+{
+    int ret = -1;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    barrier_priv_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("barrier", this, out);
+
+    priv = this->private;
+    if (!priv)
+        return 0;
+
+    gf_proc_dump_build_key(key, "xlator.features.barrier", "priv");
+    gf_proc_dump_add_section("%s", key);
+    gf_proc_dump_build_key(key, "barrier", "enabled");
+
+    LOCK(&priv->lock);
+    {
+        gf_proc_dump_write(key, "%d", priv->barrier_enabled);
+        gf_proc_dump_build_key(key, "barrier", "timeout");
+        gf_proc_dump_write(key, "%ld", priv->timeout.tv_sec);
+        if (priv->barrier_enabled) {
+            gf_proc_dump_build_key(key, "barrier", "queue_size");
+            gf_proc_dump_write(key, "%d", priv->queue_size);
+            __barrier_dump_queue(priv);
         }
-        UNLOCK (&priv->lock);
+    }
+    UNLOCK(&priv->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 struct xlator_fops fops = {
 
-        /* Barrier Class fops */
-        .rmdir          = barrier_rmdir,
-        .unlink         = barrier_unlink,
-        .rename         = barrier_rename,
-        .removexattr    = barrier_removexattr,
-        .fremovexattr   = barrier_fremovexattr,
-        .truncate       = barrier_truncate,
-        .ftruncate      = barrier_ftruncate,
-        .fsync          = barrier_fsync,
-
-        /* Writes with only O_SYNC flag */
-        .writev         = barrier_writev,
+    /* Barrier Class fops */
+    .rmdir = barrier_rmdir,
+    .unlink = barrier_unlink,
+    .rename = barrier_rename,
+    .removexattr = barrier_removexattr,
+    .fremovexattr = barrier_fremovexattr,
+    .truncate = barrier_truncate,
+    .ftruncate = barrier_ftruncate,
+    .fsync = barrier_fsync,
+
+    /* Writes with only O_SYNC flag */
+    .writev = barrier_writev,
 };
 
 struct xlator_dumpops dumpops = {
-        .priv = barrier_dump_priv,
+    .priv = barrier_dump_priv,
 };
 
 struct xlator_cbks cbks;
 
 struct volume_options options[] = {
-        { .key  = {"barrier"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "disable",
-          .op_version = {GD_OP_VERSION_3_6_0},
-          .flags = OPT_FLAG_SETTABLE,
-          .description = "When \"enabled\", blocks acknowledgements to application "
-                         "for file operations such as rmdir, rename, unlink, "
-                         "removexattr, fremovexattr, truncate, ftruncate, "
-                         "write (with O_SYNC), fsync. It is turned \"off\" by "
-                         "default."
-        },
-        { .key = {"barrier-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .default_value = BARRIER_TIMEOUT,
-          .op_version = {GD_OP_VERSION_3_6_0},
-          .flags = OPT_FLAG_SETTABLE,
-          .description = "After 'timeout' seconds since the time 'barrier' "
-                         "option was set to \"on\", acknowledgements to file "
-                         "operations are no longer blocked and previously "
-                         "blocked acknowledgements are sent to the application"
-        },
-        { .key  = {NULL} },
+    {.key = {"barrier"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "disable",
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "When \"enabled\", blocks acknowledgements to application "
+                    "for file operations such as rmdir, rename, unlink, "
+                    "removexattr, fremovexattr, truncate, ftruncate, "
+                    "write (with O_SYNC), fsync. It is turned \"off\" by "
+                    "default."},
+    {.key = {"barrier-timeout"},
+     .type = GF_OPTION_TYPE_TIME,
+     .default_value = BARRIER_TIMEOUT,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "After 'timeout' seconds since the time 'barrier' "
+                    "option was set to \"on\", acknowledgements to file "
+                    "operations are no longer blocked and previously "
+                    "blocked acknowledgements are sent to the application"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "barrier",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/barrier/src/barrier.h b/xlators/features/barrier/src/barrier.h
index 0d646f90474..1337f311f7d 100644
--- a/xlators/features/barrier/src/barrier.h
+++ b/xlators/features/barrier/src/barrier.h
@@ -12,71 +12,78 @@
 #define __BARRIER_H__
 
 #include "barrier-mem-types.h"
-#include "xlator.h"
-#include "timer.h"
-#include "call-stub.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/timer.h>
+#include <glusterfs/call-stub.h>
 
-#define BARRIER_FOP_CBK(fop_name, label, frame, this, params ...)       \
-        do {                                                            \
-                barrier_priv_t         *_priv           = NULL;         \
-                call_stub_t            *_stub           = NULL;         \
-                gf_boolean_t            _barrier_enabled= _gf_false;    \
-                struct list_head        queue           = {0, };        \
-                                                                        \
-                INIT_LIST_HEAD (&queue);                                \
-                                                                        \
-                _priv = this->private;                                  \
-                GF_ASSERT (_priv);                                      \
-                                                                        \
-                LOCK (&_priv->lock);                                    \
-                {                                                       \
-                        if (_priv->barrier_enabled) {                   \
-                                _barrier_enabled = _priv->barrier_enabled;\
-                                                                        \
-                                _stub = fop_##fop_name##_cbk_stub       \
-                                        (frame,                         \
-                                         barrier_##fop_name##_cbk_resume,\
-                                         params);                       \
-                                if (!_stub) {                           \
-                                        __barrier_disable (this, &queue);\
-                                        goto unlock;                    \
-                                }                                       \
-                                                                        \
-                                __barrier_enqueue (this, _stub);        \
-                        }                                               \
-                }                                                       \
-unlock:                                                                 \
-                UNLOCK (&_priv->lock);                                  \
-                                                                        \
-                if (_stub)                                              \
-                        goto label;                                     \
-                                                                        \
-                if (_barrier_enabled && !_stub) {                       \
-                        gf_log (this->name, GF_LOG_CRITICAL,            \
-                                "Failed to barrier FOPs, disabling "    \
-                                "barrier. FOP: %s, ERROR: %s",          \
-                                #fop_name, strerror (ENOMEM));          \
-                        barrier_dequeue_all (this, &queue);             \
-                }                                                       \
-                barrier_local_free_gfid (frame);                        \
-                STACK_UNWIND_STRICT (fop_name, frame, params);          \
-                goto label;                                             \
-        } while (0)
+#define BARRIER_FOP_CBK(fop_name, label, frame, this, params...)               \
+    do {                                                                       \
+        barrier_priv_t *_priv = NULL;                                          \
+        call_stub_t *_stub = NULL;                                             \
+        gf_boolean_t _barrier_enabled = _gf_false;                             \
+        struct list_head queue = {                                             \
+            0,                                                                 \
+        };                                                                     \
+                                                                               \
+        INIT_LIST_HEAD(&queue);                                                \
+                                                                               \
+        _priv = this->private;                                                 \
+        GF_ASSERT(_priv);                                                      \
+                                                                               \
+        LOCK(&_priv->lock);                                                    \
+        {                                                                      \
+            if (_priv->barrier_enabled) {                                      \
+                _barrier_enabled = _priv->barrier_enabled;                     \
+                                                                               \
+                _stub = fop_##fop_name##_cbk_stub(                             \
+                    frame, barrier_##fop_name##_cbk_resume, params);           \
+                if (!_stub) {                                                  \
+                    __barrier_disable(this, &queue);                           \
+                    goto unlock;                                               \
+                }                                                              \
+                                                                               \
+                __barrier_enqueue(this, _stub);                                \
+            }                                                                  \
+        }                                                                      \
+    unlock:                                                                    \
+        UNLOCK(&_priv->lock);                                                  \
+                                                                               \
+        if (_stub)                                                             \
+            goto label;                                                        \
+                                                                               \
+        if (_barrier_enabled && !_stub) {                                      \
+            gf_log(this->name, GF_LOG_CRITICAL,                                \
+                   "Failed to barrier FOPs, disabling "                        \
+                   "barrier. FOP: %s, ERROR: %s",                              \
+                   #fop_name, strerror(ENOMEM));                               \
+            barrier_dequeue_all(this, &queue);                                 \
+        }                                                                      \
+        barrier_local_free_gfid(frame);                                        \
+        STACK_UNWIND_STRICT(fop_name, frame, params);                          \
+        goto label;                                                            \
+    } while (0)
 
 typedef struct {
-        gf_timer_t       *timer;
-        gf_boolean_t      barrier_enabled;
-        gf_lock_t         lock;
-        struct list_head  queue;
-        struct timespec   timeout;
-        uint32_t          queue_size;
+    gf_timer_t *timer;
+    gf_lock_t lock;
+    struct list_head queue;
+    struct timespec timeout;
+    uint32_t queue_size;
+    gf_boolean_t barrier_enabled;
+    char _pad[3]; /* manual padding */
 } barrier_priv_t;
 
-int __barrier_enable (xlator_t *this, barrier_priv_t *priv);
-void __barrier_enqueue (xlator_t *this, call_stub_t *stub);
-void __barrier_disable (xlator_t *this, struct list_head *queue);
-void barrier_timeout (void *data);
-void barrier_dequeue_all (xlator_t *this, struct list_head *queue);
-call_stub_t *__barrier_dequeue (xlator_t *this, struct list_head *queue);
+int
+__barrier_enable(xlator_t *this, barrier_priv_t *priv);
+void
+__barrier_enqueue(xlator_t *this, call_stub_t *stub);
+void
+__barrier_disable(xlator_t *this, struct list_head *queue);
+void
+barrier_timeout(void *data);
+void
+barrier_dequeue_all(xlator_t *this, struct list_head *queue);
+call_stub_t *
+__barrier_dequeue(xlator_t *this, struct list_head *queue);
 
 #endif
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h b/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h
index 89a67cc10b1..5bc5103a27c 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h
@@ -11,7 +11,7 @@
 #ifndef _BITROT_BITD_MESSAGES_H_
 #define _BITROT_BITD_MESSAGES_H_
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
 
 /* To add new message IDs, append new identifiers at the end of the list.
  *
@@ -23,62 +23,79 @@
  * glfs-message-id.h.
  */
 
-GLFS_MSGID(BITROT_BITD,
-        BRB_MSG_FD_CREATE_FAILED,
-        BRB_MSG_READV_FAILED,
-        BRB_MSG_BLOCK_READ_FAILED,
-        BRB_MSG_CALC_CHECKSUM_FAILED,
-        BRB_MSG_NO_MEMORY,
-        BRB_MSG_GET_SIGN_FAILED,
-        BRB_MSG_SET_SIGN_FAILED,
-        BRB_MSG_OP_FAILED,
-        BRB_MSG_READ_AND_SIGN_FAILED,
-        BRB_MSG_SIGN_FAILED,
-        BRB_MSG_GET_SUBVOL_FAILED,
-        BRB_MSG_SET_TIMER_FAILED,
-        BRB_MSG_GET_INFO_FAILED,
-        BRB_MSG_PATH_FAILED,
-        BRB_MSG_MARK_BAD_FILE,
-        BRB_MSG_TRIGGER_SIGN,
-        BRB_MSG_REGISTER_FAILED,
-        BRB_MSG_CRAWLING_START,
-        BRB_MSG_SPAWN_FAILED,
-        BRB_MSG_INVALID_SUBVOL_CHILD,
-        BRB_MSG_SKIP_OBJECT,
-        BRB_MSG_NO_CHILD,
-        BRB_MSG_CHECKSUM_MISMATCH,
-        BRB_MSG_MARK_CORRUPTED,
-        BRB_MSG_CRAWLING_FINISH,
-        BRB_MSG_CALC_ERROR,
-        BRB_MSG_LOOKUP_FAILED,
-        BRB_MSG_PARTIAL_VERSION_PRESENCE,
-        BRB_MSG_MEM_ACNT_FAILED,
-        BRB_MSG_TIMER_WHEEL_UNAVAILABLE,
-        BRB_MSG_BITROT_LOADED,
-        BRB_MSG_SCALE_DOWN_FAILED,
-        BRB_MSG_SCALE_UP_FAILED,
-        BRB_MSG_SCALE_DOWN_SCRUBBER,
-        BRB_MSG_SCALING_UP_SCRUBBER,
-        BRB_MSG_UNKNOWN_THROTTLE,
-        BRB_MSG_RATE_LIMIT_INFO,
-        BRB_MSG_SCRUB_INFO,
-        BRB_MSG_CONNECTED_TO_BRICK,
-        BRB_MSG_BRICK_INFO,
-        BRB_MSG_SUBVOL_CONNECT_FAILED,
-        BRB_MSG_INVALID_SUBVOL,
-        BRB_MSG_RESCHEDULE_SCRUBBER_FAILED,
-        BRB_MSG_SCRUB_START,
-        BRB_MSG_SCRUB_FINISH,
-        BRB_MSG_SCRUB_RUNNING,
-        BRB_MSG_SCRUB_RESCHEDULED,
-        BRB_MSG_SCRUB_TUNABLE,
-        BRB_MSG_SCRUB_THREAD_CLEANUP,
-        BRB_MSG_SCRUBBER_CLEANED,
-        BRB_MSG_GENERIC_SSM_INFO,
-        BRB_MSG_ZERO_TIMEOUT_BUG,
-        BRB_MSG_BAD_OBJ_READDIR_FAIL,
-        BRB_MSG_SSM_FAILED,
-        BRB_MSG_SCRUB_WAIT_FAILED
-);
+GLFS_MSGID(BITROT_BITD, BRB_MSG_FD_CREATE_FAILED, BRB_MSG_READV_FAILED,
+           BRB_MSG_BLOCK_READ_FAILED, BRB_MSG_CALC_CHECKSUM_FAILED,
+           BRB_MSG_NO_MEMORY, BRB_MSG_GET_SIGN_FAILED, BRB_MSG_SET_SIGN_FAILED,
+           BRB_MSG_OP_FAILED, BRB_MSG_READ_AND_SIGN_FAILED, BRB_MSG_SIGN_FAILED,
+           BRB_MSG_GET_SUBVOL_FAILED, BRB_MSG_SET_TIMER_FAILED,
+           BRB_MSG_GET_INFO_FAILED, BRB_MSG_PATH_FAILED, BRB_MSG_MARK_BAD_FILE,
+           BRB_MSG_TRIGGER_SIGN, BRB_MSG_REGISTER_FAILED,
+           BRB_MSG_CRAWLING_START, BRB_MSG_SPAWN_FAILED,
+           BRB_MSG_INVALID_SUBVOL_CHILD, BRB_MSG_SKIP_OBJECT, BRB_MSG_NO_CHILD,
+           BRB_MSG_CHECKSUM_MISMATCH, BRB_MSG_MARK_CORRUPTED,
+           BRB_MSG_CRAWLING_FINISH, BRB_MSG_CALC_ERROR, BRB_MSG_LOOKUP_FAILED,
+           BRB_MSG_PARTIAL_VERSION_PRESENCE, BRB_MSG_MEM_ACNT_FAILED,
+           BRB_MSG_TIMER_WHEEL_UNAVAILABLE, BRB_MSG_BITROT_LOADED,
+           BRB_MSG_SCALE_DOWN_FAILED, BRB_MSG_SCALE_UP_FAILED,
+           BRB_MSG_SCALE_DOWN_SCRUBBER, BRB_MSG_SCALING_UP_SCRUBBER,
+           BRB_MSG_UNKNOWN_THROTTLE, BRB_MSG_RATE_LIMIT_INFO,
+           BRB_MSG_SCRUB_INFO, BRB_MSG_CONNECTED_TO_BRICK, BRB_MSG_BRICK_INFO,
+           BRB_MSG_SUBVOL_CONNECT_FAILED, BRB_MSG_INVALID_SUBVOL,
+           BRB_MSG_RESCHEDULE_SCRUBBER_FAILED, BRB_MSG_SCRUB_START,
+           BRB_MSG_SCRUB_FINISH, BRB_MSG_SCRUB_RUNNING,
+           BRB_MSG_SCRUB_RESCHEDULED, BRB_MSG_SCRUB_TUNABLE,
+           BRB_MSG_SCRUB_THREAD_CLEANUP, BRB_MSG_SCRUBBER_CLEANED,
+           BRB_MSG_GENERIC_SSM_INFO, BRB_MSG_ZERO_TIMEOUT_BUG,
+           BRB_MSG_BAD_OBJ_READDIR_FAIL, BRB_MSG_SSM_FAILED,
+           BRB_MSG_SCRUB_WAIT_FAILED, BRB_MSG_TRIGGER_SIGN_FAILED,
+           BRB_MSG_EVENT_UNHANDLED, BRB_MSG_COULD_NOT_SCHEDULE_SCRUB,
+           BRB_MSG_THREAD_CREATION_FAILED, BRB_MSG_MEM_POOL_ALLOC,
+           BRB_MSG_SAVING_HASH_FAILED);
 
+#define BRB_MSG_FD_CREATE_FAILED_STR "failed to create fd for the inode"
+#define BRB_MSG_READV_FAILED_STR "readv failed"
+#define BRB_MSG_BLOCK_READ_FAILED_STR "reading block failed"
+#define BRB_MSG_NO_MEMORY_STR "failed to allocate memory"
+#define BRB_MSG_CALC_CHECKSUM_FAILED_STR "calculating checksum failed"
+#define BRB_MSG_GET_SIGN_FAILED_STR "failed to get the signature"
+#define BRB_MSG_SET_SIGN_FAILED_STR "signing failed"
+#define BRB_MSG_OP_FAILED_STR "failed on object"
+#define BRB_MSG_TRIGGER_SIGN_FAILED_STR "Could not trigger signing"
+#define BRB_MSG_READ_AND_SIGN_FAILED_STR "reading and signing of object failed"
+#define BRB_MSG_SET_TIMER_FAILED_STR "Failed to allocate object expiry timer"
+#define BRB_MSG_GET_SUBVOL_FAILED_STR                                          \
+    "failed to get the subvolume for the brick"
+#define BRB_MSG_PATH_FAILED_STR "path failed"
+#define BRB_MSG_SKIP_OBJECT_STR "Entry is marked corrupted. skipping"
+#define BRB_MSG_PARTIAL_VERSION_PRESENCE_STR                                   \
+    "PArtial version xattr presence detected, ignoring"
+#define BRB_MSG_TRIGGER_SIGN_STR "Triggering signing"
+#define BRB_MSG_CRAWLING_START_STR                                             \
+    "Crawling brick, scanning for unsigned objects"
+#define BRB_MSG_CRAWLING_FINISH_STR "Completed crawling brick"
+#define BRB_MSG_REGISTER_FAILED_STR "Register to changelog failed"
+#define BRB_MSG_SPAWN_FAILED_STR "failed to spawn"
+#define BRB_MSG_CONNECTED_TO_BRICK_STR "Connected to brick"
+#define BRB_MSG_LOOKUP_FAILED_STR "lookup on root failed"
+#define BRB_MSG_GET_INFO_FAILED_STR "failed to get stub info"
+#define BRB_MSG_SCRUB_THREAD_CLEANUP_STR "Error cleaning up scanner thread"
+#define BRB_MSG_SCRUBBER_CLEANED_STR "clened up scrubber for brick"
+#define BRB_MSG_SUBVOL_CONNECT_FAILED_STR                                      \
+    "callback handler for subvolume failed"
+#define BRB_MSG_MEM_ACNT_FAILED_STR "Memory accounting init failed"
+#define BRB_MSG_EVENT_UNHANDLED_STR "Event unhandled for child"
+#define BRB_MSG_INVALID_SUBVOL_STR "Got event from invalid subvolume"
+#define BRB_MSG_RESCHEDULE_SCRUBBER_FAILED_STR                                 \
+    "on demand scrub schedule failed. Scrubber is not in pending state."
+#define BRB_MSG_COULD_NOT_SCHEDULE_SCRUB_STR                                   \
+    "Could not schedule ondemand scrubbing. Scrubbing will continue "          \
+    "according to old frequency."
+#define BRB_MSG_THREAD_CREATION_FAILED_STR "thread creation failed"
+#define BRB_MSG_RATE_LIMIT_INFO_STR "Rate Limit Info"
+#define BRB_MSG_MEM_POOL_ALLOC_STR "failed to allocate mem-pool for timer"
+#define BRB_MSG_NO_CHILD_STR "FATAL: no children"
+#define BRB_MSG_TIMER_WHEEL_UNAVAILABLE_STR "global timer wheel unavailable"
+#define BRB_MSG_BITROT_LOADED_STR "bit-rot xlator loaded"
+#define BRB_MSG_SAVING_HASH_FAILED_STR                                         \
+    "failed to allocate memory for saving hash of the object"
 #endif /* !_BITROT_BITD_MESSAGES_H_ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c
index 2f9aaf31a52..5cef2ffa5e5 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c
@@ -9,69 +9,70 @@
 */
 
 #include <string.h>
+#include <stdio.h>
 
 #include "bit-rot-scrub-status.h"
 
 void
-br_inc_unsigned_file_count (br_scrub_stats_t *scrub_stat)
+br_inc_unsigned_file_count(br_scrub_stats_t *scrub_stat)
 {
-        if (!scrub_stat)
-                return;
+    if (!scrub_stat)
+        return;
 
-        pthread_mutex_lock (&scrub_stat->lock);
-        {
-                scrub_stat->unsigned_files++;
-        }
-        pthread_mutex_unlock (&scrub_stat->lock);
+    pthread_mutex_lock(&scrub_stat->lock);
+    {
+        scrub_stat->unsigned_files++;
+    }
+    pthread_mutex_unlock(&scrub_stat->lock);
 }
 
 void
-br_inc_scrubbed_file (br_scrub_stats_t *scrub_stat)
+br_inc_scrubbed_file(br_scrub_stats_t *scrub_stat)
 {
-        if (!scrub_stat)
-                return;
+    if (!scrub_stat)
+        return;
 
-        pthread_mutex_lock (&scrub_stat->lock);
-        {
-                scrub_stat->scrubbed_files++;
-        }
-        pthread_mutex_unlock (&scrub_stat->lock);
+    pthread_mutex_lock(&scrub_stat->lock);
+    {
+        scrub_stat->scrubbed_files++;
+    }
+    pthread_mutex_unlock(&scrub_stat->lock);
 }
 
 void
-br_update_scrub_start_time (br_scrub_stats_t *scrub_stat, struct timeval *tv)
+br_update_scrub_start_time(br_scrub_stats_t *scrub_stat, time_t time)
 {
-        if (!scrub_stat)
-                return;
+    if (!scrub_stat)
+        return;
 
-        pthread_mutex_lock (&scrub_stat->lock);
-        {
-                scrub_stat->scrub_start_tv.tv_sec = tv->tv_sec;
-        }
-        pthread_mutex_unlock (&scrub_stat->lock);
+    pthread_mutex_lock(&scrub_stat->lock);
+    {
+        scrub_stat->scrub_start_time = time;
+    }
+    pthread_mutex_unlock(&scrub_stat->lock);
 }
 
 void
-br_update_scrub_finish_time (br_scrub_stats_t *scrub_stat, char *timestr,
-                             struct timeval *tv)
+br_update_scrub_finish_time(br_scrub_stats_t *scrub_stat, char *timestr,
+                            time_t time)
 {
-        int lst_size = 0;
+    int lst_size = 0;
 
-        if (!scrub_stat)
-                return;
+    if (!scrub_stat)
+        return;
 
-        lst_size = sizeof (scrub_stat->last_scrub_time);
-        pthread_mutex_lock (&scrub_stat->lock);
-        {
-                scrub_stat->scrub_end_tv.tv_sec = tv->tv_sec;
+    lst_size = sizeof(scrub_stat->last_scrub_time);
+    if (strlen(timestr) >= lst_size)
+        return;
 
-                scrub_stat->scrub_duration =
-                                 scrub_stat->scrub_end_tv.tv_sec -
-                                 scrub_stat->scrub_start_tv.tv_sec;
+    pthread_mutex_lock(&scrub_stat->lock);
+    {
+        scrub_stat->scrub_end_time = time;
 
-                strncpy (scrub_stat->last_scrub_time, timestr,
-                         lst_size-1);
-                scrub_stat->last_scrub_time[lst_size-1] = '\0';
-        }
-        pthread_mutex_unlock (&scrub_stat->lock);
+        scrub_stat->scrub_duration = scrub_stat->scrub_end_time -
+                                     scrub_stat->scrub_start_time;
+
+        snprintf(scrub_stat->last_scrub_time, lst_size, "%s", timestr);
+    }
+    pthread_mutex_unlock(&scrub_stat->lock);
 }
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h
index 02bd0fab04e..f022aa831eb 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h
@@ -15,34 +15,36 @@
 #include <sys/time.h>
 #include <pthread.h>
 
+#include <glusterfs/common-utils.h>
+
 struct br_scrub_stats {
-        uint64_t       scrubbed_files;       /* Total number of scrubbed file */
+    uint64_t scrubbed_files; /* Total number of scrubbed files. */
 
-        uint64_t       unsigned_files;       /* Total number of unsigned file */
+    uint64_t unsigned_files; /* Total number of unsigned files. */
 
-        uint64_t       scrub_duration;            /* Duration of last scrub */
+    uint64_t scrub_duration; /* Duration of last scrub. */
 
-        char           last_scrub_time[1024];    /*last scrub completion time */
+    char last_scrub_time[GF_TIMESTR_SIZE]; /* Last scrub completion time. */
 
-        struct         timeval scrub_start_tv;   /* Scrubbing starting time*/
+    time_t scrub_start_time; /* Scrubbing starting time. */
 
-        struct         timeval scrub_end_tv;     /* Scrubbing finishing time */
+    time_t scrub_end_time; /* Scrubbing finishing time. */
 
-        int8_t         scrub_running;           /* Scrub running or not */
+    int8_t scrub_running; /* Whether scrub running or not. */
 
-        pthread_mutex_t  lock;
+    pthread_mutex_t lock;
 };
 
 typedef struct br_scrub_stats br_scrub_stats_t;
 
 void
-br_inc_unsigned_file_count (br_scrub_stats_t *scrub_stat);
+br_inc_unsigned_file_count(br_scrub_stats_t *scrub_stat);
 void
-br_inc_scrubbed_file (br_scrub_stats_t *scrub_stat);
+br_inc_scrubbed_file(br_scrub_stats_t *scrub_stat);
 void
-br_update_scrub_start_time (br_scrub_stats_t *scrub_stat, struct timeval *tv);
+br_update_scrub_start_time(br_scrub_stats_t *scrub_stat, time_t time);
 void
-br_update_scrub_finish_time (br_scrub_stats_t *scrub_stat, char *timestr,
-                             struct timeval *tv);
+br_update_scrub_finish_time(br_scrub_stats_t *scrub_stat, char *timestr,
+                            time_t time);
 
 #endif /* __BIT_ROT_SCRUB_STATUS_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
index ee996525f76..289dd53f610 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
@@ -12,32 +12,32 @@
 #include <ctype.h>
 #include <sys/uio.h>
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "common-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
 
 #include "bit-rot-scrub.h"
 #include <pthread.h>
 #include "bit-rot-bitd-messages.h"
 #include "bit-rot-scrub-status.h"
-#include "events.h"
+#include <glusterfs/events.h>
 
 struct br_scrubbers {
-        pthread_t scrubthread;
+    pthread_t scrubthread;
 
-        struct list_head list;
+    struct list_head list;
 };
 
 struct br_fsscan_entry {
-        void *data;
+    void *data;
 
-        loc_t parent;
+    loc_t parent;
 
-        gf_dirent_t *entry;
+    gf_dirent_t *entry;
 
-        struct br_scanfs *fsscan;  /* backpointer to subvolume scanner */
+    struct br_scanfs *fsscan; /* backpointer to subvolume scanner */
 
-        struct list_head list;
+    struct list_head list;
 };
 
 /**
@@ -46,34 +46,32 @@ struct br_fsscan_entry {
  * to the dictionary value.
  */
 static int32_t
-bitd_fetch_signature (xlator_t *this, br_child_t *child,
-                      fd_t *fd, dict_t **xattr, br_isignature_out_t **sign)
+bitd_fetch_signature(xlator_t *this, br_child_t *child, fd_t *fd,
+                     dict_t **xattr, br_isignature_out_t **sign)
 {
-       int32_t ret = -1;
-
-        ret = syncop_fgetxattr (child->xl, fd, xattr,
-                               GLUSTERFS_GET_OBJECT_SIGNATURE, NULL, NULL);
-        if (ret < 0) {
-                br_log_object (this, "fgetxattr", fd->inode->gfid, -ret);
-                goto out;
-        }
-
-        ret = dict_get_ptr
-                (*xattr, GLUSTERFS_GET_OBJECT_SIGNATURE, (void **) sign);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
-                        "failed to extract signature info [GFID: %s]",
-                        uuid_utoa (fd->inode->gfid));
-                goto unref_dict;
-        }
-
-        return 0;
-
- unref_dict:
-        dict_unref (*xattr);
- out:
-        return -1;
-
+    int32_t ret = -1;
+
+    ret = syncop_fgetxattr(child->xl, fd, xattr, GLUSTERFS_GET_OBJECT_SIGNATURE,
+                           NULL, NULL);
+    if (ret < 0) {
+        br_log_object(this, "fgetxattr", fd->inode->gfid, -ret);
+        goto out;
+    }
+
+    ret = dict_get_ptr(*xattr, GLUSTERFS_GET_OBJECT_SIGNATURE, (void **)sign);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
+               "failed to extract signature info [GFID: %s]",
+               uuid_utoa(fd->inode->gfid));
+        goto unref_dict;
+    }
+
+    return 0;
+
+unref_dict:
+    dict_unref(*xattr);
+out:
+    return -1;
 }
 
 /**
@@ -86,88 +84,87 @@ bitd_fetch_signature (xlator_t *this, br_child_t *child,
  */
 
 int32_t
-bitd_scrub_post_compute_check (xlator_t *this,
-                               br_child_t *child,
-                               fd_t *fd, unsigned long version,
-                               br_isignature_out_t **signature,
-                               br_scrub_stats_t *scrub_stat,
-                               gf_boolean_t skip_stat)
+bitd_scrub_post_compute_check(xlator_t *this, br_child_t *child, fd_t *fd,
+                              unsigned long version,
+                              br_isignature_out_t **signature,
+                              br_scrub_stats_t *scrub_stat,
+                              gf_boolean_t skip_stat)
 {
-        int32_t              ret     = 0;
-        size_t               signlen = 0;
-        dict_t              *xattr   = NULL;
-        br_isignature_out_t *signptr = NULL;
-
-        ret = bitd_fetch_signature (this, child, fd, &xattr, &signptr);
-        if (ret < 0) {
-                if (!skip_stat)
-                        br_inc_unsigned_file_count (scrub_stat);
-                goto out;
-        }
-
-        /**
-         * Either the object got dirtied during the time the signature was
-         * calculated OR the version we saved during pre-compute check does
-         * not match now, implying that the object got dirtied and signed in
-         * between scrubs pre & post compute checks (checksum window).
-         *
-         * The log entry looks pretty ugly, but helps in debugging..
-         */
-        if (signptr->stale || (signptr->version != version)) {
-                if (!skip_stat)
-                        br_inc_unsigned_file_count (scrub_stat);
-                gf_msg_debug (this->name, 0, "<STAGE: POST> Object [GFID: %s] "
-                              "either has a stale signature OR underwent "
-                              "signing during checksumming {Stale: %d | "
-                              "Version: %lu,%lu}", uuid_utoa (fd->inode->gfid),
-                              (signptr->stale) ? 1 : 0, version,
-                              signptr->version);
-                ret = -1;
-                goto unref_dict;
-        }
-
-        signlen = signptr->signaturelen;
-        *signature = GF_CALLOC (1, sizeof (br_isignature_out_t) + signlen,
-                                gf_common_mt_char);
-
-        (void) memcpy (*signature, signptr,
-                       sizeof (br_isignature_out_t) + signlen);
-
- unref_dict:
-        dict_unref (xattr);
- out:
-        return ret;
+    int32_t ret = 0;
+    size_t signlen = 0;
+    dict_t *xattr = NULL;
+    br_isignature_out_t *signptr = NULL;
 
+    ret = bitd_fetch_signature(this, child, fd, &xattr, &signptr);
+    if (ret < 0) {
+        if (!skip_stat)
+            br_inc_unsigned_file_count(scrub_stat);
+        goto out;
+    }
+
+    /**
+     * Either the object got dirtied during the time the signature was
+     * calculated OR the version we saved during pre-compute check does
+     * not match now, implying that the object got dirtied and signed in
+     * between scrubs pre & post compute checks (checksum window).
+     *
+     * The log entry looks pretty ugly, but helps in debugging..
+     */
+    if (signptr->stale || (signptr->version != version)) {
+        if (!skip_stat)
+            br_inc_unsigned_file_count(scrub_stat);
+        gf_msg_debug(this->name, 0,
+                     "<STAGE: POST> Object [GFID: %s] "
+                     "either has a stale signature OR underwent "
+                     "signing during checksumming {Stale: %d | "
+                     "Version: %lu,%lu}",
+                     uuid_utoa(fd->inode->gfid), (signptr->stale) ? 1 : 0,
+                     version, signptr->version);
+        ret = -1;
+        goto unref_dict;
+    }
+
+    signlen = signptr->signaturelen;
+    *signature = GF_MALLOC(sizeof(br_isignature_out_t) + signlen,
+                           gf_common_mt_char);
+
+    (void)memcpy(*signature, signptr, sizeof(br_isignature_out_t) + signlen);
+
+    (*signature)->signaturelen = signlen;
+
+unref_dict:
+    dict_unref(xattr);
+out:
+    return ret;
 }
 
 static int32_t
-bitd_signature_staleness (xlator_t *this,
-                          br_child_t *child, fd_t *fd,
-                          int *stale, unsigned long *version,
-                          br_scrub_stats_t *scrub_stat, gf_boolean_t skip_stat)
+bitd_signature_staleness(xlator_t *this, br_child_t *child, fd_t *fd,
+                         int *stale, unsigned long *version,
+                         br_scrub_stats_t *scrub_stat, gf_boolean_t skip_stat)
 {
-        int32_t ret = -1;
-        dict_t *xattr = NULL;
-        br_isignature_out_t *signptr = NULL;
+    int32_t ret = -1;
+    dict_t *xattr = NULL;
+    br_isignature_out_t *signptr = NULL;
 
-        ret = bitd_fetch_signature (this, child, fd, &xattr, &signptr);
-        if (ret < 0) {
-                if (!skip_stat)
-                        br_inc_unsigned_file_count (scrub_stat);
-                goto out;
-        }
+    ret = bitd_fetch_signature(this, child, fd, &xattr, &signptr);
+    if (ret < 0) {
+        if (!skip_stat)
+            br_inc_unsigned_file_count(scrub_stat);
+        goto out;
+    }
 
-        /**
-         * save verison for validation in post compute stage
-         * c.f. bitd_scrub_post_compute_check()
-         */
-        *stale = signptr->stale ? 1 : 0;
-        *version = signptr->version;
+    /**
+     * save version for validation in post compute stage
+     * c.f. bitd_scrub_post_compute_check()
+     */
+    *stale = signptr->stale ? 1 : 0;
+    *version = signptr->version;
 
-        dict_unref (xattr);
+    dict_unref(xattr);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 /**
@@ -179,101 +176,102 @@ bitd_signature_staleness (xlator_t *this,
  *  - has stale signature
  */
 int32_t
-bitd_scrub_pre_compute_check (xlator_t *this, br_child_t *child,
-                              fd_t *fd, unsigned long *version,
-                              br_scrub_stats_t *scrub_stat,
-                              gf_boolean_t skip_stat)
+bitd_scrub_pre_compute_check(xlator_t *this, br_child_t *child, fd_t *fd,
+                             unsigned long *version,
+                             br_scrub_stats_t *scrub_stat,
+                             gf_boolean_t skip_stat)
 {
-        int     stale = 0;
-        int32_t ret   = -1;
-
-        if (bitd_is_bad_file (this, child, NULL, fd)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_SKIP_OBJECT,
-                        "Object [GFID: %s] is marked corrupted, skipping..",
-                        uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
-
-        ret = bitd_signature_staleness (this, child, fd, &stale, version,
-                                        scrub_stat, skip_stat);
-        if (!ret && stale) {
-                if (!skip_stat)
-                        br_inc_unsigned_file_count (scrub_stat);
-                gf_msg_debug (this->name, 0, "<STAGE: PRE> Object [GFID: %s] "
-                              "has stale signature",
-                              uuid_utoa (fd->inode->gfid));
-                ret = -1;
-        }
+    int stale = 0;
+    int32_t ret = -1;
+
+    if (bitd_is_bad_file(this, child, NULL, fd)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SKIP_OBJECT,
+               "Object [GFID: %s] is marked corrupted, skipping..",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    ret = bitd_signature_staleness(this, child, fd, &stale, version, scrub_stat,
+                                   skip_stat);
+    if (!ret && stale) {
+        if (!skip_stat)
+            br_inc_unsigned_file_count(scrub_stat);
+        gf_msg_debug(this->name, 0,
+                     "<STAGE: PRE> Object [GFID: %s] "
+                     "has stale signature",
+                     uuid_utoa(fd->inode->gfid));
+        ret = -1;
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 /* static int */
 int
-bitd_compare_ckum (xlator_t *this,
-                   br_isignature_out_t *sign,
-                   unsigned char *md, inode_t *linked_inode,
-                   gf_dirent_t *entry, fd_t *fd, br_child_t *child, loc_t *loc)
+bitd_compare_ckum(xlator_t *this, br_isignature_out_t *sign, unsigned char *md,
+                  inode_t *linked_inode, gf_dirent_t *entry, fd_t *fd,
+                  br_child_t *child, loc_t *loc)
 {
-        int   ret = -1;
-        dict_t *xattr = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, sign, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-        GF_VALIDATE_OR_GOTO (this->name, linked_inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, md, out);
-        GF_VALIDATE_OR_GOTO (this->name, entry, out);
-
-        if (strncmp
-            (sign->signature, (char *) md, strlen (sign->signature)) == 0) {
-                gf_msg_debug (this->name, 0, "%s [GFID: %s | Brick: %s] "
-                              "matches calculated checksum", loc->path,
-                              uuid_utoa (linked_inode->gfid),
-                              child->brick_path);
-                return 0;
-        }
-
-        gf_msg (this->name, GF_LOG_DEBUG, 0, BRB_MSG_CHECKSUM_MISMATCH,
-                "Object checksum mismatch: %s [GFID: %s | Brick: %s]",
-                loc->path, uuid_utoa (linked_inode->gfid), child->brick_path);
-        gf_msg (this->name, GF_LOG_ALERT, 0, BRB_MSG_CHECKSUM_MISMATCH,
-                "CORRUPTION DETECTED: Object %s {Brick: %s | GFID: %s}",
-                loc->path, child->brick_path, uuid_utoa (linked_inode->gfid));
-
-        /* Perform bad-file marking */
-        xattr = dict_new ();
-        if (!xattr) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_int32 (xattr, BITROT_OBJECT_BAD_KEY, _gf_true);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_MARK_BAD_FILE,
-                        "Error setting bad-file marker for %s [GFID: %s | "
-                        "Brick: %s]", loc->path, uuid_utoa (linked_inode->gfid),
-                        child->brick_path);
-                goto dictfree;
-        }
-
-        gf_msg (this->name, GF_LOG_ALERT, 0, BRB_MSG_MARK_CORRUPTED, "Marking"
-                " %s [GFID: %s | Brick: %s] as corrupted..", loc->path,
-                uuid_utoa (linked_inode->gfid), child->brick_path);
-        gf_event (EVENT_BITROT_BAD_FILE, "gfid=%s;path=%s;brick=%s",
-                  uuid_utoa (linked_inode->gfid), loc->path, child->brick_path);
-        ret = syncop_fsetxattr (child->xl, fd, xattr, 0, NULL, NULL);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_MARK_BAD_FILE,
-                        "Error marking object %s [GFID: %s] as corrupted",
-                        loc->path, uuid_utoa (linked_inode->gfid));
-
- dictfree:
-        dict_unref (xattr);
- out:
-        return ret;
+    int ret = -1;
+    dict_t *xattr = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, sign, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+    GF_VALIDATE_OR_GOTO(this->name, linked_inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, md, out);
+    GF_VALIDATE_OR_GOTO(this->name, entry, out);
+
+    if (strncmp(sign->signature, (char *)md, sign->signaturelen) == 0) {
+        gf_msg_debug(this->name, 0,
+                     "%s [GFID: %s | Brick: %s] "
+                     "matches calculated checksum",
+                     loc->path, uuid_utoa(linked_inode->gfid),
+                     child->brick_path);
+        return 0;
+    }
+
+    gf_msg(this->name, GF_LOG_DEBUG, 0, BRB_MSG_CHECKSUM_MISMATCH,
+           "Object checksum mismatch: %s [GFID: %s | Brick: %s]", loc->path,
+           uuid_utoa(linked_inode->gfid), child->brick_path);
+    gf_msg(this->name, GF_LOG_ALERT, 0, BRB_MSG_CHECKSUM_MISMATCH,
+           "CORRUPTION DETECTED: Object %s {Brick: %s | GFID: %s}", loc->path,
+           child->brick_path, uuid_utoa(linked_inode->gfid));
+
+    /* Perform bad-file marking */
+    xattr = dict_new();
+    if (!xattr) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_int32(xattr, BITROT_OBJECT_BAD_KEY, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_MARK_BAD_FILE,
+               "Error setting bad-file marker for %s [GFID: %s | "
+               "Brick: %s]",
+               loc->path, uuid_utoa(linked_inode->gfid), child->brick_path);
+        goto dictfree;
+    }
+
+    gf_msg(this->name, GF_LOG_ALERT, 0, BRB_MSG_MARK_CORRUPTED,
+           "Marking"
+           " %s [GFID: %s | Brick: %s] as corrupted..",
+           loc->path, uuid_utoa(linked_inode->gfid), child->brick_path);
+    gf_event(EVENT_BITROT_BAD_FILE, "gfid=%s;path=%s;brick=%s",
+             uuid_utoa(linked_inode->gfid), loc->path, child->brick_path);
+    ret = syncop_fsetxattr(child->xl, fd, xattr, 0, NULL, NULL);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_MARK_BAD_FILE,
+               "Error marking object %s [GFID: %s] as corrupted", loc->path,
+               uuid_utoa(linked_inode->gfid));
+
+dictfree:
+    dict_unref(xattr);
+out:
+    return ret;
 }
 
 /**
@@ -284,540 +282,550 @@ bitd_compare_ckum (xlator_t *this,
  * signs with SHA256).
  */
 int
-br_scrubber_scrub_begin (xlator_t *this, struct br_fsscan_entry *fsentry)
+br_scrubber_scrub_begin(xlator_t *this, struct br_fsscan_entry *fsentry)
 {
-        int32_t                ret           = -1;
-        fd_t                  *fd            = NULL;
-        loc_t                  loc           = {0, };
-        struct iatt            iatt          = {0, };
-        struct iatt            parent_buf    = {0, };
-        pid_t                  pid           = 0;
-        br_child_t            *child         = NULL;
-        unsigned char         *md            = NULL;
-        inode_t               *linked_inode  = NULL;
-        br_isignature_out_t   *sign          = NULL;
-        unsigned long          signedversion = 0;
-        gf_dirent_t           *entry         = NULL;
-        br_private_t          *priv          = NULL;
-        loc_t                 *parent        = NULL;
-        gf_boolean_t           skip_stat     = _gf_false;
-        uuid_t                 shard_root_gfid = {0,};
-
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", fsentry, out);
-
-        entry = fsentry->entry;
-        parent = &fsentry->parent;
-        child = fsentry->data;
-
-        priv = this->private;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", entry, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", parent, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", child, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", priv, out);
-
-        pid = GF_CLIENT_PID_SCRUB;
-
-        ret = br_prepare_loc (this, child, parent, entry, &loc);
-        if (!ret)
-                goto out;
-
-        syncopctx_setfspid (&pid);
-
-        ret = syncop_lookup (child->xl, &loc, &iatt, &parent_buf, NULL, NULL);
-        if (ret) {
-                br_log_object_path (this, "lookup", loc.path, -ret);
-                goto out;
-        }
-
-        linked_inode = inode_link (loc.inode, parent->inode, loc.name, &iatt);
-        if (linked_inode)
-                inode_lookup (linked_inode);
-
-        gf_msg_debug (this->name, 0, "Scrubbing object %s [GFID: %s]",
-                      entry->d_name, uuid_utoa (linked_inode->gfid));
-
-        if (iatt.ia_type != IA_IFREG) {
-                gf_msg_debug (this->name, 0, "%s is not a regular file",
-                              entry->d_name);
-                ret = 0;
-                goto unref_inode;
-        }
-
-        if (IS_DHT_LINKFILE_MODE ((&iatt))) {
-                gf_msg_debug (this->name, 0, "%s is a dht sticky bit file",
-                              entry->d_name);
-                ret = 0;
-                goto unref_inode;
-        }
-
-        /* skip updating scrub statistics for shard entries */
-        gf_uuid_parse (SHARD_ROOT_GFID, shard_root_gfid);
-        if (gf_uuid_compare (loc.pargfid, shard_root_gfid) == 0)
-                skip_stat = _gf_true;
-
-        /**
-         * open() an fd for subsequent opertaions
-         */
-        fd = fd_create (linked_inode, 0);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
-                        "failed to create fd for inode %s",
-                        uuid_utoa (linked_inode->gfid));
-                goto unref_inode;
-        }
-
-        ret = syncop_open (child->xl, &loc, O_RDWR, fd, NULL, NULL);
-        if (ret) {
-                br_log_object (this, "open", linked_inode->gfid, -ret);
-                ret = -1;
-                goto unrefd;
-        }
-
-        fd_bind (fd);
-
-        /**
-         * perform pre compute checks before initiating checksum
-         * computation
-         *  - presence of bad object
-         *  - signature staleness
-         */
-        ret = bitd_scrub_pre_compute_check (this, child, fd, &signedversion,
-                                            &priv->scrub_stat, skip_stat);
-        if (ret)
-                goto unrefd; /* skip this object */
-
-        /* if all's good, proceed to calculate the hash */
-        md = GF_CALLOC (SHA256_DIGEST_LENGTH, sizeof (*md),
-                        gf_common_mt_char);
-        if (!md)
-                goto unrefd;
-
-        ret = br_calculate_obj_checksum (md, child, fd, &iatt);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_CALC_ERROR,
-                        "error calculating hash for object [GFID: %s]",
-                        uuid_utoa (fd->inode->gfid));
-                ret = -1;
-                goto free_md;
-        }
-
-        /**
-         * perform post compute checks as an object's signature may have
-         * become stale while scrubber calculated checksum.
-         */
-        ret = bitd_scrub_post_compute_check (this, child, fd, signedversion,
-                                             &sign, &priv->scrub_stat,
-                                             skip_stat);
-        if (ret)
-                goto free_md;
-
-        ret = bitd_compare_ckum (this, sign, md,
-                                 linked_inode, entry, fd, child, &loc);
-
-        if (!skip_stat)
-                br_inc_scrubbed_file (&priv->scrub_stat);
-
-        GF_FREE (sign); /* alloced on post-compute */
-
-        /** fd_unref() takes care of closing fd.. like syncop_close() */
+    int32_t ret = -1;
+    fd_t *fd = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    struct iatt parent_buf = {
+        0,
+    };
+    pid_t pid = 0;
+    br_child_t *child = NULL;
+    unsigned char *md = NULL;
+    inode_t *linked_inode = NULL;
+    br_isignature_out_t *sign = NULL;
+    unsigned long signedversion = 0;
+    gf_dirent_t *entry = NULL;
+    br_private_t *priv = NULL;
+    loc_t *parent = NULL;
+    gf_boolean_t skip_stat = _gf_false;
+    uuid_t shard_root_gfid = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("bit-rot", fsentry, out);
+
+    entry = fsentry->entry;
+    parent = &fsentry->parent;
+    child = fsentry->data;
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", entry, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", parent, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", child, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", priv, out);
+
+    pid = GF_CLIENT_PID_SCRUB;
+
+    ret = br_prepare_loc(this, child, parent, entry, &loc);
+    if (!ret)
+        goto out;
+
+    syncopctx_setfspid(&pid);
+
+    ret = syncop_lookup(child->xl, &loc, &iatt, &parent_buf, NULL, NULL);
+    if (ret) {
+        br_log_object_path(this, "lookup", loc.path, -ret);
+        goto out;
+    }
+
+    linked_inode = inode_link(loc.inode, parent->inode, loc.name, &iatt);
+    if (linked_inode)
+        inode_lookup(linked_inode);
+
+    gf_msg_debug(this->name, 0, "Scrubbing object %s [GFID: %s]", entry->d_name,
+                 uuid_utoa(linked_inode->gfid));
+
+    if (iatt.ia_type != IA_IFREG) {
+        gf_msg_debug(this->name, 0, "%s is not a regular file", entry->d_name);
+        ret = 0;
+        goto unref_inode;
+    }
 
- free_md:
-        GF_FREE (md);
- unrefd:
-        fd_unref (fd);
- unref_inode:
-        inode_unref (linked_inode);
- out:
-        loc_wipe (&loc);
-        return ret;
+    if (IS_DHT_LINKFILE_MODE((&iatt))) {
+        gf_msg_debug(this->name, 0, "%s is a dht sticky bit file",
+                     entry->d_name);
+        ret = 0;
+        goto unref_inode;
+    }
+
+    /* skip updating scrub statistics for shard entries */
+    gf_uuid_parse(SHARD_ROOT_GFID, shard_root_gfid);
+    if (gf_uuid_compare(loc.pargfid, shard_root_gfid) == 0)
+        skip_stat = _gf_true;
+
+    /**
+     * open() an fd for subsequent operations
+     */
+    fd = fd_create(linked_inode, 0);
+    if (!fd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
+               "failed to create fd for inode %s",
+               uuid_utoa(linked_inode->gfid));
+        goto unref_inode;
+    }
+
+    ret = syncop_open(child->xl, &loc, O_RDWR, fd, NULL, NULL);
+    if (ret) {
+        br_log_object(this, "open", linked_inode->gfid, -ret);
+        ret = -1;
+        goto unrefd;
+    }
+
+    fd_bind(fd);
+
+    /**
+     * perform pre compute checks before initiating checksum
+     * computation
+     *  - presence of bad object
+     *  - signature staleness
+     */
+    ret = bitd_scrub_pre_compute_check(this, child, fd, &signedversion,
+                                       &priv->scrub_stat, skip_stat);
+    if (ret)
+        goto unrefd; /* skip this object */
+
+    /* if all's good, proceed to calculate the hash */
+    md = GF_MALLOC(SHA256_DIGEST_LENGTH, gf_common_mt_char);
+    if (!md)
+        goto unrefd;
+
+    ret = br_calculate_obj_checksum(md, child, fd, &iatt);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_CALC_ERROR,
+               "error calculating hash for object [GFID: %s]",
+               uuid_utoa(fd->inode->gfid));
+        ret = -1;
+        goto free_md;
+    }
+
+    /**
+     * perform post compute checks as an object's signature may have
+     * become stale while scrubber calculated checksum.
+     */
+    ret = bitd_scrub_post_compute_check(this, child, fd, signedversion, &sign,
+                                        &priv->scrub_stat, skip_stat);
+    if (ret)
+        goto free_md;
+
+    ret = bitd_compare_ckum(this, sign, md, linked_inode, entry, fd, child,
+                            &loc);
+
+    if (!skip_stat)
+        br_inc_scrubbed_file(&priv->scrub_stat);
+
+    GF_FREE(sign); /* allocated on post-compute */
+
+    /** fd_unref() takes care of closing fd.. like syncop_close() */
+
+free_md:
+    GF_FREE(md);
+unrefd:
+    fd_unref(fd);
+unref_inode:
+    inode_unref(linked_inode);
+out:
+    loc_wipe(&loc);
+    return ret;
 }
 
 static void
-_br_lock_cleaner (void *arg)
+_br_lock_cleaner(void *arg)
 {
-        pthread_mutex_t *mutex = arg;
+    pthread_mutex_t *mutex = arg;
 
-        pthread_mutex_unlock (mutex);
+    pthread_mutex_unlock(mutex);
 }
 
 static void
-wait_for_scrubbing (xlator_t *this, struct br_scanfs *fsscan)
+wait_for_scrubbing(xlator_t *this, struct br_scanfs *fsscan)
 {
-        br_private_t *priv = NULL;
-        struct br_scrubber *fsscrub = NULL;
+    br_private_t *priv = NULL;
+    struct br_scrubber *fsscrub = NULL;
 
-        priv = this->private;
-        fsscrub = &priv->fsscrub;
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
 
-        pthread_cleanup_push (_br_lock_cleaner, &fsscan->waitlock);
-        pthread_mutex_lock (&fsscan->waitlock);
+    pthread_cleanup_push(_br_lock_cleaner, &fsscan->waitlock);
+    pthread_mutex_lock(&fsscan->waitlock);
+    {
+        pthread_cleanup_push(_br_lock_cleaner, &fsscrub->mutex);
+        pthread_mutex_lock(&fsscrub->mutex);
         {
-                pthread_cleanup_push (_br_lock_cleaner, &fsscrub->mutex);
-                pthread_mutex_lock (&fsscrub->mutex);
-                {
-                        list_replace_init (&fsscan->queued, &fsscan->ready);
-
-                        /* wake up scrubbers */
-                        pthread_cond_broadcast (&fsscrub->cond);
-                }
-                pthread_mutex_unlock (&fsscrub->mutex);
-                pthread_cleanup_pop (0);
-
-                while (fsscan->entries != 0)
-                        pthread_cond_wait
-                                    (&fsscan->waitcond, &fsscan->waitlock);
+            list_replace_init(&fsscan->queued, &fsscan->ready);
+
+            /* wake up scrubbers */
+            pthread_cond_broadcast(&fsscrub->cond);
         }
-        pthread_mutex_unlock (&fsscan->waitlock);
-        pthread_cleanup_pop (0);
+        pthread_mutex_unlock(&fsscrub->mutex);
+        pthread_cleanup_pop(0);
+
+        while (fsscan->entries != 0)
+            pthread_cond_wait(&fsscan->waitcond, &fsscan->waitlock);
+    }
+    pthread_mutex_unlock(&fsscan->waitlock);
+    pthread_cleanup_pop(0);
 }
 
 static void
-_br_fsscan_inc_entry_count (struct br_scanfs *fsscan)
+_br_fsscan_inc_entry_count(struct br_scanfs *fsscan)
 {
-        fsscan->entries++;
+    fsscan->entries++;
 }
 
 static void
-_br_fsscan_dec_entry_count (struct br_scanfs *fsscan)
+_br_fsscan_dec_entry_count(struct br_scanfs *fsscan)
 {
-        if (--fsscan->entries == 0) {
-                pthread_mutex_lock (&fsscan->waitlock);
-                {
-                        pthread_cond_signal (&fsscan->waitcond);
-                }
-                pthread_mutex_unlock (&fsscan->waitlock);
+    if (--fsscan->entries == 0) {
+        pthread_mutex_lock(&fsscan->waitlock);
+        {
+            pthread_cond_signal(&fsscan->waitcond);
         }
+        pthread_mutex_unlock(&fsscan->waitlock);
+    }
 }
 
 static void
-_br_fsscan_collect_entry (struct br_scanfs *fsscan,
-                           struct br_fsscan_entry *fsentry)
+_br_fsscan_collect_entry(struct br_scanfs *fsscan,
+                         struct br_fsscan_entry *fsentry)
 {
-        list_add_tail (&fsentry->list, &fsscan->queued);
-        _br_fsscan_inc_entry_count (fsscan);
+    list_add_tail(&fsentry->list, &fsscan->queued);
+    _br_fsscan_inc_entry_count(fsscan);
 }
 
-#define NR_ENTRIES (1<<7) /* ..bulk scrubbing */
+#define NR_ENTRIES (1 << 7) /* ..bulk scrubbing */
 
 int
-br_fsscanner_handle_entry (xlator_t *subvol,
-                           gf_dirent_t *entry, loc_t *parent, void *data)
+br_fsscanner_handle_entry(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                          void *data)
 {
-        int32_t                 ret     = -1;
-        int                     scrub   = 0;
-        br_child_t             *child   = NULL;
-        xlator_t               *this    = NULL;
-        struct br_scanfs       *fsscan  = NULL;
-        struct br_fsscan_entry *fsentry = NULL;
+    int32_t ret = -1;
+    int scrub = 0;
+    br_child_t *child = NULL;
+    xlator_t *this = NULL;
+    struct br_scanfs *fsscan = NULL;
+    struct br_fsscan_entry *fsentry = NULL;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot", subvol, error_return);
-        GF_VALIDATE_OR_GOTO ("bit-rot", data, error_return);
+    GF_VALIDATE_OR_GOTO("bit-rot", subvol, error_return);
+    GF_VALIDATE_OR_GOTO("bit-rot", data, error_return);
 
-        child = data;
-        this = child->this;
-        fsscan = &child->fsscan;
+    child = data;
+    this = child->this;
+    fsscan = &child->fsscan;
 
-        _mask_cancellation ();
+    _mask_cancellation();
 
-        fsentry = GF_CALLOC (1, sizeof (*fsentry), gf_br_mt_br_fsscan_entry_t);
-        if (!fsentry)
-                goto error_return;
+    fsentry = GF_CALLOC(1, sizeof(*fsentry), gf_br_mt_br_fsscan_entry_t);
+    if (!fsentry)
+        goto error_return;
 
-        {
-                fsentry->data = data;
-                fsentry->fsscan = &child->fsscan;
+    {
+        fsentry->data = data;
+        fsentry->fsscan = &child->fsscan;
+
+        /* copy parent loc */
+        ret = loc_copy(&fsentry->parent, parent);
+        if (ret)
+            goto dealloc;
 
-                /* copy parent loc */
-                ret = loc_copy (&fsentry->parent, parent);
-                if (ret)
-                        goto dealloc;
+        /* copy child entry */
+        fsentry->entry = entry_copy(entry);
+        if (!fsentry->entry)
+            goto locwipe;
 
-                /* copy child entry */
-                fsentry->entry = entry_copy (entry);
-                if (!fsentry->entry)
-                        goto locwipe;
+        INIT_LIST_HEAD(&fsentry->list);
+    }
 
-                INIT_LIST_HEAD (&fsentry->list);
-        }
+    LOCK(&fsscan->entrylock);
+    {
+        _br_fsscan_collect_entry(fsscan, fsentry);
 
-        LOCK (&fsscan->entrylock);
-        {
-                _br_fsscan_collect_entry (fsscan, fsentry);
-
-                /**
-                 * need not be a equality check as entries may be pushed
-                 * back onto the scanned queue when thread(s) are cleaned.
-                 */
-                if (fsscan->entries >= NR_ENTRIES)
-                        scrub = 1;
-        }
-        UNLOCK (&fsscan->entrylock);
+        /**
+         * need not be a equality check as entries may be pushed
+         * back onto the scanned queue when thread(s) are cleaned.
+         */
+        if (fsscan->entries >= NR_ENTRIES)
+            scrub = 1;
+    }
+    UNLOCK(&fsscan->entrylock);
 
-        _unmask_cancellation ();
+    _unmask_cancellation();
 
-        if (scrub)
-                wait_for_scrubbing (this, fsscan);
+    if (scrub)
+        wait_for_scrubbing(this, fsscan);
 
-        return 0;
+    return 0;
 
- locwipe:
-        loc_wipe (&fsentry->parent);
- dealloc:
-        GF_FREE (fsentry);
- error_return:
-        return -1;
+locwipe:
+    loc_wipe(&fsentry->parent);
+dealloc:
+    GF_FREE(fsentry);
+error_return:
+    return -1;
 }
 
 int32_t
-br_fsscan_deactivate (xlator_t *this)
+br_fsscan_deactivate(xlator_t *this)
 {
-        int ret = 0;
-        br_private_t *priv = NULL;
-        br_scrub_state_t nstate = 0;
-        struct br_monitor *scrub_monitor = NULL;
-
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
-
-        ret = gf_tw_del_timer (priv->timer_wheel, scrub_monitor->timer);
-        if (ret == 0) {
-                nstate = BR_SCRUB_STATE_STALLED;
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                        "Volume is under active scrubbing. Pausing scrub..");
-        } else {
-                nstate = BR_SCRUB_STATE_PAUSED;
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                        "Scrubber paused");
-        }
-
-        _br_monitor_set_scrub_state (scrub_monitor, nstate);
-
-        return 0;
+    int ret = 0;
+    br_private_t *priv = NULL;
+    br_scrub_state_t nstate = 0;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    ret = gf_tw_del_timer(priv->timer_wheel, scrub_monitor->timer);
+    if (ret == 0) {
+        nstate = BR_SCRUB_STATE_STALLED;
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Volume is under active scrubbing. Pausing scrub..");
+    } else {
+        nstate = BR_SCRUB_STATE_PAUSED;
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Scrubber paused");
+    }
+
+    _br_monitor_set_scrub_state(scrub_monitor, nstate);
+
+    return 0;
 }
 
 static void
-br_scrubber_log_time (xlator_t *this, const char *sfx)
+br_scrubber_log_time(xlator_t *this, const char *sfx)
 {
-        char           timestr[1024] = {0,};
-        struct         timeval tv    = {0,};
-        br_private_t  *priv          = NULL;
-
-        priv = this->private;
-
-        gettimeofday (&tv, NULL);
-        gf_time_fmt (timestr, sizeof (timestr), tv.tv_sec, gf_timefmt_FT);
-
-        if (strcasecmp (sfx, "started") == 0) {
-                br_update_scrub_start_time (&priv->scrub_stat, &tv);
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_START,
-                        "Scrubbing %s at %s", sfx, timestr);
-        } else {
-                br_update_scrub_finish_time (&priv->scrub_stat, timestr, &tv);
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_FINISH,
-                        "Scrubbing %s at %s", sfx, timestr);
-        }
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    br_private_t *priv = NULL;
+    time_t now = 0;
+
+    now = gf_time();
+    priv = this->private;
+
+    gf_time_fmt(timestr, sizeof(timestr), now, gf_timefmt_FT);
+
+    if (strcasecmp(sfx, "started") == 0) {
+        br_update_scrub_start_time(&priv->scrub_stat, now);
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_START,
+               "Scrubbing %s at %s", sfx, timestr);
+    } else {
+        br_update_scrub_finish_time(&priv->scrub_stat, timestr, now);
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_FINISH,
+               "Scrubbing %s at %s", sfx, timestr);
+    }
 }
 
 static void
-br_fsscanner_log_time (xlator_t *this, br_child_t *child, const char *sfx)
+br_fsscanner_log_time(xlator_t *this, br_child_t *child, const char *sfx)
 {
-        char           timestr[1024] = {0,};
-        struct         timeval tv    = {0,};
-
-        gettimeofday (&tv, NULL);
-        gf_time_fmt (timestr, sizeof (timestr), tv.tv_sec, gf_timefmt_FT);
-
-        if (strcasecmp (sfx, "started") == 0) {
-                gf_msg_debug (this->name, 0, "Scrubbing \"%s\" %s at %s",
-                              child->brick_path, sfx, timestr);
-        } else {
-                gf_msg_debug (this->name, 0, "Scrubbing \"%s\" %s at %s",
-                              child->brick_path, sfx, timestr);
-        }
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    time_t now = 0;
+
+    now = gf_time();
+    gf_time_fmt(timestr, sizeof(timestr), now, gf_timefmt_FT);
+
+    if (strcasecmp(sfx, "started") == 0) {
+        gf_msg_debug(this->name, 0, "Scrubbing \"%s\" %s at %s",
+                     child->brick_path, sfx, timestr);
+    } else {
+        gf_msg_debug(this->name, 0, "Scrubbing \"%s\" %s at %s",
+                     child->brick_path, sfx, timestr);
+    }
 }
 
 void
-br_child_set_scrub_state (br_child_t *child, gf_boolean_t state)
+br_child_set_scrub_state(br_child_t *child, gf_boolean_t state)
 {
-        child->active_scrubbing = state;
+    child->active_scrubbing = state;
 }
 
 static void
-br_fsscanner_wait_until_kicked (xlator_t *this, br_child_t *child)
+br_fsscanner_wait_until_kicked(xlator_t *this, br_child_t *child)
 {
-        br_private_t      *priv          = NULL;
-        struct br_monitor *scrub_monitor = NULL;
-
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
-
-        pthread_cleanup_push (_br_lock_cleaner, &scrub_monitor->wakelock);
-        pthread_mutex_lock (&scrub_monitor->wakelock);
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    pthread_cleanup_push(_br_lock_cleaner, &scrub_monitor->wakelock);
+    pthread_mutex_lock(&scrub_monitor->wakelock);
+    {
+        while (!scrub_monitor->kick)
+            pthread_cond_wait(&scrub_monitor->wakecond,
+                              &scrub_monitor->wakelock);
+
+        /* Child lock is to synchronize with disconnect events */
+        pthread_cleanup_push(_br_lock_cleaner, &child->lock);
+        pthread_mutex_lock(&child->lock);
         {
-                while (!scrub_monitor->kick)
-                        pthread_cond_wait (&scrub_monitor->wakecond,
-                                           &scrub_monitor->wakelock);
-
-                /* Child lock is to synchronize with disconnect events */
-                pthread_cleanup_push (_br_lock_cleaner, &child->lock);
-                pthread_mutex_lock (&child->lock);
-                {
-                        scrub_monitor->active_child_count++;
-                        br_child_set_scrub_state (child, _gf_true);
-                }
-                pthread_mutex_unlock (&child->lock);
-                pthread_cleanup_pop (0);
-        }
-        pthread_mutex_unlock (&scrub_monitor->wakelock);
-        pthread_cleanup_pop (0);
+            scrub_monitor->active_child_count++;
+            br_child_set_scrub_state(child, _gf_true);
+        }
+        pthread_mutex_unlock(&child->lock);
+        pthread_cleanup_pop(0);
+    }
+    pthread_mutex_unlock(&scrub_monitor->wakelock);
+    pthread_cleanup_pop(0);
 }
 
 static void
-br_scrubber_entry_control (xlator_t *this)
+br_scrubber_entry_control(xlator_t *this)
 {
-        br_private_t      *priv          = NULL;
-        struct br_monitor *scrub_monitor = NULL;
-
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
-
-        LOCK (&scrub_monitor->lock);
-        {
-                /* Move the state to BR_SCRUB_STATE_ACTIVE */
-                if (scrub_monitor->state == BR_SCRUB_STATE_PENDING)
-                        scrub_monitor->state = BR_SCRUB_STATE_ACTIVE;
-                br_scrubber_log_time (this, "started");
-                priv->scrub_stat.scrub_running = 1;
-        }
-        UNLOCK (&scrub_monitor->lock);
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    LOCK(&scrub_monitor->lock);
+    {
+        /* Move the state to BR_SCRUB_STATE_ACTIVE */
+        if (scrub_monitor->state == BR_SCRUB_STATE_PENDING)
+            scrub_monitor->state = BR_SCRUB_STATE_ACTIVE;
+        br_scrubber_log_time(this, "started");
+        priv->scrub_stat.scrub_running = 1;
+    }
+    UNLOCK(&scrub_monitor->lock);
 }
 
 static void
-br_scrubber_exit_control (xlator_t *this)
+br_scrubber_exit_control(xlator_t *this)
 {
-        br_private_t      *priv          = NULL;
-        struct br_monitor *scrub_monitor = NULL;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
 
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
 
-        LOCK (&scrub_monitor->lock);
-        {
-                br_scrubber_log_time (this, "finished");
-                priv->scrub_stat.scrub_running = 0;
-
-                if (scrub_monitor->state == BR_SCRUB_STATE_ACTIVE) {
-                        (void) br_fsscan_activate (this);
-                } else {
-                        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                                "Volume waiting to get rescheduled..");
-                }
+    LOCK(&scrub_monitor->lock);
+    {
+        br_scrubber_log_time(this, "finished");
+        priv->scrub_stat.scrub_running = 0;
+
+        if (scrub_monitor->state == BR_SCRUB_STATE_ACTIVE) {
+            (void)br_fsscan_activate(this);
+        } else {
+            UNLOCK(&scrub_monitor->lock);
+            gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+                   "Volume waiting to get rescheduled..");
+            return;
         }
-        UNLOCK (&scrub_monitor->lock);
+    }
+    UNLOCK(&scrub_monitor->lock);
 }
 
 static void
-br_fsscanner_entry_control (xlator_t *this, br_child_t *child)
+br_fsscanner_entry_control(xlator_t *this, br_child_t *child)
 {
-                br_fsscanner_log_time (this, child, "started");
+    br_fsscanner_log_time(this, child, "started");
 }
 
 static void
-br_fsscanner_exit_control (xlator_t *this, br_child_t *child)
+br_fsscanner_exit_control(xlator_t *this, br_child_t *child)
 {
-        br_private_t *priv = NULL;
-        struct br_monitor *scrub_monitor = NULL;
-
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
-
-        if (!_br_is_child_connected (child)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_SCRUB_INFO,
-                        "Brick [%s] disconnected while scrubbing. Scrubbing "
-                        "might be incomplete", child->brick_path);
-        }
-
-        br_fsscanner_log_time (this, child, "finished");
-
-        pthread_cleanup_push (_br_lock_cleaner, &scrub_monitor->wakelock);
-        pthread_mutex_lock (&scrub_monitor->wakelock);
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    if (!_br_is_child_connected(child)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SCRUB_INFO,
+               "Brick [%s] disconnected while scrubbing. Scrubbing "
+               "might be incomplete",
+               child->brick_path);
+    }
+
+    br_fsscanner_log_time(this, child, "finished");
+
+    pthread_cleanup_push(_br_lock_cleaner, &scrub_monitor->wakelock);
+    pthread_mutex_lock(&scrub_monitor->wakelock);
+    {
+        scrub_monitor->active_child_count--;
+        pthread_cleanup_push(_br_lock_cleaner, &child->lock);
+        pthread_mutex_lock(&child->lock);
         {
-                scrub_monitor->active_child_count--;
-                pthread_cleanup_push (_br_lock_cleaner, &child->lock);
-                pthread_mutex_lock (&child->lock);
-                {
-                        br_child_set_scrub_state (child, _gf_false);
-                }
-                pthread_mutex_unlock (&child->lock);
-                pthread_cleanup_pop (0);
-
-                if (scrub_monitor->active_child_count == 0) {
-                        /* The last child has finished scrubbing.
-                         * Set the kick to false and  wake up other
-                         * children who are waiting for the last
-                         * child to complete scrubbing.
-                         */
-                        scrub_monitor->kick = _gf_false;
-                        pthread_cond_broadcast (&scrub_monitor->wakecond);
-
-                        /* Signal monitor thread waiting for the all
-                         * the children to finish scrubbing.
-                         */
-                        pthread_cleanup_push (_br_lock_cleaner,
-                                              &scrub_monitor->donelock);
-                        pthread_mutex_lock (&scrub_monitor->donelock);
-                        {
-                                scrub_monitor->done = _gf_true;
-                                pthread_cond_signal (&scrub_monitor->donecond);
-                        }
-                        pthread_mutex_unlock (&scrub_monitor->donelock);
-                        pthread_cleanup_pop (0);
-                } else {
-                        while (scrub_monitor->active_child_count)
-                                pthread_cond_wait (&scrub_monitor->wakecond,
-                                                   &scrub_monitor->wakelock);
-                }
+            br_child_set_scrub_state(child, _gf_false);
+        }
+        pthread_mutex_unlock(&child->lock);
+        pthread_cleanup_pop(0);
+
+        if (scrub_monitor->active_child_count == 0) {
+            /* The last child has finished scrubbing.
+             * Set the kick to false and  wake up other
+             * children who are waiting for the last
+             * child to complete scrubbing.
+             */
+            scrub_monitor->kick = _gf_false;
+            pthread_cond_broadcast(&scrub_monitor->wakecond);
+
+            /* Signal monitor thread waiting for the all
+             * the children to finish scrubbing.
+             */
+            pthread_cleanup_push(_br_lock_cleaner, &scrub_monitor->donelock);
+            pthread_mutex_lock(&scrub_monitor->donelock);
+            {
+                scrub_monitor->done = _gf_true;
+                pthread_cond_signal(&scrub_monitor->donecond);
+            }
+            pthread_mutex_unlock(&scrub_monitor->donelock);
+            pthread_cleanup_pop(0);
+        } else {
+            while (scrub_monitor->active_child_count)
+                pthread_cond_wait(&scrub_monitor->wakecond,
+                                  &scrub_monitor->wakelock);
         }
-        pthread_mutex_unlock (&scrub_monitor->wakelock);
-        pthread_cleanup_pop (0);
+    }
+    pthread_mutex_unlock(&scrub_monitor->wakelock);
+    pthread_cleanup_pop(0);
 }
 
 void *
-br_fsscanner (void *arg)
+br_fsscanner(void *arg)
 {
-        loc_t               loc     = {0,};
-        br_child_t         *child   = NULL;
-        xlator_t           *this    = NULL;
-        struct br_scanfs   *fsscan  = NULL;
-
-        child = arg;
-        this = child->this;
-        fsscan = &child->fsscan;
-
-        THIS = this;
-        loc.inode = child->table->root;
-
-        while (1) {
-                br_fsscanner_wait_until_kicked (this, child);
-                {
-                        /* precursor for scrub */
-                        br_fsscanner_entry_control (this, child);
-
-                        /* scrub */
-                        (void) syncop_ftw (child->xl,
-                                           &loc, GF_CLIENT_PID_SCRUB,
-                                           child, br_fsscanner_handle_entry);
-                        if (!list_empty (&fsscan->queued))
-                                wait_for_scrubbing (this, fsscan);
-
-                        /* scrub exit criteria */
-                        br_fsscanner_exit_control (this, child);
-                }
+    loc_t loc = {
+        0,
+    };
+    br_child_t *child = NULL;
+    xlator_t *this = NULL;
+    struct br_scanfs *fsscan = NULL;
+
+    child = arg;
+    this = child->this;
+    fsscan = &child->fsscan;
+
+    THIS = this;
+    loc.inode = child->table->root;
+
+    while (1) {
+        br_fsscanner_wait_until_kicked(this, child);
+        {
+            /* precursor for scrub */
+            br_fsscanner_entry_control(this, child);
+
+            /* scrub */
+            (void)syncop_ftw(child->xl, &loc, GF_CLIENT_PID_SCRUB, child,
+                             br_fsscanner_handle_entry);
+            if (!list_empty(&fsscan->queued))
+                wait_for_scrubbing(this, fsscan);
+
+            /* scrub exit criteria */
+            br_fsscanner_exit_control(this, child);
         }
+    }
 
-        return NULL;
+    return NULL;
 }
 
 /**
@@ -828,263 +836,268 @@ br_fsscanner (void *arg)
  * non-pending timer.
  */
 void
-br_kickstart_scanner (struct gf_tw_timer_list *timer,
-                      void *data, unsigned long calltime)
+br_kickstart_scanner(struct gf_tw_timer_list *timer, void *data,
+                     unsigned long calltime)
 {
-        xlator_t *this = NULL;
-        struct br_monitor *scrub_monitor = data;
-        br_private_t *priv = NULL;
-
-        THIS = this = scrub_monitor->this;
-        priv = this->private;
-
-        /* Reset scrub statistics */
-        priv->scrub_stat.scrubbed_files = 0;
-        priv->scrub_stat.unsigned_files = 0;
-
-        /* Moves state from PENDING to ACTIVE */
-        (void) br_scrubber_entry_control (this);
-
-        /* kickstart scanning.. */
-        pthread_mutex_lock (&scrub_monitor->wakelock);
-        {
-                scrub_monitor->kick = _gf_true;
-                GF_ASSERT (scrub_monitor->active_child_count == 0);
-                pthread_cond_broadcast (&scrub_monitor->wakecond);
-        }
-        pthread_mutex_unlock (&scrub_monitor->wakelock);
-
-        return;
+    xlator_t *this = NULL;
+    struct br_monitor *scrub_monitor = data;
+    br_private_t *priv = NULL;
+
+    THIS = this = scrub_monitor->this;
+    priv = this->private;
+
+    /* Reset scrub statistics */
+    priv->scrub_stat.scrubbed_files = 0;
+    priv->scrub_stat.unsigned_files = 0;
+
+    /* Moves state from PENDING to ACTIVE */
+    (void)br_scrubber_entry_control(this);
+
+    /* kickstart scanning.. */
+    pthread_mutex_lock(&scrub_monitor->wakelock);
+    {
+        scrub_monitor->kick = _gf_true;
+        GF_ASSERT(scrub_monitor->active_child_count == 0);
+        pthread_cond_broadcast(&scrub_monitor->wakecond);
+    }
+    pthread_mutex_unlock(&scrub_monitor->wakelock);
+
+    return;
 }
 
 static uint32_t
-br_fsscan_calculate_delta (uint32_t times)
+br_fsscan_calculate_delta(uint32_t times)
 {
-        return times;
+    return times;
 }
 
-#define BR_SCRUB_ONDEMAND   (1)
-#define BR_SCRUB_MINUTE     (60)
-#define BR_SCRUB_HOURLY     (60 * 60)
-#define BR_SCRUB_DAILY      (1 * 24 * 60 * 60)
-#define BR_SCRUB_WEEKLY     (7 * 24 * 60 * 60)
-#define BR_SCRUB_BIWEEKLY   (14 * 24 * 60 * 60)
-#define BR_SCRUB_MONTHLY    (30 * 24 * 60 * 60)
+#define BR_SCRUB_ONDEMAND (1)
+#define BR_SCRUB_MINUTE (60)
+#define BR_SCRUB_HOURLY (60 * 60)
+#define BR_SCRUB_DAILY (1 * 24 * 60 * 60)
+#define BR_SCRUB_WEEKLY (7 * 24 * 60 * 60)
+#define BR_SCRUB_BIWEEKLY (14 * 24 * 60 * 60)
+#define BR_SCRUB_MONTHLY (30 * 24 * 60 * 60)
 
 static unsigned int
-br_fsscan_calculate_timeout (scrub_freq_t freq)
+br_fsscan_calculate_timeout(scrub_freq_t freq)
 {
-        uint32_t timo = 0;
+    uint32_t timo = 0;
 
-        switch (freq) {
+    switch (freq) {
         case BR_FSSCRUB_FREQ_MINUTE:
-                timo = br_fsscan_calculate_delta (BR_SCRUB_MINUTE);
-                break;
+            timo = br_fsscan_calculate_delta(BR_SCRUB_MINUTE);
+            break;
         case BR_FSSCRUB_FREQ_HOURLY:
-                timo = br_fsscan_calculate_delta (BR_SCRUB_HOURLY);
-                break;
+            timo = br_fsscan_calculate_delta(BR_SCRUB_HOURLY);
+            break;
         case BR_FSSCRUB_FREQ_DAILY:
-                timo = br_fsscan_calculate_delta (BR_SCRUB_DAILY);
-                break;
+            timo = br_fsscan_calculate_delta(BR_SCRUB_DAILY);
+            break;
         case BR_FSSCRUB_FREQ_WEEKLY:
-                timo = br_fsscan_calculate_delta (BR_SCRUB_WEEKLY);
-                break;
+            timo = br_fsscan_calculate_delta(BR_SCRUB_WEEKLY);
+            break;
         case BR_FSSCRUB_FREQ_BIWEEKLY:
-                timo = br_fsscan_calculate_delta (BR_SCRUB_BIWEEKLY);
-                break;
+            timo = br_fsscan_calculate_delta(BR_SCRUB_BIWEEKLY);
+            break;
         case BR_FSSCRUB_FREQ_MONTHLY:
-                timo = br_fsscan_calculate_delta (BR_SCRUB_MONTHLY);
-                break;
+            timo = br_fsscan_calculate_delta(BR_SCRUB_MONTHLY);
+            break;
         default:
-                timo = 0;
-        }
+            timo = 0;
+    }
 
-        return timo;
+    return timo;
 }
 
 int32_t
-br_fsscan_schedule (xlator_t *this)
+br_fsscan_schedule(xlator_t *this)
 {
-        uint32_t timo = 0;
-        br_private_t *priv = NULL;
-        struct timeval tv = {0,};
-        char timestr[1024] = {0,};
-        struct br_scrubber *fsscrub = NULL;
-        struct gf_tw_timer_list *timer = NULL;
-        struct br_monitor *scrub_monitor = NULL;
-
-        priv = this->private;
-        fsscrub = &priv->fsscrub;
-        scrub_monitor = &priv->scrub_monitor;
-
-        (void) gettimeofday (&tv, NULL);
-        scrub_monitor->boot = tv.tv_sec;
-
-        timo = br_fsscan_calculate_timeout (fsscrub->frequency);
-        if (timo == 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
-                        "BUG: Zero schedule timeout");
-                goto error_return;
-        }
-
-        scrub_monitor->timer = GF_CALLOC (1, sizeof (*scrub_monitor->timer),
-                                   gf_br_stub_mt_br_scanner_freq_t);
-        if (!scrub_monitor->timer)
-                goto error_return;
-
-        timer = scrub_monitor->timer;
-        INIT_LIST_HEAD (&timer->entry);
-
-        timer->data = scrub_monitor;
-        timer->expires = timo;
-        timer->function = br_kickstart_scanner;
-
-        gf_tw_add_timer (priv->timer_wheel, timer);
-        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PENDING);
-
-        gf_time_fmt (timestr, sizeof (timestr),
-                     (scrub_monitor->boot + timo), gf_timefmt_FT);
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, "Scrubbing is "
-                "scheduled to run at %s", timestr);
-
-        return 0;
-
- error_return:
-        return -1;
+    uint32_t timo = 0;
+    br_private_t *priv = NULL;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    struct br_scrubber *fsscrub = NULL;
+    struct gf_tw_timer_list *timer = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    scrub_monitor->boot = gf_time();
+
+    timo = br_fsscan_calculate_timeout(fsscrub->frequency);
+    if (timo == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
+               "BUG: Zero schedule timeout");
+        goto error_return;
+    }
+
+    scrub_monitor->timer = GF_CALLOC(1, sizeof(*scrub_monitor->timer),
+                                     gf_br_stub_mt_br_scanner_freq_t);
+    if (!scrub_monitor->timer)
+        goto error_return;
+
+    timer = scrub_monitor->timer;
+    INIT_LIST_HEAD(&timer->entry);
+
+    timer->data = scrub_monitor;
+    timer->expires = timo;
+    timer->function = br_kickstart_scanner;
+
+    gf_tw_add_timer(priv->timer_wheel, timer);
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PENDING);
+
+    gf_time_fmt(timestr, sizeof(timestr), (scrub_monitor->boot + timo),
+                gf_timefmt_FT);
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+           "Scrubbing is "
+           "scheduled to run at %s",
+           timestr);
+
+    return 0;
+
+error_return:
+    return -1;
 }
 
 int32_t
-br_fsscan_activate (xlator_t *this)
+br_fsscan_activate(xlator_t *this)
 {
-        uint32_t            timo    = 0;
-        char timestr[1024]          = {0,};
-        struct timeval      now     = {0,};
-        br_private_t       *priv    = NULL;
-        struct br_scrubber *fsscrub = NULL;
-        struct br_monitor  *scrub_monitor = NULL;
-
-        priv = this->private;
-        fsscrub = &priv->fsscrub;
-        scrub_monitor = &priv->scrub_monitor;
-
-        (void) gettimeofday (&now, NULL);
-        timo = br_fsscan_calculate_timeout (fsscrub->frequency);
-        if (timo == 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
-                        "BUG: Zero schedule timeout");
-                return -1;
-        }
+    uint32_t timo = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    time_t now = 0;
+    br_private_t *priv = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    now = gf_time();
+    timo = br_fsscan_calculate_timeout(fsscrub->frequency);
+    if (timo == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
+               "BUG: Zero schedule timeout");
+        return -1;
+    }
 
-        pthread_mutex_lock (&scrub_monitor->donelock);
-        {
-                scrub_monitor->done = _gf_false;
-        }
-        pthread_mutex_unlock (&scrub_monitor->donelock);
+    pthread_mutex_lock(&scrub_monitor->donelock);
+    {
+        scrub_monitor->done = _gf_false;
+    }
+    pthread_mutex_unlock(&scrub_monitor->donelock);
 
-        gf_time_fmt (timestr, sizeof (timestr),
-                     (now.tv_sec + timo), gf_timefmt_FT);
-        (void) gf_tw_mod_timer (priv->timer_wheel, scrub_monitor->timer, timo);
+    gf_time_fmt(timestr, sizeof(timestr), now + timo, gf_timefmt_FT);
+    (void)gf_tw_mod_timer(priv->timer_wheel, scrub_monitor->timer, timo);
 
-        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PENDING);
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO, "Scrubbing is "
-                "rescheduled to run at %s", timestr);
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PENDING);
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+           "Scrubbing is "
+           "rescheduled to run at %s",
+           timestr);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-br_fsscan_reschedule (xlator_t *this)
+br_fsscan_reschedule(xlator_t *this)
 {
-        int32_t             ret     = 0;
-        uint32_t            timo    = 0;
-        char timestr[1024]          = {0,};
-        struct timeval      now     = {0,};
-        br_private_t       *priv    = NULL;
-        struct br_scrubber *fsscrub = NULL;
-        struct br_monitor  *scrub_monitor = NULL;
-
-        priv = this->private;
-        fsscrub = &priv->fsscrub;
-        scrub_monitor = &priv->scrub_monitor;
-
-        if (!fsscrub->frequency_reconf)
-                return 0;
-
-        (void) gettimeofday (&now, NULL);
-        timo = br_fsscan_calculate_timeout (fsscrub->frequency);
-        if (timo == 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
-                        "BUG: Zero schedule timeout");
-                return -1;
-        }
+    int32_t ret = 0;
+    uint32_t timo = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    time_t now = 0;
+    br_private_t *priv = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    if (!fsscrub->frequency_reconf)
+        return 0;
 
-        gf_time_fmt (timestr, sizeof (timestr),
-                     (now.tv_sec + timo), gf_timefmt_FT);
+    now = gf_time();
+    timo = br_fsscan_calculate_timeout(fsscrub->frequency);
+    if (timo == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
+               "BUG: Zero schedule timeout");
+        return -1;
+    }
 
-        pthread_mutex_lock (&scrub_monitor->donelock);
-        {
-                scrub_monitor->done = _gf_false;
-        }
-        pthread_mutex_unlock (&scrub_monitor->donelock);
-
-        ret = gf_tw_mod_timer_pending (priv->timer_wheel, scrub_monitor->timer, timo);
-        if (ret == 0)
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                        "Scrubber is currently running and would be "
-                        "rescheduled after completion");
-        else {
-                _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PENDING);
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                        "Scrubbing rescheduled to run at %s", timestr);
-        }
+    gf_time_fmt(timestr, sizeof(timestr), now + timo, gf_timefmt_FT);
 
-        return 0;
+    pthread_mutex_lock(&scrub_monitor->donelock);
+    {
+        scrub_monitor->done = _gf_false;
+    }
+    pthread_mutex_unlock(&scrub_monitor->donelock);
+
+    ret = gf_tw_mod_timer_pending(priv->timer_wheel, scrub_monitor->timer,
+                                  timo);
+    if (ret == 0)
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Scrubber is currently running and would be "
+               "rescheduled after completion");
+    else {
+        _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PENDING);
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Scrubbing rescheduled to run at %s", timestr);
+    }
+
+    return 0;
 }
 
 int32_t
-br_fsscan_ondemand (xlator_t *this)
+br_fsscan_ondemand(xlator_t *this)
 {
-        int32_t             ret     = 0;
-        uint32_t            timo    = 0;
-        char timestr[1024]          = {0,};
-        struct timeval      now     = {0,};
-        br_private_t       *priv    = NULL;
-        struct br_monitor  *scrub_monitor = NULL;
-
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
-
-        (void) gettimeofday (&now, NULL);
-
-        timo = BR_SCRUB_ONDEMAND;
-
-        gf_time_fmt (timestr, sizeof (timestr),
-                     (now.tv_sec + timo), gf_timefmt_FT);
-
-        pthread_mutex_lock (&scrub_monitor->donelock);
-        {
-                scrub_monitor->done = _gf_false;
-        }
-        pthread_mutex_unlock (&scrub_monitor->donelock);
-
-        ret = gf_tw_mod_timer_pending (priv->timer_wheel, scrub_monitor->timer,
-                                       timo);
-        if (ret == 0)
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                        "Scrubber is currently running and would be "
-                        "rescheduled after completion");
-        else {
-                _br_monitor_set_scrub_state (scrub_monitor,
-                                             BR_SCRUB_STATE_PENDING);
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                        "Ondemand Scrubbing scheduled to run at %s", timestr);
-        }
-
-        return 0;
+    int32_t ret = 0;
+    uint32_t timo = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    time_t now = 0;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    now = gf_time();
+    timo = BR_SCRUB_ONDEMAND;
+    gf_time_fmt(timestr, sizeof(timestr), now + timo, gf_timefmt_FT);
+
+    pthread_mutex_lock(&scrub_monitor->donelock);
+    {
+        scrub_monitor->done = _gf_false;
+    }
+    pthread_mutex_unlock(&scrub_monitor->donelock);
+
+    ret = gf_tw_mod_timer_pending(priv->timer_wheel, scrub_monitor->timer,
+                                  timo);
+    if (ret == 0)
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Scrubber is currently running and would be "
+               "rescheduled after completion");
+    else {
+        _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PENDING);
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Ondemand Scrubbing scheduled to run at %s", timestr);
+    }
+
+    return 0;
 }
 
-#define BR_SCRUB_THREAD_SCALE_LAZY       0
-#define BR_SCRUB_THREAD_SCALE_NORMAL     0.4
+#define BR_SCRUB_THREAD_SCALE_LAZY 0
+#define BR_SCRUB_THREAD_SCALE_NORMAL 0.4
 #define BR_SCRUB_THREAD_SCALE_AGGRESSIVE 1.0
 
 #ifndef M_E
@@ -1097,111 +1110,105 @@ br_fsscan_ondemand (xlator_t *this)
  * the scale based on the number of processor cores too.
  */
 static unsigned int
-br_scrubber_calc_scale (xlator_t *this,
-                        br_private_t *priv, scrub_throttle_t throttle)
+br_scrubber_calc_scale(xlator_t *this, br_private_t *priv,
+                       scrub_throttle_t throttle)
 {
-        unsigned int scale = 0;
+    unsigned int scale = 0;
 
-        switch (throttle) {
+    switch (throttle) {
         case BR_SCRUB_THROTTLE_VOID:
         case BR_SCRUB_THROTTLE_STALLED:
-                scale = 0;
-                break;
+            scale = 0;
+            break;
         case BR_SCRUB_THROTTLE_LAZY:
-                scale = priv->child_count *
-                              pow (M_E, BR_SCRUB_THREAD_SCALE_LAZY);
-                break;
+            scale = priv->child_count * pow(M_E, BR_SCRUB_THREAD_SCALE_LAZY);
+            break;
         case BR_SCRUB_THROTTLE_NORMAL:
-                scale = priv->child_count *
-                              pow (M_E, BR_SCRUB_THREAD_SCALE_NORMAL);
-                break;
+            scale = priv->child_count * pow(M_E, BR_SCRUB_THREAD_SCALE_NORMAL);
+            break;
         case BR_SCRUB_THROTTLE_AGGRESSIVE:
-                scale = priv->child_count *
-                              pow (M_E, BR_SCRUB_THREAD_SCALE_AGGRESSIVE);
-                break;
+            scale = priv->child_count *
+                    pow(M_E, BR_SCRUB_THREAD_SCALE_AGGRESSIVE);
+            break;
         default:
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_UNKNOWN_THROTTLE,
-                        "Unknown throttle %d", throttle);
-        }
-
-        return scale;
+            gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_UNKNOWN_THROTTLE,
+                   "Unknown throttle %d", throttle);
+    }
 
+    return scale;
 }
 
 static br_child_t *
-_br_scrubber_get_next_child (struct br_scrubber *fsscrub)
+_br_scrubber_get_next_child(struct br_scrubber *fsscrub)
 {
-        br_child_t *child = NULL;
+    br_child_t *child = NULL;
 
-        child = list_first_entry (&fsscrub->scrublist, br_child_t, list);
-        list_rotate_left (&fsscrub->scrublist);
+    child = list_first_entry(&fsscrub->scrublist, br_child_t, list);
+    list_rotate_left(&fsscrub->scrublist);
 
-        return child;
+    return child;
 }
 
 static void
-_br_scrubber_get_entry (br_child_t *child, struct br_fsscan_entry **fsentry)
+_br_scrubber_get_entry(br_child_t *child, struct br_fsscan_entry **fsentry)
 {
-        struct br_scanfs *fsscan = &child->fsscan;
+    struct br_scanfs *fsscan = &child->fsscan;
 
-        if (list_empty (&fsscan->ready))
-                return;
-        *fsentry = list_first_entry
-                            (&fsscan->ready, struct br_fsscan_entry, list);
-        list_del_init (&(*fsentry)->list);
+    if (list_empty(&fsscan->ready))
+        return;
+    *fsentry = list_first_entry(&fsscan->ready, struct br_fsscan_entry, list);
+    list_del_init(&(*fsentry)->list);
 }
 
 static void
-_br_scrubber_find_scrubbable_entry (struct br_scrubber *fsscrub,
-                                     struct br_fsscan_entry **fsentry)
+_br_scrubber_find_scrubbable_entry(struct br_scrubber *fsscrub,
+                                   struct br_fsscan_entry **fsentry)
 {
-        br_child_t *child = NULL;
-        br_child_t *firstchild = NULL;
+    br_child_t *child = NULL;
+    br_child_t *firstchild = NULL;
 
-        while (1) {
-                while (list_empty (&fsscrub->scrublist))
-                        pthread_cond_wait (&fsscrub->cond, &fsscrub->mutex);
+    while (1) {
+        while (list_empty(&fsscrub->scrublist))
+            pthread_cond_wait(&fsscrub->cond, &fsscrub->mutex);
 
-                firstchild = NULL;
-                for (child = _br_scrubber_get_next_child (fsscrub);
-                     child != firstchild;
-                     child = _br_scrubber_get_next_child (fsscrub)) {
+        firstchild = NULL;
+        for (child = _br_scrubber_get_next_child(fsscrub); child != firstchild;
+             child = _br_scrubber_get_next_child(fsscrub)) {
+            if (!firstchild)
+                firstchild = child;
 
-                        if (!firstchild)
-                                firstchild = child;
-
-                        _br_scrubber_get_entry (child, fsentry);
-                        if (*fsentry)
-                                break;
-                }
+            _br_scrubber_get_entry(child, fsentry);
+            if (*fsentry)
+                break;
+        }
 
-                if (*fsentry)
-                        break;
+        if (*fsentry)
+            break;
 
-                /* nothing to work on.. wait till available */
-                pthread_cond_wait (&fsscrub->cond, &fsscrub->mutex);
-        }
+        /* nothing to work on.. wait till available */
+        pthread_cond_wait(&fsscrub->cond, &fsscrub->mutex);
+    }
 }
 
 static void
-br_scrubber_pick_entry (struct br_scrubber *fsscrub,
-                        struct br_fsscan_entry **fsentry)
+br_scrubber_pick_entry(struct br_scrubber *fsscrub,
+                       struct br_fsscan_entry **fsentry)
 {
-        pthread_cleanup_push (_br_lock_cleaner, &fsscrub->mutex);
+    pthread_cleanup_push(_br_lock_cleaner, &fsscrub->mutex);
 
-        pthread_mutex_lock (&fsscrub->mutex);
-        {
-                *fsentry = NULL;
-                _br_scrubber_find_scrubbable_entry (fsscrub, fsentry);
-        }
-        pthread_mutex_unlock (&fsscrub->mutex);
+    pthread_mutex_lock(&fsscrub->mutex);
+    {
+        *fsentry = NULL;
+        _br_scrubber_find_scrubbable_entry(fsscrub, fsentry);
+    }
+    pthread_mutex_unlock(&fsscrub->mutex);
 
-        pthread_cleanup_pop (0);
+    pthread_cleanup_pop(0);
 }
 
 struct br_scrub_entry {
-        gf_boolean_t scrubbed;
-        struct br_fsscan_entry *fsentry;
+    gf_boolean_t scrubbed;
+    struct br_fsscan_entry *fsentry;
 };
 
 /**
@@ -1211,682 +1218,702 @@ struct br_scrub_entry {
  * in the ->pending queue or when an object is undergoing scrubbing.
  */
 static void
-br_scrubber_entry_handle (void *arg)
+br_scrubber_entry_handle(void *arg)
 {
-        struct br_scanfs       *fsscan  = NULL;
-        struct br_scrub_entry  *sentry  = NULL;
-        struct br_fsscan_entry *fsentry = NULL;
+    struct br_scanfs *fsscan = NULL;
+    struct br_scrub_entry *sentry = NULL;
+    struct br_fsscan_entry *fsentry = NULL;
 
-        sentry = arg;
+    sentry = arg;
 
-        fsentry = sentry->fsentry;
-        fsscan  = fsentry->fsscan;
+    fsentry = sentry->fsentry;
+    fsscan = fsentry->fsscan;
 
-        LOCK (&fsscan->entrylock);
-        {
-                if (sentry->scrubbed) {
-                        _br_fsscan_dec_entry_count (fsscan);
-
-                        /* cleanup ->entry */
-                        fsentry->data   = NULL;
-                        fsentry->fsscan = NULL;
-                        loc_wipe (&fsentry->parent);
-                        gf_dirent_entry_free (fsentry->entry);
-
-                        GF_FREE (sentry->fsentry);
-                } else {
-                        /* (re)queue the entry again for scrub */
-                        _br_fsscan_collect_entry (fsscan, sentry->fsentry);
-                }
+    LOCK(&fsscan->entrylock);
+    {
+        if (sentry->scrubbed) {
+            _br_fsscan_dec_entry_count(fsscan);
+
+            /* cleanup ->entry */
+            fsentry->data = NULL;
+            fsentry->fsscan = NULL;
+            loc_wipe(&fsentry->parent);
+            gf_dirent_entry_free(fsentry->entry);
+
+            GF_FREE(sentry->fsentry);
+        } else {
+            /* (re)queue the entry again for scrub */
+            _br_fsscan_collect_entry(fsscan, sentry->fsentry);
         }
-        UNLOCK (&fsscan->entrylock);
+    }
+    UNLOCK(&fsscan->entrylock);
 }
 
 static void
-br_scrubber_scrub_entry (xlator_t *this, struct br_fsscan_entry *fsentry)
+br_scrubber_scrub_entry(xlator_t *this, struct br_fsscan_entry *fsentry)
 {
-        struct br_scrub_entry sentry = {0, };
-
-        sentry.scrubbed = 0;
-        sentry.fsentry = fsentry;
-
-        pthread_cleanup_push (br_scrubber_entry_handle, &sentry);
-        {
-                (void) br_scrubber_scrub_begin (this, fsentry);
-                sentry.scrubbed = 1;
-        }
-        pthread_cleanup_pop (1);
+    struct br_scrub_entry sentry = {
+        0,
+    };
+
+    sentry.scrubbed = 0;
+    sentry.fsentry = fsentry;
+
+    pthread_cleanup_push(br_scrubber_entry_handle, &sentry);
+    {
+        (void)br_scrubber_scrub_begin(this, fsentry);
+        sentry.scrubbed = 1;
+    }
+    pthread_cleanup_pop(1);
 }
 
-void *br_scrubber_proc (void *arg)
+void *
+br_scrubber_proc(void *arg)
 {
-        xlator_t *this = NULL;
-        struct br_scrubber *fsscrub = NULL;
-        struct br_fsscan_entry *fsentry = NULL;
+    xlator_t *this = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    struct br_fsscan_entry *fsentry = NULL;
 
-        fsscrub = arg;
-        THIS = this = fsscrub->this;
+    fsscrub = arg;
+    THIS = this = fsscrub->this;
 
-        while (1) {
-                br_scrubber_pick_entry (fsscrub, &fsentry);
-                br_scrubber_scrub_entry (this, fsentry);
-                sleep (1);
-        }
+    while (1) {
+        br_scrubber_pick_entry(fsscrub, &fsentry);
+        br_scrubber_scrub_entry(this, fsentry);
+        sleep(1);
+    }
 
-        return NULL;
+    return NULL;
 }
 
 static int32_t
-br_scrubber_scale_up (xlator_t *this,
-                      struct br_scrubber *fsscrub,
-                      unsigned int v1, unsigned int v2)
+br_scrubber_scale_up(xlator_t *this, struct br_scrubber *fsscrub,
+                     unsigned int v1, unsigned int v2)
 {
-        int i = 0;
-        int32_t ret = -1;
-        int diff = 0;
-        struct br_scrubbers *scrub = NULL;
-
-        diff = (int)(v2 - v1);
-
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCALING_UP_SCRUBBER,
-                "Scaling up scrubbers [%d => %d]", v1, v2);
-
-        for (i = 0; i < diff; i++) {
-                scrub = GF_CALLOC (diff, sizeof (*scrub),
-                                   gf_br_mt_br_scrubber_t);
-                if (!scrub)
-                        break;
-
-                INIT_LIST_HEAD (&scrub->list);
-                ret = gf_thread_create (&scrub->scrubthread,
-                                        NULL, br_scrubber_proc, fsscrub,
-                                        "brsproc");
-                if (ret)
-                        break;
-
-                fsscrub->nr_scrubbers++;
-                list_add_tail (&scrub->list, &fsscrub->scrubbers);
-        }
+    int i = 0;
+    int32_t ret = -1;
+    int diff = 0;
+    struct br_scrubbers *scrub = NULL;
 
-        if ((i != diff) && !scrub)
-                goto error_return;
+    diff = (int)(v2 - v1);
 
-        if (i != diff) /* degraded scaling.. */
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_SCALE_UP_FAILED,
-                        "Could not fully scale up to %d scrubber(s). Spawned "
-                        "%d/%d [total scrubber(s): %d]", v2, i, diff, (v1 + i));
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCALING_UP_SCRUBBER,
+           "Scaling up scrubbers [%d => %d]", v1, v2);
 
-        return 0;
+    for (i = 0; i < diff; i++) {
+        scrub = GF_CALLOC(diff, sizeof(*scrub), gf_br_mt_br_scrubber_t);
+        if (!scrub)
+            break;
 
- error_return:
-        return -1;
+        INIT_LIST_HEAD(&scrub->list);
+        ret = gf_thread_create(&scrub->scrubthread, NULL, br_scrubber_proc,
+                               fsscrub, "brsproc");
+        if (ret)
+            break;
+
+        fsscrub->nr_scrubbers++;
+        list_add_tail(&scrub->list, &fsscrub->scrubbers);
+    }
+
+    if ((i != diff) && !scrub)
+        goto error_return;
+
+    if (i != diff) /* degraded scaling.. */
+        gf_msg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SCALE_UP_FAILED,
+               "Could not fully scale up to %d scrubber(s). Spawned "
+               "%d/%d [total scrubber(s): %d]",
+               v2, i, diff, (v1 + i));
+
+    return 0;
+
+error_return:
+    return -1;
 }
 
 static int32_t
-br_scrubber_scale_down (xlator_t *this,
-                        struct br_scrubber *fsscrub,
-                        unsigned int v1, unsigned int v2)
+br_scrubber_scale_down(xlator_t *this, struct br_scrubber *fsscrub,
+                       unsigned int v1, unsigned int v2)
 {
-        int i = 0;
-        int diff = 0;
-        int32_t ret = -1;
-        struct br_scrubbers *scrub = NULL;
-
-        diff = (int)(v1 - v2);
-
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCALE_DOWN_SCRUBBER,
-                "Scaling down scrubbers [%d => %d]", v1, v2);
+    int i = 0;
+    int diff = 0;
+    int32_t ret = -1;
+    struct br_scrubbers *scrub = NULL;
 
-        for (i = 0 ; i < diff; i++) {
-                scrub = list_first_entry
-                            (&fsscrub->scrubbers, struct br_scrubbers, list);
+    diff = (int)(v1 - v2);
 
-                list_del_init (&scrub->list);
-                ret = gf_thread_cleanup_xint (scrub->scrubthread);
-                if (ret)
-                        break;
-                GF_FREE (scrub);
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCALE_DOWN_SCRUBBER,
+           "Scaling down scrubbers [%d => %d]", v1, v2);
 
-                fsscrub->nr_scrubbers--;
-        }
+    for (i = 0; i < diff; i++) {
+        scrub = list_first_entry(&fsscrub->scrubbers, struct br_scrubbers,
+                                 list);
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRB_MSG_SCALE_DOWN_FAILED, "Could not fully scale down "
-                        "to %d scrubber(s). Terminated %d/%d [total "
-                        "scrubber(s): %d]", v1, i, diff, (v2 - i));
-                ret = 0;
-        }
+        list_del_init(&scrub->list);
+        ret = gf_thread_cleanup_xint(scrub->scrubthread);
+        if (ret)
+            break;
+        GF_FREE(scrub);
+
+        fsscrub->nr_scrubbers--;
+    }
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SCALE_DOWN_FAILED,
+               "Could not fully scale down "
+               "to %d scrubber(s). Terminated %d/%d [total "
+               "scrubber(s): %d]",
+               v1, i, diff, (v2 - i));
+        ret = 0;
+    }
 
-        return ret;
+    return ret;
 }
 
 static int32_t
-br_scrubber_configure (xlator_t *this, br_private_t *priv,
-                       struct br_scrubber *fsscrub, scrub_throttle_t nthrottle)
+br_scrubber_configure(xlator_t *this, br_private_t *priv,
+                      struct br_scrubber *fsscrub, scrub_throttle_t nthrottle)
 {
-        int32_t ret = 0;
-        unsigned int v1 = 0;
-        unsigned int v2 = 0;
+    int32_t ret = 0;
+    unsigned int v1 = 0;
+    unsigned int v2 = 0;
 
-        v1 = fsscrub->nr_scrubbers;
-        v2 = br_scrubber_calc_scale (this, priv, nthrottle);
+    v1 = fsscrub->nr_scrubbers;
+    v2 = br_scrubber_calc_scale(this, priv, nthrottle);
 
-        if (v1 == v2)
-                return 0;
+    if (v1 == v2)
+        return 0;
 
-        if (v1 > v2)
-                ret = br_scrubber_scale_down (this, fsscrub, v1, v2);
-        else
-                ret = br_scrubber_scale_up (this, fsscrub, v1, v2);
+    if (v1 > v2)
+        ret = br_scrubber_scale_down(this, fsscrub, v1, v2);
+    else
+        ret = br_scrubber_scale_up(this, fsscrub, v1, v2);
 
-        return ret;
+    return ret;
 }
 
 static int32_t
-br_scrubber_fetch_option (xlator_t *this,
-                          char *opt, dict_t *options, char **value)
+br_scrubber_fetch_option(xlator_t *this, char *opt, dict_t *options,
+                         char **value)
 {
-        if (options)
-                GF_OPTION_RECONF (opt, *value, options, str, error_return);
-        else
-                GF_OPTION_INIT (opt, *value, str, error_return);
+    if (options)
+        GF_OPTION_RECONF(opt, *value, options, str, error_return);
+    else
+        GF_OPTION_INIT(opt, *value, str, error_return);
 
-        return 0;
+    return 0;
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 /* internal "throttle" override */
-#define BR_SCRUB_STALLED  "STALLED"
+#define BR_SCRUB_STALLED "STALLED"
 
 /* TODO: token buket spec */
 static int32_t
-br_scrubber_handle_throttle (xlator_t *this, br_private_t *priv,
-                             dict_t *options, gf_boolean_t scrubstall)
+br_scrubber_handle_throttle(xlator_t *this, br_private_t *priv, dict_t *options,
+                            gf_boolean_t scrubstall)
 {
-        int32_t ret = 0;
-        char *tmp = NULL;
-        struct br_scrubber *fsscrub = NULL;
-        scrub_throttle_t nthrottle = BR_SCRUB_THROTTLE_VOID;
-
-        fsscrub = &priv->fsscrub;
-        fsscrub->throttle_reconf = _gf_false;
-
-        ret = br_scrubber_fetch_option (this, "scrub-throttle", options, &tmp);
-        if (ret)
-                goto error_return;
-
-        if (scrubstall)
-                tmp = BR_SCRUB_STALLED;
-
-        if (strcasecmp (tmp, "lazy") == 0)
-                nthrottle = BR_SCRUB_THROTTLE_LAZY;
-        else if (strcasecmp (tmp, "normal") == 0)
-                nthrottle = BR_SCRUB_THROTTLE_NORMAL;
-        else if (strcasecmp (tmp, "aggressive") == 0)
-                nthrottle = BR_SCRUB_THROTTLE_AGGRESSIVE;
-        else if (strcasecmp (tmp, BR_SCRUB_STALLED) == 0)
-                nthrottle = BR_SCRUB_THROTTLE_STALLED;
-        else
-                goto error_return;
-
-        /* on failure old throttling value is preserved */
-        ret = br_scrubber_configure (this, priv, fsscrub, nthrottle);
-        if (ret)
-                goto error_return;
-
-        if (fsscrub->throttle != nthrottle)
-                fsscrub->throttle_reconf = _gf_true;
-
-        fsscrub->throttle = nthrottle;
-        return 0;
-
- error_return:
-        return -1;
+    int32_t ret = 0;
+    char *tmp = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    scrub_throttle_t nthrottle = BR_SCRUB_THROTTLE_VOID;
+
+    fsscrub = &priv->fsscrub;
+    fsscrub->throttle_reconf = _gf_false;
+
+    ret = br_scrubber_fetch_option(this, "scrub-throttle", options, &tmp);
+    if (ret)
+        goto error_return;
+
+    if (scrubstall)
+        tmp = BR_SCRUB_STALLED;
+
+    if (strcasecmp(tmp, "lazy") == 0)
+        nthrottle = BR_SCRUB_THROTTLE_LAZY;
+    else if (strcasecmp(tmp, "normal") == 0)
+        nthrottle = BR_SCRUB_THROTTLE_NORMAL;
+    else if (strcasecmp(tmp, "aggressive") == 0)
+        nthrottle = BR_SCRUB_THROTTLE_AGGRESSIVE;
+    else if (strcasecmp(tmp, BR_SCRUB_STALLED) == 0)
+        nthrottle = BR_SCRUB_THROTTLE_STALLED;
+    else
+        goto error_return;
+
+    /* on failure old throttling value is preserved */
+    ret = br_scrubber_configure(this, priv, fsscrub, nthrottle);
+    if (ret)
+        goto error_return;
+
+    if (fsscrub->throttle != nthrottle)
+        fsscrub->throttle_reconf = _gf_true;
+
+    fsscrub->throttle = nthrottle;
+    return 0;
+
+error_return:
+    return -1;
 }
 
 static int32_t
-br_scrubber_handle_stall (xlator_t *this, br_private_t *priv,
-                          dict_t *options, gf_boolean_t *scrubstall)
+br_scrubber_handle_stall(xlator_t *this, br_private_t *priv, dict_t *options,
+                         gf_boolean_t *scrubstall)
 {
-        int32_t ret = 0;
-        char *tmp = NULL;
+    int32_t ret = 0;
+    char *tmp = NULL;
 
-        ret = br_scrubber_fetch_option (this, "scrub-state", options, &tmp);
-        if (ret)
-                goto error_return;
+    ret = br_scrubber_fetch_option(this, "scrub-state", options, &tmp);
+    if (ret)
+        goto error_return;
 
-        if (strcasecmp (tmp, "pause") == 0) /* anything else is active */
-                *scrubstall = _gf_true;
+    if (strcasecmp(tmp, "pause") == 0) /* anything else is active */
+        *scrubstall = _gf_true;
 
-        return 0;
+    return 0;
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 static int32_t
-br_scrubber_handle_freq (xlator_t *this, br_private_t *priv,
-                         dict_t *options, gf_boolean_t scrubstall)
+br_scrubber_handle_freq(xlator_t *this, br_private_t *priv, dict_t *options,
+                        gf_boolean_t scrubstall)
 {
-        int32_t ret  = -1;
-        char *tmp = NULL;
-        scrub_freq_t frequency = BR_FSSCRUB_FREQ_HOURLY;
-        struct br_scrubber *fsscrub = NULL;
-
-        fsscrub = &priv->fsscrub;
-        fsscrub->frequency_reconf = _gf_true;
-
-        ret = br_scrubber_fetch_option (this, "scrub-freq", options, &tmp);
-        if (ret)
-                goto error_return;
-
-        if (scrubstall)
-                tmp = BR_SCRUB_STALLED;
-
-        if (strcasecmp (tmp, "hourly") == 0) {
-                frequency = BR_FSSCRUB_FREQ_HOURLY;
-        } else if (strcasecmp (tmp, "daily") == 0) {
-                frequency = BR_FSSCRUB_FREQ_DAILY;
-        } else if (strcasecmp (tmp, "weekly") == 0) {
-                frequency = BR_FSSCRUB_FREQ_WEEKLY;
-        } else if (strcasecmp (tmp, "biweekly") == 0) {
-                frequency = BR_FSSCRUB_FREQ_BIWEEKLY;
-        } else if (strcasecmp (tmp, "monthly") == 0) {
-                frequency = BR_FSSCRUB_FREQ_MONTHLY;
-        } else if (strcasecmp (tmp, "minute") == 0) {
-                frequency = BR_FSSCRUB_FREQ_MINUTE;
-        } else if (strcasecmp (tmp, BR_SCRUB_STALLED) == 0) {
-                frequency = BR_FSSCRUB_FREQ_STALLED;
-        } else
-                goto error_return;
-
-        if (fsscrub->frequency == frequency)
-                fsscrub->frequency_reconf = _gf_false;
-        else
-                fsscrub->frequency = frequency;
-
-        return 0;
-
- error_return:
-        return -1;
+    int32_t ret = -1;
+    char *tmp = NULL;
+    scrub_freq_t frequency = BR_FSSCRUB_FREQ_HOURLY;
+    struct br_scrubber *fsscrub = NULL;
+
+    fsscrub = &priv->fsscrub;
+    fsscrub->frequency_reconf = _gf_true;
+
+    ret = br_scrubber_fetch_option(this, "scrub-freq", options, &tmp);
+    if (ret)
+        goto error_return;
+
+    if (scrubstall)
+        tmp = BR_SCRUB_STALLED;
+
+    if (strcasecmp(tmp, "hourly") == 0) {
+        frequency = BR_FSSCRUB_FREQ_HOURLY;
+    } else if (strcasecmp(tmp, "daily") == 0) {
+        frequency = BR_FSSCRUB_FREQ_DAILY;
+    } else if (strcasecmp(tmp, "weekly") == 0) {
+        frequency = BR_FSSCRUB_FREQ_WEEKLY;
+    } else if (strcasecmp(tmp, "biweekly") == 0) {
+        frequency = BR_FSSCRUB_FREQ_BIWEEKLY;
+    } else if (strcasecmp(tmp, "monthly") == 0) {
+        frequency = BR_FSSCRUB_FREQ_MONTHLY;
+    } else if (strcasecmp(tmp, "minute") == 0) {
+        frequency = BR_FSSCRUB_FREQ_MINUTE;
+    } else if (strcasecmp(tmp, BR_SCRUB_STALLED) == 0) {
+        frequency = BR_FSSCRUB_FREQ_STALLED;
+    } else
+        goto error_return;
+
+    if (fsscrub->frequency == frequency)
+        fsscrub->frequency_reconf = _gf_false;
+    else
+        fsscrub->frequency = frequency;
+
+    return 0;
+
+error_return:
+    return -1;
 }
 
-static void br_scrubber_log_option (xlator_t *this,
-                                    br_private_t *priv, gf_boolean_t scrubstall)
+static void
+br_scrubber_log_option(xlator_t *this, br_private_t *priv,
+                       gf_boolean_t scrubstall)
 {
-        struct br_scrubber *fsscrub = &priv->fsscrub;
-        char *scrub_throttle_str[] = {
-                [BR_SCRUB_THROTTLE_LAZY]       = "lazy",
-                [BR_SCRUB_THROTTLE_NORMAL]     = "normal",
-                [BR_SCRUB_THROTTLE_AGGRESSIVE] = "aggressive",
-        };
-
-        char *scrub_freq_str[] = {
-                [BR_FSSCRUB_FREQ_HOURLY]   = "hourly",
-                [BR_FSSCRUB_FREQ_DAILY]    = "daily",
-                [BR_FSSCRUB_FREQ_WEEKLY]   = "weekly",
-                [BR_FSSCRUB_FREQ_BIWEEKLY] = "biweekly",
-                [BR_FSSCRUB_FREQ_MONTHLY]  = "monthly (30 days)",
-                [BR_FSSCRUB_FREQ_MINUTE]  = "every minute",
-        };
-
-        if (scrubstall)
-                return; /* logged as pause */
-
-        if (fsscrub->frequency_reconf || fsscrub->throttle_reconf) {
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_TUNABLE,
-                        "SCRUB TUNABLES:: [Frequency: %s, Throttle: %s]",
-                        scrub_freq_str[fsscrub->frequency],
-                        scrub_throttle_str[fsscrub->throttle]);
-        }
+    struct br_scrubber *fsscrub = &priv->fsscrub;
+    char *scrub_throttle_str[] = {
+        [BR_SCRUB_THROTTLE_LAZY] = "lazy",
+        [BR_SCRUB_THROTTLE_NORMAL] = "normal",
+        [BR_SCRUB_THROTTLE_AGGRESSIVE] = "aggressive",
+        [BR_SCRUB_THROTTLE_STALLED] = "stalled",
+    };
+
+    char *scrub_freq_str[] = {
+        [0] = "",
+        [BR_FSSCRUB_FREQ_HOURLY] = "hourly",
+        [BR_FSSCRUB_FREQ_DAILY] = "daily",
+        [BR_FSSCRUB_FREQ_WEEKLY] = "weekly",
+        [BR_FSSCRUB_FREQ_BIWEEKLY] = "biweekly",
+        [BR_FSSCRUB_FREQ_MONTHLY] = "monthly (30 days)",
+        [BR_FSSCRUB_FREQ_MINUTE] = "every minute",
+    };
+
+    if (scrubstall)
+        return; /* logged as pause */
+
+    if (fsscrub->frequency_reconf || fsscrub->throttle_reconf) {
+        if (fsscrub->throttle == BR_SCRUB_THROTTLE_VOID)
+            return;
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_TUNABLE,
+               "SCRUB TUNABLES:: [Frequency: %s, Throttle: %s]",
+               scrub_freq_str[fsscrub->frequency],
+               scrub_throttle_str[fsscrub->throttle]);
+    }
 }
 
 int32_t
-br_scrubber_handle_options (xlator_t *this, br_private_t *priv, dict_t *options)
+br_scrubber_handle_options(xlator_t *this, br_private_t *priv, dict_t *options)
 {
-        int32_t ret = 0;
-        gf_boolean_t scrubstall = _gf_false; /* not as dangerous as it sounds */
+    int32_t ret = 0;
+    gf_boolean_t scrubstall = _gf_false; /* not as dangerous as it sounds */
 
-        ret = br_scrubber_handle_stall (this, priv, options, &scrubstall);
-        if (ret)
-                goto error_return;
+    ret = br_scrubber_handle_stall(this, priv, options, &scrubstall);
+    if (ret)
+        goto error_return;
 
-        ret = br_scrubber_handle_throttle (this, priv, options, scrubstall);
-        if (ret)
-                goto error_return;
+    ret = br_scrubber_handle_throttle(this, priv, options, scrubstall);
+    if (ret)
+        goto error_return;
 
-        ret = br_scrubber_handle_freq (this, priv, options, scrubstall);
-        if (ret)
-                goto error_return;
+    ret = br_scrubber_handle_freq(this, priv, options, scrubstall);
+    if (ret)
+        goto error_return;
 
-        br_scrubber_log_option (this, priv, scrubstall);
+    br_scrubber_log_option(this, priv, scrubstall);
 
-        return 0;
+    return 0;
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 inode_t *
-br_lookup_bad_obj_dir (xlator_t *this, br_child_t *child, uuid_t gfid)
+br_lookup_bad_obj_dir(xlator_t *this, br_child_t *child, uuid_t gfid)
 {
-        struct  iatt statbuf   = {0, };
-        inode_table_t *table = NULL;
-        int32_t      ret     = -1;
-        loc_t        loc     = {0, };
-        inode_t     *linked_inode = NULL;
-        int32_t      op_errno = 0;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-scrubber", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-
-        table = child->table;
-
-        loc.inode = inode_new (table);
-        if (!loc.inode) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        BRB_MSG_NO_MEMORY, "failed to allocate a new inode for"
-                        "bad object directory");
-                goto out;
-        }
-
-        gf_uuid_copy (loc.gfid, gfid);
-
-        ret = syncop_lookup (child->xl, &loc, &statbuf, NULL, NULL, NULL);
-        if (ret < 0) {
-                op_errno = -ret;
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRB_MSG_LOOKUP_FAILED, "failed to lookup the bad "
-                        "objects directory (gfid: %s (%s))", uuid_utoa (gfid),
-                        strerror (op_errno));
-                goto out;
-        }
-
-        linked_inode = inode_link (loc.inode, NULL, NULL, &statbuf);
-        if (linked_inode)
-                inode_lookup (linked_inode);
+    struct iatt statbuf = {
+        0,
+    };
+    inode_table_t *table = NULL;
+    int32_t ret = -1;
+    loc_t loc = {
+        0,
+    };
+    inode_t *linked_inode = NULL;
+    int32_t op_errno = 0;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-scrubber", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+
+    table = child->table;
+
+    loc.inode = inode_new(table);
+    if (!loc.inode) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
+               "failed to allocate a new inode for"
+               "bad object directory");
+        goto out;
+    }
+
+    gf_uuid_copy(loc.gfid, gfid);
+
+    ret = syncop_lookup(child->xl, &loc, &statbuf, NULL, NULL, NULL);
+    if (ret < 0) {
+        op_errno = -ret;
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_LOOKUP_FAILED,
+               "failed to lookup the bad "
+               "objects directory (gfid: %s (%s))",
+               uuid_utoa(gfid), strerror(op_errno));
+        goto out;
+    }
+
+    linked_inode = inode_link(loc.inode, NULL, NULL, &statbuf);
+    if (linked_inode)
+        inode_lookup(linked_inode);
 
 out:
-        loc_wipe (&loc);
-        return linked_inode;
+    loc_wipe(&loc);
+    return linked_inode;
 }
 
 int32_t
-br_read_bad_object_dir (xlator_t *this, br_child_t *child, fd_t *fd,
-                        dict_t *dict)
+br_read_bad_object_dir(xlator_t *this, br_child_t *child, fd_t *fd,
+                       dict_t *dict)
 {
-        gf_dirent_t  entries;
-        gf_dirent_t *entry   = NULL;
-        int32_t      ret     = -1;
-        off_t        offset  = 0;
-        int32_t      count   = 0;
-        char         key[PATH_MAX] = {0, };
-        dict_t      *out_dict = NULL;
-
-        INIT_LIST_HEAD (&entries.list);
-
-        while ((ret = syncop_readdir (child->xl, fd, 131072, offset, &entries,
-                                      NULL, &out_dict))) {
-                if (ret < 0)
-                        goto out;
-
-                list_for_each_entry (entry, &entries.list, list) {
-                         offset = entry->d_off;
-
-                         snprintf (key, sizeof (key), "quarantine-%d", count);
-
-                        /*
-                         * ignore the dict_set errors for now. The intention is
-                         * to get as many bad objects as possible instead of
-                         * erroring out at the first failure.
-                         */
-                         ret = dict_set_dynstr_with_alloc (dict, key,
-                                                           entry->d_name);
-                        if (!ret)
-                                count++;
-
-                        if (out_dict) {
-                                dict_copy (out_dict, dict);
-                                dict_unref (out_dict);
-                                out_dict = NULL;
-                        }
-                }
-
-                gf_dirent_free (&entries);
-        }
-
-        ret = count;
-        ret = dict_set_int32 (dict, "count", count);
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    int32_t ret = -1;
+    off_t offset = 0;
+    int32_t count = 0;
+    char key[32] = {
+        0,
+    };
+    dict_t *out_dict = NULL;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    while ((ret = syncop_readdir(child->xl, fd, 131072, offset, &entries, NULL,
+                                 &out_dict))) {
+        if (ret < 0)
+            goto out;
+
+        list_for_each_entry(entry, &entries.list, list)
+        {
+            offset = entry->d_off;
 
-out:
-        return ret;
-}
+            snprintf(key, sizeof(key), "quarantine-%d", count);
 
-int32_t
-br_get_bad_objects_from_child (xlator_t *this, dict_t *dict, br_child_t *child)
-{
-        inode_t     *inode   = NULL;
-        inode_table_t *table = NULL;
-        fd_t        *fd      = NULL;
-        int32_t      ret     = -1;
-        loc_t        loc     = {0, };
-        int32_t      op_errno = 0;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-scrubber", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-
-        table = child->table;
-
-        inode = inode_find (table, BR_BAD_OBJ_CONTAINER);
-        if (!inode) {
-                inode = br_lookup_bad_obj_dir (this, child,
-                                               BR_BAD_OBJ_CONTAINER);
-                if (!inode)
-                        goto out;
-        }
+            /*
+             * ignore the dict_set errors for now. The intention is
+             * to get as many bad objects as possible instead of
+             * erroring out at the first failure.
+             */
+            ret = dict_set_dynstr_with_alloc(dict, key, entry->d_name);
+            if (!ret)
+                count++;
 
-        fd = fd_create (inode, 0);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        BRB_MSG_FD_CREATE_FAILED, "fd creation for the bad "
-                        "objects directory failed (gfid: %s)",
-                        uuid_utoa (BR_BAD_OBJ_CONTAINER));
-                goto out;
+            if (out_dict) {
+                dict_copy(out_dict, dict);
+                dict_unref(out_dict);
+                out_dict = NULL;
+            }
         }
 
-        loc.inode = inode;
-        gf_uuid_copy (loc.gfid, inode->gfid);
-
-        ret = syncop_opendir (child->xl, &loc, fd, NULL, NULL);
-        if (ret < 0) {
-                op_errno = -ret;
-                fd_unref (fd);
-                fd = NULL;
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        BRB_MSG_FD_CREATE_FAILED, "failed to open the bad "
-                        "objects directory %s",
-                        uuid_utoa (BR_BAD_OBJ_CONTAINER));
-                goto out;
-        }
+        gf_dirent_free(&entries);
+    }
 
-        fd_bind (fd);
+    ret = count;
+    ret = dict_set_int32_sizen(dict, "count", count);
 
-        ret = br_read_bad_object_dir (this, child, fd, dict);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRB_MSG_BAD_OBJ_READDIR_FAIL, "readdir of the bad "
-                        "objects directory (%s) failed ",
-                        uuid_utoa (BR_BAD_OBJ_CONTAINER));
-                goto out;
-        }
+out:
+    return ret;
+}
 
-        ret = 0;
+int32_t
+br_get_bad_objects_from_child(xlator_t *this, dict_t *dict, br_child_t *child)
+{
+    inode_t *inode = NULL;
+    inode_table_t *table = NULL;
+    fd_t *fd = NULL;
+    int32_t ret = -1;
+    loc_t loc = {
+        0,
+    };
+    int32_t op_errno = 0;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-scrubber", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    table = child->table;
+
+    inode = inode_find(table, BR_BAD_OBJ_CONTAINER);
+    if (!inode) {
+        inode = br_lookup_bad_obj_dir(this, child, BR_BAD_OBJ_CONTAINER);
+        if (!inode)
+            goto out;
+    }
+
+    fd = fd_create(inode, 0);
+    if (!fd) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_FD_CREATE_FAILED,
+               "fd creation for the bad "
+               "objects directory failed (gfid: %s)",
+               uuid_utoa(BR_BAD_OBJ_CONTAINER));
+        goto out;
+    }
+
+    loc.inode = inode;
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = syncop_opendir(child->xl, &loc, fd, NULL, NULL);
+    if (ret < 0) {
+        op_errno = -ret;
+        fd_unref(fd);
+        fd = NULL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_FD_CREATE_FAILED,
+               "failed to open the bad "
+               "objects directory %s",
+               uuid_utoa(BR_BAD_OBJ_CONTAINER));
+        goto out;
+    }
+
+    fd_bind(fd);
+
+    ret = br_read_bad_object_dir(this, child, fd, dict);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_BAD_OBJ_READDIR_FAIL,
+               "readdir of the bad "
+               "objects directory (%s) failed ",
+               uuid_utoa(BR_BAD_OBJ_CONTAINER));
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        loc_wipe (&loc);
-        if (fd)
-                fd_unref (fd);
-        return ret;
+    loc_wipe(&loc);
+    if (fd)
+        fd_unref(fd);
+    return ret;
 }
 
 int32_t
-br_collect_bad_objects_of_child (xlator_t *this, br_child_t *child,
-                                 dict_t *dict, dict_t *child_dict,
-                                 int32_t total_count)
+br_collect_bad_objects_of_child(xlator_t *this, br_child_t *child, dict_t *dict,
+                                dict_t *child_dict, int32_t total_count)
 {
-
-        int32_t    ret = -1;
-        int32_t    count = 0;
-        char       key[PATH_MAX] = {0, };
-        char       main_key[PATH_MAX] = {0, };
-        int32_t     j = 0;
-        int32_t    tmp_count = 0;
-        char       *entry = NULL;
-        char       tmp[PATH_MAX]  = {0, };
-        char       *path = NULL;
-
-        ret = dict_get_int32 (child_dict, "count", &count);
+    int32_t ret = -1;
+    int32_t count = 0;
+    char key[32] = {
+        0,
+    };
+    char main_key[32] = {
+        0,
+    };
+    int32_t j = 0;
+    int32_t tmp_count = 0;
+    char *entry = NULL;
+    char tmp[PATH_MAX] = {
+        0,
+    };
+    char *path = NULL;
+    int32_t len = 0;
+
+    ret = dict_get_int32_sizen(child_dict, "count", &count);
+    if (ret)
+        goto out;
+
+    tmp_count = total_count;
+
+    for (j = 0; j < count; j++) {
+        len = snprintf(key, sizeof(key), "quarantine-%d", j);
+        ret = dict_get_strn(child_dict, key, len, &entry);
         if (ret)
-                goto out;
-
-        tmp_count = total_count;
-
-        for (j = 0; j < count; j++) {
-                snprintf (key, PATH_MAX, "quarantine-%d", j);
-                ret = dict_get_str (child_dict, key, &entry);
-                if (ret)
-                        continue;
-
-                ret = dict_get_str (child_dict, entry, &path);
-                snprintf (tmp, PATH_MAX, "%s ==> BRICK: %s\n path: %s",
-                          entry, child->brick_path, path);
-                snprintf (main_key, PATH_MAX, "quarantine-%d",
-                          tmp_count);
-
-                ret = dict_set_dynstr_with_alloc (dict, main_key, tmp);
-                if (!ret)
-                        tmp_count++;
-                path = NULL;
+            continue;
+
+        ret = dict_get_str(child_dict, entry, &path);
+        len = snprintf(tmp, PATH_MAX, "%s ==> BRICK: %s\n path: %s", entry,
+                       child->brick_path, path);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            continue;
         }
+        snprintf(main_key, sizeof(main_key), "quarantine-%d", tmp_count);
 
-        ret = tmp_count;
+        ret = dict_set_dynstr_with_alloc(dict, main_key, tmp);
+        if (!ret)
+            tmp_count++;
+        path = NULL;
+    }
+
+    ret = tmp_count;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-br_collect_bad_objects_from_children (xlator_t *this, dict_t *dict)
+br_collect_bad_objects_from_children(xlator_t *this, dict_t *dict)
 {
-        int32_t    ret = -1;
-        dict_t    *child_dict = NULL;
-        int32_t    i = 0;
-        int32_t    total_count = 0;
-        br_child_t *child = NULL;
-        br_private_t *priv = NULL;
-        dict_t     *tmp_dict = NULL;
-
-        priv = this->private;
-        tmp_dict = dict;
-
-        for (i = 0; i < priv->child_count; i++) {
-                child = &priv->children[i];
-                GF_ASSERT (child);
-                if (!_br_is_child_connected (child))
-                        continue;
-
-                child_dict = dict_new ();
-                if (!child_dict) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                BRB_MSG_NO_MEMORY, "failed to allocate dict");
-                        continue;
-                }
-                ret = br_get_bad_objects_from_child (this, child_dict, child);
-                /*
-                 * Continue asking the remaining children for the list of
-                 * bad objects even though getting the list from one of them
-                 * fails.
-                 */
-                if (ret) {
-                        dict_unref (child_dict);
-                        continue;
-                }
-
-                ret = br_collect_bad_objects_of_child (this, child, tmp_dict,
-                                                       child_dict, total_count);
-                if (ret < 0) {
-                        dict_unref (child_dict);
-                        continue;
-                }
-
-                total_count = ret;
-                dict_unref (child_dict);
-                child_dict = NULL;
+    int32_t ret = -1;
+    dict_t *child_dict = NULL;
+    int32_t i = 0;
+    int32_t total_count = 0;
+    br_child_t *child = NULL;
+    br_private_t *priv = NULL;
+    dict_t *tmp_dict = NULL;
+
+    priv = this->private;
+    tmp_dict = dict;
+
+    for (i = 0; i < priv->child_count; i++) {
+        child = &priv->children[i];
+        GF_ASSERT(child);
+        if (!_br_is_child_connected(child))
+            continue;
+
+        child_dict = dict_new();
+        if (!child_dict) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
+                   "failed to allocate dict");
+            continue;
+        }
+        ret = br_get_bad_objects_from_child(this, child_dict, child);
+        /*
+         * Continue asking the remaining children for the list of
+         * bad objects even though getting the list from one of them
+         * fails.
+         */
+        if (ret) {
+            dict_unref(child_dict);
+            continue;
         }
 
-        ret = dict_set_int32 (tmp_dict, "total-count", total_count);
+        ret = br_collect_bad_objects_of_child(this, child, tmp_dict, child_dict,
+                                              total_count);
+        if (ret < 0) {
+            dict_unref(child_dict);
+            continue;
+        }
+
+        total_count = ret;
+        dict_unref(child_dict);
+        child_dict = NULL;
+    }
 
-        return ret;
+    ret = dict_set_int32(tmp_dict, "total-count", total_count);
+
+    return ret;
 }
 
 int32_t
-br_get_bad_objects_list (xlator_t *this, dict_t **dict)
+br_get_bad_objects_list(xlator_t *this, dict_t **dict)
 {
-        int32_t      ret     = -1;
-        dict_t      *tmp_dict = NULL;
+    int32_t ret = -1;
+    dict_t *tmp_dict = NULL;
 
-        GF_VALIDATE_OR_GOTO ("bir-rot-scrubber", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+    GF_VALIDATE_OR_GOTO("bir-rot-scrubber", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
 
-        tmp_dict = *dict;
+    tmp_dict = *dict;
+    if (!tmp_dict) {
+        tmp_dict = dict_new();
         if (!tmp_dict) {
-                tmp_dict = dict_new ();
-                if (!tmp_dict) {
-                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                                BRB_MSG_NO_MEMORY, "failed to allocate dict");
-                        goto out;
-                }
-                *dict = tmp_dict;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
+                   "failed to allocate dict");
+            goto out;
         }
+        *dict = tmp_dict;
+    }
 
-        ret = br_collect_bad_objects_from_children (this, tmp_dict);
+    ret = br_collect_bad_objects_from_children(this, tmp_dict);
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-wait_for_scrub_to_finish (xlator_t *this)
+wait_for_scrub_to_finish(xlator_t *this)
 {
-        int                  ret               = -1;
-        br_private_t         *priv             = NULL;
-        struct br_monitor    *scrub_monitor    = NULL;
-
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", scrub_monitor, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
-                "Waiting for all children to start and finish scrub");
-
-        pthread_mutex_lock (&scrub_monitor->donelock);
-        {
-                while (!scrub_monitor->done)
-                        pthread_cond_wait (&scrub_monitor->donecond,
-                                           &scrub_monitor->donelock);
-        }
-        pthread_mutex_unlock (&scrub_monitor->donelock);
-        ret = 0;
+    int ret = -1;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", scrub_monitor, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+           "Waiting for all children to start and finish scrub");
+
+    pthread_mutex_lock(&scrub_monitor->donelock);
+    {
+        while (!scrub_monitor->done)
+            pthread_cond_wait(&scrub_monitor->donecond,
+                              &scrub_monitor->donelock);
+    }
+    pthread_mutex_unlock(&scrub_monitor->donelock);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -1894,153 +1921,150 @@ out:
  * thread that takes care of state machine.
  */
 void *
-br_monitor_thread (void *arg)
+br_monitor_thread(void *arg)
 {
-        int32_t              ret               = 0;
-        xlator_t            *this              = NULL;
-        br_private_t        *priv              = NULL;
-        struct br_monitor   *scrub_monitor     = NULL;
-
-        this = arg;
-        priv = this->private;
-
-        /*
-         * Since, this is the topmost xlator, THIS has to be set by bit-rot
-         * xlator itself (STACK_WIND wont help in this case). Also it has
-         * to be done for each thread that gets spawned. Otherwise, a new
-         * thread will get global_xlator's pointer when it does "THIS".
-         */
-        THIS = this;
-
-        scrub_monitor = &priv->scrub_monitor;
-
-        pthread_mutex_lock (&scrub_monitor->mutex);
-        {
-                while (!scrub_monitor->inited)
-                        pthread_cond_wait (&scrub_monitor->cond,
-                                           &scrub_monitor->mutex);
-        }
-        pthread_mutex_unlock (&scrub_monitor->mutex);
-
-        /* this needs to be serialized with reconfigure() */
-        pthread_mutex_lock (&priv->lock);
-        {
-                ret = br_scrub_state_machine (this, _gf_false);
-        }
-        pthread_mutex_unlock (&priv->lock);
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    this = arg;
+    priv = this->private;
+
+    /*
+     * Since, this is the topmost xlator, THIS has to be set by bit-rot
+     * xlator itself (STACK_WIND won't help in this case). Also it has
+     * to be done for each thread that gets spawned. Otherwise, a new
+     * thread will get global_xlator's pointer when it does "THIS".
+     */
+    THIS = this;
+
+    scrub_monitor = &priv->scrub_monitor;
+
+    pthread_mutex_lock(&scrub_monitor->mutex);
+    {
+        while (!scrub_monitor->inited)
+            pthread_cond_wait(&scrub_monitor->cond, &scrub_monitor->mutex);
+    }
+    pthread_mutex_unlock(&scrub_monitor->mutex);
+
+    /* this needs to be serialized with reconfigure() */
+    pthread_mutex_lock(&priv->lock);
+    {
+        ret = br_scrub_state_machine(this, _gf_false);
+    }
+    pthread_mutex_unlock(&priv->lock);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, BRB_MSG_SSM_FAILED,
+               "Scrub state machine failed");
+        goto out;
+    }
+
+    while (1) {
+        /* Wait for all children to finish scrubbing */
+        ret = wait_for_scrub_to_finish(this);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        BRB_MSG_SSM_FAILED,
-                        "Scrub state machine failed");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, BRB_MSG_SCRUB_WAIT_FAILED,
+                   "Scrub wait failed");
+            goto out;
         }
 
-        while (1) {
-                /* Wait for all children to finish scrubbing */
-                ret = wait_for_scrub_to_finish (this);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                BRB_MSG_SCRUB_WAIT_FAILED,
-                                "Scrub wait failed");
-                        goto out;
-                }
-
-                /* scrub exit criteria: Move the state to PENDING */
-                br_scrubber_exit_control (this);
-        }
+        /* scrub exit criteria: Move the state to PENDING */
+        br_scrubber_exit_control(this);
+    }
 
 out:
-        return NULL;
+    return NULL;
 }
 
 static void
-br_set_scrub_state (struct br_monitor *scrub_monitor, br_scrub_state_t state)
+br_set_scrub_state(struct br_monitor *scrub_monitor, br_scrub_state_t state)
 {
-        LOCK (&scrub_monitor->lock);
-        {
-                _br_monitor_set_scrub_state (scrub_monitor, state);
-        }
-        UNLOCK (&scrub_monitor->lock);
+    LOCK(&scrub_monitor->lock);
+    {
+        _br_monitor_set_scrub_state(scrub_monitor, state);
+    }
+    UNLOCK(&scrub_monitor->lock);
 }
 
 int32_t
-br_scrubber_monitor_init (xlator_t *this, br_private_t *priv)
+br_scrubber_monitor_init(xlator_t *this, br_private_t *priv)
 {
-        struct br_monitor *scrub_monitor = NULL;
-        int                ret           = 0;
+    struct br_monitor *scrub_monitor = NULL;
+    int ret = 0;
 
-        scrub_monitor = &priv->scrub_monitor;
+    scrub_monitor = &priv->scrub_monitor;
 
-        LOCK_INIT (&scrub_monitor->lock);
-        scrub_monitor->this = this;
+    LOCK_INIT(&scrub_monitor->lock);
+    scrub_monitor->this = this;
 
-        scrub_monitor->inited = _gf_false;
-        pthread_mutex_init (&scrub_monitor->mutex, NULL);
-        pthread_cond_init (&scrub_monitor->cond, NULL);
+    scrub_monitor->inited = _gf_false;
+    pthread_mutex_init(&scrub_monitor->mutex, NULL);
+    pthread_cond_init(&scrub_monitor->cond, NULL);
 
-        scrub_monitor->kick = _gf_false;
-        scrub_monitor->active_child_count = 0;
-        pthread_mutex_init (&scrub_monitor->wakelock, NULL);
-        pthread_cond_init (&scrub_monitor->wakecond, NULL);
+    scrub_monitor->kick = _gf_false;
+    scrub_monitor->active_child_count = 0;
+    pthread_mutex_init(&scrub_monitor->wakelock, NULL);
+    pthread_cond_init(&scrub_monitor->wakecond, NULL);
 
-        scrub_monitor->done = _gf_false;
-        pthread_mutex_init (&scrub_monitor->donelock, NULL);
-        pthread_cond_init (&scrub_monitor->donecond, NULL);
-
-        /* Set the state to INACTIVE */
-        br_set_scrub_state (&priv->scrub_monitor, BR_SCRUB_STATE_INACTIVE);
-
-        /* Start the monitor thread */
-        ret = gf_thread_create (&scrub_monitor->thread, NULL,
-                                br_monitor_thread, this, "brmon");
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        BRB_MSG_SPAWN_FAILED, "monitor thread creation failed");
-                ret = -1;
-                goto err;
-        }
+    scrub_monitor->done = _gf_false;
+    pthread_mutex_init(&scrub_monitor->donelock, NULL);
+    pthread_cond_init(&scrub_monitor->donecond, NULL);
 
-        return 0;
+    /* Set the state to INACTIVE */
+    br_set_scrub_state(&priv->scrub_monitor, BR_SCRUB_STATE_INACTIVE);
+
+    /* Start the monitor thread */
+    ret = gf_thread_create(&scrub_monitor->thread, NULL, br_monitor_thread,
+                           this, "brmon");
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, BRB_MSG_SPAWN_FAILED,
+               "monitor thread creation failed");
+        ret = -1;
+        goto err;
+    }
+
+    return 0;
 err:
-        pthread_mutex_destroy (&scrub_monitor->mutex);
-        pthread_cond_destroy (&scrub_monitor->cond);
+    pthread_mutex_destroy(&scrub_monitor->mutex);
+    pthread_cond_destroy(&scrub_monitor->cond);
 
-        pthread_mutex_destroy (&scrub_monitor->wakelock);
-        pthread_cond_destroy (&scrub_monitor->wakecond);
+    pthread_mutex_destroy(&scrub_monitor->wakelock);
+    pthread_cond_destroy(&scrub_monitor->wakecond);
 
-        pthread_mutex_destroy (&scrub_monitor->donelock);
-        pthread_cond_destroy (&scrub_monitor->donecond);
+    pthread_mutex_destroy(&scrub_monitor->donelock);
+    pthread_cond_destroy(&scrub_monitor->donecond);
 
-        LOCK_DESTROY (&scrub_monitor->lock);
+    LOCK_DESTROY(&scrub_monitor->lock);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-br_scrubber_init (xlator_t *this, br_private_t *priv)
+br_scrubber_init(xlator_t *this, br_private_t *priv)
 {
-        struct br_scrubber *fsscrub = NULL;
-        int                 ret     = 0;
+    struct br_scrubber *fsscrub = NULL;
+    int ret = 0;
 
-        priv->tbf = tbf_init (NULL, 0);
-        if (!priv->tbf)
-                return -1;
+    priv->tbf = tbf_init(NULL, 0);
+    if (!priv->tbf)
+        return -1;
 
-        ret = br_scrubber_monitor_init (this, priv);
-        if (ret)
-                return -1;
+    ret = br_scrubber_monitor_init(this, priv);
+    if (ret)
+        return -1;
 
-        fsscrub = &priv->fsscrub;
+    fsscrub = &priv->fsscrub;
 
-        fsscrub->this = this;
-        fsscrub->throttle = BR_SCRUB_THROTTLE_VOID;
+    fsscrub->this = this;
+    fsscrub->throttle = BR_SCRUB_THROTTLE_VOID;
 
-        pthread_mutex_init (&fsscrub->mutex, NULL);
-        pthread_cond_init (&fsscrub->cond, NULL);
+    pthread_mutex_init(&fsscrub->mutex, NULL);
+    pthread_cond_init(&fsscrub->cond, NULL);
 
-        fsscrub->nr_scrubbers = 0;
-        INIT_LIST_HEAD (&fsscrub->scrubbers);
-        INIT_LIST_HEAD (&fsscrub->scrublist);
+    fsscrub->nr_scrubbers = 0;
+    INIT_LIST_HEAD(&fsscrub->scrubbers);
+    INIT_LIST_HEAD(&fsscrub->scrublist);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
index 8cc88ec153e..4e5f67bc021 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
@@ -11,27 +11,36 @@
 #ifndef __BIT_ROT_SCRUB_H__
 #define __BIT_ROT_SCRUB_H__
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "bit-rot.h"
 
-void *br_fsscanner (void *);
+void *
+br_fsscanner(void *);
 
-int32_t br_fsscan_schedule (xlator_t *);
-int32_t br_fsscan_reschedule (xlator_t *);
-int32_t br_fsscan_activate (xlator_t *);
-int32_t br_fsscan_deactivate (xlator_t *);
-int32_t br_fsscan_ondemand (xlator_t *);
+int32_t
+br_fsscan_schedule(xlator_t *);
+int32_t
+br_fsscan_reschedule(xlator_t *);
+int32_t
+br_fsscan_activate(xlator_t *);
+int32_t
+br_fsscan_deactivate(xlator_t *);
+int32_t
+br_fsscan_ondemand(xlator_t *);
 
-int32_t br_scrubber_handle_options (xlator_t *, br_private_t *, dict_t *);
+int32_t
+br_scrubber_handle_options(xlator_t *, br_private_t *, dict_t *);
 
 int32_t
-br_scrubber_monitor_init (xlator_t *, br_private_t *);
+br_scrubber_monitor_init(xlator_t *, br_private_t *);
 
-int32_t br_scrubber_init (xlator_t *, br_private_t *);
+int32_t
+br_scrubber_init(xlator_t *, br_private_t *);
 
-int32_t br_collect_bad_objects_from_children (xlator_t *this, dict_t *dict);
+int32_t
+br_collect_bad_objects_from_children(xlator_t *this, dict_t *dict);
 
 void
-br_child_set_scrub_state (br_child_t *, gf_boolean_t);
+br_child_set_scrub_state(br_child_t *, gf_boolean_t);
 
 #endif /* __BIT_ROT_SCRUB_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c
index af887a1ff8b..753e31a3b23 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c
@@ -12,78 +12,79 @@
 #include "bit-rot-scrub.h"
 #include "bit-rot-bitd-messages.h"
 
-int br_scrub_ssm_noop (xlator_t *this)
+int
+br_scrub_ssm_noop(xlator_t *this)
 {
-        return 0;
+    return 0;
 }
 
 int
-br_scrub_ssm_state_pause (xlator_t *this)
+br_scrub_ssm_state_pause(xlator_t *this)
 {
-        br_private_t        *priv               = NULL;
-        struct br_monitor   *scrub_monitor      = NULL;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
 
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
 
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
-                "Scrubber paused");
-        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_PAUSED);
-        return 0;
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
+           "Scrubber paused");
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PAUSED);
+    return 0;
 }
 
 int
-br_scrub_ssm_state_ipause (xlator_t *this)
+br_scrub_ssm_state_ipause(xlator_t *this)
 {
-        br_private_t        *priv               = NULL;
-        struct br_monitor   *scrub_monitor      = NULL;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
 
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
 
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
-                "Scrubber paused");
-        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_IPAUSED);
-        return 0;
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
+           "Scrubber paused");
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_IPAUSED);
+    return 0;
 }
 
 int
-br_scrub_ssm_state_active (xlator_t *this)
+br_scrub_ssm_state_active(xlator_t *this)
 {
-        br_private_t        *priv               = NULL;
-        struct br_monitor   *scrub_monitor      = NULL;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
 
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
 
-        if (scrub_monitor->done) {
-                (void) br_fsscan_activate (this);
-        } else {
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
-                        "Scrubbing resumed");
-                _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_ACTIVE);
-        }
+    if (scrub_monitor->done) {
+        (void)br_fsscan_activate(this);
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
+               "Scrubbing resumed");
+        _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_ACTIVE);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-br_scrub_ssm_state_stall (xlator_t *this)
+br_scrub_ssm_state_stall(xlator_t *this)
 {
-        br_private_t        *priv               = NULL;
-        struct br_monitor   *scrub_monitor      = NULL;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
 
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
 
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
-                "Volume is under active scrubbing. Pausing scrub..");
-        _br_monitor_set_scrub_state (scrub_monitor, BR_SCRUB_STATE_STALLED);
-        return 0;
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
+           "Volume is under active scrubbing. Pausing scrub..");
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_STALLED);
+    return 0;
 }
 
-static br_scrub_ssm_call *
-br_scrub_ssm[BR_SCRUB_MAXSTATES][BR_SCRUB_MAXEVENTS] = {
+static br_scrub_ssm_call *br_scrub_ssm[BR_SCRUB_MAXSTATES][BR_SCRUB_MAXEVENTS] =
+    {
         /* INACTIVE */
         {br_fsscan_schedule, br_scrub_ssm_state_ipause, br_scrub_ssm_noop},
         /* PENDING  */
@@ -99,25 +100,25 @@ br_scrub_ssm[BR_SCRUB_MAXSTATES][BR_SCRUB_MAXEVENTS] = {
 };
 
 int32_t
-br_scrub_state_machine (xlator_t *this, gf_boolean_t scrub_ondemand)
+br_scrub_state_machine(xlator_t *this, gf_boolean_t scrub_ondemand)
 {
-        br_private_t       *priv      = NULL;
-        br_scrub_ssm_call  *call      = NULL;
-        struct br_scrubber *fsscrub   = NULL;
-        br_scrub_state_t    currstate = 0;
-        br_scrub_event_t    event     = 0;
-        struct br_monitor  *scrub_monitor = NULL;
-
-        priv = this->private;
-        fsscrub = &priv->fsscrub;
-        scrub_monitor = &priv->scrub_monitor;
-
-        currstate = scrub_monitor->state;
-        if (scrub_ondemand)
-                event = BR_SCRUB_EVENT_ONDEMAND;
-        else
-                event = _br_child_get_scrub_event (fsscrub);
-
-        call = br_scrub_ssm[currstate][event];
-        return call (this);
+    br_private_t *priv = NULL;
+    br_scrub_ssm_call *call = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    br_scrub_state_t currstate = 0;
+    br_scrub_event_t event = 0;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    currstate = scrub_monitor->state;
+    if (scrub_ondemand)
+        event = BR_SCRUB_EVENT_ONDEMAND;
+    else
+        event = _br_child_get_scrub_event(fsscrub);
+
+    call = br_scrub_ssm[currstate][event];
+    return call(this);
 }
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h
index 8609477180b..37b45a42eac 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h
@@ -11,27 +11,28 @@
 #ifndef __BIT_ROT_SSM_H__
 #define __BIT_ROT_SSM_H__
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
 typedef enum br_scrub_state {
-        BR_SCRUB_STATE_INACTIVE = 0,
-        BR_SCRUB_STATE_PENDING,
-        BR_SCRUB_STATE_ACTIVE,
-        BR_SCRUB_STATE_PAUSED,
-        BR_SCRUB_STATE_IPAUSED,
-        BR_SCRUB_STATE_STALLED,
-        BR_SCRUB_MAXSTATES,
+    BR_SCRUB_STATE_INACTIVE = 0,
+    BR_SCRUB_STATE_PENDING,
+    BR_SCRUB_STATE_ACTIVE,
+    BR_SCRUB_STATE_PAUSED,
+    BR_SCRUB_STATE_IPAUSED,
+    BR_SCRUB_STATE_STALLED,
+    BR_SCRUB_MAXSTATES,
 } br_scrub_state_t;
 
 typedef enum br_scrub_event {
-        BR_SCRUB_EVENT_SCHEDULE = 0,
-        BR_SCRUB_EVENT_PAUSE,
-        BR_SCRUB_EVENT_ONDEMAND,
-        BR_SCRUB_MAXEVENTS,
+    BR_SCRUB_EVENT_SCHEDULE = 0,
+    BR_SCRUB_EVENT_PAUSE,
+    BR_SCRUB_EVENT_ONDEMAND,
+    BR_SCRUB_MAXEVENTS,
 } br_scrub_event_t;
 
 struct br_monitor;
 
-int32_t br_scrub_state_machine (xlator_t *, gf_boolean_t);
+int32_t
+br_scrub_state_machine(xlator_t *, gf_boolean_t);
 
 #endif /* __BIT_ROT_SSM_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c
index a9b70b61f8a..a2f1c343a1d 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.c
@@ -9,84 +9,81 @@
 */
 
 #include <ctype.h>
-#include <sys/uio.h>
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "compat-errno.h"
+#include <glusterfs/logging.h>
+#include <glusterfs/compat-errno.h>
 
 #include "bit-rot.h"
 #include "bit-rot-scrub.h"
 #include <pthread.h>
 #include "bit-rot-bitd-messages.h"
 
-#define BR_HASH_CALC_READ_SIZE  (128 * 1024)
+#define BR_HASH_CALC_READ_SIZE (128 * 1024)
 
-typedef int32_t (br_child_handler)(xlator_t *, br_child_t *);
+typedef int32_t(br_child_handler)(xlator_t *, br_child_t *);
 
 struct br_child_event {
-        xlator_t *this;
+    xlator_t *this;
 
-        br_child_t *child;
+    br_child_t *child;
 
-        br_child_handler *call;
+    br_child_handler *call;
 
-        struct list_head list;
+    struct list_head list;
 };
 
 static int
-br_find_child_index (xlator_t *this, xlator_t *child)
+br_find_child_index(xlator_t *this, xlator_t *child)
 {
-        br_private_t *priv   = NULL;
-        int           i      = -1;
-        int           index  = -1;
+    br_private_t *priv = NULL;
+    int i = -1;
+    int index = -1;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (child == priv->children[i].xl) {
-                        index = i;
-                        break;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (child == priv->children[i].xl) {
+            index = i;
+            break;
         }
+    }
 
 out:
-        return index;
+    return index;
 }
 
 br_child_t *
-br_get_child_from_brick_path (xlator_t *this, char *brick_path)
+br_get_child_from_brick_path(xlator_t *this, char *brick_path)
 {
-        br_private_t *priv  = NULL;
-        br_child_t   *child = NULL;
-        br_child_t   *tmp   = NULL;
-        int           i     = 0;
+    br_private_t *priv = NULL;
+    br_child_t *child = NULL;
+    br_child_t *tmp = NULL;
+    int i = 0;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, brick_path, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, brick_path, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        pthread_mutex_lock (&priv->lock);
-        {
-                for (i = 0; i < priv->child_count; i++) {
-                        tmp = &priv->children[i];
-                        if (!strcmp (tmp->brick_path, brick_path)) {
-                                child = tmp;
-                                break;
-                        }
-                }
+    pthread_mutex_lock(&priv->lock);
+    {
+        for (i = 0; i < priv->child_count; i++) {
+            tmp = &priv->children[i];
+            if (!strcmp(tmp->brick_path, brick_path)) {
+                child = tmp;
+                break;
+            }
         }
-        pthread_mutex_unlock (&priv->lock);
+    }
+    pthread_mutex_unlock(&priv->lock);
 
 out:
-        return child;
+    return child;
 }
 
 /**
@@ -94,18 +91,18 @@ out:
  * needed -- later.
  */
 void *
-br_brick_init (void *xl, struct gf_brick_spec *brick)
+br_brick_init(void *xl, struct gf_brick_spec *brick)
 {
-        return brick;
+    return brick;
 }
 
 /**
  * and cleanup things here when allocated br_brick_init().
  */
 void
-br_brick_fini (void *xl, char *brick, void *data)
+br_brick_fini(void *xl, char *brick, void *data)
 {
-        return;
+    return;
 }
 
 /**
@@ -117,109 +114,109 @@ br_brick_fini (void *xl, char *brick, void *data)
  *      change stub to handle this change.
  */
 static br_isignature_t *
-br_prepare_signature (const unsigned char *sign,
-                      unsigned long hashlen,
-                      int8_t hashtype, br_object_t *object)
+br_prepare_signature(const unsigned char *sign, unsigned long hashlen,
+                     int8_t hashtype, br_object_t *object)
 {
-        br_isignature_t *signature = NULL;
+    br_isignature_t *signature = NULL;
 
-        /* TODO: use mem-pool */
-        signature = GF_CALLOC (1, signature_size (hashlen + 1),
-                               gf_br_stub_mt_signature_t);
-        if (!signature)
-                return NULL;
+    /* TODO: use mem-pool */
+    signature = GF_CALLOC(1, signature_size(hashlen + 1),
+                          gf_br_stub_mt_signature_t);
+    if (!signature)
+        return NULL;
 
-        /* object version */
-        signature->signedversion = object->signedversion;
+    /* object version */
+    signature->signedversion = object->signedversion;
 
-        /* signature length & type */
-        signature->signaturelen = hashlen;
-        signature->signaturetype = hashtype;
+    /* signature length & type */
+    signature->signaturelen = hashlen;
+    signature->signaturetype = hashtype;
 
-        /* signature itself */
-        memcpy (signature->signature, (char *)sign, hashlen);
-        signature->signature[hashlen+1] = '\0';
+    /* signature itself */
+    memcpy(signature->signature, (char *)sign, hashlen);
+    signature->signature[hashlen + 1] = '\0';
 
-        return signature;
+    return signature;
 }
 
 gf_boolean_t
-bitd_is_bad_file (xlator_t *this, br_child_t *child, loc_t *loc, fd_t *fd)
+bitd_is_bad_file(xlator_t *this, br_child_t *child, loc_t *loc, fd_t *fd)
 {
-        int32_t       ret      = -1;
-        dict_t       *xattr    = NULL;
-        inode_t      *inode    = NULL;
-        gf_boolean_t  bad_file = _gf_false;
+    int32_t ret = -1;
+    dict_t *xattr = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t bad_file = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
 
-        inode = (loc) ? loc->inode : fd->inode;
+    inode = (loc) ? loc->inode : fd->inode;
 
-        if (fd)
-                ret = syncop_fgetxattr (child->xl, fd, &xattr,
-                                        BITROT_OBJECT_BAD_KEY, NULL, NULL);
-        else if (loc)
-                ret = syncop_getxattr (child->xl, loc,
-                                       &xattr, BITROT_OBJECT_BAD_KEY, NULL,
-                                       NULL);
+    if (fd)
+        ret = syncop_fgetxattr(child->xl, fd, &xattr, BITROT_OBJECT_BAD_KEY,
+                               NULL, NULL);
+    else if (loc)
+        ret = syncop_getxattr(child->xl, loc, &xattr, BITROT_OBJECT_BAD_KEY,
+                              NULL, NULL);
 
-        if (!ret) {
-                gf_msg_debug (this->name, 0, "[GFID: %s] is marked corrupted",
-                              uuid_utoa (inode->gfid));
-                bad_file = _gf_true;
-        }
+    if (!ret) {
+        gf_msg_debug(this->name, 0, "[GFID: %s] is marked corrupted",
+                     uuid_utoa(inode->gfid));
+        bad_file = _gf_true;
+    }
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
 out:
-        return bad_file;
+    return bad_file;
 }
 
 /**
  * Do a lookup on the gfid present within the object.
  */
 static int32_t
-br_object_lookup (xlator_t *this, br_object_t *object,
-                  struct iatt *iatt, inode_t **linked_inode)
+br_object_lookup(xlator_t *this, br_object_t *object, struct iatt *iatt,
+                 inode_t **linked_inode)
 {
-	int      ret          = -EINVAL;
-	loc_t    loc          = {0, };
-	inode_t *inode        = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, object, out);
-
-	inode = inode_find (object->child->table, object->gfid);
-
-        if (inode)
-                loc.inode = inode;
-        else
-                loc.inode = inode_new (object->child->table);
-
-	if (!loc.inode) {
-                ret = -ENOMEM;
-		goto out;
-        }
-
-	gf_uuid_copy (loc.gfid, object->gfid);
-
-	ret = syncop_lookup (object->child->xl, &loc, iatt, NULL, NULL, NULL);
-	if (ret < 0)
-		goto out;
-
-        /*
-         * The file might have been deleted by the application
-         * after getting the event, but before doing a lookup.
-         * So use linked_inode after inode_link is done.
-         */
-	*linked_inode = inode_link (loc.inode, NULL, NULL, iatt);
-	if (*linked_inode)
-		inode_lookup (*linked_inode);
+    int ret = -EINVAL;
+    loc_t loc = {
+        0,
+    };
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, object, out);
+
+    inode = inode_find(object->child->table, object->gfid);
+
+    if (inode)
+        loc.inode = inode;
+    else
+        loc.inode = inode_new(object->child->table);
+
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_copy(loc.gfid, object->gfid);
+
+    ret = syncop_lookup(object->child->xl, &loc, iatt, NULL, NULL, NULL);
+    if (ret < 0)
+        goto out;
+
+    /*
+     * The file might have been deleted by the application
+     * after getting the event, but before doing a lookup.
+     * So use linked_inode after inode_link is done.
+     */
+    *linked_inode = inode_link(loc.inode, NULL, NULL, iatt);
+    if (*linked_inode)
+        inode_lookup(*linked_inode);
 
 out:
-	loc_wipe (&loc);
-	return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
 /**
@@ -228,43 +225,44 @@ out:
  * passing xdata -- may be use frame->root->pid itself.
  */
 static int32_t
-br_object_open (xlator_t *this,
-                br_object_t *object, inode_t *inode, fd_t **openfd)
+br_object_open(xlator_t *this, br_object_t *object, inode_t *inode,
+               fd_t **openfd)
 {
-        int32_t      ret   = -1;
-        fd_t        *fd   = NULL;
-        loc_t        loc   = {0, };
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, object, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        ret = -EINVAL;
-        fd = fd_create (inode, 0);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
-                        "failed to create fd for the inode %s",
-                        uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-        ret = syncop_open (object->child->xl, &loc, O_RDONLY, fd, NULL, NULL);
-	if (ret) {
-                br_log_object (this, "open", inode->gfid, -ret);
-		fd_unref (fd);
-		fd = NULL;
-	} else {
-		fd_bind (fd);
-                *openfd = fd;
-	}
-
-        loc_wipe (&loc);
+    int32_t ret = -1;
+    fd_t *fd = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, object, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    ret = -EINVAL;
+    fd = fd_create(inode, 0);
+    if (!fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        goto out;
+    }
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = syncop_open(object->child->xl, &loc, O_RDONLY, fd, NULL, NULL);
+    if (ret) {
+        br_log_object(this, "open", inode->gfid, -ret);
+        fd_unref(fd);
+        fd = NULL;
+    } else {
+        fd_bind(fd);
+        *openfd = fd;
+    }
+
+    loc_wipe(&loc);
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -272,287 +270,282 @@ out:
  * and return the buffer.
  */
 static int32_t
-br_object_read_block_and_sign (xlator_t *this, fd_t *fd, br_child_t *child,
-                               off_t offset, size_t size, SHA256_CTX *sha256)
+br_object_read_block_and_sign(xlator_t *this, fd_t *fd, br_child_t *child,
+                              off_t offset, size_t size, SHA256_CTX *sha256)
 {
-        int32_t        ret    = -1;
-        tbf_t      *tbf    = NULL;
-        struct iovec  *iovec  = NULL;
-        struct iobref *iobref = NULL;
-        br_private_t  *priv   = NULL;
-        int            count  = 0;
-        int            i      = 0;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        priv = this->private;
-
-        GF_VALIDATE_OR_GOTO (this->name, priv->tbf, out);
-        tbf = priv->tbf;
-
-        ret = syncop_readv (child->xl, fd,
-                            size, offset, 0, &iovec, &count, &iobref, NULL,
-                            NULL, NULL);
+    int32_t ret = -1;
+    tbf_t *tbf = NULL;
+    struct iovec *iovec = NULL;
+    struct iobref *iobref = NULL;
+    br_private_t *priv = NULL;
+    int count = 0;
+    int i = 0;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO(this->name, priv->tbf, out);
+    tbf = priv->tbf;
+
+    ret = syncop_readv(child->xl, fd, size, offset, 0, &iovec, &count, &iobref,
+                       NULL, NULL, NULL);
+
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, BRB_MSG_READV_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        ret = -1;
+        goto out;
+    }
 
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno, BRB_MSG_READV_FAILED,
-                        "readv on %s failed", uuid_utoa (fd->inode->gfid));
-                ret = -1;
-                goto out;
-        }
+    if (ret == 0)
+        goto out;
 
-        if (ret == 0)
-                goto out;
-
-        for (i = 0; i < count; i++) {
-                TBF_THROTTLE_BEGIN (tbf, TBF_OP_HASH, iovec[i].iov_len);
-                {
-                        SHA256_Update (sha256, (const unsigned char *)
-                                       (iovec[i].iov_base), iovec[i].iov_len);
-                }
-                TBF_THROTTLE_BEGIN (tbf, TBF_OP_HASH, iovec[i].iov_len);
+    for (i = 0; i < count; i++) {
+        TBF_THROTTLE_BEGIN(tbf, TBF_OP_HASH, iovec[i].iov_len);
+        {
+            SHA256_Update(sha256, (const unsigned char *)(iovec[i].iov_base),
+                          iovec[i].iov_len);
         }
+        TBF_THROTTLE_BEGIN(tbf, TBF_OP_HASH, iovec[i].iov_len);
+    }
 
- out:
-        if (iovec)
-                GF_FREE (iovec);
+out:
+    if (iovec)
+        GF_FREE(iovec);
 
-        if (iobref)
-                iobref_unref (iobref);
+    if (iobref)
+        iobref_unref(iobref);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-br_calculate_obj_checksum (unsigned char *md,
-                           br_child_t *child, fd_t *fd, struct iatt *iatt)
+br_calculate_obj_checksum(unsigned char *md, br_child_t *child, fd_t *fd,
+                          struct iatt *iatt)
 {
-        int32_t   ret    = -1;
-        off_t     offset = 0;
-        size_t    block  = BR_HASH_CALC_READ_SIZE;
-        xlator_t *this   = NULL;
-
-        SHA256_CTX       sha256;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", child, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", iatt, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", fd, out);
+    int32_t ret = -1;
+    off_t offset = 0;
+    size_t block = BR_HASH_CALC_READ_SIZE;
+    xlator_t *this = NULL;
 
-        this = child->this;
+    SHA256_CTX sha256;
 
-        SHA256_Init (&sha256);
+    GF_VALIDATE_OR_GOTO("bit-rot", child, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", iatt, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", fd, out);
 
-        while (1) {
-                ret = br_object_read_block_and_sign (this, fd, child,
-                                                     offset, block, &sha256);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRB_MSG_BLOCK_READ_FAILED, "reading block with "
-                                "offset %lu of object %s failed", offset,
-                                uuid_utoa (fd->inode->gfid));
-                        break;
-                }
+    this = child->this;
 
-                if (ret == 0)
-                        break;
+    SHA256_Init(&sha256);
 
-                offset += ret;
+    while (1) {
+        ret = br_object_read_block_and_sign(this, fd, child, offset, block,
+                                            &sha256);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_BLOCK_READ_FAILED,
+                    "offset=%" PRIu64, offset, "object-gfid=%s",
+                    uuid_utoa(fd->inode->gfid), NULL);
+            break;
         }
 
         if (ret == 0)
-                SHA256_Final (md, &sha256);
+            break;
 
- out:
-        return ret;
+        offset += ret;
+    }
+
+    if (ret == 0)
+        SHA256_Final(md, &sha256);
+
+out:
+    return ret;
 }
 
 static int32_t
-br_object_checksum (unsigned char *md,
-                    br_object_t *object, fd_t *fd, struct iatt *iatt)
+br_object_checksum(unsigned char *md, br_object_t *object, fd_t *fd,
+                   struct iatt *iatt)
 {
-        return br_calculate_obj_checksum (md, object->child, fd,  iatt);
+    return br_calculate_obj_checksum(md, object->child, fd, iatt);
 }
 
 static int32_t
-br_object_read_sign (inode_t *linked_inode, fd_t *fd, br_object_t *object,
-                     struct iatt *iatt)
+br_object_read_sign(inode_t *linked_inode, fd_t *fd, br_object_t *object,
+                    struct iatt *iatt)
 {
-        int32_t          ret           = -1;
-        xlator_t        *this          = NULL;
-        dict_t          *xattr         = NULL;
-        unsigned char   *md            = NULL;
-        br_isignature_t *sign          = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", object, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", linked_inode, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", fd, out);
-
-        this = object->this;
-
-        md = GF_CALLOC (SHA256_DIGEST_LENGTH, sizeof (*md), gf_common_mt_char);
-        if (!md) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
-                        "failed to allocate memory for saving hash of the "
-                        "object %s", uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
-
-        ret = br_object_checksum (md, object, fd, iatt);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRB_MSG_CALC_CHECKSUM_FAILED, "calculating checksum "
-                        "for the object %s failed",
-                        uuid_utoa (linked_inode->gfid));
-                goto free_signature;
-        }
-
-        sign = br_prepare_signature (md, SHA256_DIGEST_LENGTH,
-                                     BR_SIGNATURE_TYPE_SHA256, object);
-        if (!sign) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
-                        "failed to get the signature for the object %s",
-                        uuid_utoa (fd->inode->gfid));
-                goto free_signature;
-        }
-
-        xattr = dict_for_key_value
-                (GLUSTERFS_SET_OBJECT_SIGNATURE,
-                 (void *)sign, signature_size (SHA256_DIGEST_LENGTH), _gf_true);
-
-        if (!xattr) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_SIGN_FAILED,
-                        "dict allocation for signing failed for the object %s",
-                        uuid_utoa (fd->inode->gfid));
-                goto free_isign;
-        }
-
-        ret = syncop_fsetxattr (object->child->xl, fd, xattr, 0, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_SIGN_FAILED,
-                        "fsetxattr of signature to the object %s failed",
-                        uuid_utoa (fd->inode->gfid));
-                goto unref_dict;
-        }
-
-        ret = 0;
-
- unref_dict:
-        dict_unref (xattr);
- free_isign:
-        GF_FREE (sign);
- free_signature:
-        GF_FREE (md);
- out:
-        return ret;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    dict_t *xattr = NULL;
+    unsigned char *md = NULL;
+    br_isignature_t *sign = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", object, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", linked_inode, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", fd, out);
+
+    this = object->this;
+
+    md = GF_MALLOC(SHA256_DIGEST_LENGTH, gf_common_mt_char);
+    if (!md) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_SAVING_HASH_FAILED,
+                "object-gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+
+    ret = br_object_checksum(md, object, fd, iatt);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_CALC_CHECKSUM_FAILED,
+                "object-gfid=%s", uuid_utoa(linked_inode->gfid), NULL);
+        goto free_signature;
+    }
+
+    sign = br_prepare_signature(md, SHA256_DIGEST_LENGTH,
+                                BR_SIGNATURE_TYPE_SHA256, object);
+    if (!sign) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
+                "object-gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto free_signature;
+    }
+
+    xattr = dict_for_key_value(GLUSTERFS_SET_OBJECT_SIGNATURE, (void *)sign,
+                               signature_size(SHA256_DIGEST_LENGTH), _gf_true);
+
+    if (!xattr) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_SIGN_FAILED,
+                "dict-allocation object-gfid=%s", uuid_utoa(fd->inode->gfid),
+                NULL);
+        goto free_isign;
+    }
+
+    ret = syncop_fsetxattr(object->child->xl, fd, xattr, 0, NULL, NULL);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_SIGN_FAILED,
+                "fsetxattr object-gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto unref_dict;
+    }
+
+    ret = 0;
+
+unref_dict:
+    dict_unref(xattr);
+free_isign:
+    GF_FREE(sign);
+free_signature:
+    GF_FREE(md);
+out:
+    return ret;
 }
 
-static int br_object_sign_softerror (int32_t op_errno)
+static int
+br_object_sign_softerror(int32_t op_errno)
 {
-        return ((op_errno == ENOENT) || (op_errno == ESTALE)
-                || (op_errno == ENODATA));
+    return ((op_errno == ENOENT) || (op_errno == ESTALE) ||
+            (op_errno == ENODATA));
 }
 
 void
-br_log_object (xlator_t *this, char *op, uuid_t gfid, int32_t op_errno)
+br_log_object(xlator_t *this, char *op, uuid_t gfid, int32_t op_errno)
 {
-        int softerror = br_object_sign_softerror (op_errno);
-        if (softerror) {
-                gf_msg_debug (this->name, 0, "%s() failed on object %s "
-                              "[reason: %s]", op, uuid_utoa (gfid),
-                              strerror (op_errno));
-        } else {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno, BRB_MSG_OP_FAILED,
-                        "%s() failed on object %s", op, uuid_utoa (gfid));
-        }
+    int softerror = br_object_sign_softerror(op_errno);
+    if (softerror) {
+        gf_msg_debug(this->name, 0,
+                     "%s() failed on object %s "
+                     "[reason: %s]",
+                     op, uuid_utoa(gfid), strerror(op_errno));
+    } else {
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_OP_FAILED, "op=%s",
+                op, "gfid=%s", uuid_utoa(gfid), NULL);
+    }
 }
 
 void
-br_log_object_path (xlator_t *this, char *op,
-                    const char *path, int32_t op_errno)
+br_log_object_path(xlator_t *this, char *op, const char *path, int32_t op_errno)
 {
-        int softerror = br_object_sign_softerror (op_errno);
-        if (softerror) {
-                gf_msg_debug (this->name, 0, "%s() failed on object %s "
-                              "[reason: %s]", op, path, strerror (op_errno));
-        } else {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno, BRB_MSG_OP_FAILED,
-                        "%s() failed on object %s", op, path);
-        }
+    int softerror = br_object_sign_softerror(op_errno);
+    if (softerror) {
+        gf_msg_debug(this->name, 0,
+                     "%s() failed on object %s "
+                     "[reason: %s]",
+                     op, path, strerror(op_errno));
+    } else {
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_OP_FAILED, "op=%s",
+                op, "path=%s", path, NULL);
+    }
 }
 
 static void
-br_trigger_sign (xlator_t *this, br_child_t *child,
-                 inode_t *linked_inode, loc_t *loc, gf_boolean_t need_reopen)
+br_trigger_sign(xlator_t *this, br_child_t *child, inode_t *linked_inode,
+                loc_t *loc, gf_boolean_t need_reopen)
 {
-        fd_t     *fd   = NULL;
-        int32_t   ret  = -1;
-        uint32_t  val  = 0;
-        dict_t   *dict = NULL;
-        pid_t     pid  = GF_CLIENT_PID_BITD;
-
-        syncopctx_setfspid (&pid);
-
-        val = (need_reopen == _gf_true) ? BR_OBJECT_REOPEN : BR_OBJECT_RESIGN;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_set_uint32 (dict, BR_REOPEN_SIGN_HINT_KEY, val);
-        if (ret)
-                goto cleanup_dict;
-
-        ret = -1;
-        fd = fd_create (linked_inode, 0);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
-                        "Failed to create fd [GFID %s]",
-                        uuid_utoa (linked_inode->gfid));
-                goto cleanup_dict;
-        }
-
-        ret = syncop_open (child->xl, loc, O_RDWR, fd, NULL, NULL);
-	if (ret) {
-                br_log_object (this, "open", linked_inode->gfid, -ret);
-                goto unref_fd;
-	}
-
-        fd_bind (fd);
-
-        ret = syncop_fsetxattr (child->xl, fd, dict, 0, NULL, NULL);
-        if (ret)
-                br_log_object (this, "fsetxattr", linked_inode->gfid, -ret);
-
-        /* passthough: fd_unref() */
-
- unref_fd:
-        fd_unref (fd);
- cleanup_dict:
-        dict_unref (dict);
- out:
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_TRIGGER_SIGN,
-                        "Could not trigger signingd for %s (reopen hint: %d)",
-                        uuid_utoa (linked_inode->gfid), val);
-        }
+    fd_t *fd = NULL;
+    int32_t ret = -1;
+    uint32_t val = 0;
+    dict_t *dict = NULL;
+    pid_t pid = GF_CLIENT_PID_BITD;
+
+    syncopctx_setfspid(&pid);
+
+    val = (need_reopen == _gf_true) ? BR_OBJECT_REOPEN : BR_OBJECT_RESIGN;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_uint32(dict, BR_REOPEN_SIGN_HINT_KEY, val);
+    if (ret)
+        goto cleanup_dict;
+
+    ret = -1;
+    fd = fd_create(linked_inode, 0);
+    if (!fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
+                "gfid=%s", uuid_utoa(linked_inode->gfid), NULL);
+        goto cleanup_dict;
+    }
+
+    ret = syncop_open(child->xl, loc, O_RDWR, fd, NULL, NULL);
+    if (ret) {
+        br_log_object(this, "open", linked_inode->gfid, -ret);
+        goto unref_fd;
+    }
+
+    fd_bind(fd);
+
+    ret = syncop_fsetxattr(child->xl, fd, dict, 0, NULL, NULL);
+    if (ret)
+        br_log_object(this, "fsetxattr", linked_inode->gfid, -ret);
+
+    /* passthough: fd_unref() */
+
+unref_fd:
+    fd_unref(fd);
+cleanup_dict:
+    dict_unref(dict);
+out:
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRB_MSG_TRIGGER_SIGN_FAILED,
+                "gfid=%s", uuid_utoa(linked_inode->gfid), "reopen-hint-val=%d",
+                val, NULL);
+    }
 }
 
 static void
-br_object_resign (xlator_t *this,
-                  br_object_t *object, inode_t *linked_inode)
+br_object_resign(xlator_t *this, br_object_t *object, inode_t *linked_inode)
 {
-        loc_t loc = {0, };
+    loc_t loc = {
+        0,
+    };
 
-        loc.inode = inode_ref (linked_inode);
-        gf_uuid_copy (loc.gfid, linked_inode->gfid);
+    loc.inode = inode_ref(linked_inode);
+    gf_uuid_copy(loc.gfid, linked_inode->gfid);
 
-        br_trigger_sign (this, object->child, linked_inode, &loc, _gf_false);
+    br_trigger_sign(this, object->child, linked_inode, &loc, _gf_false);
 
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 }
 
 /**
@@ -560,122 +553,123 @@ br_object_resign (xlator_t *this,
  * some form of priority scheduling and/or read burstness to avoid starving
  * (or kicking) client I/O's.
  */
-static int32_t br_sign_object (br_object_t *object)
+static int32_t
+br_sign_object(br_object_t *object)
 {
-        int32_t         ret           = -1;
-        inode_t        *linked_inode  = NULL;
-        xlator_t       *this          = NULL;
-        fd_t           *fd            = NULL;
-        struct iatt     iatt          = {0, };
-        pid_t           pid           = GF_CLIENT_PID_BITD;
-        br_sign_state_t sign_info     = BR_SIGN_NORMAL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", object, out);
-
-        this = object->this;
-
-        /**
-         * FIXME: This is required as signing an object is restricted to
-         * clients with special frame->root->pid. Change the way client
-         * pid is set.
-         */
-        syncopctx_setfspid (&pid);
-
-        ret = br_object_lookup (this, object, &iatt, &linked_inode);
-        if (ret) {
-                br_log_object (this, "lookup", object->gfid, -ret);
-                goto out;
-        }
-
-        /**
-         * For fd's that have notified for reopening, we send an explicit
-         * open() followed by a dummy write() call. This triggers the
-         * actual signing of the object.
-         */
-        sign_info = ntohl (object->sign_info);
-        if (sign_info == BR_SIGN_REOPEN_WAIT) {
-                br_object_resign (this, object, linked_inode);
-                goto unref_inode;
-        }
-
-        ret = br_object_open (this, object, linked_inode, &fd);
-        if (!fd) {
-                br_log_object (this, "open", object->gfid, -ret);
-                goto unref_inode;
-        }
-
-        /**
-         * we have an open file descriptor on the object. from here on,
-         * do not be generous to file operation errors.
-         */
-        gf_msg_debug (this->name, 0, "Signing object [%s]",
-                      uuid_utoa (linked_inode->gfid));
-
-        ret = br_object_read_sign (linked_inode, fd, object, &iatt);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRB_MSG_READ_AND_SIGN_FAILED, "reading and signing of "
-                        "the object %s failed", uuid_utoa (linked_inode->gfid));
-                goto unref_fd;
-        }
-
-        ret = 0;
-
- unref_fd:
-        fd_unref (fd);
- unref_inode:
-        inode_unref (linked_inode);
- out:
-        return ret;
+    int32_t ret = -1;
+    inode_t *linked_inode = NULL;
+    xlator_t *this = NULL;
+    fd_t *fd = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    pid_t pid = GF_CLIENT_PID_BITD;
+    br_sign_state_t sign_info = BR_SIGN_NORMAL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", object, out);
+
+    this = object->this;
+
+    /**
+     * FIXME: This is required as signing an object is restricted to
+     * clients with special frame->root->pid. Change the way client
+     * pid is set.
+     */
+    syncopctx_setfspid(&pid);
+
+    ret = br_object_lookup(this, object, &iatt, &linked_inode);
+    if (ret) {
+        br_log_object(this, "lookup", object->gfid, -ret);
+        goto out;
+    }
+
+    /**
+     * For fd's that have notified for reopening, we send an explicit
+     * open() followed by a dummy write() call. This triggers the
+     * actual signing of the object.
+     */
+    sign_info = ntohl(object->sign_info);
+    if (sign_info == BR_SIGN_REOPEN_WAIT) {
+        br_object_resign(this, object, linked_inode);
+        goto unref_inode;
+    }
+
+    ret = br_object_open(this, object, linked_inode, &fd);
+    if (!fd) {
+        br_log_object(this, "open", object->gfid, -ret);
+        goto unref_inode;
+    }
+
+    /**
+     * we have an open file descriptor on the object. from here on,
+     * do not be generous to file operation errors.
+     */
+    gf_msg_debug(this->name, 0, "Signing object [%s]",
+                 uuid_utoa(linked_inode->gfid));
+
+    ret = br_object_read_sign(linked_inode, fd, object, &iatt);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_READ_AND_SIGN_FAILED,
+                "gfid=%s", uuid_utoa(linked_inode->gfid), NULL);
+        goto unref_fd;
+    }
+
+    ret = 0;
+
+unref_fd:
+    fd_unref(fd);
+unref_inode:
+    inode_unref(linked_inode);
+out:
+    return ret;
 }
 
-static br_object_t *__br_pick_object (br_private_t *priv)
+static br_object_t *
+__br_pick_object(br_private_t *priv)
 {
-        br_object_t *object = NULL;
+    br_object_t *object = NULL;
 
-        while (list_empty (&priv->obj_queue->objects)) {
-                pthread_cond_wait (&priv->object_cond, &priv->lock);
-        }
+    while (list_empty(&priv->obj_queue->objects)) {
+        pthread_cond_wait(&priv->object_cond, &priv->lock);
+    }
 
-        object = list_first_entry
-                (&priv->obj_queue->objects, br_object_t, list);
-        list_del_init (&object->list);
+    object = list_first_entry(&priv->obj_queue->objects, br_object_t, list);
+    list_del_init(&object->list);
 
-        return object;
+    return object;
 }
 
 /**
  * This is the place where the signing of the objects is triggered.
  */
 void *
-br_process_object (void *arg)
+br_process_object(void *arg)
 {
-        xlator_t     *this   = NULL;
-        br_object_t  *object = NULL;
-        br_private_t *priv   = NULL;
-        int32_t       ret    = -1;
-
-        this = arg;
-        priv = this->private;
-
-        THIS = this;
-
-        for (;;) {
-                pthread_mutex_lock (&priv->lock);
-                {
-                        object = __br_pick_object (priv);
-                }
-                pthread_mutex_unlock (&priv->lock);
-
-                ret = br_sign_object (object);
-                if (ret && !br_object_sign_softerror (-ret))
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRB_MSG_SIGN_FAILED, "SIGNING FAILURE [%s]",
-                                uuid_utoa (object->gfid));
-                GF_FREE (object);
+    xlator_t *this = NULL;
+    br_object_t *object = NULL;
+    br_private_t *priv = NULL;
+    int32_t ret = -1;
+
+    this = arg;
+    priv = this->private;
+
+    THIS = this;
+
+    for (;;) {
+        pthread_mutex_lock(&priv->lock);
+        {
+            object = __br_pick_object(priv);
         }
+        pthread_mutex_unlock(&priv->lock);
 
-        return NULL;
+        ret = br_sign_object(object);
+        if (ret && !br_object_sign_softerror(-ret))
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_SIGN_FAILED,
+                    "gfid=%s", uuid_utoa(object->gfid), NULL);
+        GF_FREE(object);
+    }
+
+    return NULL;
 }
 
 /**
@@ -692,98 +686,97 @@ br_process_object (void *arg)
  * NOTE: use call_time to instrument signing time in br_sign_object().
  */
 void
-br_add_object_to_queue (struct gf_tw_timer_list *timer,
-                        void *data, unsigned long call_time)
+br_add_object_to_queue(struct gf_tw_timer_list *timer, void *data,
+                       unsigned long call_time)
 {
-        br_object_t   *object = NULL;
-        xlator_t      *this   = NULL;
-        br_private_t  *priv   = NULL;
-
-        object = data;
-        this   = object->this;
-        priv   = this->private;
-
-        THIS = this;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                list_add_tail (&object->list, &priv->obj_queue->objects);
-                pthread_cond_broadcast (&priv->object_cond);
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        if (timer)
-                mem_put (timer);
-        return;
+    br_object_t *object = NULL;
+    xlator_t *this = NULL;
+    br_private_t *priv = NULL;
+
+    object = data;
+    this = object->this;
+    priv = this->private;
+
+    THIS = this;
+
+    pthread_mutex_lock(&priv->lock);
+    {
+        list_add_tail(&object->list, &priv->obj_queue->objects);
+        pthread_cond_broadcast(&priv->object_cond);
+    }
+    pthread_mutex_unlock(&priv->lock);
+
+    if (timer)
+        mem_put(timer);
+    return;
 }
 
 static br_object_t *
-br_initialize_object (xlator_t *this, br_child_t *child, changelog_event_t *ev)
+br_initialize_object(xlator_t *this, br_child_t *child, changelog_event_t *ev)
 {
-        br_object_t *object = NULL;
+    br_object_t *object = NULL;
 
-        object = GF_CALLOC (1, sizeof (*object), gf_br_mt_br_object_t);
-        if (!object)
-                goto out;
-        INIT_LIST_HEAD (&object->list);
+    object = GF_CALLOC(1, sizeof(*object), gf_br_mt_br_object_t);
+    if (!object)
+        goto out;
+    INIT_LIST_HEAD(&object->list);
 
-        object->this  = this;
-        object->child = child;
-        gf_uuid_copy (object->gfid, ev->u.releasebr.gfid);
+    object->this = this;
+    object->child = child;
+    gf_uuid_copy(object->gfid, ev->u.releasebr.gfid);
 
-        /* NOTE: it's BE, but no worry */
-        object->signedversion = ev->u.releasebr.version;
-        object->sign_info = ev->u.releasebr.sign_info;
+    /* NOTE: it's BE, but no worry */
+    object->signedversion = ev->u.releasebr.version;
+    object->sign_info = ev->u.releasebr.sign_info;
 
 out:
-        return object;
+    return object;
 }
 
 static struct gf_tw_timer_list *
-br_initialize_timer (xlator_t *this, br_object_t *object, br_child_t *child,
-                     changelog_event_t *ev)
+br_initialize_timer(xlator_t *this, br_object_t *object, br_child_t *child,
+                    changelog_event_t *ev)
 {
-        br_private_t  *priv = NULL;
-        struct gf_tw_timer_list *timer = NULL;
+    br_private_t *priv = NULL;
+    struct gf_tw_timer_list *timer = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        timer = mem_get0 (child->timer_pool);
-        if (!timer)
-                goto out;
-        INIT_LIST_HEAD (&timer->entry);
+    timer = mem_get0(child->timer_pool);
+    if (!timer)
+        goto out;
+    INIT_LIST_HEAD(&timer->entry);
 
-        timer->expires = priv->expiry_time;
-        if (!timer->expires)
-                timer->expires = 1;
+    timer->expires = priv->expiry_time;
+    if (!timer->expires)
+        timer->expires = 1;
 
-        timer->data     = object;
-        timer->function = br_add_object_to_queue;
-        gf_tw_add_timer (priv->timer_wheel, timer);
+    timer->data = object;
+    timer->function = br_add_object_to_queue;
+    gf_tw_add_timer(priv->timer_wheel, timer);
 
 out:
-        return timer;
+    return timer;
 }
 
 static int32_t
-br_schedule_object_reopen (xlator_t *this, br_object_t *object,
-                           br_child_t *child, changelog_event_t *ev)
+br_schedule_object_reopen(xlator_t *this, br_object_t *object,
+                          br_child_t *child, changelog_event_t *ev)
 {
-        struct gf_tw_timer_list *timer = NULL;
-
-        timer = br_initialize_timer (this, object, child, ev);
-        if (!timer)
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_TIMER_FAILED,
-                        "Failed to allocate object expiry timer [GFID: %s]",
-                        uuid_utoa (object->gfid));
-        return timer ? 0 : -1;
+    struct gf_tw_timer_list *timer = NULL;
+
+    timer = br_initialize_timer(this, object, child, ev);
+    if (!timer)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_TIMER_FAILED,
+                "gfid=%s", uuid_utoa(object->gfid), NULL);
+    return timer ? 0 : -1;
 }
 
 static int32_t
-br_object_quicksign (xlator_t *this, br_object_t *object)
+br_object_quicksign(xlator_t *this, br_object_t *object)
 {
-        br_add_object_to_queue (NULL, object, 0ULL);
-        return 0;
+    br_add_object_to_queue(NULL, object, 0ULL);
+    return 0;
 }
 
 /**
@@ -796,148 +789,146 @@ br_object_quicksign (xlator_t *this, br_object_t *object)
  * object as a single alloc and bifurcate their respective pointers.
  */
 void
-br_brick_callback (void *xl, char *brick,
-                   void *data, changelog_event_t *ev)
+br_brick_callback(void *xl, char *brick, void *data, changelog_event_t *ev)
 {
-        int32_t          ret       = 0;
-        uuid_t           gfid      = {0,};
-        xlator_t        *this      = NULL;
-        br_object_t     *object    = NULL;
-        br_child_t      *child     = NULL;
-        br_sign_state_t  sign_info = BR_SIGN_INVALID;
-
-        this = xl;
-
-        GF_VALIDATE_OR_GOTO (this->name, ev, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        GF_ASSERT (ev->ev_type == CHANGELOG_OP_TYPE_BR_RELEASE);
-        GF_ASSERT (!gf_uuid_is_null (ev->u.releasebr.gfid));
-
-        gf_uuid_copy (gfid, ev->u.releasebr.gfid);
-
-        gf_msg_debug (this->name, 0, "RELEASE EVENT [GFID %s]",
-                      uuid_utoa (gfid));
-
-        child = br_get_child_from_brick_path (this, brick);
-        if (!child) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SUBVOL_FAILED,
-                        "failed to get the subvolume for the brick %s", brick);
-                goto out;
-        }
-
-        object = br_initialize_object (this, child, ev);
-        if (!object) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
-                        "failed to allocate object memory [GFID: %s]",
-                        uuid_utoa (gfid));
-                goto out;
-        }
-
-        /* sanity check */
-        sign_info = ntohl (object->sign_info);
-        GF_ASSERT (sign_info != BR_SIGN_NORMAL);
-
-        if (sign_info == BR_SIGN_REOPEN_WAIT)
-                ret = br_schedule_object_reopen (this, object, child, ev);
-        else
-                ret = br_object_quicksign (this, object);
-
-        if (ret)
-                goto free_object;
-
-        gf_msg_debug (this->name, 0, "->callback: brick [%s], type [%d]\n",
-                      brick, ev->ev_type);
-        return;
-
- free_object:
-        GF_FREE (object);
- out:
-        return;
+    int32_t ret = 0;
+    uuid_t gfid = {
+        0,
+    };
+    xlator_t *this = NULL;
+    br_object_t *object = NULL;
+    br_child_t *child = NULL;
+    br_sign_state_t sign_info = BR_SIGN_INVALID;
+
+    this = xl;
+
+    GF_VALIDATE_OR_GOTO(this->name, ev, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    GF_ASSERT(ev->ev_type == CHANGELOG_OP_TYPE_BR_RELEASE);
+    GF_ASSERT(!gf_uuid_is_null(ev->u.releasebr.gfid));
+
+    gf_uuid_copy(gfid, ev->u.releasebr.gfid);
+
+    gf_msg_debug(this->name, 0, "RELEASE EVENT [GFID %s]", uuid_utoa(gfid));
+
+    child = br_get_child_from_brick_path(this, brick);
+    if (!child) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SUBVOL_FAILED,
+                "brick=%s", brick, NULL);
+        goto out;
+    }
+
+    object = br_initialize_object(this, child, ev);
+    if (!object) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
+                "object-gfid=%s", uuid_utoa(gfid), NULL);
+        goto out;
+    }
+
+    /* sanity check */
+    sign_info = ntohl(object->sign_info);
+    GF_ASSERT(sign_info != BR_SIGN_NORMAL);
+
+    if (sign_info == BR_SIGN_REOPEN_WAIT)
+        ret = br_schedule_object_reopen(this, object, child, ev);
+    else
+        ret = br_object_quicksign(this, object);
+
+    if (ret)
+        goto free_object;
+
+    gf_msg_debug(this->name, 0, "->callback: brick [%s], type [%d]\n", brick,
+                 ev->ev_type);
+    return;
+
+free_object:
+    GF_FREE(object);
+out:
+    return;
 }
 
 void
-br_fill_brick_spec (struct gf_brick_spec *brick, char *path)
+br_fill_brick_spec(struct gf_brick_spec *brick, char *path)
 {
-        brick->brick_path = gf_strdup (path);
-        brick->filter = CHANGELOG_OP_TYPE_BR_RELEASE;
-
-        brick->init         = br_brick_init;
-        brick->fini         = br_brick_fini;
-        brick->callback     = br_brick_callback;
-        brick->connected    = NULL;
-        brick->disconnected = NULL;
+    brick->brick_path = gf_strdup(path);
+    brick->filter = CHANGELOG_OP_TYPE_BR_RELEASE;
+
+    brick->init = br_brick_init;
+    brick->fini = br_brick_fini;
+    brick->callback = br_brick_callback;
+    brick->connected = NULL;
+    brick->disconnected = NULL;
 }
 
 static gf_boolean_t
-br_check_object_need_sign (xlator_t *this, dict_t *xattr, br_child_t *child)
+br_check_object_need_sign(xlator_t *this, dict_t *xattr, br_child_t *child)
 {
-        int32_t              ret       = -1;
-        gf_boolean_t         need_sign = _gf_false;
-        br_isignature_out_t *sign      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, xattr, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-
-        ret = dict_get_ptr (xattr, GLUSTERFS_GET_OBJECT_SIGNATURE,
-                            (void **)&sign);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
-                        "failed to get object signature info");
-                goto out;
-        }
+    int32_t ret = -1;
+    gf_boolean_t need_sign = _gf_false;
+    br_isignature_out_t *sign = NULL;
 
-        /* Object has been opened and hence dirty. Do not sign it */
-        if (sign->stale)
-                need_sign = _gf_true;
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, xattr, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
 
-out:
-        return need_sign;
-}
+    ret = dict_get_ptr(xattr, GLUSTERFS_GET_OBJECT_SIGNATURE, (void **)&sign);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
+                "object-info", NULL);
+        goto out;
+    }
 
+    /* Object has been opened and hence dirty. Do not sign it */
+    if (sign->stale)
+        need_sign = _gf_true;
 
+out:
+    return need_sign;
+}
 
 int32_t
-br_prepare_loc (xlator_t *this, br_child_t *child, loc_t *parent,
-                gf_dirent_t *entry, loc_t *loc)
+br_prepare_loc(xlator_t *this, br_child_t *child, loc_t *parent,
+               gf_dirent_t *entry, loc_t *loc)
 {
-        int32_t  ret   = -1;
-        inode_t *inode = NULL;
-
-        inode = inode_grep (child->table, parent->inode, entry->d_name);
-        if (!inode)
-                loc->inode = inode_new (child->table);
-        else {
-                loc->inode = inode;
-                if (loc->inode->ia_type != IA_IFREG) {
-                        gf_msg_debug (this->name, 0, "%s is not a regular "
-                                      "file", entry->d_name);
-                        ret = 0;
-                        goto out;
-                }
-        }
-
-        loc->parent = inode_ref (parent->inode);
-        gf_uuid_copy (loc->pargfid, parent->inode->gfid);
-
-        ret = inode_path (parent->inode, entry->d_name, (char **)&loc->path);
-        if (ret < 0 || !loc->path) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_PATH_FAILED,
-                        "inode_path on %s (parent: %s) failed", entry->d_name,
-                        uuid_utoa (parent->inode->gfid));
-                goto out;
-        }
-
-        loc->name = strrchr (loc->path, '/');
-        if (loc->name)
-                loc->name++;
-
-        ret = 1;
+    int32_t ret = -1;
+    inode_t *inode = NULL;
+
+    inode = inode_grep(child->table, parent->inode, entry->d_name);
+    if (!inode)
+        loc->inode = inode_new(child->table);
+    else {
+        loc->inode = inode;
+        if (loc->inode->ia_type != IA_IFREG) {
+            gf_msg_debug(this->name, 0,
+                         "%s is not a regular "
+                         "file",
+                         entry->d_name);
+            ret = 0;
+            goto out;
+        }
+    }
+
+    loc->parent = inode_ref(parent->inode);
+    gf_uuid_copy(loc->pargfid, parent->inode->gfid);
+
+    ret = inode_path(parent->inode, entry->d_name, (char **)&loc->path);
+    if (ret < 0 || !loc->path) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_PATH_FAILED,
+                "inode_path=%s", entry->d_name, "parent-gfid=%s",
+                uuid_utoa(parent->inode->gfid), NULL);
+        goto out;
+    }
+
+    loc->name = strrchr(loc->path, '/');
+    if (loc->name)
+        loc->name++;
+
+    ret = 1;
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -947,154 +938,174 @@ out:
  * last run for whatever reason (node crashes, reboots, etc..) become
  * candidates for signing. This allows the signature to "catch up" with
  * the current state of the object. Triggering signing is easy: perform
- * an open() followed by a close() therby resulting in call boomerang.
+ * an open() followed by a close() thereby resulting in call boomerang.
  * (though not back to itself :))
  */
 int
-bitd_oneshot_crawl (xlator_t *subvol,
-                    gf_dirent_t *entry, loc_t *parent, void *data)
+bitd_oneshot_crawl(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                   void *data)
 {
-        int           op_errno     = 0;
-        br_child_t   *child        = NULL;
-        xlator_t     *this         = NULL;
-        loc_t         loc          = {0, };
-        struct iatt   iatt         = {0, };
-        struct iatt   parent_buf   = {0, };
-        dict_t       *xattr        = NULL;
-        int32_t       ret          = -1;
-        inode_t      *linked_inode = NULL;
-        gf_boolean_t  need_signing = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", subvol, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", data, out);
-
-        child = data;
-        this = child->this;
-
-        ret = br_prepare_loc (this, child, parent, entry, &loc);
-        if (!ret)
-                goto out;
-
-        ret = syncop_lookup (child->xl, &loc, &iatt, &parent_buf, NULL, NULL);
-        if (ret) {
-                br_log_object_path (this, "lookup", loc.path, -ret);
-                goto out;
-        }
-
-        linked_inode = inode_link (loc.inode, parent->inode, loc.name, &iatt);
-        if (linked_inode)
-                inode_lookup (linked_inode);
-
-        if (iatt.ia_type != IA_IFREG) {
-                gf_msg_debug (this->name, 0,  "%s is not a regular file, "
-                              "skipping..", entry->d_name);
-                ret = 0;
-                goto unref_inode;
-        }
+    int op_errno = 0;
+    br_child_t *child = NULL;
+    xlator_t *this = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    struct iatt parent_buf = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int32_t ret = -1;
+    inode_t *linked_inode = NULL;
+    gf_boolean_t need_signing = _gf_false;
+    gf_boolean_t need_reopen = _gf_true;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", subvol, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", data, out);
+
+    child = data;
+    this = child->this;
+
+    ret = br_prepare_loc(this, child, parent, entry, &loc);
+    if (!ret)
+        goto out;
+
+    ret = syncop_lookup(child->xl, &loc, &iatt, &parent_buf, NULL, NULL);
+    if (ret) {
+        br_log_object_path(this, "lookup", loc.path, -ret);
+        goto out;
+    }
+
+    linked_inode = inode_link(loc.inode, parent->inode, loc.name, &iatt);
+    if (linked_inode)
+        inode_lookup(linked_inode);
+
+    if (iatt.ia_type != IA_IFREG) {
+        gf_msg_debug(this->name, 0,
+                     "%s is not a regular file, "
+                     "skipping..",
+                     entry->d_name);
+        ret = 0;
+        goto unref_inode;
+    }
+
+    /**
+     * As of now, 2 cases  are possible and handled.
+     * 1) GlusterFS is upgraded from a previous version which does not
+     *    have any idea about bit-rot and have data in the filesystem.
+     *    In this case syncop_getxattr fails with ENODATA and the object
+     *    is signed. (In real, when crawler sends lookup, bit-rot-stub
+     *    creates the xattrs before returning lookup reply)
+     * 2) Bit-rot was not enabled or BitD was does for some reasons, during
+     *    which some files were created, but since BitD was down, were not
+     *    signed.
+     * If the file was just created and was being written some data when
+     * the down BitD came up, then bit-rot stub should be intelligent to
+     * identify this case (by comparing the ongoing version or by checking
+     * if there are any fds present for that inode) and handle properly.
+     */
+
+    if (bitd_is_bad_file(this, child, &loc, NULL)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SKIP_OBJECT, "path=%s",
+                loc.path, NULL);
+        goto unref_inode;
+    }
+
+    ret = syncop_getxattr(child->xl, &loc, &xattr,
+                          GLUSTERFS_GET_OBJECT_SIGNATURE, NULL, NULL);
+    if (ret < 0) {
+        op_errno = -ret;
+        br_log_object(this, "getxattr", linked_inode->gfid, op_errno);
 
         /**
-         * As of now, 2 cases  are possible and handled.
-         * 1) GlusterFS is upgraded from a previous version which does not
-         *    have any idea about bit-rot and have data in the filesystem.
-         *    In this case syncop_getxattr fails with ENODATA and the object
-         *    is signed. (In real, when crawler sends lookup, bit-rot-stub
-         *    creates the xattrs before returning lookup reply)
-         * 2) Bit-rot was not enabled or BitD was dows for some reasons, during
-         *    which some files were created, but since BitD was down, were not
-         *    signed.
-         * If the file was just created and was being written some data when
-         * the down BitD came up, then bit-rot stub should be intelligent to
-         * identify this case (by comparing the ongoing version or by checking
-         * if there are any fds present for that inode) and handle properly.
+         * No need to sign the zero byte objects as the signing
+         * happens upon first modification of the object.
          */
+        if (op_errno == ENODATA && (iatt.ia_size != 0))
+            need_signing = _gf_true;
+        if (op_errno == EINVAL)
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    BRB_MSG_PARTIAL_VERSION_PRESENCE, "gfid=%s",
+                    uuid_utoa(linked_inode->gfid), NULL);
+    } else {
+        need_signing = br_check_object_need_sign(this, xattr, child);
 
-        if (bitd_is_bad_file (this, child, &loc, NULL)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_SKIP_OBJECT,
-                        "Entry [%s] is marked corrupted.. skipping.", loc.path);
-                goto unref_inode;
-        }
-
-        ret = syncop_getxattr (child->xl, &loc, &xattr,
-                               GLUSTERFS_GET_OBJECT_SIGNATURE, NULL, NULL);
-        if (ret < 0) {
-                op_errno = -ret;
-                br_log_object (this, "getxattr", linked_inode->gfid, op_errno);
-
-                /**
-                 * No need to sign the zero byte objects as the signing
-                 * happens upon first modification of the object.
-                 */
-                if (op_errno == ENODATA && (iatt.ia_size != 0))
-                        need_signing = _gf_true;
-                if (op_errno == EINVAL)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                BRB_MSG_PARTIAL_VERSION_PRESENCE, "Partial "
-                                "version xattr presence detected, ignoring "
-                                "[GFID: %s]", uuid_utoa (linked_inode->gfid));
-        } else {
-                need_signing = br_check_object_need_sign (this, xattr, child);
+        /*
+         * If we are here means, bitrot daemon has started. Is it just
+         * a simple restart of the daemon or is it started because the
+         * feature is enabled is something hard to determine. Hence,
+         * if need_signing is false (because bit-rot version and signature
+         * are present), then still go ahead and sign it.
+         */
+        if (!need_signing) {
+            need_signing = _gf_true;
+            need_reopen = _gf_true;
         }
+    }
 
-        if (!need_signing)
-                goto unref_dict;
+    if (!need_signing)
+        goto unref_dict;
 
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_TRIGGER_SIGN,
-                "Triggering signing for %s [GFID: %s | Brick: %s]",
-                loc.path, uuid_utoa (linked_inode->gfid), child->brick_path);
-        br_trigger_sign (this, child, linked_inode, &loc, _gf_true);
+    gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_TRIGGER_SIGN, "path=%s",
+            loc.path, "gfid=%s", uuid_utoa(linked_inode->gfid), "Brick-path=%s",
+            child->brick_path, NULL);
+    br_trigger_sign(this, child, linked_inode, &loc, need_reopen);
 
-        ret = 0;
+    ret = 0;
 
- unref_dict:
-        if (xattr)
-                dict_unref (xattr);
- unref_inode:
-        inode_unref (linked_inode);
- out:
-        loc_wipe (&loc);
+unref_dict:
+    if (xattr)
+        dict_unref(xattr);
+unref_inode:
+    inode_unref(linked_inode);
+out:
+    loc_wipe(&loc);
 
-        return ret;
+    return ret;
 }
 
 #define BR_CRAWL_THROTTLE_COUNT 50
-#define BR_CRAWL_THROTTLE_ZZZ   5
+#define BR_CRAWL_THROTTLE_ZZZ 5
 
 void *
-br_oneshot_signer (void *arg)
+br_oneshot_signer(void *arg)
 {
-        loc_t       loc   = {0,};
-        xlator_t   *this  = NULL;
-        br_child_t *child = NULL;
+    loc_t loc = {
+        0,
+    };
+    xlator_t *this = NULL;
+    br_child_t *child = NULL;
 
-        child = arg;
-        this = child->this;
+    child = arg;
+    this = child->this;
 
-        THIS = this;
+    THIS = this;
 
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_CRAWLING_START,
-                "Crawling brick [%s], scanning for unsigned objects",
-                child->brick_path);
+    gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_CRAWLING_START, "brick-path=%s",
+            child->brick_path, NULL);
 
-        loc.inode = child->table->root;
-        (void) syncop_ftw_throttle
-                         (child->xl, &loc,
-                         GF_CLIENT_PID_BITD, child, bitd_oneshot_crawl,
-                         BR_CRAWL_THROTTLE_COUNT, BR_CRAWL_THROTTLE_ZZZ);
+    loc.inode = child->table->root;
+    (void)syncop_ftw_throttle(child->xl, &loc, GF_CLIENT_PID_BITD, child,
+                              bitd_oneshot_crawl, BR_CRAWL_THROTTLE_COUNT,
+                              BR_CRAWL_THROTTLE_ZZZ);
 
-        gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_CRAWLING_FINISH,
-                "Completed crawling brick [%s]", child->brick_path);
+    gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_CRAWLING_FINISH,
+            "brick-path=%s", child->brick_path, NULL);
 
-        return NULL;
+    return NULL;
 }
 
 static void
-br_set_child_state (br_child_t *child, br_child_state_t state)
+br_set_child_state(br_child_t *child, br_child_state_t state)
 {
-        pthread_mutex_lock (&child->lock);
-        {
-                _br_set_child_state (child, state);
-        }
-        pthread_mutex_unlock (&child->lock);
+    pthread_mutex_lock(&child->lock);
+    {
+        _br_set_child_state(child, state);
+    }
+    pthread_mutex_unlock(&child->lock);
 }
 
 /**
@@ -1109,160 +1120,157 @@ br_set_child_state (br_child_t *child, br_child_state_t state)
  * notifications.
  */
 static int32_t
-br_enact_signer (xlator_t *this, br_child_t *child, br_stub_init_t *stub)
+br_enact_signer(xlator_t *this, br_child_t *child, br_stub_init_t *stub)
 {
-        int32_t ret = 0;
-        br_private_t *priv = NULL;
-        struct gf_brick_spec *brick = NULL;
-
-        priv = this->private;
-
-        brick = GF_CALLOC (1, sizeof (struct gf_brick_spec),
-                           gf_common_mt_gf_brick_spec_t);
-        if (!brick)
-                goto error_return;
-
-        br_fill_brick_spec (brick, stub->export);
-        ret = gf_changelog_register_generic
-                         (brick, 1, 1, this->ctx->cmd_args.log_file, -1, this);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        BRB_MSG_REGISTER_FAILED, "Register to changelog "
-                        "failed");
-                goto dealloc;
-        }
-
-        child->threadrunning = 0;
-        ret = gf_thread_create (&child->thread, NULL, br_oneshot_signer, child,
-                                "brosign");
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_SPAWN_FAILED,
-                        "failed to spawn FS crawler thread");
-        else
-                child->threadrunning = 1;
-
-        /* it's OK to continue, "old" objects would be signed when modified */
-        list_add_tail (&child->list, &priv->signing);
-        return 0;
-
- dealloc:
-        GF_FREE (brick);
- error_return:
-        return -1;
+    int32_t ret = 0;
+    br_private_t *priv = NULL;
+    struct gf_brick_spec *brick = NULL;
+
+    priv = this->private;
+
+    brick = GF_CALLOC(1, sizeof(struct gf_brick_spec),
+                      gf_common_mt_gf_brick_spec_t);
+    if (!brick)
+        goto error_return;
+
+    br_fill_brick_spec(brick, stub->export);
+    ret = gf_changelog_register_generic(brick, 1, 1,
+                                        this->ctx->cmd_args.log_file, -1, this);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, BRB_MSG_REGISTER_FAILED, NULL);
+        goto dealloc;
+    }
+
+    child->threadrunning = 0;
+    ret = gf_thread_create(&child->thread, NULL, br_oneshot_signer, child,
+                           "brosign");
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SPAWN_FAILED,
+                "FS-crawler-thread", NULL);
+    else
+        child->threadrunning = 1;
+
+    /* it's OK to continue, "old" objects would be signed when modified */
+    list_add_tail(&child->list, &priv->signing);
+    return 0;
+
+dealloc:
+    GF_FREE(brick);
+error_return:
+    return -1;
 }
 
 static int32_t
-br_launch_scrubber (xlator_t *this, br_child_t *child,
-                    struct br_scanfs *fsscan, struct br_scrubber *fsscrub)
+br_launch_scrubber(xlator_t *this, br_child_t *child, struct br_scanfs *fsscan,
+                   struct br_scrubber *fsscrub)
 {
-        int32_t ret = -1;
-        br_private_t *priv = NULL;
-        struct br_monitor *scrub_monitor = NULL;
-
-        priv = this->private;
-
-        scrub_monitor = &priv->scrub_monitor;
-        ret = gf_thread_create (&child->thread, NULL, br_fsscanner, child,
-                                "brfsscan");
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ALERT, 0, BRB_MSG_SPAWN_FAILED,
-                        "failed to spawn bitrot scrubber daemon [Brick: %s]",
-                        child->brick_path);
-                goto error_return;
-        }
-
-        /* Signal monitor to kick off state machine*/
-        pthread_mutex_lock (&scrub_monitor->mutex);
-        {
-                if (!scrub_monitor->inited)
-                        pthread_cond_signal (&scrub_monitor->cond);
-                scrub_monitor->inited = _gf_true;
-        }
-        pthread_mutex_unlock (&scrub_monitor->mutex);
+    int32_t ret = -1;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+
+    scrub_monitor = &priv->scrub_monitor;
+    ret = gf_thread_create(&child->thread, NULL, br_fsscanner, child,
+                           "brfsscan");
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ALERT, 0, BRB_MSG_SPAWN_FAILED,
+                "bitrot-scrubber-daemon Brick-path=%s", child->brick_path,
+                NULL);
+        goto error_return;
+    }
+
+    /* Signal monitor to kick off state machine*/
+    pthread_mutex_lock(&scrub_monitor->mutex);
+    {
+        if (!scrub_monitor->inited)
+            pthread_cond_signal(&scrub_monitor->cond);
+        scrub_monitor->inited = _gf_true;
+    }
+    pthread_mutex_unlock(&scrub_monitor->mutex);
+
+    /**
+     * Everything has been setup.. add this subvolume to scrubbers
+     * list.
+     */
+    pthread_mutex_lock(&fsscrub->mutex);
+    {
+        list_add_tail(&child->list, &fsscrub->scrublist);
+        pthread_cond_broadcast(&fsscrub->cond);
+    }
+    pthread_mutex_unlock(&fsscrub->mutex);
+
+    return 0;
 
-        /**
-         * Everything has been setup.. add this subvolume to scrubbers
-         * list.
-         */
-        pthread_mutex_lock (&fsscrub->mutex);
-        {
-                list_add_tail (&child->list, &fsscrub->scrublist);
-                pthread_cond_broadcast (&fsscrub->cond);
-        }
-        pthread_mutex_unlock (&fsscrub->mutex);
-
-        return 0;
-
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 static int32_t
-br_enact_scrubber (xlator_t *this, br_child_t *child)
+br_enact_scrubber(xlator_t *this, br_child_t *child)
 {
-        int32_t ret = 0;
-        br_private_t *priv = NULL;
-        struct br_scanfs *fsscan = NULL;
-        struct br_scrubber *fsscrub = NULL;
+    int32_t ret = 0;
+    br_private_t *priv = NULL;
+    struct br_scanfs *fsscan = NULL;
+    struct br_scrubber *fsscrub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        fsscan = &child->fsscan;
-        fsscrub = &priv->fsscrub;
+    fsscan = &child->fsscan;
+    fsscrub = &priv->fsscrub;
 
-        /**
-         * if this child already witnesses a successful connection earlier
-         * there's no need to initialize mutexes, condvars, etc..
-         */
-        if (_br_child_witnessed_connection (child))
-                return br_launch_scrubber (this, child, fsscan, fsscrub);
+    /**
+     * if this child already witnesses a successful connection earlier
+     * there's no need to initialize mutexes, condvars, etc..
+     */
+    if (_br_child_witnessed_connection(child))
+        return br_launch_scrubber(this, child, fsscan, fsscrub);
 
-        LOCK_INIT (&fsscan->entrylock);
-        pthread_mutex_init (&fsscan->waitlock, NULL);
-        pthread_cond_init (&fsscan->waitcond, NULL);
+    LOCK_INIT(&fsscan->entrylock);
+    pthread_mutex_init(&fsscan->waitlock, NULL);
+    pthread_cond_init(&fsscan->waitcond, NULL);
 
-        fsscan->entries = 0;
-        INIT_LIST_HEAD (&fsscan->queued);
-        INIT_LIST_HEAD (&fsscan->ready);
+    fsscan->entries = 0;
+    INIT_LIST_HEAD(&fsscan->queued);
+    INIT_LIST_HEAD(&fsscan->ready);
 
-        ret = br_launch_scrubber (this, child, fsscan, fsscrub);
-        if (ret)
-                goto error_return;
+    ret = br_launch_scrubber(this, child, fsscan, fsscrub);
+    if (ret)
+        goto error_return;
 
-        return 0;
+    return 0;
 
- error_return:
-        LOCK_DESTROY (&fsscan->entrylock);
-        pthread_mutex_destroy (&fsscan->waitlock);
-        pthread_cond_destroy (&fsscan->waitcond);
+error_return:
+    LOCK_DESTROY(&fsscan->entrylock);
+    pthread_mutex_destroy(&fsscan->waitlock);
+    pthread_cond_destroy(&fsscan->waitcond);
 
-        return -1;
+    return -1;
 }
 
 static int32_t
-br_child_enaction (xlator_t *this, br_child_t *child, br_stub_init_t *stub)
+br_child_enaction(xlator_t *this, br_child_t *child, br_stub_init_t *stub)
 {
-        int32_t ret = -1;
-        br_private_t *priv = this->private;
+    int32_t ret = -1;
+    br_private_t *priv = this->private;
 
-        pthread_mutex_lock (&child->lock);
-        {
-                if (priv->iamscrubber)
-                        ret = br_enact_scrubber (this, child);
-                else
-                        ret = br_enact_signer (this, child, stub);
-
-                if (!ret) {
-                        child->witnessed = 1;
-                        _br_set_child_state (child, BR_CHILD_STATE_CONNECTED);
-                        gf_msg (this->name, GF_LOG_INFO,
-                                0, BRB_MSG_CONNECTED_TO_BRICK,
-                                "Connected to brick %s..", child->brick_path);
-                }
+    pthread_mutex_lock(&child->lock);
+    {
+        if (priv->iamscrubber)
+            ret = br_enact_scrubber(this, child);
+        else
+            ret = br_enact_signer(this, child, stub);
+
+        if (!ret) {
+            child->witnessed = 1;
+            _br_set_child_state(child, BR_CHILD_STATE_CONNECTED);
+            gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_CONNECTED_TO_BRICK,
+                    "brick-path=%s", child->brick_path, NULL);
         }
-        pthread_mutex_unlock (&child->lock);
+    }
+    pthread_mutex_unlock(&child->lock);
 
-        return ret;
+    return ret;
 }
 
 /**
@@ -1273,129 +1281,130 @@ br_child_enaction (xlator_t *this, br_child_t *child, br_stub_init_t *stub)
  * process either acts as a signer or a scrubber.
  */
 int32_t
-br_brick_connect (xlator_t *this, br_child_t *child)
+br_brick_connect(xlator_t *this, br_child_t *child)
 {
-        int32_t         ret      = -1;
-        loc_t           loc      = {0, };
-        struct iatt     buf      = {0, };
-        struct iatt     parent   = {0, };
-        br_stub_init_t *stub     = NULL;
-        dict_t         *xattr    = NULL;
-        int             op_errno = 0;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        br_child_set_scrub_state (child, _gf_false);
-        br_set_child_state (child, BR_CHILD_STATE_INITIALIZING);
-
-        loc.inode = inode_ref (child->table->root);
-        gf_uuid_copy (loc.gfid, loc.inode->gfid);
-        loc.path = gf_strdup ("/");
-
-        ret = syncop_lookup (child->xl, &loc, &buf, &parent, NULL, NULL);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        BRB_MSG_LOOKUP_FAILED, "lookup on root failed");
-                goto wipeloc;
-        }
-
-        ret = syncop_getxattr (child->xl, &loc, &xattr,
-                               GLUSTERFS_GET_BR_STUB_INIT_TIME, NULL, NULL);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        BRB_MSG_GET_INFO_FAILED, "failed to get stub info");
-                goto wipeloc;
-        }
-
-        ret = dict_get_ptr (xattr, GLUSTERFS_GET_BR_STUB_INIT_TIME,
-                            (void **)&stub);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_INFO_FAILED,
-                        "failed to extract stub information");
-                goto free_dict;
-        }
-
-        memcpy (child->brick_path, stub->export, strlen (stub->export) + 1);
-        child->tv.tv_sec = ntohl (stub->timebuf[0]);
-        child->tv.tv_usec = ntohl (stub->timebuf[1]);
-
-        ret = br_child_enaction (this, child, stub);
-
- free_dict:
-        dict_unref (xattr);
- wipeloc:
-        loc_wipe (&loc);
- out:
-        if (ret)
-                br_set_child_state (child, BR_CHILD_STATE_CONNFAILED);
-        return ret;
+    int32_t ret = -1;
+    loc_t loc = {
+        0,
+    };
+    struct iatt buf = {
+        0,
+    };
+    struct iatt parent = {
+        0,
+    };
+    br_stub_init_t *stub = NULL;
+    dict_t *xattr = NULL;
+    int op_errno = 0;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    br_child_set_scrub_state(child, _gf_false);
+    br_set_child_state(child, BR_CHILD_STATE_INITIALIZING);
+
+    loc.inode = inode_ref(child->table->root);
+    gf_uuid_copy(loc.gfid, loc.inode->gfid);
+    loc.path = gf_strdup("/");
+
+    ret = syncop_lookup(child->xl, &loc, &buf, &parent, NULL, NULL);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_LOOKUP_FAILED,
+                NULL);
+        goto wipeloc;
+    }
+
+    ret = syncop_getxattr(child->xl, &loc, &xattr,
+                          GLUSTERFS_GET_BR_STUB_INIT_TIME, NULL, NULL);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_GET_INFO_FAILED,
+                NULL);
+        goto wipeloc;
+    }
+
+    ret = dict_get_ptr(xattr, GLUSTERFS_GET_BR_STUB_INIT_TIME, (void **)&stub);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_INFO_FAILED, NULL);
+        goto free_dict;
+    }
+
+    memcpy(child->brick_path, stub->export, strlen(stub->export) + 1);
+    child->tv.tv_sec = ntohl(stub->timebuf[0]);
+    child->tv.tv_usec = ntohl(stub->timebuf[1]);
+
+    ret = br_child_enaction(this, child, stub);
+
+free_dict:
+    dict_unref(xattr);
+wipeloc:
+    loc_wipe(&loc);
+out:
+    if (ret)
+        br_set_child_state(child, BR_CHILD_STATE_CONNFAILED);
+    return ret;
 }
 
 /* TODO: cleanup signer */
 static int32_t
-br_cleanup_signer (xlator_t *this, br_child_t *child)
+br_cleanup_signer(xlator_t *this, br_child_t *child)
 {
-        return 0;
+    return 0;
 }
 
 static int32_t
-br_cleanup_scrubber (xlator_t *this, br_child_t *child)
+br_cleanup_scrubber(xlator_t *this, br_child_t *child)
 {
-        int32_t ret = 0;
-        br_private_t *priv = NULL;
-        struct br_scrubber *fsscrub = NULL;
-        struct br_monitor *scrub_monitor = NULL;
-
-        priv    = this->private;
-        fsscrub = &priv->fsscrub;
-        scrub_monitor = &priv->scrub_monitor;
-
-        if (_br_is_child_scrub_active (child)) {
-                scrub_monitor->active_child_count--;
-                br_child_set_scrub_state (child, _gf_false);
-        }
-
-        /**
-         * 0x0: child (brick) goes out of rotation
-         *
-         * This is fully safe w.r.t. entries for this child being actively
-         * scrubbed. Each of the scrubber thread(s) would finish scrubbing
-         * the entry (probably failing due to disconnection) and either
-         * putting the entry back into the queue or continuing further.
-         * Either way, pending entries for this child's queue need not be
-         * drained; entries just sit there in the queued/ready list to be
-         * consumed later upon re-connection.
-         */
-        pthread_mutex_lock (&fsscrub->mutex);
-        {
-                list_del_init (&child->list);
-        }
-        pthread_mutex_unlock (&fsscrub->mutex);
-
-        /**
-         * 0x1: cleanup scanner thread
-         *
-         * The pending timer needs to be removed _after_ cleaning up the
-         * filesystem scanner (scheduling the next scrub time is not a
-         * cancellation point).
-         */
-        ret = gf_thread_cleanup_xint (child->thread);
-        if (ret)
-                gf_msg (this->name, GF_LOG_INFO,
-                        0, BRB_MSG_SCRUB_THREAD_CLEANUP,
-                        "Error cleaning up scanner thread");
-
-        gf_msg (this->name, GF_LOG_INFO,
-                0, BRB_MSG_SCRUBBER_CLEANED,
-                "Cleaned up scrubber for brick [%s]", child->brick_path);
-
-        return 0;
+    int32_t ret = 0;
+    br_private_t *priv = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    if (_br_is_child_scrub_active(child)) {
+        scrub_monitor->active_child_count--;
+        br_child_set_scrub_state(child, _gf_false);
+    }
+
+    /**
+     * 0x0: child (brick) goes out of rotation
+     *
+     * This is fully safe w.r.t. entries for this child being actively
+     * scrubbed. Each of the scrubber thread(s) would finish scrubbing
+     * the entry (probably failing due to disconnection) and either
+     * putting the entry back into the queue or continuing further.
+     * Either way, pending entries for this child's queue need not be
+     * drained; entries just sit there in the queued/ready list to be
+     * consumed later upon re-connection.
+     */
+    pthread_mutex_lock(&fsscrub->mutex);
+    {
+        list_del_init(&child->list);
+    }
+    pthread_mutex_unlock(&fsscrub->mutex);
+
+    /**
+     * 0x1: cleanup scanner thread
+     *
+     * The pending timer needs to be removed _after_ cleaning up the
+     * filesystem scanner (scheduling the next scrub time is not a
+     * cancellation point).
+     */
+    ret = gf_thread_cleanup_xint(child->thread);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_THREAD_CLEANUP, NULL);
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUBBER_CLEANED,
+            "brick-path=%s", child->brick_path, NULL);
+
+    return 0;
 }
 
 /**
@@ -1404,38 +1413,38 @@ br_cleanup_scrubber (xlator_t *this, br_child_t *child)
  * the inode table, it's just reused taking care of stale inodes)
  */
 int32_t
-br_brick_disconnect (xlator_t *this, br_child_t *child)
+br_brick_disconnect(xlator_t *this, br_child_t *child)
 {
-        int32_t ret = 0;
-        struct br_monitor *scrub_monitor = NULL;
-        br_private_t *priv = this->private;
+    int32_t ret = 0;
+    struct br_monitor *scrub_monitor = NULL;
+    br_private_t *priv = this->private;
+
+    scrub_monitor = &priv->scrub_monitor;
+
+    /* Lock order should be wakelock and then child lock to
+     * dead locks.
+     */
+    pthread_mutex_lock(&scrub_monitor->wakelock);
+    {
+        pthread_mutex_lock(&child->lock);
+        {
+            if (!_br_is_child_connected(child))
+                goto unblock;
 
-        scrub_monitor = &priv->scrub_monitor;
+            /* child is on death row.. */
+            _br_set_child_state(child, BR_CHILD_STATE_DISCONNECTED);
 
-        /* Lock order should be wakelock and then child lock to
-         * dead locks.
-         */
-        pthread_mutex_lock (&scrub_monitor->wakelock);
-        {
-                pthread_mutex_lock (&child->lock);
-                {
-                        if (!_br_is_child_connected (child))
-                                goto unblock;
-
-                        /* child is on death row.. */
-                        _br_set_child_state (child, BR_CHILD_STATE_DISCONNECTED);
-
-                        if (priv->iamscrubber)
-                                ret = br_cleanup_scrubber (this, child);
-                        else
-                                ret = br_cleanup_signer (this, child);
-                }
- unblock:
-                pthread_mutex_unlock (&child->lock);
+            if (priv->iamscrubber)
+                ret = br_cleanup_scrubber(this, child);
+            else
+                ret = br_cleanup_signer(this, child);
         }
-        pthread_mutex_unlock (&scrub_monitor->wakelock);
+    unblock:
+        pthread_mutex_unlock(&child->lock);
+    }
+    pthread_mutex_unlock(&scrub_monitor->wakelock);
 
-         return ret;
+    return ret;
 }
 
 /**
@@ -1444,338 +1453,341 @@ br_brick_disconnect (xlator_t *this, br_child_t *child)
  * information regarding that brick (such as brick path).
  */
 void *
-br_handle_events (void *arg)
+br_handle_events(void *arg)
 {
-        int32_t       ret   = 0;
-        xlator_t     *this  = NULL;
-        br_private_t *priv  = NULL;
-        br_child_t *child = NULL;
-        struct br_child_event *childev = NULL;
-
-        this = arg;
-        priv = this->private;
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    br_private_t *priv = NULL;
+    br_child_t *child = NULL;
+    struct br_child_event *childev = NULL;
+
+    this = arg;
+    priv = this->private;
+
+    /*
+     * Since, this is the topmost xlator, THIS has to be set by bit-rot
+     * xlator itself (STACK_WIND won't help in this case). Also it has
+     * to be done for each thread that gets spawned. Otherwise, a new
+     * thread will get global_xlator's pointer when it does "THIS".
+     */
+    THIS = this;
+
+    while (1) {
+        pthread_mutex_lock(&priv->lock);
+        {
+            while (list_empty(&priv->bricks))
+                pthread_cond_wait(&priv->cond, &priv->lock);
 
-        /*
-         * Since, this is the topmost xlator, THIS has to be set by bit-rot
-         * xlator itself (STACK_WIND wont help in this case). Also it has
-         * to be done for each thread that gets spawned. Otherwise, a new
-         * thread will get global_xlator's pointer when it does "THIS".
-         */
-        THIS = this;
-
-        while (1) {
-                pthread_mutex_lock (&priv->lock);
-                {
-                        while (list_empty (&priv->bricks))
-                                pthread_cond_wait (&priv->cond, &priv->lock);
-
-                        childev = list_first_entry
-                                   (&priv->bricks, struct br_child_event, list);
-                        list_del_init (&childev->list);
-                }
-                pthread_mutex_unlock (&priv->lock);
-
-                child = childev->child;
-                ret = childev->call (this, child);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRB_MSG_SUBVOL_CONNECT_FAILED,
-                                "callback handler for subvolume [%s] failed",
-                                child->xl->name);
-                GF_FREE (childev);
+            childev = list_first_entry(&priv->bricks, struct br_child_event,
+                                       list);
+            list_del_init(&childev->list);
         }
+        pthread_mutex_unlock(&priv->lock);
 
-        return NULL;
+        child = childev->child;
+        ret = childev->call(this, child);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SUBVOL_CONNECT_FAILED,
+                    "name=%s", child->xl->name, NULL);
+        GF_FREE(childev);
+    }
+
+    return NULL;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int32_t ret = -1;
-
-        if (!this)
-                return ret;
+    int32_t ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_br_stub_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRB_MSG_MEM_ACNT_FAILED,
-                        "Memory accounting init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_br_stub_mt_end + 1);
 
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRB_MSG_MEM_ACNT_FAILED, NULL);
         return ret;
+    }
+
+    return ret;
 }
 
 static void
-_br_qchild_event (xlator_t *this, br_child_t *child, br_child_handler *call)
+_br_qchild_event(xlator_t *this, br_child_t *child, br_child_handler *call)
 {
-        br_private_t *priv = NULL;
-        struct br_child_event *childev = NULL;
+    br_private_t *priv = NULL;
+    struct br_child_event *childev = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        childev = GF_CALLOC (1, sizeof (*childev), gf_br_mt_br_child_event_t);
-        if (!childev) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
-                        "Event unhandled for child.. [Brick: %s]",
-                        child->xl->name);
-                return;
-        }
+    childev = GF_CALLOC(1, sizeof(*childev), gf_br_mt_br_child_event_t);
+    if (!childev) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_EVENT_UNHANDLED,
+                "Brick-name=%s", child->xl->name, NULL);
+        return;
+    }
 
-        INIT_LIST_HEAD (&childev->list);
-        childev->this  = this;
-        childev->child = child;
-        childev->call  = call;
+    INIT_LIST_HEAD(&childev->list);
+    childev->this = this;
+    childev->child = child;
+    childev->call = call;
 
-        list_add_tail (&childev->list, &priv->bricks);
+    list_add_tail(&childev->list, &priv->bricks);
 }
 
 int
-br_scrubber_status_get (xlator_t *this, dict_t **dict)
+br_scrubber_status_get(xlator_t *this, dict_t **dict)
 {
-        int                    ret          = -1;
-        br_private_t          *priv         = NULL;
-        struct br_scrub_stats *scrub_stats  = NULL;
-
-        priv = this->private;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", priv, out);
-
-        scrub_stats = &priv->scrub_stat;
-
-        ret = br_get_bad_objects_list (this, dict);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to collect corrupt "
-                              "files");
-        }
-
-        ret = dict_set_int8 (*dict, "scrub-running",
-                             scrub_stats->scrub_running);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed setting scrub_running "
-                              "entry to the dictionary");
-        }
-
-        ret = dict_set_uint64 (*dict, "scrubbed-files",
-                               scrub_stats->scrubbed_files);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to setting scrubbed file "
-                              "entry to the dictionary");
-        }
-
-        ret = dict_set_uint64 (*dict, "unsigned-files",
-                               scrub_stats->unsigned_files);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to set unsigned file count"
-                              " entry to the dictionary");
-        }
-
-        ret = dict_set_uint64 (*dict, "scrub-duration",
-                               scrub_stats->scrub_duration);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to set scrub duration"
-                              " entry to the dictionary");
-        }
-
-        ret = dict_set_dynstr_with_alloc (*dict, "last-scrub-time",
-                                          scrub_stats->last_scrub_time);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "Failed to set "
-                                      "last scrub time value");
-        }
+    int ret = -1;
+    br_private_t *priv = NULL;
+    struct br_scrub_stats *scrub_stats = NULL;
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", priv, out);
+
+    scrub_stats = &priv->scrub_stat;
+
+    ret = br_get_bad_objects_list(this, dict);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to collect corrupt "
+                     "files");
+    }
+
+    ret = dict_set_int8(*dict, "scrub-running", scrub_stats->scrub_running);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed setting scrub_running "
+                     "entry to the dictionary");
+    }
+
+    ret = dict_set_uint64(*dict, "scrubbed-files", scrub_stats->scrubbed_files);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to setting scrubbed file "
+                     "entry to the dictionary");
+    }
+
+    ret = dict_set_uint64(*dict, "unsigned-files", scrub_stats->unsigned_files);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to set unsigned file count"
+                     " entry to the dictionary");
+    }
+
+    ret = dict_set_uint64(*dict, "scrub-duration", scrub_stats->scrub_duration);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to set scrub duration"
+                     " entry to the dictionary");
+    }
+
+    ret = dict_set_dynstr_with_alloc(*dict, "last-scrub-time",
+                                     scrub_stats->last_scrub_time);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to set "
+                     "last scrub time value");
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-notify (xlator_t *this, int32_t event, void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int           idx    = -1;
-        int           ret    = -1;
-        xlator_t     *subvol = NULL;
-        br_child_t   *child  = NULL;
-        br_private_t *priv   = NULL;
-        dict_t       *output = NULL;
-        va_list       ap;
-        struct br_monitor  *scrub_monitor = NULL;
+    int idx = -1;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    br_child_t *child = NULL;
+    br_private_t *priv = NULL;
+    dict_t *output = NULL;
+    va_list ap;
+    struct br_monitor *scrub_monitor = NULL;
 
-        subvol = (xlator_t *)data;
-        priv = this->private;
-        scrub_monitor = &priv->scrub_monitor;
+    subvol = (xlator_t *)data;
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
 
-        gf_msg_trace (this->name, 0, "Notification received: %d", event);
+    gf_msg_trace(this->name, 0, "Notification received: %d", event);
 
-        idx = br_find_child_index (this, subvol);
+    idx = br_find_child_index(this, subvol);
 
-        switch (event) {
+    switch (event) {
         case GF_EVENT_CHILD_UP:
-                if (idx < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRB_MSG_INVALID_SUBVOL, "Got event %d from "
-                                "invalid subvolume", event);
-                        goto out;
-                }
-
-                pthread_mutex_lock (&priv->lock);
-                {
-                        child = &priv->children[idx];
-                        if (child->child_up == 1)
-                                goto unblock_0;
-                        priv->up_children++;
-
-                        child->child_up = 1;
-                        child->xl = subvol;
-                        if (!child->table)
-                                child->table = inode_table_new (4096, subvol);
-
-                        _br_qchild_event (this, child, br_brick_connect);
-                        pthread_cond_signal (&priv->cond);
-                }
+            if (idx < 0) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_INVALID_SUBVOL,
+                        "event=%d", event, NULL);
+                goto out;
+            }
+
+            pthread_mutex_lock(&priv->lock);
+            {
+                child = &priv->children[idx];
+                if (child->child_up == 1)
+                    goto unblock_0;
+                priv->up_children++;
+
+                child->child_up = 1;
+                child->xl = subvol;
+                if (!child->table)
+                    child->table = inode_table_new(4096, subvol);
+
+                _br_qchild_event(this, child, br_brick_connect);
+                pthread_cond_signal(&priv->cond);
+            }
         unblock_0:
-                pthread_mutex_unlock (&priv->lock);
+            pthread_mutex_unlock(&priv->lock);
 
-                if (priv->up_children == priv->child_count)
-                        default_notify (this, event, data);
-                break;
+            if (priv->up_children == priv->child_count)
+                default_notify(this, event, data);
+            break;
 
         case GF_EVENT_CHILD_DOWN:
-                if (idx < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRB_MSG_INVALID_SUBVOL_CHILD,
-                                "Got event %d from invalid subvolume", event);
-                        goto out;
-                }
-
-                pthread_mutex_lock (&priv->lock);
-                {
-                        child = &priv->children[idx];
-                        if (child->child_up == 0)
-                                goto unblock_1;
-
-                        child->child_up = 0;
-                        priv->up_children--;
-
-                        _br_qchild_event (this, child, br_brick_disconnect);
-                        pthread_cond_signal (&priv->cond);
-                }
+            if (idx < 0) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_INVALID_SUBVOL,
+                        "event=%d", event, NULL);
+                goto out;
+            }
+
+            pthread_mutex_lock(&priv->lock);
+            {
+                child = &priv->children[idx];
+                if (child->child_up == 0)
+                    goto unblock_1;
+
+                child->child_up = 0;
+                priv->up_children--;
+
+                _br_qchild_event(this, child, br_brick_disconnect);
+                pthread_cond_signal(&priv->cond);
+            }
         unblock_1:
-                pthread_mutex_unlock (&priv->lock);
+            pthread_mutex_unlock(&priv->lock);
 
-                if (priv->up_children == 0)
-                        default_notify (this, event, data);
-                break;
+            if (priv->up_children == 0)
+                default_notify(this, event, data);
+            break;
 
         case GF_EVENT_SCRUB_STATUS:
-                gf_msg_debug (this->name, GF_LOG_INFO, "BitRot scrub status "
-                              "called");
-                va_start (ap, data);
-                output = va_arg (ap, dict_t *);
-                va_end (ap);
-
-                ret = br_scrubber_status_get (this, &output);
-                gf_msg_debug (this->name, 0, "returning %d", ret);
-                break;
+            gf_msg_debug(this->name, GF_LOG_INFO,
+                         "BitRot scrub status "
+                         "called");
+            va_start(ap, data);
+            output = va_arg(ap, dict_t *);
+            va_end(ap);
+
+            ret = br_scrubber_status_get(this, &output);
+            gf_msg_debug(this->name, 0, "returning %d", ret);
+            break;
 
         case GF_EVENT_SCRUB_ONDEMAND:
-                gf_log (this->name, GF_LOG_INFO, "BitRot scrub ondemand "
-                              "called");
-
-                if (scrub_monitor->state != BR_SCRUB_STATE_PENDING) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRB_MSG_RESCHEDULE_SCRUBBER_FAILED,
-                                "on demand scrub schedule failed. Scrubber is "
-                                "not in pending state. Current state is %d",
-                                 scrub_monitor->state);
-                        return -2;
-                }
-
-                /* Needs synchronization with reconfigure thread */
-                pthread_mutex_lock (&priv->lock);
-                {
-                        ret = br_scrub_state_machine (this, _gf_true);
-                }
-                pthread_mutex_unlock (&priv->lock);
-
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRB_MSG_RESCHEDULE_SCRUBBER_FAILED,
-                                "Could not schedule ondemand scrubbing. "
-                                "Scrubbing will continue according to "
-                                "old frequency.");
-                }
-                gf_msg_debug (this->name, 0, "returning %d", ret);
-                break;
+            gf_log(this->name, GF_LOG_INFO,
+                   "BitRot scrub ondemand "
+                   "called");
+
+            if (scrub_monitor->state != BR_SCRUB_STATE_PENDING) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        BRB_MSG_RESCHEDULE_SCRUBBER_FAILED, "Current-state=%d",
+                        scrub_monitor->state, NULL);
+                return -2;
+            }
+
+            /* Needs synchronization with reconfigure thread */
+            pthread_mutex_lock(&priv->lock);
+            {
+                ret = br_scrub_state_machine(this, _gf_true);
+            }
+            pthread_mutex_unlock(&priv->lock);
+
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        BRB_MSG_COULD_NOT_SCHEDULE_SCRUB, NULL);
+            }
+            gf_msg_debug(this->name, 0, "returning %d", ret);
+            break;
         default:
-                default_notify (this, event, data);
-        }
+            default_notify(this, event, data);
+    }
 
- out:
-        return 0;
+out:
+    return 0;
 }
 
-/**
- * Initialize signer specific structures, spawn worker threads.
- */
-
 static void
-br_fini_signer (xlator_t *this, br_private_t *priv)
+br_fini_signer(xlator_t *this, br_private_t *priv)
 {
-        int i = 0;
+    int i = 0;
 
-        for (; i < BR_WORKERS; i++) {
-                (void) gf_thread_cleanup_xint (priv->obj_queue->workers[i]);
-        }
+    if (priv == NULL)
+        return;
+
+    for (; i < priv->signer_th_count; i++) {
+        (void)gf_thread_cleanup_xint(priv->obj_queue->workers[i]);
+    }
+    GF_FREE(priv->obj_queue->workers);
 
-        pthread_cond_destroy (&priv->object_cond);
+    pthread_cond_destroy(&priv->object_cond);
 }
 
+/**
+ * Initialize signer specific structures, spawn worker threads.
+ */
+
 static int32_t
-br_init_signer (xlator_t *this, br_private_t *priv)
+br_init_signer(xlator_t *this, br_private_t *priv)
 {
-        int i = 0;
-        int32_t ret = -1;
-
-        /* initialize gfchangelog xlator context */
-        ret = gf_changelog_init (this);
-        if (ret)
-                goto out;
-
-        pthread_cond_init (&priv->object_cond, NULL);
-
-        priv->obj_queue = GF_CALLOC (1, sizeof (*priv->obj_queue),
-                                     gf_br_mt_br_ob_n_wk_t);
-        if (!priv->obj_queue)
-                goto cleanup_cond;
-        INIT_LIST_HEAD (&priv->obj_queue->objects);
-
-        for (i = 0; i < BR_WORKERS; i++) {
-                ret = gf_thread_create (&priv->obj_queue->workers[i], NULL,
-                                        br_process_object, this, "brpobj");
-                if (ret != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, -ret,
-                                BRB_MSG_SPAWN_FAILED, "thread creation"
-                                " failed");
-                        ret = -1;
-                        goto cleanup_threads;
-                }
+    int i = 0;
+    int32_t ret = -1;
+
+    /* initialize gfchangelog xlator context */
+    ret = gf_changelog_init(this);
+    if (ret)
+        goto out;
+
+    pthread_cond_init(&priv->object_cond, NULL);
+
+    priv->obj_queue = GF_CALLOC(1, sizeof(*priv->obj_queue),
+                                gf_br_mt_br_ob_n_wk_t);
+    if (!priv->obj_queue)
+        goto cleanup_cond;
+    INIT_LIST_HEAD(&priv->obj_queue->objects);
+
+    priv->obj_queue->workers = GF_CALLOC(
+        priv->signer_th_count, sizeof(pthread_t), gf_br_mt_br_worker_t);
+    if (!priv->obj_queue->workers)
+        goto cleanup_obj_queue;
+
+    for (i = 0; i < priv->signer_th_count; i++) {
+        ret = gf_thread_create(&priv->obj_queue->workers[i], NULL,
+                               br_process_object, this, "brpobj");
+        if (ret != 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    BRB_MSG_THREAD_CREATION_FAILED, NULL);
+            ret = -1;
+            goto cleanup_threads;
         }
+    }
 
-        return 0;
+    return 0;
 
- cleanup_threads:
-        for (i--; i >= 0; i--) {
-                (void) gf_thread_cleanup_xint (priv->obj_queue->workers[i]);
-        }
+cleanup_threads:
+    for (i--; i >= 0; i--) {
+        (void)gf_thread_cleanup_xint(priv->obj_queue->workers[i]);
+    }
+    GF_FREE(priv->obj_queue->workers);
 
-        GF_FREE (priv->obj_queue);
+cleanup_obj_queue:
+    GF_FREE(priv->obj_queue);
 
- cleanup_cond:
-        /* that's explicit */
-        pthread_cond_destroy (&priv->object_cond);
- out:
-        return -1;
+cleanup_cond:
+    /* that's explicit */
+    pthread_cond_destroy(&priv->object_cond);
+out:
+    return -1;
 }
 
 /**
@@ -1784,360 +1796,358 @@ br_init_signer (xlator_t *this, br_private_t *priv)
  * throttle.
  */
 static int32_t
-br_rate_limit_signer (xlator_t *this, int child_count, int numbricks)
+br_rate_limit_signer(xlator_t *this, int child_count, int numbricks)
 {
-        br_private_t *priv = NULL;
-        tbf_opspec_t spec = {0,};
-
-        priv = this->private;
-
-        spec.op       = TBF_OP_HASH;
-        spec.rate     = 0;
-        spec.maxlimit = 0;
-
-/**
- * OK. Most implementations of TBF I've come across generate tokens
- * every second (UML, etc..) and some chose sub-second granularity
- * (blk-iothrottle cgroups). TBF algorithm itself does not enforce
- * any logic for choosing generation interval and it seems pretty
- * logical as one could jack up token count per interval w.r.t.
- * generation rate.
- *
- * Value used here is chosen based on a series of test(s) performed
- * to balance object signing time and not maxing out on all available
- * CPU cores. It's obvious to have seconds granularity and jack up
- * token count per interval, thereby achieving close to similar
- * results. Let's stick to this as it seems to be working fine for
- * the set of ops that are throttled.
- **/
-        spec.token_gen_interval = 600000; /* In usec */
-
+    br_private_t *priv = NULL;
+    tbf_opspec_t spec = {
+        0,
+    };
+
+    priv = this->private;
+
+    spec.op = TBF_OP_HASH;
+    spec.rate = 0;
+    spec.maxlimit = 0;
+
+    /**
+     * OK. Most implementations of TBF I've come across generate tokens
+     * every second (UML, etc..) and some chose sub-second granularity
+     * (blk-iothrottle cgroups). TBF algorithm itself does not enforce
+     * any logic for choosing generation interval and it seems pretty
+     * logical as one could jack up token count per interval w.r.t.
+     * generation rate.
+     *
+     * Value used here is chosen based on a series of test(s) performed
+     * to balance object signing time and not maxing out on all available
+     * CPU cores. It's obvious to have seconds granularity and jack up
+     * token count per interval, thereby achieving close to similar
+     * results. Let's stick to this as it seems to be working fine for
+     * the set of ops that are throttled.
+     **/
+    spec.token_gen_interval = 600000; /* In usec */
 
 #ifdef BR_RATE_LIMIT_SIGNER
 
-        double contribution = 0;
-        contribution = ((double)1 - ((double)child_count / (double)numbricks));
-        if (contribution == 0)
-                contribution = 1;
-        spec.rate = BR_HASH_CALC_READ_SIZE * contribution;
-        spec.maxlimit = BR_WORKERS * BR_HASH_CALC_READ_SIZE;
+    double contribution = 0;
+    contribution = ((double)1 - ((double)child_count / (double)numbricks));
+    if (contribution == 0)
+        contribution = 1;
+    spec.rate = BR_HASH_CALC_READ_SIZE * contribution;
+    spec.maxlimit = priv->signer_th_count * BR_HASH_CALC_READ_SIZE;
 
 #endif
 
-        if (!spec.rate)
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_RATE_LIMIT_INFO,
-                        "[Rate Limit Info] \"FULL THROTTLE\"");
-        else
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_RATE_LIMIT_INFO,
-                        "[Rate Limit Info] \"tokens/sec (rate): %lu, "
-                        "maxlimit: %lu\"", spec.rate, spec.maxlimit);
+    if (!spec.rate)
+        gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_RATE_LIMIT_INFO,
+                "FULL THROTTLE", NULL);
+    else
+        gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_RATE_LIMIT_INFO,
+                "tokens/sec-rate=%lu", spec.rate, "maxlimit=%lu", spec.maxlimit,
+                NULL);
 
-        priv->tbf = tbf_init (&spec, 1);
-        return priv->tbf ? 0 : -1;
+    priv->tbf = tbf_init(&spec, 1);
+    return priv->tbf ? 0 : -1;
 }
 
 static int32_t
-br_signer_handle_options (xlator_t *this, br_private_t *priv, dict_t *options)
+br_signer_handle_options(xlator_t *this, br_private_t *priv, dict_t *options)
 {
-        if (options)
-                GF_OPTION_RECONF ("expiry-time", priv->expiry_time,
-                                  options, uint32, error_return);
-        else
-                GF_OPTION_INIT ("expiry-time", priv->expiry_time,
-                                uint32, error_return);
-
-        return 0;
+    if (options) {
+        GF_OPTION_RECONF("expiry-time", priv->expiry_time, options, uint32,
+                         error_return);
+        GF_OPTION_RECONF("signer-threads", priv->signer_th_count, options,
+                         uint32, error_return);
+    } else {
+        GF_OPTION_INIT("expiry-time", priv->expiry_time, uint32, error_return);
+        GF_OPTION_INIT("signer-threads", priv->signer_th_count, uint32,
+                       error_return);
+    }
+
+    return 0;
 
 error_return:
-        return -1;
+    return -1;
 }
 
 static int32_t
-br_signer_init (xlator_t *this, br_private_t *priv)
+br_signer_init(xlator_t *this, br_private_t *priv)
 {
-        int32_t ret = 0;
-        int numbricks = 0;
+    int32_t ret = 0;
+    int numbricks = 0;
 
-        GF_OPTION_INIT ("expiry-time", priv->expiry_time, uint32, error_return);
-        GF_OPTION_INIT ("brick-count", numbricks, int32, error_return);
+    GF_OPTION_INIT("expiry-time", priv->expiry_time, uint32, error_return);
+    GF_OPTION_INIT("brick-count", numbricks, int32, error_return);
+    GF_OPTION_INIT("signer-threads", priv->signer_th_count, uint32,
+                   error_return);
 
-        ret = br_rate_limit_signer (this, priv->child_count, numbricks);
-        if (ret)
-                goto error_return;
-
-        ret = br_init_signer (this, priv);
-        if (ret)
-                goto cleanup_tbf;
+    ret = br_rate_limit_signer(this, priv->child_count, numbricks);
+    if (ret)
+        goto error_return;
 
-        return 0;
+    ret = br_init_signer(this, priv);
+    if (ret)
+        goto cleanup_tbf;
 
- cleanup_tbf:
-        /* cleanup TBF */
- error_return:
-        return -1;
+    return 0;
 
+cleanup_tbf:
+    /* cleanup TBF */
+error_return:
+    return -1;
 }
 
 static void
-br_free_scrubber_monitor (xlator_t *this, br_private_t *priv)
+br_free_scrubber_monitor(xlator_t *this, br_private_t *priv)
 {
-        struct br_monitor *scrub_monitor = &priv->scrub_monitor;
+    struct br_monitor *scrub_monitor = &priv->scrub_monitor;
 
-        if (scrub_monitor->timer) {
-                (void) gf_tw_del_timer (priv->timer_wheel, scrub_monitor->timer);
+    if (scrub_monitor->timer) {
+        (void)gf_tw_del_timer(priv->timer_wheel, scrub_monitor->timer);
 
-                GF_FREE (scrub_monitor->timer);
-                scrub_monitor->timer = NULL;
-        }
+        GF_FREE(scrub_monitor->timer);
+        scrub_monitor->timer = NULL;
+    }
 
-        (void) gf_thread_cleanup_xint (scrub_monitor->thread);
+    (void)gf_thread_cleanup_xint(scrub_monitor->thread);
 
-        /* Clean up cond and mutex variables */
-        pthread_mutex_destroy (&scrub_monitor->mutex);
-        pthread_cond_destroy (&scrub_monitor->cond);
+    /* Clean up cond and mutex variables */
+    pthread_mutex_destroy(&scrub_monitor->mutex);
+    pthread_cond_destroy(&scrub_monitor->cond);
 
-        pthread_mutex_destroy (&scrub_monitor->wakelock);
-        pthread_cond_destroy (&scrub_monitor->wakecond);
+    pthread_mutex_destroy(&scrub_monitor->wakelock);
+    pthread_cond_destroy(&scrub_monitor->wakecond);
 
-        pthread_mutex_destroy (&scrub_monitor->donelock);
-        pthread_cond_destroy (&scrub_monitor->donecond);
+    pthread_mutex_destroy(&scrub_monitor->donelock);
+    pthread_cond_destroy(&scrub_monitor->donecond);
 
-        LOCK_DESTROY (&scrub_monitor->lock);
+    LOCK_DESTROY(&scrub_monitor->lock);
 }
 
 static void
-br_free_children (xlator_t *this, br_private_t *priv, int count)
+br_free_children(xlator_t *this, br_private_t *priv, int count)
 {
-        br_child_t *child = NULL;
+    br_child_t *child = NULL;
 
-        for (--count; count >= 0; count--) {
-                child = &priv->children[count];
-                mem_pool_destroy (child->timer_pool);
-                pthread_mutex_destroy (&child->lock);
-        }
+    for (--count; count >= 0; count--) {
+        child = &priv->children[count];
+        mem_pool_destroy(child->timer_pool);
+        pthread_mutex_destroy(&child->lock);
+    }
 
-        GF_FREE (priv->children);
-        priv->children = NULL;
+    GF_FREE(priv->children);
+    priv->children = NULL;
 }
 
 static int
-br_init_children (xlator_t *this, br_private_t *priv)
+br_init_children(xlator_t *this, br_private_t *priv)
 {
-        int i = 0;
-        br_child_t *child = NULL;
-        xlator_list_t *trav = NULL;
-
-        priv->child_count = xlator_subvolume_count (this);
-        priv->children = GF_CALLOC (priv->child_count, sizeof (*priv->children),
-                                    gf_br_mt_br_child_t);
-        if (!priv->children)
-                goto err;
-
-        trav = this->children;
-        while (trav) {
-                child = &priv->children[i];
-
-                pthread_mutex_init (&child->lock, NULL);
-                child->witnessed = 0;
-
-                br_set_child_state (child, BR_CHILD_STATE_DISCONNECTED);
-
-                child->this = this;
-                child->xl = trav->xlator;
-
-                child->timer_pool = mem_pool_new
-                                    (struct gf_tw_timer_list,  4096);
-                if (!child->timer_pool) {
-                        gf_msg (this->name, GF_LOG_ERROR,
-                                ENOMEM, BRB_MSG_NO_MEMORY,
-                                "failed to allocate mem-pool for timer");
-                        errno = ENOMEM;
-                        goto freechild;
-                }
-
-                INIT_LIST_HEAD (&child->list);
-
-                i++;
-                trav = trav->next;
+    int i = 0;
+    br_child_t *child = NULL;
+    xlator_list_t *trav = NULL;
+
+    priv->child_count = xlator_subvolume_count(this);
+    priv->children = GF_CALLOC(priv->child_count, sizeof(*priv->children),
+                               gf_br_mt_br_child_t);
+    if (!priv->children)
+        goto err;
+
+    trav = this->children;
+    while (trav) {
+        child = &priv->children[i];
+
+        pthread_mutex_init(&child->lock, NULL);
+        child->witnessed = 0;
+
+        br_set_child_state(child, BR_CHILD_STATE_DISCONNECTED);
+
+        child->this = this;
+        child->xl = trav->xlator;
+
+        child->timer_pool = mem_pool_new(struct gf_tw_timer_list, 4096);
+        if (!child->timer_pool) {
+            gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_MEM_POOL_ALLOC,
+                    NULL);
+            errno = ENOMEM;
+            goto freechild;
         }
 
-        return 0;
+        INIT_LIST_HEAD(&child->list);
+
+        i++;
+        trav = trav->next;
+    }
 
- freechild:
-        br_free_children (this, priv, i);
- err:
-        return -1;
+    return 0;
+
+freechild:
+    br_free_children(this, priv, i);
+err:
+    return -1;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int32_t       ret  = -1;
-	br_private_t *priv = NULL;
-
-	if (!this->children) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRB_MSG_NO_CHILD,
-                        "FATAL: no children");
-		goto out;
-	}
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_br_mt_br_private_t);
-        if (!priv) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
-                        "failed to allocate memory (->priv)");
-                goto out;
-        }
+    int32_t ret = -1;
+    br_private_t *priv = NULL;
 
-        GF_OPTION_INIT ("scrubber", priv->iamscrubber, bool, free_priv);
+    if (!this->children) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_NO_CHILD, NULL);
+        goto out;
+    }
 
-        ret = br_init_children (this, priv);
-        if (ret)
-                goto free_priv;
+    priv = GF_CALLOC(1, sizeof(*priv), gf_br_mt_br_private_t);
+    if (!priv) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
 
-        pthread_mutex_init (&priv->lock, NULL);
-        pthread_cond_init (&priv->cond, NULL);
+    GF_OPTION_INIT("scrubber", priv->iamscrubber, bool, free_priv);
 
-        INIT_LIST_HEAD (&priv->bricks);
-        INIT_LIST_HEAD (&priv->signing);
+    ret = br_init_children(this, priv);
+    if (ret)
+        goto free_priv;
 
-        priv->timer_wheel = glusterfs_ctx_tw_get (this->ctx);
-        if (!priv->timer_wheel) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRB_MSG_TIMER_WHEEL_UNAVAILABLE,
-                        "global timer wheel unavailable");
-                goto cleanup;
-        }
+    pthread_mutex_init(&priv->lock, NULL);
+    pthread_cond_init(&priv->cond, NULL);
 
-	this->private = priv;
+    INIT_LIST_HEAD(&priv->bricks);
+    INIT_LIST_HEAD(&priv->signing);
 
-        if (!priv->iamscrubber) {
-                ret = br_signer_init (this, priv);
-                if (!ret)
-                        ret = br_signer_handle_options (this, priv, NULL);
-        } else {
-                ret = br_scrubber_init (this, priv);
-                if (!ret)
-                        ret = br_scrubber_handle_options (this, priv, NULL);
-        }
+    priv->timer_wheel = glusterfs_ctx_tw_get(this->ctx);
+    if (!priv->timer_wheel) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_TIMER_WHEEL_UNAVAILABLE,
+                NULL);
+        goto cleanup;
+    }
 
-        if (ret)
-                goto cleanup;
+    this->private = priv;
 
-        ret = gf_thread_create (&priv->thread, NULL, br_handle_events, this,
-                                "brhevent");
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        BRB_MSG_SPAWN_FAILED, "thread creation failed");
-                ret = -1;
-        }
+    if (!priv->iamscrubber) {
+        ret = br_signer_init(this, priv);
+        if (!ret)
+            ret = br_signer_handle_options(this, priv, NULL);
+    } else {
+        ret = br_scrubber_init(this, priv);
+        if (!ret)
+            ret = br_scrubber_handle_options(this, priv, NULL);
+    }
 
-        if (!ret) {
-                gf_msg (this->name, GF_LOG_INFO, 0, BRB_MSG_BITROT_LOADED,
-                        "bit-rot xlator loaded in \"%s\" mode",
-                        (priv->iamscrubber) ? "SCRUBBER" : "SIGNER");
-                return 0;
-        }
+    if (ret)
+        goto cleanup;
+
+    ret = gf_thread_create(&priv->thread, NULL, br_handle_events, this,
+                           "brhevent");
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, BRB_MSG_THREAD_CREATION_FAILED,
+                NULL);
+        ret = -1;
+    }
+
+    if (!ret) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_BITROT_LOADED, "mode=%s",
+                (priv->iamscrubber) ? "SCRUBBER" : "SIGNER", NULL);
+        return 0;
+    }
 
- cleanup:
-        (void) pthread_cond_destroy (&priv->cond);
-        (void) pthread_mutex_destroy (&priv->lock);
+cleanup:
+    (void)pthread_cond_destroy(&priv->cond);
+    (void)pthread_mutex_destroy(&priv->lock);
 
-        br_free_children (this, priv, priv->child_count);
+    br_free_children(this, priv, priv->child_count);
 
- free_priv:
-        GF_FREE (priv);
- out:
-        this->private = NULL;
-        return -1;
+free_priv:
+    GF_FREE(priv);
+out:
+    this->private = NULL;
+    return -1;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-	br_private_t *priv = this->private;
+    br_private_t *priv = this->private;
 
-        if (!priv)
-                return;
+    if (!priv)
+        return;
 
-        if (!priv->iamscrubber)
-                br_fini_signer (this, priv);
-        else
-                (void) br_free_scrubber_monitor (this, priv);
+    if (!priv->iamscrubber)
+        br_fini_signer(this, priv);
+    else
+        (void)br_free_scrubber_monitor(this, priv);
 
-        br_free_children (this, priv, priv->child_count);
+    br_free_children(this, priv, priv->child_count);
 
-        this->private = NULL;
-	GF_FREE (priv);
+    this->private = NULL;
+    GF_FREE(priv);
 
-        glusterfs_ctx_tw_put (this->ctx);
+    glusterfs_ctx_tw_put(this->ctx);
 
-	return;
+    return;
 }
 
 static void
-br_reconfigure_monitor (xlator_t *this)
+br_reconfigure_monitor(xlator_t *this)
 {
-        int32_t ret = 0;
-
-        ret = br_scrub_state_machine (this, _gf_false);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRB_MSG_RESCHEDULE_SCRUBBER_FAILED,
-                        "Could not reschedule scrubber for the volume. Scrubbing "
-                        "will continue according to old frequency.");
-        }
+    int32_t ret = 0;
+
+    ret = br_scrub_state_machine(this, _gf_false);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_COULD_NOT_SCHEDULE_SCRUB,
+                NULL);
+    }
 }
 
 static int
-br_reconfigure_scrubber (xlator_t *this, dict_t *options)
+br_reconfigure_scrubber(xlator_t *this, dict_t *options)
 {
-        int32_t       ret   = -1;
-        br_private_t *priv  = NULL;
+    int32_t ret = -1;
+    br_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        pthread_mutex_lock (&priv->lock);
-        {
-                ret = br_scrubber_handle_options (this, priv, options);
-        }
-        pthread_mutex_unlock (&priv->lock);
+    pthread_mutex_lock(&priv->lock);
+    {
+        ret = br_scrubber_handle_options(this, priv, options);
+    }
+    pthread_mutex_unlock(&priv->lock);
 
-        if (ret)
-                goto err;
+    if (ret)
+        goto err;
 
-        /* change state for all _up_ subvolume(s) */
-        pthread_mutex_lock (&priv->lock);
-        {
-                br_reconfigure_monitor (this);
-        }
-        pthread_mutex_unlock (&priv->lock);
+    /* change state for all _up_ subvolume(s) */
+    pthread_mutex_lock(&priv->lock);
+    {
+        br_reconfigure_monitor(this);
+    }
+    pthread_mutex_unlock(&priv->lock);
 
- err:
-        return ret;
+err:
+    return ret;
 }
 
 static int
-br_reconfigure_signer (xlator_t *this, dict_t *options)
+br_reconfigure_signer(xlator_t *this, dict_t *options)
 {
-        br_private_t *priv = this->private;
+    br_private_t *priv = this->private;
 
-        return br_signer_handle_options (this, priv, options);
+    return br_signer_handle_options(this, priv, options);
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int ret = 0;
-        br_private_t *priv = NULL;
+    int ret = 0;
+    br_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->iamscrubber)
-                ret = br_reconfigure_scrubber (this, options);
-        else
-                ret = br_reconfigure_signer (this, options);
+    if (priv->iamscrubber)
+        ret = br_reconfigure_scrubber(this, options);
+    else
+        ret = br_reconfigure_signer(this, options);
 
-        return ret;
+    return ret;
 }
 
 struct xlator_fops fops;
@@ -2145,49 +2155,78 @@ struct xlator_fops fops;
 struct xlator_cbks cbks;
 
 struct volume_options options[] = {
-        { .key = {"expiry-time"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = SIGNING_TIMEOUT,
-          .op_version = {GD_OP_VERSION_3_7_0},
-          .flags = OPT_FLAG_SETTABLE,
-          .description = "Waiting time for an object on which it waits "
-                         "before it is signed",
-        },
-        { .key = {"brick-count"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Total number of bricks for the current node for "
-                         "all volumes in the trusted storage pool.",
-        },
-        { .key = {"scrubber", "scrub"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "false",
-          .op_version = {GD_OP_VERSION_3_7_0},
-          .flags = OPT_FLAG_SETTABLE | OPT_FLAG_FORCE,
-          .description = "option to run as a scrubber",
-        },
-        { .key = {"scrub-throttle"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "lazy",
-          .op_version = {GD_OP_VERSION_3_7_0},
-          .flags = OPT_FLAG_SETTABLE,
-          .description = "Scrub-throttle value is a measure of how fast "
-                         "or slow the scrubber scrubs the filesystem for "
-                         "volume <VOLNAME>",
-        },
-        { .key = {"scrub-freq"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "biweekly",
-          .op_version = {GD_OP_VERSION_3_7_0},
-          .flags = OPT_FLAG_SETTABLE,
-          .description = "Scrub frequency for volume <VOLNAME>",
-        },
-        { .key = {"scrub-state"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "active",
-          .op_version = {GD_OP_VERSION_4_0_0},
-          .flags = OPT_FLAG_SETTABLE,
-          .description = "Pause/Resume scrub. Upon resume, scrubber "
-                         "continues from where it left off.",
-        },
-	{ .key  = {NULL} },
+    {
+        .key = {"expiry-time"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = SIGNING_TIMEOUT,
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Waiting time for an object on which it waits "
+                       "before it is signed",
+    },
+    {
+        .key = {"brick-count"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Total number of bricks for the current node for "
+                       "all volumes in the trusted storage pool.",
+    },
+    {
+        .key = {"scrubber", "scrub"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_FORCE,
+        .description = "option to run as a scrubber",
+    },
+    {
+        .key = {"scrub-throttle"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "lazy",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Scrub-throttle value is a measure of how fast "
+                       "or slow the scrubber scrubs the filesystem for "
+                       "volume <VOLNAME>",
+    },
+    {
+        .key = {"scrub-freq"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "biweekly",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Scrub frequency for volume <VOLNAME>",
+    },
+    {
+        .key = {"scrub-state"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "active",
+        .op_version = {GD_OP_VERSION_4_0_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Pause/Resume scrub. Upon resume, scrubber "
+                       "continues from where it left off.",
+    },
+    {
+        .key = {"signer-threads"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = BR_WORKERS,
+        .op_version = {GD_OP_VERSION_8_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Number of signing process threads. As a best "
+                       "practice, set this to the number of processor cores",
+    },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "bit-rot",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.h b/xlators/features/bit-rot/src/bitd/bit-rot.h
index c2c0c225792..8ac7dcdac3d 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.h
@@ -11,17 +11,17 @@
 #ifndef __BIT_ROT_H__
 #define __BIT_ROT_H__
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
-#include "syncop.h"
-#include "syncop-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/syncop-utils.h>
 #include "changelog.h"
 #include "timer-wheel.h"
 
-#include "throttle-tbf.h"
+#include <glusterfs/throttle-tbf.h>
 #include "bit-rot-ssm.h"
 
 #include "bit-rot-common.h"
@@ -30,278 +30,273 @@
 
 #include <openssl/sha.h>
 
-/**
- * TODO: make this configurable. As a best practice, set this to the
- * number of processor cores.
- */
-#define BR_WORKERS 4
-
 typedef enum scrub_throttle {
-        BR_SCRUB_THROTTLE_VOID       = -1,
-        BR_SCRUB_THROTTLE_LAZY       = 0,
-        BR_SCRUB_THROTTLE_NORMAL     = 1,
-        BR_SCRUB_THROTTLE_AGGRESSIVE = 2,
-        BR_SCRUB_THROTTLE_STALLED    = 3,
+    BR_SCRUB_THROTTLE_VOID = -1,
+    BR_SCRUB_THROTTLE_LAZY = 0,
+    BR_SCRUB_THROTTLE_NORMAL = 1,
+    BR_SCRUB_THROTTLE_AGGRESSIVE = 2,
+    BR_SCRUB_THROTTLE_STALLED = 3,
 } scrub_throttle_t;
 
 typedef enum scrub_freq {
-        BR_FSSCRUB_FREQ_HOURLY = 1,
-        BR_FSSCRUB_FREQ_DAILY,
-        BR_FSSCRUB_FREQ_WEEKLY,
-        BR_FSSCRUB_FREQ_BIWEEKLY,
-        BR_FSSCRUB_FREQ_MONTHLY,
-        BR_FSSCRUB_FREQ_MINUTE,
-        BR_FSSCRUB_FREQ_STALLED,
+    BR_FSSCRUB_FREQ_HOURLY = 1,
+    BR_FSSCRUB_FREQ_DAILY,
+    BR_FSSCRUB_FREQ_WEEKLY,
+    BR_FSSCRUB_FREQ_BIWEEKLY,
+    BR_FSSCRUB_FREQ_MONTHLY,
+    BR_FSSCRUB_FREQ_MINUTE,
+    BR_FSSCRUB_FREQ_STALLED,
 } scrub_freq_t;
 
-#define signature_size(hl) (sizeof (br_isignature_t) + hl + 1)
+#define signature_size(hl) (sizeof(br_isignature_t) + hl + 1)
 
 struct br_scanfs {
-        gf_lock_t entrylock;
+    gf_lock_t entrylock;
 
-        pthread_mutex_t waitlock;
-        pthread_cond_t  waitcond;
+    pthread_mutex_t waitlock;
+    pthread_cond_t waitcond;
 
-        unsigned int     entries;
-        struct list_head queued;
-        struct list_head ready;
+    unsigned int entries;
+    struct list_head queued;
+    struct list_head ready;
 };
 
 /* just need three states to track child status */
 typedef enum br_child_state {
-        BR_CHILD_STATE_CONNECTED = 1,
-        BR_CHILD_STATE_INITIALIZING,
-        BR_CHILD_STATE_CONNFAILED,
-        BR_CHILD_STATE_DISCONNECTED,
+    BR_CHILD_STATE_CONNECTED = 1,
+    BR_CHILD_STATE_INITIALIZING,
+    BR_CHILD_STATE_CONNFAILED,
+    BR_CHILD_STATE_DISCONNECTED,
 } br_child_state_t;
 
 struct br_child {
-        pthread_mutex_t lock;         /* protects child state */
-        char witnessed;               /* witnessed at least one succesfull
-                                         connection */
-        br_child_state_t c_state;     /* current state of this child */
+    pthread_mutex_t lock;     /* protects child state */
+    char witnessed;           /* witnessed at least one successful
+                                 connection */
+    br_child_state_t c_state; /* current state of this child */
 
-        char child_up;                /* Indicates whether this child is
-                                         up or not */
-        xlator_t *xl;                 /* client xlator corresponding to
-                                         this child */
-        inode_table_t *table;         /* inode table for this child */
-        char brick_path[PATH_MAX];    /* brick export directory of this
-                                         child */
-        struct list_head list;        /* hook to attach to the list of
-                                         UP children */
-        xlator_t *this;               /* Bit rot xlator */
+    char child_up;             /* Indicates whether this child is
+                                  up or not */
+    xlator_t *xl;              /* client xlator corresponding to
+                                  this child */
+    inode_table_t *table;      /* inode table for this child */
+    char brick_path[PATH_MAX]; /* brick export directory of this
+                                  child */
+    struct list_head list;     /* hook to attach to the list of
+                                  UP children */
+    xlator_t *this;            /* Bit rot xlator */
 
-        pthread_t thread;             /* initial crawler for unsigned
-                                         object(s) or scrub crawler */
-        int threadrunning;            /* active thread */
+    pthread_t thread;  /* initial crawler for unsigned
+                          object(s) or scrub crawler */
+    int threadrunning; /* active thread */
 
-        struct mem_pool *timer_pool;  /* timer-wheel's timer mem-pool */
+    struct mem_pool *timer_pool; /* timer-wheel's timer mem-pool */
 
-        struct timeval tv;
+    struct timeval tv;
 
-        struct br_scanfs fsscan;      /* per subvolume FS scanner */
+    struct br_scanfs fsscan; /* per subvolume FS scanner */
 
-        gf_boolean_t active_scrubbing; /* Actively scrubbing or not */
+    gf_boolean_t active_scrubbing; /* Actively scrubbing or not */
 };
 
 typedef struct br_child br_child_t;
 
 struct br_obj_n_workers {
-        struct list_head objects;         /* queue of objects expired from the
-                                             timer wheel and ready to be picked
-                                             up for signing */
-        pthread_t workers[BR_WORKERS];    /* Threads which pick up the objects
-                                             from the above queue and start
-                                             signing each object */
+    struct list_head objects; /* queue of objects expired from the
+                                 timer wheel and ready to be picked
+                                 up for signing */
+    pthread_t *workers;       /* Threads which pick up the objects
+                                 from the above queue and start
+                                 signing each object */
 };
 
 struct br_scrubber {
-        xlator_t *this;
+    xlator_t *this;
 
-        scrub_throttle_t throttle;
+    scrub_throttle_t throttle;
 
-        /**
-         * frequency of scanning for this subvolume. this should
-         * normally be per-child, but since all childs follow the
-         * same frequency for a volume, this option ends up here
-         * instead of br_child_t.
-         */
-        scrub_freq_t frequency;
+    /**
+     * frequency of scanning for this subvolume. this should
+     * normally be per-child, but since all children follow the
+     * same frequency for a volume, this option ends up here
+     * instead of br_child_t.
+     */
+    scrub_freq_t frequency;
 
-        gf_boolean_t frequency_reconf;
-        gf_boolean_t throttle_reconf;
+    gf_boolean_t frequency_reconf;
+    gf_boolean_t throttle_reconf;
 
-        pthread_mutex_t mutex;
-        pthread_cond_t  cond;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
 
-        unsigned int nr_scrubbers;
-        struct list_head scrubbers;
+    unsigned int nr_scrubbers;
+    struct list_head scrubbers;
 
-        /**
-         * list of "rotatable" subvolume(s) undergoing scrubbing
-         */
-        struct list_head scrublist;
+    /**
+     * list of "rotatable" subvolume(s) undergoing scrubbing
+     */
+    struct list_head scrublist;
 };
 
 struct br_monitor {
-        gf_lock_t lock;
-        pthread_t thread;         /* Monitor thread */
-
-        gf_boolean_t  inited;
-        pthread_mutex_t mutex;
-        pthread_cond_t cond;      /* Thread starts and will be waiting on cond.
-                                     First child which is up wakes this up */
-
-        xlator_t *this;
-        /* scheduler */
-        uint32_t boot;
-
-        int32_t active_child_count; /* Number of children currently scrubbing */
-        gf_boolean_t kick;          /* This variable tracks the scrubber is
-                                     * kicked or not. Both 'kick' and
-                                     * 'active_child_count' uses the same pair
-                                     * of mutex-cond variable, i.e, wakelock and
-                                     * wakecond. */
-
-        pthread_mutex_t wakelock;
-        pthread_cond_t  wakecond;
-
-        gf_boolean_t done;
-        pthread_mutex_t donelock;
-        pthread_cond_t  donecond;
-
-        struct gf_tw_timer_list *timer;
-        br_scrub_state_t state;   /* current scrub state */
+    gf_lock_t lock;
+    pthread_t thread; /* Monitor thread */
+
+    gf_boolean_t inited;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond; /* Thread starts and will be waiting on cond.
+                            First child which is up wakes this up */
+
+    xlator_t *this;
+    /* scheduler */
+    uint32_t boot;
+
+    int32_t active_child_count; /* Number of children currently scrubbing */
+    gf_boolean_t kick;          /* This variable tracks the scrubber is
+                                 * kicked or not. Both 'kick' and
+                                 * 'active_child_count' uses the same pair
+                                 * of mutex-cond variable, i.e, wakelock and
+                                 * wakecond. */
+
+    pthread_mutex_t wakelock;
+    pthread_cond_t wakecond;
+
+    gf_boolean_t done;
+    pthread_mutex_t donelock;
+    pthread_cond_t donecond;
+
+    struct gf_tw_timer_list *timer;
+    br_scrub_state_t state; /* current scrub state */
 };
 
 typedef struct br_obj_n_workers br_obj_n_workers_t;
 
 typedef struct br_private br_private_t;
 
-typedef void (*br_scrubbed_file_update) (br_private_t *priv);
+typedef void (*br_scrubbed_file_update)(br_private_t *priv);
 
 struct br_private {
-        pthread_mutex_t lock;
+    pthread_mutex_t lock;
+
+    struct list_head bricks; /* list of bricks from which enents
+                                have been received */
 
-        struct list_head bricks;          /* list of bricks from which enents
-                                             have been received */
+    struct list_head signing;
 
-        struct list_head signing;
+    pthread_cond_t object_cond; /* handling signing of objects */
+    int child_count;
+    br_child_t *children; /* list of subvolumes */
+    int up_children;
 
-        pthread_cond_t object_cond;       /* handling signing of objects */
-        int child_count;
-        br_child_t *children;             /* list of subvolumes */
-        int up_children;
+    pthread_cond_t cond; /* handling CHILD_UP notifications */
+    pthread_t thread;    /* thread for connecting each UP
+                            child with changelog */
 
-        pthread_cond_t cond;              /* handling CHILD_UP notifications */
-        pthread_t thread;                 /* thread for connecting each UP
-                                             child with changelog */
+    struct tvec_base *timer_wheel; /* timer wheel where the objects which
+                                      changelog has sent sits and waits
+                                      for expiry */
+    br_obj_n_workers_t *obj_queue; /* place holder for all the objects
+                                      that are expired from timer wheel
+                                      and ready to be picked up for
+                                      signing and the workers which sign
+                                      the objects */
 
-        struct tvec_base *timer_wheel;    /* timer wheel where the objects which
-                                             changelog has sent sits and waits
-                                             for expiry */
-        br_obj_n_workers_t *obj_queue;    /* place holder for all the objects
-                                             that are expired from timer wheel
-                                             and ready to be picked up for
-                                             signing and the workers which sign
-                                             the objects */
+    uint32_t expiry_time; /* objects "wait" time */
 
-        uint32_t expiry_time;              /* objects "wait" time */
+    uint32_t signer_th_count; /* Number of signing process threads */
 
-        tbf_t *tbf;                    /* token bucket filter */
+    tbf_t *tbf; /* token bucket filter */
 
-        gf_boolean_t iamscrubber;         /* function as a fs scrubber */
+    gf_boolean_t iamscrubber; /* function as a fs scrubber */
 
-        struct br_scrub_stats scrub_stat; /* statistics of scrub*/
+    struct br_scrub_stats scrub_stat; /* statistics of scrub*/
 
-        struct br_scrubber fsscrub;       /* scrubbers for this subvolume */
+    struct br_scrubber fsscrub; /* scrubbers for this subvolume */
 
-        struct br_monitor scrub_monitor;  /* scrubber monitor */
+    struct br_monitor scrub_monitor; /* scrubber monitor */
 };
 
 struct br_object {
-        xlator_t *this;
+    xlator_t *this;
 
-        uuid_t gfid;
+    uuid_t gfid;
 
-        unsigned long signedversion;    /* version aginst which this object will
-                                           be signed */
-        br_child_t *child;              /* object's subvolume */
+    unsigned long signedversion; /* version against which this object will
+                                    be signed */
+    br_child_t *child;           /* object's subvolume */
 
-        int sign_info;
+    int sign_info;
 
-        struct list_head list;          /* hook to add to the queue once the
-                                           object is expired from timer wheel */
-        void *data;
+    struct list_head list; /* hook to add to the queue once the
+                              object is expired from timer wheel */
+    void *data;
 };
 
 typedef struct br_object br_object_t;
-typedef int32_t (br_scrub_ssm_call) (xlator_t *);
+typedef int32_t(br_scrub_ssm_call)(xlator_t *);
 
 void
-br_log_object (xlator_t *, char *, uuid_t, int32_t);
+br_log_object(xlator_t *, char *, uuid_t, int32_t);
 
 void
-br_log_object_path (xlator_t *, char *, const char *, int32_t);
+br_log_object_path(xlator_t *, char *, const char *, int32_t);
 
 int32_t
-br_calculate_obj_checksum (unsigned char *,
-                           br_child_t *, fd_t *, struct iatt *);
+br_calculate_obj_checksum(unsigned char *, br_child_t *, fd_t *, struct iatt *);
 
 int32_t
-br_prepare_loc (xlator_t *, br_child_t *, loc_t *, gf_dirent_t *, loc_t *);
+br_prepare_loc(xlator_t *, br_child_t *, loc_t *, gf_dirent_t *, loc_t *);
 
 gf_boolean_t
-bitd_is_bad_file (xlator_t *, br_child_t *, loc_t *, fd_t *);
+bitd_is_bad_file(xlator_t *, br_child_t *, loc_t *, fd_t *);
 
 static inline void
-_br_set_child_state (br_child_t *child, br_child_state_t state)
+_br_set_child_state(br_child_t *child, br_child_state_t state)
 {
-        child->c_state = state;
+    child->c_state = state;
 }
 
 static inline int
-_br_is_child_connected (br_child_t *child)
+_br_is_child_connected(br_child_t *child)
 {
-        return (child->c_state == BR_CHILD_STATE_CONNECTED);
+    return (child->c_state == BR_CHILD_STATE_CONNECTED);
 }
 
 static inline int
-_br_is_child_scrub_active (br_child_t *child)
+_br_is_child_scrub_active(br_child_t *child)
 {
-        return child->active_scrubbing;
+    return child->active_scrubbing;
 }
 
 static inline int
-_br_child_failed_conn (br_child_t *child)
+_br_child_failed_conn(br_child_t *child)
 {
-        return (child->c_state == BR_CHILD_STATE_CONNFAILED);
+    return (child->c_state == BR_CHILD_STATE_CONNFAILED);
 }
 
 static inline int
-_br_child_witnessed_connection (br_child_t *child)
+_br_child_witnessed_connection(br_child_t *child)
 {
-        return (child->witnessed == 1);
+    return (child->witnessed == 1);
 }
 
 /* scrub state */
 static inline void
-_br_monitor_set_scrub_state (struct br_monitor *scrub_monitor,
-                           br_scrub_state_t state)
+_br_monitor_set_scrub_state(struct br_monitor *scrub_monitor,
+                            br_scrub_state_t state)
 {
-        scrub_monitor->state = state;
+    scrub_monitor->state = state;
 }
 
 static inline br_scrub_event_t
-_br_child_get_scrub_event (struct br_scrubber *fsscrub)
+_br_child_get_scrub_event(struct br_scrubber *fsscrub)
 {
-        return (fsscrub->frequency == BR_FSSCRUB_FREQ_STALLED)
-             ? BR_SCRUB_EVENT_PAUSE : BR_SCRUB_EVENT_SCHEDULE;
+    return (fsscrub->frequency == BR_FSSCRUB_FREQ_STALLED)
+               ? BR_SCRUB_EVENT_PAUSE
+               : BR_SCRUB_EVENT_SCHEDULE;
 }
 
 int32_t
-br_get_bad_objects_list (xlator_t *this, dict_t **dict);
-
+br_get_bad_objects_list(xlator_t *this, dict_t **dict);
 
 #endif /* __BIT_ROT_H__ */
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-common.h b/xlators/features/bit-rot/src/stub/bit-rot-common.h
index 2afc9f47c29..20561aa7764 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-common.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-common.h
@@ -11,74 +11,74 @@
 #ifndef __BIT_ROT_COMMON_H__
 #define __BIT_ROT_COMMON_H__
 
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "bit-rot-object-version.h"
 
-#define BR_VXATTR_VERSION   (1 << 0)
+#define BR_VXATTR_VERSION (1 << 0)
 #define BR_VXATTR_SIGNATURE (1 << 1)
 
 #define BR_VXATTR_SIGN_MISSING (BR_VXATTR_SIGNATURE)
-#define BR_VXATTR_ALL_MISSING                           \
-        (BR_VXATTR_VERSION | BR_VXATTR_SIGNATURE)
+#define BR_VXATTR_ALL_MISSING (BR_VXATTR_VERSION | BR_VXATTR_SIGNATURE)
 
-#define BR_BAD_OBJ_CONTAINER (uuid_t){0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8}
+#define BR_BAD_OBJ_CONTAINER                                                   \
+    (uuid_t) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8 }
 
 typedef enum br_vxattr_state {
-        BR_VXATTR_STATUS_FULL     = 0,
-        BR_VXATTR_STATUS_MISSING  = 1,
-        BR_VXATTR_STATUS_UNSIGNED = 2,
-        BR_VXATTR_STATUS_INVALID  = 3,
+    BR_VXATTR_STATUS_FULL = 0,
+    BR_VXATTR_STATUS_MISSING = 1,
+    BR_VXATTR_STATUS_UNSIGNED = 2,
+    BR_VXATTR_STATUS_INVALID = 3,
 } br_vxattr_status_t;
 
 typedef enum br_sign_state {
-        BR_SIGN_INVALID     = -1,
-        BR_SIGN_NORMAL      = 0,
-        BR_SIGN_REOPEN_WAIT = 1,
-        BR_SIGN_QUICK       = 2,
+    BR_SIGN_INVALID = -1,
+    BR_SIGN_NORMAL = 0,
+    BR_SIGN_REOPEN_WAIT = 1,
+    BR_SIGN_QUICK = 2,
 } br_sign_state_t;
 
 static inline br_vxattr_status_t
-br_version_xattr_state (dict_t *xattr, br_version_t **obuf,
-                        br_signature_t **sbuf, gf_boolean_t *objbad)
+br_version_xattr_state(dict_t *xattr, br_version_t **obuf,
+                       br_signature_t **sbuf, gf_boolean_t *objbad)
 {
-        int32_t             ret    = 0;
-        int32_t             vxattr = 0;
-        br_vxattr_status_t  status;
-        void               *data   = NULL;
-
-        /**
-         * The key being present in the dict indicates the xattr was set on
-         * disk. The presence of xattr itself as of now is suffecient to say
-         * the the object is bad.
-         */
-        *objbad = _gf_false;
-        ret = dict_get_bin (xattr, BITROT_OBJECT_BAD_KEY, (void **)&data);
-        if (!ret)
-                *objbad = _gf_true;
-
-        ret = dict_get_bin (xattr, BITROT_CURRENT_VERSION_KEY, (void **)obuf);
-        if (ret)
-                vxattr |= BR_VXATTR_VERSION;
-
-        ret = dict_get_bin (xattr, BITROT_SIGNING_VERSION_KEY, (void **)sbuf);
-        if (ret)
-                vxattr |= BR_VXATTR_SIGNATURE;
-
-        switch (vxattr) {
+    int32_t ret = 0;
+    int32_t vxattr = 0;
+    br_vxattr_status_t status;
+    void *data = NULL;
+
+    /**
+     * The key being present in the dict indicates the xattr was set on
+     * disk. The presence of xattr itself as of now is suffecient to say
+     * the the object is bad.
+     */
+    *objbad = _gf_false;
+    ret = dict_get_bin(xattr, BITROT_OBJECT_BAD_KEY, (void **)&data);
+    if (!ret)
+        *objbad = _gf_true;
+
+    ret = dict_get_bin(xattr, BITROT_CURRENT_VERSION_KEY, (void **)obuf);
+    if (ret)
+        vxattr |= BR_VXATTR_VERSION;
+
+    ret = dict_get_bin(xattr, BITROT_SIGNING_VERSION_KEY, (void **)sbuf);
+    if (ret)
+        vxattr |= BR_VXATTR_SIGNATURE;
+
+    switch (vxattr) {
         case 0:
-                status = BR_VXATTR_STATUS_FULL;
-                break;
+            status = BR_VXATTR_STATUS_FULL;
+            break;
         case BR_VXATTR_SIGN_MISSING:
-                status = BR_VXATTR_STATUS_UNSIGNED;
-                break;
+            status = BR_VXATTR_STATUS_UNSIGNED;
+            break;
         case BR_VXATTR_ALL_MISSING:
-                status = BR_VXATTR_STATUS_MISSING;
-                break;
+            status = BR_VXATTR_STATUS_MISSING;
+            break;
         default:
-                status = BR_VXATTR_STATUS_INVALID;
-        }
+            status = BR_VXATTR_STATUS_INVALID;
+    }
 
-        return status;
+    return status;
 }
 
 /**
@@ -86,13 +86,13 @@ br_version_xattr_state (dict_t *xattr, br_version_t **obuf,
  * signing.
  */
 typedef struct br_isignature_in {
-        int8_t signaturetype;            /* signature type            */
+    int8_t signaturetype; /* signature type            */
 
-        unsigned long signedversion;     /* version against which the
-                                            object was signed         */
+    unsigned long signedversion; /* version against which the
+                                    object was signed         */
 
-        size_t signaturelen;             /* signature length          */
-        char signature[0];               /* object signature          */
+    size_t signaturelen; /* signature length          */
+    char signature[0];   /* object signature          */
 } br_isignature_t;
 
 /**
@@ -100,80 +100,79 @@ typedef struct br_isignature_in {
  * verification.
  */
 typedef struct br_isignature_out {
-        char stale;                      /* stale signature?          */
+    char stale; /* stale signature?          */
 
-        unsigned long version;           /* current signed version    */
+    unsigned long version; /* current signed version    */
 
-        uint32_t time[2];                /* time when the object
-                                            got dirtied               */
+    uint32_t time[2]; /* time when the object
+                         got dirtied               */
 
-        int8_t signaturetype;            /* hash type                 */
-        size_t signaturelen;             /* signature length          */
-        char   signature[0];             /* signature (hash)          */
+    int8_t signaturetype; /* hash type                 */
+    size_t signaturelen;  /* signature length          */
+    char signature[0];    /* signature (hash)          */
 } br_isignature_out_t;
 
 typedef struct br_stub_init {
-        uint32_t timebuf[2];
-        char export[PATH_MAX];
+    uint32_t timebuf[2];
+    char export[PATH_MAX];
 } br_stub_init_t;
 
 typedef enum {
-        BR_SIGNATURE_TYPE_VOID   = -1,   /* object is not signed       */
-        BR_SIGNATURE_TYPE_ZERO   = 0,    /* min boundary               */
-        BR_SIGNATURE_TYPE_SHA256 = 1,    /* signed with SHA256         */
-        BR_SIGNATURE_TYPE_MAX    = 2,    /* max boundary               */
+    BR_SIGNATURE_TYPE_VOID = -1,  /* object is not signed       */
+    BR_SIGNATURE_TYPE_ZERO = 0,   /* min boundary               */
+    BR_SIGNATURE_TYPE_SHA256 = 1, /* signed with SHA256         */
+    BR_SIGNATURE_TYPE_MAX = 2,    /* max boundary               */
 } br_signature_type;
 
 /* BitRot stub start time (virtual xattr) */
-#define GLUSTERFS_GET_BR_STUB_INIT_TIME  "trusted.glusterfs.bit-rot.stub-init"
+#define GLUSTERFS_GET_BR_STUB_INIT_TIME "trusted.glusterfs.bit-rot.stub-init"
 
 /* signing/reopen hint */
 #define BR_OBJECT_RESIGN 0
-#define BR_OBJECT_REOPEN  1
-#define BR_REOPEN_SIGN_HINT_KEY  "trusted.glusterfs.bit-rot.reopen-hint"
+#define BR_OBJECT_REOPEN 1
+#define BR_REOPEN_SIGN_HINT_KEY "trusted.glusterfs.bit-rot.reopen-hint"
 
 static inline int
-br_is_signature_type_valid (int8_t signaturetype)
+br_is_signature_type_valid(int8_t signaturetype)
 {
-        return ((signaturetype > BR_SIGNATURE_TYPE_ZERO)
-                 && (signaturetype < BR_SIGNATURE_TYPE_MAX));
+    return ((signaturetype > BR_SIGNATURE_TYPE_ZERO) &&
+            (signaturetype < BR_SIGNATURE_TYPE_MAX));
 }
 
 static inline void
-br_set_default_ongoingversion (br_version_t *buf, uint32_t *tv)
+br_set_default_ongoingversion(br_version_t *buf, uint32_t *tv)
 {
-        buf->ongoingversion = BITROT_DEFAULT_CURRENT_VERSION;
-        buf->timebuf[0] = tv[0];
-        buf->timebuf[1] = tv[1];
+    buf->ongoingversion = BITROT_DEFAULT_CURRENT_VERSION;
+    buf->timebuf[0] = tv[0];
+    buf->timebuf[1] = tv[1];
 }
 
 static inline void
-br_set_default_signature (br_signature_t *buf, size_t *size)
+br_set_default_signature(br_signature_t *buf, size_t *size)
 {
-        buf->signaturetype = (int8_t) BR_SIGNATURE_TYPE_VOID;
-        buf->signedversion = BITROT_DEFAULT_SIGNING_VERSION;
+    buf->signaturetype = (int8_t)BR_SIGNATURE_TYPE_VOID;
+    buf->signedversion = BITROT_DEFAULT_SIGNING_VERSION;
 
-        *size = sizeof (br_signature_t); /* no signature */
+    *size = sizeof(br_signature_t); /* no signature */
 }
 
 static inline void
-br_set_ongoingversion (br_version_t *buf,
-                       unsigned long version, uint32_t *tv)
+br_set_ongoingversion(br_version_t *buf, unsigned long version, uint32_t *tv)
 {
-        buf->ongoingversion = version;
-        buf->timebuf[0] = tv[0];
-        buf->timebuf[1] = tv[1];
+    buf->ongoingversion = version;
+    buf->timebuf[0] = tv[0];
+    buf->timebuf[1] = tv[1];
 }
 
 static inline void
-br_set_signature (br_signature_t *buf,
-                  br_isignature_t *sign, size_t signaturelen, size_t *size)
+br_set_signature(br_signature_t *buf, br_isignature_t *sign,
+                 size_t signaturelen, size_t *size)
 {
-        buf->signaturetype  = sign->signaturetype;
-        buf->signedversion  = ntohl (sign->signedversion);
+    buf->signaturetype = sign->signaturetype;
+    buf->signedversion = ntohl(sign->signedversion);
 
-        memcpy (buf->signature, sign->signature, signaturelen);
-        *size = sizeof (br_signature_t) + signaturelen;
+    memcpy(buf->signature, sign->signature, signaturelen);
+    *size = sizeof(br_signature_t) + signaturelen;
 }
 
 #endif /* __BIT_ROT_COMMON_H__ */
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-object-version.h b/xlators/features/bit-rot/src/stub/bit-rot-object-version.h
index 1f2497aebe9..7ae6a5200df 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-object-version.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-object-version.h
@@ -15,16 +15,16 @@
  * on-disk formats for ongoing version and object signature.
  */
 typedef struct br_version {
-        unsigned long ongoingversion;
-        uint32_t timebuf[2];
+    unsigned long ongoingversion;
+    uint32_t timebuf[2];
 } br_version_t;
 
-typedef struct __attribute__ ((__packed__)) br_signature {
-        int8_t signaturetype;
+typedef struct __attribute__((__packed__)) br_signature {
+    int8_t signaturetype;
 
-        unsigned long signedversion;
+    unsigned long signedversion;
 
-        char signature[0];
+    char signature[0];
 } br_signature_t;
 
 #endif
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub-helpers.c b/xlators/features/bit-rot/src/stub/bit-rot-stub-helpers.c
index 24aa9aa5ff4..8ac13a09941 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub-helpers.c
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub-helpers.c
@@ -11,89 +11,88 @@
 #include "bit-rot-stub.h"
 
 br_stub_fd_t *
-br_stub_fd_new (void)
+br_stub_fd_new(void)
 {
-        br_stub_fd_t    *br_stub_fd = NULL;
+    br_stub_fd_t *br_stub_fd = NULL;
 
-        br_stub_fd = GF_CALLOC (1, sizeof (*br_stub_fd),
-                                gf_br_stub_mt_br_stub_fd_t);
+    br_stub_fd = GF_CALLOC(1, sizeof(*br_stub_fd), gf_br_stub_mt_br_stub_fd_t);
 
-        return br_stub_fd;
+    return br_stub_fd;
 }
 
 int
-__br_stub_fd_ctx_set (xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd)
+__br_stub_fd_ctx_set(xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd)
 {
-        uint64_t    value = 0;
-        int         ret   = -1;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, br_stub_fd, out);
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, br_stub_fd, out);
 
-        value = (uint64_t)(long) br_stub_fd;
+    value = (uint64_t)(long)br_stub_fd;
 
-        ret = __fd_ctx_set (fd, this, value);
+    ret = __fd_ctx_set(fd, this, value);
 
 out:
-        return ret;
+    return ret;
 }
 
 br_stub_fd_t *
-__br_stub_fd_ctx_get (xlator_t *this, fd_t *fd)
+__br_stub_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        br_stub_fd_t *br_stub_fd = NULL;
-        uint64_t  value  = 0;
-        int       ret    = -1;
+    br_stub_fd_t *br_stub_fd = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        ret = __fd_ctx_get (fd, this, &value);
-        if (ret)
-                return NULL;
+    ret = __fd_ctx_get(fd, this, &value);
+    if (ret)
+        return NULL;
 
-        br_stub_fd = (br_stub_fd_t *) ((long) value);
+    br_stub_fd = (br_stub_fd_t *)((long)value);
 
 out:
-        return br_stub_fd;
+    return br_stub_fd;
 }
 
 br_stub_fd_t *
-br_stub_fd_ctx_get (xlator_t *this, fd_t *fd)
+br_stub_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        br_stub_fd_t *br_stub_fd = NULL;
+    br_stub_fd_t *br_stub_fd = NULL;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        LOCK (&fd->lock);
-        {
-                br_stub_fd = __br_stub_fd_ctx_get (this, fd);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        br_stub_fd = __br_stub_fd_ctx_get(this, fd);
+    }
+    UNLOCK(&fd->lock);
 
 out:
-        return br_stub_fd;
+    return br_stub_fd;
 }
 
 int32_t
-br_stub_fd_ctx_set (xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd)
+br_stub_fd_ctx_set(xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd)
 {
-        int32_t    ret = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, br_stub_fd, out);
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, br_stub_fd, out);
 
-        LOCK (&fd->lock);
-        {
-                ret = __br_stub_fd_ctx_set (this, fd, br_stub_fd);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        ret = __br_stub_fd_ctx_set(this, fd, br_stub_fd);
+    }
+    UNLOCK(&fd->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -101,114 +100,112 @@ out:
  * @gfid: gfid of the bad object being added to the bad objects directory
  */
 int
-br_stub_add (xlator_t *this, uuid_t gfid)
+br_stub_add(xlator_t *this, uuid_t gfid)
 {
-        char              gfid_path[BR_PATH_MAX_PLUS] = {0};
-        char              bad_gfid_path[BR_PATH_MAX_PLUS] = {0};
-        int               ret = 0;
-        br_stub_private_t *priv = NULL;
-        struct stat       st = {0};
-
-        priv = this->private;
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !gf_uuid_is_null (gfid),
-                                       out, errno, EINVAL);
+    char gfid_path[BR_PATH_MAX_PLUS] = {0};
+    char bad_gfid_path[BR_PATH_MAX_PLUS] = {0};
+    int ret = 0;
+    br_stub_private_t *priv = NULL;
+    struct stat st = {0};
+
+    priv = this->private;
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, !gf_uuid_is_null(gfid), out,
+                                  errno, EINVAL);
+
+    snprintf(gfid_path, sizeof(gfid_path), "%s/%s", priv->stub_basepath,
+             uuid_utoa(gfid));
+
+    ret = sys_stat(gfid_path, &st);
+    if (!ret)
+        goto out;
+    snprintf(bad_gfid_path, sizeof(bad_gfid_path), "%s/stub-%s",
+             priv->stub_basepath, uuid_utoa(priv->bad_object_dir_gfid));
+
+    ret = sys_link(bad_gfid_path, gfid_path);
+    if (ret) {
+        if ((errno != ENOENT) && (errno != EMLINK) && (errno != EEXIST))
+            goto out;
 
-        snprintf (gfid_path, sizeof (gfid_path), "%s/%s",
-                  priv->stub_basepath, uuid_utoa (gfid));
-
-        ret = sys_stat (gfid_path, &st);
-        if (!ret)
-                goto out;
-        snprintf (bad_gfid_path, sizeof (bad_gfid_path), "%s/stub-%s",
-                  priv->stub_basepath, uuid_utoa (priv->bad_object_dir_gfid));
-
-        ret = sys_link (bad_gfid_path, gfid_path);
-        if (ret) {
-                if ((errno != ENOENT) && (errno != EMLINK) && (errno != EEXIST))
-                        goto out;
-
-                /*
-                 * Continue with success. At least we'll have half of the
-                 * functionality, in the sense, object is marked bad and
-                 * would be inaccessible. It's only scrub status that would
-                 * show up less number of objects. That's fine as we'll have
-                 * the log files that will have the missing information.
-                 */
-                gf_msg (this->name, GF_LOG_WARNING, errno, BRS_MSG_LINK_FAIL,
-                        "failed to record  gfid [%s]", uuid_utoa (gfid));
-        }
+        /*
+         * Continue with success. At least we'll have half of the
+         * functionality, in the sense, object is marked bad and
+         * would be inaccessible. It's only scrub status that would
+         * show up less number of objects. That's fine as we'll have
+         * the log files that will have the missing information.
+         */
+        gf_smsg(this->name, GF_LOG_WARNING, errno, BRS_MSG_LINK_FAIL, "gfid=%s",
+                uuid_utoa(gfid), NULL);
+    }
 
-        return 0;
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int
-br_stub_del (xlator_t *this, uuid_t gfid)
+br_stub_del(xlator_t *this, uuid_t gfid)
 {
-        int32_t      op_errno __attribute__((unused)) = 0;
-        br_stub_private_t *priv = NULL;
-        int          ret = 0;
-        char         gfid_path[BR_PATH_MAX_PLUS] = {0};
-
-        priv = this->private;
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !gf_uuid_is_null (gfid),
-                                       out, op_errno, EINVAL);
-        snprintf (gfid_path, sizeof (gfid_path), "%s/%s",
-                  priv->stub_basepath, uuid_utoa (gfid));
-        ret = sys_unlink (gfid_path);
-        if (ret && (errno != ENOENT)) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        BRS_MSG_BAD_OBJ_UNLINK_FAIL,
-                        "%s: failed to delete bad object link from quarantine "
-                        "directory", gfid_path);
-                ret = -errno;
-                goto out;
-        }
-
-        ret = 0;
+    int32_t op_errno __attribute__((unused)) = 0;
+    br_stub_private_t *priv = NULL;
+    int ret = 0;
+    char gfid_path[BR_PATH_MAX_PLUS] = {0};
+
+    priv = this->private;
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, !gf_uuid_is_null(gfid), out,
+                                  op_errno, EINVAL);
+    snprintf(gfid_path, sizeof(gfid_path), "%s/%s", priv->stub_basepath,
+             uuid_utoa(gfid));
+    ret = sys_unlink(gfid_path);
+    if (ret && (errno != ENOENT)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, BRS_MSG_BAD_OBJ_UNLINK_FAIL,
+                "path=%s", gfid_path, NULL);
+        ret = -errno;
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-br_stub_check_stub_directory (xlator_t *this, char *fullpath)
+br_stub_check_stub_directory(xlator_t *this, char *fullpath)
 {
-        int         ret         = 0;
-        struct stat st          = {0,};
-        char  oldpath[BR_PATH_MAX_PLUS] = {0};
-        br_stub_private_t    *priv = NULL;
-
-        priv = this->private;
-
-        snprintf (oldpath, sizeof (oldpath), "%s/%s",
-                  priv->export, OLD_BR_STUB_QUARANTINE_DIR);
-
-        ret = sys_stat (fullpath, &st);
-        if (!ret && !S_ISDIR (st.st_mode))
-                goto error_return;
-        if (ret) {
-                if (errno != ENOENT)
-                        goto error_return;
-                ret =  sys_stat (oldpath, &st);
-                if (ret)
-                        ret = mkdir_p (fullpath, 0600, _gf_true);
-                else
-                        ret = sys_rename (oldpath, fullpath);
-        }
-
+    int ret = 0;
+    struct stat st = {
+        0,
+    };
+    char oldpath[BR_PATH_MAX_PLUS] = {0};
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+
+    snprintf(oldpath, sizeof(oldpath), "%s/%s", priv->export,
+             OLD_BR_STUB_QUARANTINE_DIR);
+
+    ret = sys_stat(fullpath, &st);
+    if (!ret && !S_ISDIR(st.st_mode))
+        goto error_return;
+    if (ret) {
+        if (errno != ENOENT)
+            goto error_return;
+        ret = sys_stat(oldpath, &st);
         if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        BRS_MSG_BAD_OBJECT_DIR_FAIL,
-                        "failed to create stub directory [%s]", fullpath);
-        return ret;
+            ret = mkdir_p(fullpath, 0600, _gf_true);
+        else
+            ret = sys_rename(oldpath, fullpath);
+    }
+
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, BRS_MSG_BAD_OBJECT_DIR_FAIL,
+                "create-path=%s", fullpath, NULL);
+    return ret;
 
 error_return:
-        gf_msg (this->name, GF_LOG_ERROR, errno,
-                BRS_MSG_BAD_OBJECT_DIR_FAIL,
-                "Failed to verify stub directory [%s]", fullpath);
-        return -1;
+    gf_smsg(this->name, GF_LOG_ERROR, errno, BRS_MSG_BAD_OBJECT_DIR_FAIL,
+            "verify-path=%s", fullpath, NULL);
+    return -1;
 }
 
 /**
@@ -216,393 +213,399 @@ error_return:
  * directory.
  */
 static int
-br_stub_check_stub_file (xlator_t *this, char *path)
+br_stub_check_stub_file(xlator_t *this, char *path)
 {
-        int ret = 0;
-        int fd = -1;
-        struct stat st = {0,};
-
-        ret = sys_stat (path, &st);
-        if (!ret && !S_ISREG (st.st_mode))
-                goto error_return;
-        if (ret) {
-                if (errno != ENOENT)
-                        goto error_return;
-                fd = sys_creat (path, 0);
-                if (fd < 0)
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                BRS_MSG_BAD_OBJECT_DIR_FAIL,
-                                "Failed ot create stub file [%s]", path);
-        }
-
-        if (fd >= 0) {
-                sys_close (fd);
-                ret = 0;
-        }
+    int ret = 0;
+    int fd = -1;
+    struct stat st = {
+        0,
+    };
+
+    ret = sys_stat(path, &st);
+    if (!ret && !S_ISREG(st.st_mode))
+        goto error_return;
+    if (ret) {
+        if (errno != ENOENT)
+            goto error_return;
+        fd = sys_creat(path, 0);
+        if (fd < 0)
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    BRS_MSG_BAD_OBJECT_DIR_FAIL, "create-path=%s", path, NULL);
+    }
+
+    if (fd >= 0) {
+        sys_close(fd);
+        ret = 0;
+    }
 
-        return ret;
+    return ret;
 
 error_return:
-        gf_msg (this->name, GF_LOG_ERROR, errno,
-                BRS_MSG_BAD_OBJECT_DIR_FAIL, "Failed ot verify stub file [%s]", path);
-        return -1;
+    gf_smsg(this->name, GF_LOG_ERROR, errno, BRS_MSG_BAD_OBJECT_DIR_FAIL,
+            "verify-path=%s", path, NULL);
+    return -1;
 }
 
 int
-br_stub_dir_create (xlator_t *this, br_stub_private_t *priv)
+br_stub_dir_create(xlator_t *this, br_stub_private_t *priv)
 {
-        int          ret = -1;
-        char         fullpath[BR_PATH_MAX_PLUS] = {0,};
-        char         stub_gfid_path[BR_PATH_MAX_PLUS] = {0,};
-
-        gf_uuid_copy (priv->bad_object_dir_gfid, BR_BAD_OBJ_CONTAINER);
-
-        strncpy (fullpath, priv->stub_basepath, sizeof (fullpath));
-
-        snprintf (stub_gfid_path, sizeof (stub_gfid_path), "%s/stub-%s",
-                  priv->stub_basepath, uuid_utoa (priv->bad_object_dir_gfid));
-
-        ret = br_stub_check_stub_directory (this, fullpath);
-        if (ret)
-                goto out;
-        ret = br_stub_check_stub_file (this, stub_gfid_path);
-        if (ret)
-                goto out;
-
-        return 0;
+    int ret = -1;
+    char fullpath[BR_PATH_MAX_PLUS] = {
+        0,
+    };
+    char stub_gfid_path[BR_PATH_MAX_PLUS] = {
+        0,
+    };
+
+    gf_uuid_copy(priv->bad_object_dir_gfid, BR_BAD_OBJ_CONTAINER);
+
+    if (snprintf(fullpath, sizeof(fullpath), "%s", priv->stub_basepath) >=
+        sizeof(fullpath))
+        goto out;
+
+    if (snprintf(stub_gfid_path, sizeof(stub_gfid_path), "%s/stub-%s",
+                 priv->stub_basepath, uuid_utoa(priv->bad_object_dir_gfid)) >=
+        sizeof(stub_gfid_path))
+        goto out;
+
+    ret = br_stub_check_stub_directory(this, fullpath);
+    if (ret)
+        goto out;
+    ret = br_stub_check_stub_file(this, stub_gfid_path);
+    if (ret)
+        goto out;
+
+    return 0;
 
 out:
-        return -1;
+    return -1;
 }
 
 call_stub_t *
-__br_stub_dequeue (struct list_head *callstubs)
+__br_stub_dequeue(struct list_head *callstubs)
 {
-        call_stub_t *stub = NULL;
+    call_stub_t *stub = NULL;
 
-        if (!list_empty (callstubs)) {
-                stub = list_entry (callstubs->next, call_stub_t, list);
-                list_del_init (&stub->list);
-        }
+    if (!list_empty(callstubs)) {
+        stub = list_entry(callstubs->next, call_stub_t, list);
+        list_del_init(&stub->list);
+    }
 
-        return stub;
+    return stub;
 }
 
 void
-__br_stub_enqueue (struct list_head *callstubs, call_stub_t *stub)
+__br_stub_enqueue(struct list_head *callstubs, call_stub_t *stub)
 {
-        list_add_tail (&stub->list, callstubs);
+    list_add_tail(&stub->list, callstubs);
 }
 
 void
-br_stub_worker_enqueue (xlator_t *this, call_stub_t *stub)
+br_stub_worker_enqueue(xlator_t *this, call_stub_t *stub)
 {
-        br_stub_private_t    *priv = NULL;
-
-        priv = this->private;
-        pthread_mutex_lock (&priv->container.bad_lock);
-        {
-                __br_stub_enqueue (&priv->container.bad_queue, stub);
-                pthread_cond_signal (&priv->container.bad_cond);
-        }
-        pthread_mutex_unlock (&priv->container.bad_lock);
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+    pthread_mutex_lock(&priv->container.bad_lock);
+    {
+        __br_stub_enqueue(&priv->container.bad_queue, stub);
+        pthread_cond_signal(&priv->container.bad_cond);
+    }
+    pthread_mutex_unlock(&priv->container.bad_lock);
 }
 
 void *
-br_stub_worker (void *data)
+br_stub_worker(void *data)
 {
-        br_stub_private_t     *priv = NULL;
-        xlator_t         *this = NULL;
-        call_stub_t      *stub = NULL;
-
+    br_stub_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    call_stub_t *stub = NULL;
 
-        THIS = data;
-        this = data;
-        priv = this->private;
+    THIS = data;
+    this = data;
+    priv = this->private;
 
-        for (;;) {
-                pthread_mutex_lock (&priv->container.bad_lock);
-                {
-                        while (list_empty (&priv->container.bad_queue)) {
-                              (void) pthread_cond_wait (&priv->container.bad_cond,
-                                                        &priv->container.bad_lock);
-                        }
-
-                        stub = __br_stub_dequeue (&priv->container.bad_queue);
-                }
-                pthread_mutex_unlock (&priv->container.bad_lock);
+    for (;;) {
+        pthread_mutex_lock(&priv->container.bad_lock);
+        {
+            while (list_empty(&priv->container.bad_queue)) {
+                (void)pthread_cond_wait(&priv->container.bad_cond,
+                                        &priv->container.bad_lock);
+            }
 
-                if (stub) /* guard against spurious wakeups */
-                        call_resume (stub);
+            stub = __br_stub_dequeue(&priv->container.bad_queue);
         }
+        pthread_mutex_unlock(&priv->container.bad_lock);
 
-        return NULL;
+        if (stub) /* guard against spurious wakeups */
+            call_resume(stub);
+    }
+
+    return NULL;
 }
 
 int32_t
-br_stub_lookup_wrapper (call_frame_t *frame, xlator_t *this,
-                      loc_t *loc, dict_t *xattr_req)
+br_stub_lookup_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xattr_req)
 {
-        br_stub_private_t *priv        = NULL;
-        struct stat        lstatbuf    = {0};
-        int                ret         = 0;
-        int32_t            op_errno    = EINVAL;
-        int32_t            op_ret      = -1;
-        struct iatt        stbuf       = {0, };
-        struct iatt        postparent  = {0,};
-        dict_t            *xattr       = NULL;
-        gf_boolean_t       ver_enabled = _gf_false;
-
-        BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
-        priv = this->private;
-        BR_STUB_VER_COND_GOTO (priv, (!ver_enabled), done);
-
-        VALIDATE_OR_GOTO (loc, done);
-        if (gf_uuid_compare (loc->gfid, priv->bad_object_dir_gfid))
-                goto done;
-
-        ret = sys_lstat (priv->stub_basepath, &lstatbuf);
-        if (ret) {
-                gf_msg_debug (this->name, errno, "Stat failed on stub bad "
-                              "object dir");
-                op_errno = errno;
-                goto done;
-        } else if (!S_ISDIR (lstatbuf.st_mode)) {
-                gf_msg_debug (this->name, errno, "bad object container is not "
-                              "a directory");
-                op_errno = ENOTDIR;
-                goto done;
-        }
-
-        iatt_from_stat (&stbuf, &lstatbuf);
-        gf_uuid_copy (stbuf.ia_gfid, priv->bad_object_dir_gfid);
-
-        op_ret = op_errno = 0;
-        xattr = dict_new ();
-        if (!xattr) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-        }
+    br_stub_private_t *priv = NULL;
+    struct stat lstatbuf = {0};
+    int ret = 0;
+    int32_t op_errno = EINVAL;
+    int32_t op_ret = -1;
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    gf_boolean_t ver_enabled = _gf_false;
+
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), done);
+
+    VALIDATE_OR_GOTO(loc, done);
+    if (gf_uuid_compare(loc->gfid, priv->bad_object_dir_gfid))
+        goto done;
+
+    ret = sys_lstat(priv->stub_basepath, &lstatbuf);
+    if (ret) {
+        gf_msg_debug(this->name, errno,
+                     "Stat failed on stub bad "
+                     "object dir");
+        op_errno = errno;
+        goto done;
+    } else if (!S_ISDIR(lstatbuf.st_mode)) {
+        gf_msg_debug(this->name, errno,
+                     "bad object container is not "
+                     "a directory");
+        op_errno = ENOTDIR;
+        goto done;
+    }
+
+    iatt_from_stat(&stbuf, &lstatbuf);
+    gf_uuid_copy(stbuf.ia_gfid, priv->bad_object_dir_gfid);
+
+    op_ret = op_errno = 0;
+    xattr = dict_new();
+    if (!xattr) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+    }
 
 done:
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno,
-                             loc->inode, &stbuf, xattr, &postparent);
-        if (xattr)
-                dict_unref (xattr);
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, loc->inode, &stbuf,
+                        xattr, &postparent);
+    if (xattr)
+        dict_unref(xattr);
+    return 0;
 }
 
 static int
-is_bad_gfid_file_current (char *filename, uuid_t gfid)
+is_bad_gfid_file_current(char *filename, uuid_t gfid)
 {
-        char current_stub_gfid[GF_UUID_BUF_SIZE + 16] = {0, };
+    char current_stub_gfid[GF_UUID_BUF_SIZE + 16] = {
+        0,
+    };
 
-        snprintf (current_stub_gfid, sizeof current_stub_gfid,
-                  "stub-%s", uuid_utoa(gfid));
-        return (!strcmp(filename, current_stub_gfid));
+    snprintf(current_stub_gfid, sizeof current_stub_gfid, "stub-%s",
+             uuid_utoa(gfid));
+    return (!strcmp(filename, current_stub_gfid));
 }
 
 static void
-check_delete_stale_bad_file (xlator_t *this, char *filename)
+check_delete_stale_bad_file(xlator_t *this, char *filename)
 {
-        int             ret = 0;
-        struct stat     st = {0};
-        char            filepath[BR_PATH_MAX_PLUS] = {0};
-        br_stub_private_t    *priv = NULL;
+    int ret = 0;
+    struct stat st = {0};
+    char filepath[BR_PATH_MAX_PLUS] = {0};
+    br_stub_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (is_bad_gfid_file_current (filename, priv->bad_object_dir_gfid))
-                return;
+    if (is_bad_gfid_file_current(filename, priv->bad_object_dir_gfid))
+        return;
 
-        snprintf (filepath, sizeof (filepath), "%s/%s",
-                  priv->stub_basepath, filename);
+    snprintf(filepath, sizeof(filepath), "%s/%s", priv->stub_basepath,
+             filename);
 
-        ret = sys_stat (filepath, &st);
-        if (!ret && st.st_nlink == 1)
-                sys_unlink (filepath);
+    ret = sys_stat(filepath, &st);
+    if (!ret && st.st_nlink == 1)
+        sys_unlink(filepath);
 }
 
 static int
-br_stub_fill_readdir (fd_t *fd, br_stub_fd_t *fctx, DIR *dir, off_t off,
-                      size_t size, gf_dirent_t *entries)
+br_stub_fill_readdir(fd_t *fd, br_stub_fd_t *fctx, DIR *dir, off_t off,
+                     size_t size, gf_dirent_t *entries)
 {
-        off_t          in_case = -1;
-        off_t          last_off = 0;
-        size_t         filled = 0;
-        int            count = 0;
-        int32_t        this_size      = -1;
-        gf_dirent_t   *this_entry     = NULL;
-        xlator_t      *this           = NULL;
-        struct dirent *entry          = NULL;
-        struct dirent  scratch[2]     = {{0,},};
-
-        this = THIS;
-        if (!off) {
-                rewinddir (dir);
-        } else {
-                seekdir (dir, off);
+    off_t in_case = -1;
+    off_t last_off = 0;
+    size_t filled = 0;
+    int count = 0;
+    int32_t this_size = -1;
+    gf_dirent_t *this_entry = NULL;
+    xlator_t *this = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+
+    this = THIS;
+    if (!off) {
+        rewinddir(dir);
+    } else {
+        seekdir(dir, off);
 #ifndef GF_LINUX_HOST_OS
-                if ((u_long)telldir(dir) != off &&
-                    off != fctx->bad_object.dir_eof) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL,
-                                "seekdir(0x%llx) failed on dir=%p: "
-				"Invalid argument (offset reused from "
-				"another DIR * structure?)", off, dir);
-                        errno = EINVAL;
-                        count = -1;
-                        goto out;
-                }
-#endif /* GF_LINUX_HOST_OS */
+        if ((u_long)telldir(dir) != off && off != fctx->bad_object.dir_eof) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL, "off=(0x%llx)", off,
+                    "dir=%p", dir, NULL);
+            errno = EINVAL;
+            count = -1;
+            goto out;
         }
-
-        while (filled <= size) {
-                in_case = (u_long)telldir (dir);
-
-                if (in_case == -1) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                BRS_MSG_BAD_OBJECT_DIR_TELL_FAIL,
-                                "telldir failed on dir=%p: %s",
-                                dir, strerror (errno));
-                        goto out;
-                }
-
-                errno = 0;
-                entry = sys_readdir (dir, scratch);
-                if (!entry || errno != 0) {
-                        if (errno == EBADF) {
-                                gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                                        BRS_MSG_BAD_OBJECT_DIR_READ_FAIL,
-                                        "readdir failed on dir=%p: %s",
-                                        dir, strerror (errno));
-                                goto out;
-                        }
-                        break;
-                }
-
-                if (!strcmp (entry->d_name, ".") ||
-                    !strcmp (entry->d_name, ".."))
-                        continue;
-
-                if (!strncmp (entry->d_name, "stub-",
-                              strlen ("stub-"))) {
-                        check_delete_stale_bad_file (this, entry->d_name);
-                        continue;
-                }
-
-                this_size = max (sizeof (gf_dirent_t),
-                                 sizeof (gfs3_dirplist))
-                        + strlen (entry->d_name) + 1;
-
-                if (this_size + filled > size) {
-                        seekdir (dir, in_case);
-#ifndef GF_LINUX_HOST_OS
-                        if ((u_long)telldir(dir) != in_case &&
-                            in_case != fctx->bad_object.dir_eof) {
-				gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                        BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL,
-					"seekdir(0x%llx) failed on dir=%p: "
-					"Invalid argument (offset reused from "
-					"another DIR * structure?)",
-					in_case, dir);
-				errno = EINVAL;
-				count = -1;
-				goto out;
-                        }
 #endif /* GF_LINUX_HOST_OS */
-                        break;
-                }
-
-                this_entry = gf_dirent_for_name (entry->d_name);
-
-                if (!this_entry) {
-                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
-                                BRS_MSG_NO_MEMORY,
-                                "could not create gf_dirent for entry %s: (%s)",
-                                entry->d_name, strerror (errno));
-                        goto out;
-                }
-                /*
-                 * we store the offset of next entry here, which is
-                 * probably not intended, but code using syncop_readdir()
-                 * (glfs-heal.c, afr-self-heald.c, pump.c) rely on it
-                 * for directory read resumption.
-                 */
-                last_off = (u_long)telldir(dir);
-                this_entry->d_off = last_off;
-                this_entry->d_ino = entry->d_ino;
-
-                list_add_tail (&this_entry->list, &entries->list);
-
-                filled += this_size;
-                count++;
-        }
+    }
+
+    while (filled <= size) {
+        in_case = (u_long)telldir(dir);
 
-        if ((!sys_readdir (dir, scratch) && (errno == 0))) {
-                /* Indicate EOF */
-                errno = ENOENT;
-                /* Remember EOF offset for later detection */
-                fctx->bad_object.dir_eof = last_off;
+        if (in_case == -1) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_BAD_OBJECT_DIR_TELL_FAIL, "dir=%p", dir, "err=%s",
+                    strerror(errno), NULL);
+            goto out;
         }
-out:
-        return count;
-}
 
-int32_t
-br_stub_readdir_wrapper (call_frame_t *frame, xlator_t *this,
-                         fd_t *fd, size_t size, off_t off, dict_t *xdata)
-{
-        br_stub_fd_t         *fctx           = NULL;
-        DIR                  *dir            = NULL;
-        int                   ret            = -1;
-        int32_t               op_ret         = -1;
-        int32_t               op_errno       = 0;
-        int                   count          = 0;
-        gf_dirent_t           entries;
-        gf_boolean_t          xdata_unref    = _gf_false;
-        dict_t               *dict           = NULL;
-
-        INIT_LIST_HEAD (&entries.list);
-
-        fctx = br_stub_fd_ctx_get (this, fd);
-        if (!fctx) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_GET_FD_CONTEXT_FAILED,
-                        "pfd is NULL, fd=%p", fd);
-                op_errno = -ret;
-                goto done;
+        errno = 0;
+        entry = sys_readdir(dir, scratch);
+        if (!entry || errno != 0) {
+            if (errno == EBADF) {
+                gf_smsg(THIS->name, GF_LOG_WARNING, 0,
+                        BRS_MSG_BAD_OBJECT_DIR_READ_FAIL, "dir=%p", dir,
+                        "err=%s", strerror(errno), NULL);
+                goto out;
+            }
+            break;
         }
 
-        dir = fctx->bad_object.dir;
+        if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+            continue;
 
-        if (!dir) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_BAD_HANDLE_DIR_NULL,
-                        "dir is NULL for fd=%p", fd);
-                op_errno = EINVAL;
-                goto done;
+        if (!strncmp(entry->d_name, "stub-", strlen("stub-"))) {
+            check_delete_stale_bad_file(this, entry->d_name);
+            continue;
         }
 
-        count = br_stub_fill_readdir (fd, fctx, dir, off, size, &entries);
+        this_size = max(sizeof(gf_dirent_t), sizeof(gfs3_dirplist)) +
+                    strlen(entry->d_name) + 1;
 
-        /* pick ENOENT to indicate EOF */
-        op_errno = errno;
-        op_ret = count;
+        if (this_size + filled > size) {
+            seekdir(dir, in_case);
+#ifndef GF_LINUX_HOST_OS
+            if ((u_long)telldir(dir) != in_case &&
+                in_case != fctx->bad_object.dir_eof) {
+                gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                        BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL, "in_case=(0x%llx)",
+                        in_case, "dir=%p", dir, NULL);
+                errno = EINVAL;
+                count = -1;
+                goto out;
+            }
+#endif /* GF_LINUX_HOST_OS */
+            break;
+        }
+
+        this_entry = gf_dirent_for_name(entry->d_name);
 
-        dict = xdata;
-        (void) br_stub_bad_objects_path (this, fd, &entries, &dict);
-        if (!xdata && dict) {
-                xdata = dict;
-                xdata_unref = _gf_true;
+        if (!this_entry) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_CREATE_GF_DIRENT_FAILED, "entry-name=%s",
+                    entry->d_name, "err=%s", strerror(errno), NULL);
+            goto out;
         }
+        /*
+         * we store the offset of next entry here, which is
+         * probably not intended, but code using syncop_readdir()
+         * (glfs-heal.c, afr-self-heald.c, pump.c) rely on it
+         * for directory read resumption.
+         */
+        last_off = (u_long)telldir(dir);
+        this_entry->d_off = last_off;
+        this_entry->d_ino = entry->d_ino;
+
+        list_add_tail(&this_entry->list, &entries->list);
+
+        filled += this_size;
+        count++;
+    }
+
+    if ((!sys_readdir(dir, scratch) && (errno == 0))) {
+        /* Indicate EOF */
+        errno = ENOENT;
+        /* Remember EOF offset for later detection */
+        fctx->bad_object.dir_eof = last_off;
+    }
+out:
+    return count;
+}
+
+int32_t
+br_stub_readdir_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        size_t size, off_t off, dict_t *xdata)
+{
+    br_stub_fd_t *fctx = NULL;
+    DIR *dir = NULL;
+    int ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int count = 0;
+    gf_dirent_t entries;
+    gf_boolean_t xdata_unref = _gf_false;
+    dict_t *dict = NULL;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    fctx = br_stub_fd_ctx_get(this, fd);
+    if (!fctx) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_GET_FD_CONTEXT_FAILED,
+                "fd=%p", fd, NULL);
+        op_errno = -ret;
+        goto done;
+    }
+
+    dir = fctx->bad_object.dir;
+
+    if (!dir) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_BAD_HANDLE_DIR_NULL,
+                "fd=%p", fd, NULL);
+        op_errno = EINVAL;
+        goto done;
+    }
+
+    count = br_stub_fill_readdir(fd, fctx, dir, off, size, &entries);
+
+    /* pick ENOENT to indicate EOF */
+    op_errno = errno;
+    op_ret = count;
+
+    dict = xdata;
+    (void)br_stub_bad_objects_path(this, fd, &entries, &dict);
+    if (!xdata && dict) {
+        xdata = dict;
+        xdata_unref = _gf_true;
+    }
 
 done:
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries, xdata);
-        gf_dirent_free (&entries);
-        if (xdata_unref)
-                dict_unref (xdata);
-        return 0;
+    STACK_UNWIND_STRICT(readdir, frame, op_ret, op_errno, &entries, xdata);
+    gf_dirent_free(&entries);
+    if (xdata_unref)
+        dict_unref(xdata);
+    return 0;
 }
 
 /**
@@ -638,176 +641,156 @@ done:
  *    path will not be printed in scrub and only the gfid will be there.
  **/
 int
-br_stub_bad_objects_path (xlator_t *this, fd_t *fd, gf_dirent_t *entries,
-                          dict_t **dict)
+br_stub_bad_objects_path(xlator_t *this, fd_t *fd, gf_dirent_t *entries,
+                         dict_t **dict)
 {
-        gf_dirent_t     *entry    = NULL;
-        inode_t         *inode    = NULL;
-        char            *hpath    = NULL;
-        uuid_t           gfid     = {0};
-        int              ret      = -1;
-        dict_t          *tmp_dict = NULL;
-        char             str_gfid[64] = {0};
-
-        if (list_empty(&entries->list))
-                return 0;
+    gf_dirent_t *entry = NULL;
+    inode_t *inode = NULL;
+    char *hpath = NULL;
+    uuid_t gfid = {0};
+    int ret = -1;
+    dict_t *tmp_dict = NULL;
+    char str_gfid[64] = {0};
+
+    if (list_empty(&entries->list))
+        return 0;
 
-        tmp_dict = *dict;
+    tmp_dict = *dict;
 
+    if (!tmp_dict) {
+        tmp_dict = dict_new();
+        /*
+         * If the allocation of dict fails then no need treat it
+         * it as a error. This path (or function) is executed when
+         * "gluster volume bitrot <volume name> scrub status" is
+         * executed, to get the list of the corrupted objects.
+         * And the motive of this function is to get the paths of
+         * the corrupted objects. If the dict allocation fails, then
+         * the scrub status will only show the gfids of those corrupted
+         * objects (which is the behavior as of the time of this patch
+         * being worked upon). So just return and only the gfids will
+         * be shown.
+         */
         if (!tmp_dict) {
-                tmp_dict = dict_new ();
-                /*
-                 * If the allocation of dict fails then no need treat it
-                 * it as a error. This path (or function) is executed when
-                 * "gluster volume bitrot <volume name> scrub status" is
-                 * executed, to get the list of the corrupted objects.
-                 * And the motive of this function is to get the paths of
-                 * the corrupted objects. If the dict allocation fails, then
-                 * the scrub status will only show the gfids of those corrupted
-                 * objects (which is the behavior as of the time of this patch
-                 * being worked upon). So just return and only the gfids will
-                 * be shown.
-                 */
-                if (!tmp_dict) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_NO_MEMORY,
-                                "failed to allocate new dict for saving the paths "
-                                "of the corrupted objects. Scrub status will only "
-                                "display the gfid");
-                        goto out;
-                }
-        }
-
-        list_for_each_entry (entry, &entries->list, list) {
-                gf_uuid_clear (gfid);
-                gf_uuid_parse (entry->d_name, gfid);
-
-                inode = inode_find (fd->inode->table, gfid);
-
-                /* No need to check the return value here.
-                 * Because @hpath is examined.
-                 */
-                (void) br_stub_get_path_of_gfid (this, fd->inode, inode,
-                                                 gfid, &hpath);
-
-                if (hpath) {
-                        gf_msg_debug (this->name, 0, "path of the corrupted "
-                                      "object (gfid: %s) is %s",
-                                      uuid_utoa (gfid), hpath);
-                        br_stub_entry_xattr_fill (this, hpath, entry, tmp_dict);
-                } else
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                BRS_MSG_PATH_GET_FAILED,
-                                "failed to get the path for the inode %s",
-                                uuid_utoa_r (gfid, str_gfid));
-
-                inode = NULL;
-                hpath = NULL;
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ALLOC_FAILED, NULL);
+            goto out;
         }
+    }
 
-         ret = 0;
-         *dict = tmp_dict;
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        gf_uuid_clear(gfid);
+        gf_uuid_parse(entry->d_name, gfid);
 
-out:
-         return ret;
- }
+        inode = inode_find(fd->inode->table, gfid);
 
-int
-br_stub_get_path_of_gfid (xlator_t *this, inode_t *parent, inode_t *inode,
-                          uuid_t gfid, char **path)
-{
-        int32_t    ret = -1;
-        char       gfid_str[64] = {0};
-
-        GF_VALIDATE_OR_GOTO ("bitrot-stub", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, parent, out);
-        GF_VALIDATE_OR_GOTO (this->name, path, out);
-
-        /* No need to validate the @inode for hard resolution. Because inode
-         * can be NULL and if it is NULL, then syncop_gfid_to_path_hard will
-         * allocate a new inode and proceed. So no need to bother about
-         * @inode. Because we need it only to send a syncop_getxattr call
-         * from inside syncop_gfid_to_path_hard. And getxattr fetches the
-         * path from the backend.
+        /* No need to check the return value here.
+         * Because @hpath is examined.
          */
-        ret = syncop_gfid_to_path_hard (parent->table, FIRST_CHILD (this), gfid,
-                                        inode, path, _gf_true);
+        (void)br_stub_get_path_of_gfid(this, fd->inode, inode, gfid, &hpath);
 
-        /*
-         * This is to handle those corrupted files which does not contain
-         * the gfid2path xattr in the backend (because they were created
-         * when the option was OFF OR it was upgraded from a version before
-         * gfid2path was brought in.
-         * Ideally posix should be returning ret < 0 i.e. error if the
-         * gfid2path xattr is not present. But for some reason it is
-         * returning success and path as "". THis is causing problems.
-         * For now handling it by adding extra checks. But the better way
-         * is to make posix return error if gfid2path xattr is absent.
-         * When that is done remove below if block and also this entire
-         * comment.
-         */
-        if (ret >= 0 && !strlen (*path)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRS_MSG_PATH_GET_FAILED,
-                        "path for the object %s is %s. Going for in memory path",
-                        uuid_utoa_r (gfid, gfid_str), *path);
-                ret = -1;
-        }
+        if (hpath) {
+            gf_msg_debug(this->name, 0,
+                         "path of the corrupted "
+                         "object (gfid: %s) is %s",
+                         uuid_utoa(gfid), hpath);
+            br_stub_entry_xattr_fill(this, hpath, entry, tmp_dict);
+        } else
+            gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_PATH_GET_FAILED,
+                    "gfid=%s", uuid_utoa_r(gfid, str_gfid), NULL);
 
-        /*
-         * Try with soft resolution of path if hard resolve fails. Because
-         * checking the xattr on disk to get the path of a inode (or gfid)
-         * is dependent on whether that option is enabled in the posix
-         * xlator or not. If it is not enabled, then hard resolution by
-         * checking the on disk xattr fails.
-         *
-         * Thus in such situations fall back to the soft resolution which
-         * mainly depends on the inode_path() function. And for using
-         * inode_path, @inode has to be linked i.e. a successful lookup should
-         * have happened on the gfid (or the path) to link the inode to the
-         * inode table. And if @inode is NULL, means, the inode has not been
-         * found in the inode table and better not to do inode_path() on the
-         * inode which has not been linked.
-         */
-        if (ret < 0 && inode)
-                ret = syncop_gfid_to_path_hard (parent->table,
-                                                FIRST_CHILD (this), gfid, inode,
-                                                path, _gf_false);
+        inode = NULL;
+        hpath = NULL;
+    }
+
+    ret = 0;
+    *dict = tmp_dict;
 
 out:
-        return ret;
+    return ret;
 }
 
+int
+br_stub_get_path_of_gfid(xlator_t *this, inode_t *parent, inode_t *inode,
+                         uuid_t gfid, char **path)
+{
+    int32_t ret = -1;
+    char gfid_str[64] = {0};
+
+    GF_VALIDATE_OR_GOTO("bitrot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent, out);
+    GF_VALIDATE_OR_GOTO(this->name, path, out);
+
+    /* Above, No need to validate the @inode for hard resolution. Because
+     * inode can be NULL and if it is NULL, then syncop_gfid_to_path_hard
+     * will allocate a new inode and proceed. So no need to bother about
+     * @inode. Because we need it only to send a syncop_getxattr call
+     * from inside syncop_gfid_to_path_hard. And getxattr fetches the
+     * path from the backend.
+     */
+
+    ret = syncop_gfid_to_path_hard(parent->table, FIRST_CHILD(this), gfid,
+                                   inode, path, _gf_true);
+    if (ret < 0)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_PATH_GET_FAILED,
+                "gfid=%s", uuid_utoa_r(gfid, gfid_str), NULL);
+
+    /*
+     * Try with soft resolution of path if hard resolve fails. Because
+     * checking the xattr on disk to get the path of a inode (or gfid)
+     * is dependent on whether that option is enabled in the posix
+     * xlator or not. If it is not enabled, then hard resolution by
+     * checking the on disk xattr fails.
+     *
+     * Thus in such situations fall back to the soft resolution which
+     * mainly depends on the inode_path() function. And for using
+     * inode_path, @inode has to be linked i.e. a successful lookup should
+     * have happened on the gfid (or the path) to link the inode to the
+     * inode table. And if @inode is NULL, means, the inode has not been
+     * found in the inode table and better not to do inode_path() on the
+     * inode which has not been linked.
+     */
+    if (ret < 0 && inode) {
+        ret = syncop_gfid_to_path_hard(parent->table, FIRST_CHILD(this), gfid,
+                                       inode, path, _gf_false);
+        if (ret < 0)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_PATH_GET_FAILED,
+                    "from-memory  gfid=%s", uuid_utoa_r(gfid, gfid_str), NULL);
+    }
+
+out:
+    return ret;
+}
 
 /**
-* NOTE: If the file has multiple hardlinks (in gluster volume
-* namespace), the path would be one of the hardlinks. Its upto
-* the user to find the remaining hardlinks (using find -samefile)
-* and remove them.
-**/
+ * NOTE: If the file has multiple hardlinks (in gluster volume
+ * namespace), the path would be one of the hardlinks. Its up to
+ * the user to find the remaining hardlinks (using find -samefile)
+ * and remove them.
+ **/
 void
-br_stub_entry_xattr_fill (xlator_t *this, char *hpath, gf_dirent_t *entry,
-                          dict_t *dict)
+br_stub_entry_xattr_fill(xlator_t *this, char *hpath, gf_dirent_t *entry,
+                         dict_t *dict)
 {
-                int32_t    ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, hpath, out);
-
-        /*
-         * Use the entry->d_name (which is nothing but the gfid of the
-         * corrupted object) as the key. And the value will be the actual
-         * path of that object (or file).
-         *
-         * ALso ignore the dict_set errors. scrubber will get the gfid of
-         * the corrupted object for sure. So, for now lets just log the
-         * dict_set_dynstr failure and move on.
-         */
-
-        ret = dict_set_dynstr (dict, entry->d_name, hpath);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRS_MSG_DICT_SET_FAILED,
-                        "failed to set the actual path %s as the value in the "
-                        "dict for the corrupted object %s", hpath,
-                        entry->d_name);
+    int32_t ret = -1;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, hpath, out);
+
+    /*
+     * Use the entry->d_name (which is nothing but the gfid of the
+     * corrupted object) as the key. And the value will be the actual
+     * path of that object (or file).
+     *
+     * ALso ignore the dict_set errors. scrubber will get the gfid of
+     * the corrupted object for sure. So, for now lets just log the
+     * dict_set_dynstr failure and move on.
+     */
+
+    ret = dict_set_dynstr(dict, entry->d_name, hpath);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_DICT_SET_FAILED,
+                "path=%s", hpath, "object-name=%s", entry->d_name, NULL);
 out:
-        return;
+    return;
 }
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h b/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
index 1652a3ebf8d..9d93caf069f 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
@@ -11,25 +11,26 @@
 #ifndef _BR_MEM_TYPES_H
 #define _BR_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum br_mem_types {
-        gf_br_stub_mt_private_t   = gf_common_mt_end + 1,
-        gf_br_stub_mt_version_t,
-        gf_br_stub_mt_inode_ctx_t,
-        gf_br_stub_mt_signature_t,
-        gf_br_mt_br_private_t,
-        gf_br_mt_br_child_t,
-        gf_br_mt_br_object_t,
-        gf_br_mt_br_ob_n_wk_t,
-        gf_br_mt_br_scrubber_t,
-        gf_br_mt_br_fsscan_entry_t,
-        gf_br_stub_mt_br_stub_fd_t,
-        gf_br_stub_mt_br_scanner_freq_t,
-        gf_br_stub_mt_sigstub_t,
-        gf_br_mt_br_child_event_t,
-        gf_br_stub_mt_misc,
-        gf_br_stub_mt_end,
+    gf_br_stub_mt_private_t = gf_common_mt_end + 1,
+    gf_br_stub_mt_version_t,
+    gf_br_stub_mt_inode_ctx_t,
+    gf_br_stub_mt_signature_t,
+    gf_br_mt_br_private_t,
+    gf_br_mt_br_child_t,
+    gf_br_mt_br_object_t,
+    gf_br_mt_br_ob_n_wk_t,
+    gf_br_mt_br_scrubber_t,
+    gf_br_mt_br_fsscan_entry_t,
+    gf_br_stub_mt_br_stub_fd_t,
+    gf_br_stub_mt_br_scanner_freq_t,
+    gf_br_stub_mt_sigstub_t,
+    gf_br_mt_br_child_event_t,
+    gf_br_stub_mt_misc,
+    gf_br_mt_br_worker_t,
+    gf_br_stub_mt_end,
 };
 
 #endif
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h b/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h
index 2ade4f03997..6c15a166f18 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h
@@ -11,7 +11,7 @@
 #ifndef _BITROT_STUB_MESSAGES_H_
 #define _BITROT_STUB_MESSAGES_H_
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
 
 /* To add new message IDs, append new identifiers at the end of the list.
  *
@@ -23,40 +23,95 @@
  * glfs-message-id.h.
  */
 
-GLFS_MSGID(BITROT_STUB,
-        BRS_MSG_NO_MEMORY,
-        BRS_MSG_SET_EVENT_FAILED,
-        BRS_MSG_MEM_ACNT_FAILED,
-        BRS_MSG_CREATE_FRAME_FAILED,
-        BRS_MSG_SET_CONTEXT_FAILED,
-        BRS_MSG_CHANGE_VERSION_FAILED,
-        BRS_MSG_ADD_FD_TO_LIST_FAILED,
-        BRS_MSG_SET_FD_CONTEXT_FAILED,
-        BRS_MSG_CREATE_ANONYMOUS_FD_FAILED,
-        BRS_MSG_NO_CHILD,
-        BRS_MSG_STUB_ALLOC_FAILED,
-        BRS_MSG_GET_INODE_CONTEXT_FAILED,
-        BRS_MSG_CANCEL_SIGN_THREAD_FAILED,
-        BRS_MSG_ADD_FD_TO_INODE,
-        BRS_MSG_SIGN_VERSION_ERROR,
-        BRS_MSG_BAD_OBJ_MARK_FAIL,
-        BRS_MSG_NON_SCRUB_BAD_OBJ_MARK,
-        BRS_MSG_REMOVE_INTERNAL_XATTR,
-        BRS_MSG_SET_INTERNAL_XATTR,
-        BRS_MSG_BAD_OBJECT_ACCESS,
-        BRS_MSG_BAD_CONTAINER_FAIL,
-        BRS_MSG_BAD_OBJECT_DIR_FAIL,
-        BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL,
-        BRS_MSG_BAD_OBJECT_DIR_TELL_FAIL,
-        BRS_MSG_BAD_OBJECT_DIR_READ_FAIL,
-        BRS_MSG_GET_FD_CONTEXT_FAILED,
-        BRS_MSG_BAD_HANDLE_DIR_NULL,
-        BRS_MSG_BAD_OBJ_THREAD_FAIL,
-        BRS_MSG_BAD_OBJ_DIR_CLOSE_FAIL,
-        BRS_MSG_LINK_FAIL,
-        BRS_MSG_BAD_OBJ_UNLINK_FAIL,
-        BRS_MSG_DICT_SET_FAILED,
-        BRS_MSG_PATH_GET_FAILED
-);
+GLFS_MSGID(BITROT_STUB, BRS_MSG_NO_MEMORY, BRS_MSG_SET_EVENT_FAILED,
+           BRS_MSG_MEM_ACNT_FAILED, BRS_MSG_CREATE_FRAME_FAILED,
+           BRS_MSG_SET_CONTEXT_FAILED, BRS_MSG_CHANGE_VERSION_FAILED,
+           BRS_MSG_ADD_FD_TO_LIST_FAILED, BRS_MSG_SET_FD_CONTEXT_FAILED,
+           BRS_MSG_CREATE_ANONYMOUS_FD_FAILED, BRS_MSG_NO_CHILD,
+           BRS_MSG_STUB_ALLOC_FAILED, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+           BRS_MSG_CANCEL_SIGN_THREAD_FAILED, BRS_MSG_ADD_FD_TO_INODE,
+           BRS_MSG_SIGN_VERSION_ERROR, BRS_MSG_BAD_OBJ_MARK_FAIL,
+           BRS_MSG_NON_SCRUB_BAD_OBJ_MARK, BRS_MSG_REMOVE_INTERNAL_XATTR,
+           BRS_MSG_SET_INTERNAL_XATTR, BRS_MSG_BAD_OBJECT_ACCESS,
+           BRS_MSG_BAD_CONTAINER_FAIL, BRS_MSG_BAD_OBJECT_DIR_FAIL,
+           BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL, BRS_MSG_BAD_OBJECT_DIR_TELL_FAIL,
+           BRS_MSG_BAD_OBJECT_DIR_READ_FAIL, BRS_MSG_GET_FD_CONTEXT_FAILED,
+           BRS_MSG_BAD_HANDLE_DIR_NULL, BRS_MSG_BAD_OBJ_THREAD_FAIL,
+           BRS_MSG_BAD_OBJ_DIR_CLOSE_FAIL, BRS_MSG_LINK_FAIL,
+           BRS_MSG_BAD_OBJ_UNLINK_FAIL, BRS_MSG_DICT_SET_FAILED,
+           BRS_MSG_PATH_GET_FAILED, BRS_MSG_NULL_LOCAL,
+           BRS_MSG_SPAWN_SIGN_THRD_FAILED, BRS_MSG_KILL_SIGN_THREAD,
+           BRS_MSG_NON_BITD_PID, BRS_MSG_SIGN_PREPARE_FAIL,
+           BRS_MSG_USING_DEFAULT_THREAD_SIZE, BRS_MSG_ALLOC_MEM_FAILED,
+           BRS_MSG_DICT_ALLOC_FAILED, BRS_MSG_CREATE_GF_DIRENT_FAILED,
+           BRS_MSG_ALLOC_FAILED, BRS_MSG_PATH_XATTR_GET_FAILED,
+           BRS_MSG_VERSION_PREPARE_FAIL);
 
+#define BRS_MSG_MEM_ACNT_FAILED_STR "Memory accounting init failed"
+#define BRS_MSG_BAD_OBJ_THREAD_FAIL_STR "pthread_init failed"
+#define BRS_MSG_USING_DEFAULT_THREAD_SIZE_STR "Using default thread stack size"
+#define BRS_MSG_NO_CHILD_STR "FATAL: no children"
+#define BRS_MSG_SPAWN_SIGN_THRD_FAILED_STR                                     \
+    "failed to create the new thread for signer"
+#define BRS_MSG_BAD_CONTAINER_FAIL_STR                                         \
+    "failed to launch the thread for storing bad gfids"
+#define BRS_MSG_CANCEL_SIGN_THREAD_FAILED_STR                                  \
+    "Could not cancel sign serializer thread"
+#define BRS_MSG_KILL_SIGN_THREAD_STR "killed the signer thread"
+#define BRS_MSG_GET_INODE_CONTEXT_FAILED_STR                                   \
+    "failed to init the inode context for the inode"
+#define BRS_MSG_ADD_FD_TO_INODE_STR "failed to add fd to the inode"
+#define BRS_MSG_NO_MEMORY_STR "local allocation failed"
+#define BRS_MSG_BAD_OBJECT_ACCESS_STR "bad object accessed. Returning"
+#define BRS_MSG_SIGN_VERSION_ERROR_STR "Signing version exceeds current version"
+#define BRS_MSG_NON_BITD_PID_STR                                               \
+    "PID from where signature request came, does not belong to bit-rot "       \
+    "daemon. Unwinding the fop"
+#define BRS_MSG_SIGN_PREPARE_FAIL_STR                                          \
+    "failed to prepare the signature. Unwinding the fop"
+#define BRS_MSG_VERSION_PREPARE_FAIL_STR                                       \
+    "failed to prepare the version. Unwinding the fop"
+#define BRS_MSG_STUB_ALLOC_FAILED_STR "failed to allocate stub fop, Unwinding"
+#define BRS_MSG_BAD_OBJ_MARK_FAIL_STR "failed to mark object as bad"
+#define BRS_MSG_NON_SCRUB_BAD_OBJ_MARK_STR                                     \
+    "bad object marking is not from the scrubber"
+#define BRS_MSG_ALLOC_MEM_FAILED_STR "failed to allocate memory"
+#define BRS_MSG_SET_INTERNAL_XATTR_STR "called on the internal xattr"
+#define BRS_MSG_REMOVE_INTERNAL_XATTR_STR "removexattr called on internal xattr"
+#define BRS_MSG_CREATE_ANONYMOUS_FD_FAILED_STR                                 \
+    "failed to create anonymous fd for the inode"
+#define BRS_MSG_ADD_FD_TO_LIST_FAILED_STR "failed add fd to the list"
+#define BRS_MSG_SET_FD_CONTEXT_FAILED_STR                                      \
+    "failed to set the fd context for the file"
+#define BRS_MSG_NULL_LOCAL_STR "local is NULL"
+#define BRS_MSG_DICT_ALLOC_FAILED_STR                                          \
+    "dict allocation failed: cannot send IPC FOP to changelog"
+#define BRS_MSG_SET_EVENT_FAILED_STR "cannot set release event in dict"
+#define BRS_MSG_CREATE_FRAME_FAILED_STR "create_frame() failure"
+#define BRS_MSG_BAD_OBJ_DIR_CLOSE_FAIL_STR "closedir error"
+#define BRS_MSG_LINK_FAIL_STR "failed to record gfid"
+#define BRS_MSG_BAD_OBJ_UNLINK_FAIL_STR                                        \
+    "failed to delete bad object link from quaratine directory"
+#define BRS_MSG_BAD_OBJECT_DIR_FAIL_STR "failed stub directory"
+#define BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL_STR                                   \
+    "seekdir failed. Invalid argument (offset reused from another DIR * "      \
+    "structure)"
+#define BRS_MSG_BAD_OBJECT_DIR_TELL_FAIL_STR "telldir failed on dir"
+#define BRS_MSG_BAD_OBJECT_DIR_READ_FAIL_STR "readdir failed on dir"
+#define BRS_MSG_CREATE_GF_DIRENT_FAILED_STR "could not create gf_dirent"
+#define BRS_MSG_GET_FD_CONTEXT_FAILED_STR "pfd is NULL"
+#define BRS_MSG_BAD_HANDLE_DIR_NULL_STR "dir if NULL"
+#define BRS_MSG_ALLOC_FAILED_STR                                               \
+    "failed to allocate new dict for saving the paths of the corrupted "       \
+    "objects. Scrub status will only display the gfid"
+#define BRS_MSG_PATH_GET_FAILED_STR "failed to get the path"
+#define BRS_MSG_PATH_XATTR_GET_FAILED_STR                                      \
+    "failed to get the path xattr from disk for the gfid. Trying to get path " \
+    "from the memory"
+#define BRS_MSG_DICT_SET_FAILED_STR                                            \
+    "failed to set the actual path as the value in the dict for the "          \
+    "corrupted object"
+#define BRS_MSG_SET_CONTEXT_FAILED_STR                                         \
+    "could not set fd context for release callback"
+#define BRS_MSG_CHANGE_VERSION_FAILED_STR "change version failed"
 #endif /* !_BITROT_STUB_MESSAGES_H_ */
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub.c b/xlators/features/bit-rot/src/stub/bit-rot-stub.c
index 830357070d6..447dd47ff41 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub.c
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub.c
@@ -12,180 +12,193 @@
 #include <sys/uio.h>
 #include <signal.h>
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
 #include "changelog.h"
-#include "compat-errno.h"
-#include "call-stub.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/call-stub.h>
 
 #include "bit-rot-stub.h"
 #include "bit-rot-stub-mem-types.h"
 #include "bit-rot-stub-messages.h"
 #include "bit-rot-common.h"
 
-#define BR_STUB_REQUEST_COOKIE  0x1
+#define BR_STUB_REQUEST_COOKIE 0x1
 
-void *br_stub_signth (void *);
+void
+br_stub_lock_cleaner(void *arg)
+{
+    pthread_mutex_t *clean_mutex = arg;
+
+    pthread_mutex_unlock(clean_mutex);
+    return;
+}
+
+void *
+br_stub_signth(void *);
 
 struct br_stub_signentry {
-        unsigned long v;
+    unsigned long v;
 
-        call_stub_t *stub;
+    call_stub_t *stub;
 
-        struct list_head list;
+    struct list_head list;
 };
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int32_t ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_br_stub_mt_end + 1);
-
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRS_MSG_MEM_ACNT_FAILED,
-                        "Memory accounting init failed");
-                return ret;
-        }
+    int32_t ret = -1;
 
+    if (!this)
         return ret;
-}
-
-int32_t
-br_stub_bad_object_container_init (xlator_t *this, br_stub_private_t *priv)
-{
-        pthread_attr_t  w_attr;
-        int32_t         ret          = -1;
-
-        ret = pthread_cond_init(&priv->container.bad_cond, NULL);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_BAD_OBJ_THREAD_FAIL,
-                        "pthread_cond_init failed (%d)", ret);
-                goto out;
-        }
-
-        ret = pthread_mutex_init(&priv->container.bad_lock, NULL);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_BAD_OBJ_THREAD_FAIL,
-                        "pthread_mutex_init failed (%d)", ret);
-                goto cleanup_cond;
-        }
 
-        ret = pthread_attr_init (&w_attr);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_BAD_OBJ_THREAD_FAIL,
-                        "pthread_attr_init failed (%d)", ret);
-                goto cleanup_lock;
-        }
-
-        ret = pthread_attr_setstacksize (&w_attr, BAD_OBJECT_THREAD_STACK_SIZE);
-        if (ret == EINVAL) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_BAD_OBJ_THREAD_FAIL,
-                        "Using default thread stack size");
-        }
+    ret = xlator_mem_acct_init(this, gf_br_stub_mt_end + 1);
 
-        INIT_LIST_HEAD (&priv->container.bad_queue);
-        ret = br_stub_dir_create (this, priv);
-        if (ret < 0)
-                goto cleanup_lock;
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_MEM_ACNT_FAILED, NULL);
+        return ret;
+    }
 
-        ret = gf_thread_create (&priv->container.thread, &w_attr,
-                                br_stub_worker, this, "brswrker");
-        if (ret)
-                goto cleanup_attr;
+    return ret;
+}
 
-        return 0;
+int
+br_stub_bad_object_container_init(xlator_t *this, br_stub_private_t *priv)
+{
+    pthread_attr_t w_attr;
+    int ret = -1;
+
+    ret = pthread_cond_init(&priv->container.bad_cond, NULL);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_THREAD_FAIL,
+                "cond_init ret=%d", ret, NULL);
+        goto out;
+    }
+
+    ret = pthread_mutex_init(&priv->container.bad_lock, NULL);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_THREAD_FAIL,
+                "mutex_init ret=%d", ret, NULL);
+        goto cleanup_cond;
+    }
+
+    ret = pthread_attr_init(&w_attr);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_THREAD_FAIL,
+                "attr_init ret=%d", ret, NULL);
+        goto cleanup_lock;
+    }
+
+    ret = pthread_attr_setstacksize(&w_attr, BAD_OBJECT_THREAD_STACK_SIZE);
+    if (ret == EINVAL) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0,
+                BRS_MSG_USING_DEFAULT_THREAD_SIZE, NULL);
+    }
+
+    INIT_LIST_HEAD(&priv->container.bad_queue);
+    ret = br_stub_dir_create(this, priv);
+    if (ret < 0)
+        goto cleanup_lock;
+
+    ret = gf_thread_create(&priv->container.thread, &w_attr, br_stub_worker,
+                           this, "brswrker");
+    if (ret)
+        goto cleanup_attr;
+
+    return 0;
 
 cleanup_attr:
-        pthread_attr_destroy (&w_attr);
+    pthread_attr_destroy(&w_attr);
 cleanup_lock:
-        pthread_mutex_destroy (&priv->container.bad_lock);
+    pthread_mutex_destroy(&priv->container.bad_lock);
 cleanup_cond:
-        pthread_cond_destroy (&priv->container.bad_cond);
+    pthread_cond_destroy(&priv->container.bad_cond);
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int32_t ret = 0;
-        char *tmp = NULL;
-        struct timeval tv = {0,};
-        br_stub_private_t *priv = NULL;
+    int ret = 0;
+    char *tmp = NULL;
+    struct timeval tv = {
+        0,
+    };
+    br_stub_private_t *priv = NULL;
 
-        if (!this->children) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_NO_CHILD,
-                        "FATAL: no children");
-                goto error_return;
-        }
+    if (!this->children) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_NO_CHILD, NULL);
+        goto error_return;
+    }
 
-        priv = GF_CALLOC (1, sizeof (*priv), gf_br_stub_mt_private_t);
-        if (!priv)
-                goto error_return;
+    priv = GF_CALLOC(1, sizeof(*priv), gf_br_stub_mt_private_t);
+    if (!priv)
+        goto error_return;
 
-        priv->local_pool = mem_pool_new (br_stub_local_t, 512);
-        if (!priv->local_pool)
-                goto free_priv;
+    priv->local_pool = mem_pool_new(br_stub_local_t, 512);
+    if (!priv->local_pool)
+        goto free_priv;
 
-        GF_OPTION_INIT ("bitrot", priv->do_versioning, bool, free_mempool);
+    GF_OPTION_INIT("bitrot", priv->do_versioning, bool, free_mempool);
 
-        GF_OPTION_INIT ("export", tmp, str, free_mempool);
-        strncpy (priv->export, tmp, PATH_MAX-1);
-        priv->export[PATH_MAX-1] = '\0';
+    GF_OPTION_INIT("export", tmp, str, free_mempool);
 
-        (void) snprintf (priv->stub_basepath, sizeof (priv->stub_basepath),
-                         "%s/%s", priv->export, BR_STUB_QUARANTINE_DIR);
+    if (snprintf(priv->export, PATH_MAX, "%s", tmp) >= PATH_MAX)
+        goto free_mempool;
 
-        (void) gettimeofday (&tv, NULL);
+    if (snprintf(priv->stub_basepath, sizeof(priv->stub_basepath), "%s/%s",
+                 priv->export,
+                 BR_STUB_QUARANTINE_DIR) >= sizeof(priv->stub_basepath))
+        goto free_mempool;
 
-        /* boot time is in network endian format */
-        priv->boot[0] = htonl (tv.tv_sec);
-        priv->boot[1] = htonl (tv.tv_usec);
+    (void)gettimeofday(&tv, NULL);
 
-        pthread_mutex_init (&priv->lock, NULL);
-        pthread_cond_init (&priv->cond, NULL);
-        INIT_LIST_HEAD (&priv->squeue);
+    /* boot time is in network endian format */
+    priv->boot[0] = htonl(tv.tv_sec);
+    priv->boot[1] = htonl(tv.tv_usec);
 
-        /* Thread creations need 'this' to be passed so that THIS can be
-         * assigned inside the thread. So setting this->private here.
-         */
-        this->private = priv;
+    pthread_mutex_init(&priv->lock, NULL);
+    pthread_cond_init(&priv->cond, NULL);
+    INIT_LIST_HEAD(&priv->squeue);
 
-        ret = gf_thread_create (&priv->signth, NULL, br_stub_signth, this,
-                                "brssign");
-        if (ret != 0)
-                goto cleanup_lock;
+    /* Thread creations need 'this' to be passed so that THIS can be
+     * assigned inside the thread. So setting this->private here.
+     */
+    this->private = priv;
+    if (!priv->do_versioning)
+        return 0;
 
-        ret = br_stub_bad_object_container_init (this, priv);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_CONTAINER_FAIL,
-                        "failed to launch the thread for storing bad gfids");
-                goto cleanup_lock;
-        }
+    ret = gf_thread_create(&priv->signth, NULL, br_stub_signth, this,
+                           "brssign");
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SPAWN_SIGN_THRD_FAILED,
+                NULL);
+        goto cleanup_lock;
+    }
 
-        gf_msg_debug (this->name, 0, "bit-rot stub loaded");
+    ret = br_stub_bad_object_container_init(this, priv);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_CONTAINER_FAIL, NULL);
+        goto cleanup_lock;
+    }
 
-        return 0;
+    gf_msg_debug(this->name, 0, "bit-rot stub loaded");
 
- cleanup_lock:
-        pthread_cond_destroy (&priv->cond);
-        pthread_mutex_destroy (&priv->lock);
- free_mempool:
-        mem_pool_destroy (priv->local_pool);
- free_priv:
-        GF_FREE (priv);
-        this->private = NULL;
- error_return:
-        return -1;
+    return 0;
+
+cleanup_lock:
+    pthread_cond_destroy(&priv->cond);
+    pthread_mutex_destroy(&priv->lock);
+free_mempool:
+    mem_pool_destroy(priv->local_pool);
+    priv->local_pool = NULL;
+free_priv:
+    GF_FREE(priv);
+    this->private = NULL;
+error_return:
+    return -1;
 }
 
 /* TODO:
@@ -196,184 +209,234 @@ init (xlator_t *this)
  *    incremented upon modification.
  * So object versioning is tied to bitrot daemon's signing. In future, object
  * versioning might be necessary for other things as well apart from bit-rot
- * detection (well thats the objective of bringing in object-versioning :)).
+ * detection (well that's the objective of bringing in object-versioning :)).
  * In that case, better to make versioning a new option and letting it to be
  * enabled despite bit-rot detection is not needed.
  * Ex: ICAP.
  */
 int32_t
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int32_t            ret  = -1;
-        br_stub_private_t *priv = NULL;
+    int32_t ret = -1;
+    br_stub_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        GF_OPTION_RECONF ("bitrot", priv->do_versioning, options, bool, out);
+    GF_OPTION_RECONF("bitrot", priv->do_versioning, options, bool, err);
+    if (priv->do_versioning && !priv->signth) {
+        ret = gf_thread_create(&priv->signth, NULL, br_stub_signth, this,
+                               "brssign");
+        if (ret != 0) {
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    BRS_MSG_SPAWN_SIGN_THRD_FAILED, NULL);
+            goto err;
+        }
 
-        ret = 0;
-out:
-        return ret;
-}
+        ret = br_stub_bad_object_container_init(this, priv);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_CONTAINER_FAIL,
+                    NULL);
+            goto err;
+        }
+    } else {
+        if (priv->signth) {
+            if (gf_thread_cleanup_xint(priv->signth)) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        BRS_MSG_CANCEL_SIGN_THREAD_FAILED, NULL);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, BRS_MSG_KILL_SIGN_THREAD,
+                        NULL);
+                priv->signth = 0;
+            }
+        }
+
+        if (priv->container.thread) {
+            if (gf_thread_cleanup_xint(priv->container.thread)) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        BRS_MSG_CANCEL_SIGN_THREAD_FAILED, NULL);
+            }
+            priv->container.thread = 0;
+        }
+    }
+
+    ret = 0;
+    return ret;
+err:
+    if (priv->signth) {
+        if (gf_thread_cleanup_xint(priv->signth)) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_CANCEL_SIGN_THREAD_FAILED, NULL);
+        }
+        priv->signth = 0;
+    }
 
+    if (priv->container.thread) {
+        if (gf_thread_cleanup_xint(priv->container.thread)) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_CANCEL_SIGN_THREAD_FAILED, NULL);
+        }
+        priv->container.thread = 0;
+    }
+    ret = -1;
+    return ret;
+}
 
 int
-notify (xlator_t *this, int event, void *data, ...)
+notify(xlator_t *this, int event, void *data, ...)
 {
-        br_stub_private_t *priv = NULL;
+    br_stub_private_t *priv = NULL;
 
-        if (!this)
-                return 0;
-
-        priv = this->private;
-        if (!priv)
-                return 0;
+    if (!this)
+        return 0;
 
-        default_notify (this, event, data);
+    priv = this->private;
+    if (!priv)
         return 0;
+
+    default_notify(this, event, data);
+    return 0;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        int32_t ret = 0;
-	br_stub_private_t *priv = this->private;
-        struct br_stub_signentry *sigstub = NULL;
-        call_stub_t *stub = NULL;
+    int32_t ret = 0;
+    br_stub_private_t *priv = this->private;
+    struct br_stub_signentry *sigstub = NULL;
+    call_stub_t *stub = NULL;
 
-        if (!priv)
-                return;
+    if (!priv)
+        return;
 
-        ret = gf_thread_cleanup_xint (priv->signth);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_CANCEL_SIGN_THREAD_FAILED,
-                        "Could not cancel sign serializer thread");
-                goto out;
-        }
-        priv->signth = 0;
+    if (!priv->do_versioning)
+        goto cleanup;
 
-        while (!list_empty (&priv->squeue)) {
-                sigstub = list_first_entry (&priv->squeue,
-                                            struct br_stub_signentry, list);
-                list_del_init (&sigstub->list);
+    ret = gf_thread_cleanup_xint(priv->signth);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_CANCEL_SIGN_THREAD_FAILED,
+                NULL);
+        goto out;
+    }
+    priv->signth = 0;
 
-                call_stub_destroy (sigstub->stub);
-                GF_FREE (sigstub);
-        }
+    while (!list_empty(&priv->squeue)) {
+        sigstub = list_first_entry(&priv->squeue, struct br_stub_signentry,
+                                   list);
+        list_del_init(&sigstub->list);
 
-        pthread_mutex_destroy (&priv->lock);
-        pthread_cond_destroy (&priv->cond);
+        call_stub_destroy(sigstub->stub);
+        GF_FREE(sigstub);
+    }
 
-        ret = gf_thread_cleanup_xint (priv->container.thread);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_CANCEL_SIGN_THREAD_FAILED,
-                        "Could not cancel sign serializer thread");
-                goto out;
-        }
+    ret = gf_thread_cleanup_xint(priv->container.thread);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_CANCEL_SIGN_THREAD_FAILED,
+                NULL);
+        goto out;
+    }
 
-        priv->container.thread = 0;
+    priv->container.thread = 0;
 
-        while (!list_empty (&priv->container.bad_queue)) {
-                stub = list_first_entry (&priv->container.bad_queue, call_stub_t,
-                                         list);
-                list_del_init (&stub->list);
-                call_stub_destroy (stub);
-        }
+    while (!list_empty(&priv->container.bad_queue)) {
+        stub = list_first_entry(&priv->container.bad_queue, call_stub_t, list);
+        list_del_init(&stub->list);
+        call_stub_destroy(stub);
+    }
 
-        if (priv->local_pool) {
-                mem_pool_destroy (priv->local_pool);
-                priv->local_pool = NULL;
-        }
+    pthread_mutex_destroy(&priv->container.bad_lock);
+    pthread_cond_destroy(&priv->container.bad_cond);
 
-        pthread_mutex_destroy (&priv->container.bad_lock);
-        pthread_cond_destroy (&priv->container.bad_cond);
+cleanup:
+    pthread_mutex_destroy(&priv->lock);
+    pthread_cond_destroy(&priv->cond);
 
-        this->private = NULL;
-	GF_FREE (priv);
+    if (priv->local_pool) {
+        mem_pool_destroy(priv->local_pool);
+        priv->local_pool = NULL;
+    }
 
- out:
-	return;
+    this->private = NULL;
+    GF_FREE(priv);
+
+out:
+    return;
 }
 
 static int
-br_stub_alloc_versions (br_version_t **obuf,
-                        br_signature_t **sbuf, size_t signaturelen)
+br_stub_alloc_versions(br_version_t **obuf, br_signature_t **sbuf,
+                       size_t signaturelen)
 {
-        void *mem = NULL;
-        size_t size = 0;
+    void *mem = NULL;
+    size_t size = 0;
 
-        if (obuf)
-                size += sizeof (br_version_t);
-        if (sbuf)
-                size += sizeof (br_signature_t) + signaturelen;
+    if (obuf)
+        size += sizeof(br_version_t);
+    if (sbuf)
+        size += sizeof(br_signature_t) + signaturelen;
 
-        mem = GF_CALLOC (1, size, gf_br_stub_mt_version_t);
-        if (!mem)
-                goto error_return;
+    mem = GF_CALLOC(1, size, gf_br_stub_mt_version_t);
+    if (!mem)
+        goto error_return;
 
-        if (obuf) {
-                *obuf = (br_version_t *)mem;
-                mem = ((char *)mem + sizeof (br_version_t));
-        }
-        if (sbuf) {
-                *sbuf = (br_signature_t *)mem;
-        }
+    if (obuf) {
+        *obuf = (br_version_t *)mem;
+        mem = ((char *)mem + sizeof(br_version_t));
+    }
+    if (sbuf) {
+        *sbuf = (br_signature_t *)mem;
+    }
 
-        return 0;
+    return 0;
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 static void
-br_stub_dealloc_versions (void *mem)
+br_stub_dealloc_versions(void *mem)
 {
-        GF_FREE (mem);
+    GF_FREE(mem);
 }
 
 static br_stub_local_t *
-br_stub_alloc_local (xlator_t *this)
+br_stub_alloc_local(xlator_t *this)
 {
-        br_stub_private_t *priv = this->private;
+    br_stub_private_t *priv = this->private;
 
-        return mem_get0 (priv->local_pool);
+    return mem_get0(priv->local_pool);
 }
 
 static void
-br_stub_dealloc_local (br_stub_local_t *ptr)
+br_stub_dealloc_local(br_stub_local_t *ptr)
 {
-        if (!ptr)
-                return;
+    if (!ptr)
+        return;
 
-        mem_put (ptr);
+    mem_put(ptr);
 }
 
 static int
-br_stub_prepare_version_request (xlator_t *this, dict_t *dict,
+br_stub_prepare_version_request(xlator_t *this, dict_t *dict,
                                 br_version_t *obuf, unsigned long oversion)
 {
-        br_stub_private_t *priv = NULL;
+    br_stub_private_t *priv = NULL;
 
-        priv = this->private;
-        br_set_ongoingversion (obuf, oversion, priv->boot);
+    priv = this->private;
+    br_set_ongoingversion(obuf, oversion, priv->boot);
 
-        return dict_set_static_bin (dict, BITROT_CURRENT_VERSION_KEY,
-                                    (void *)obuf, sizeof (br_version_t));
+    return dict_set_bin(dict, BITROT_CURRENT_VERSION_KEY, (void *)obuf,
+                        sizeof(br_version_t));
 }
 
 static int
-br_stub_prepare_signing_request (dict_t *dict,
-                                 br_signature_t *sbuf,
-                                 br_isignature_t *sign, size_t signaturelen)
+br_stub_prepare_signing_request(dict_t *dict, br_signature_t *sbuf,
+                                br_isignature_t *sign, size_t signaturelen)
 {
-        size_t size = 0;
+    size_t size = 0;
 
-        br_set_signature (sbuf, sign, signaturelen, &size);
+    br_set_signature(sbuf, sign, signaturelen, &size);
 
-        return dict_set_static_bin (dict, BITROT_SIGNING_VERSION_KEY,
-                                    (void *)sbuf, size);
+    return dict_set_bin(dict, BITROT_SIGNING_VERSION_KEY, (void *)sbuf, size);
 }
 
 /**
@@ -383,253 +446,245 @@ br_stub_prepare_signing_request (dict_t *dict,
  * initializes the transient inode version.
  */
 static int
-br_stub_init_inode_versions (xlator_t *this, fd_t *fd, inode_t *inode,
-                             unsigned long version, gf_boolean_t markdirty,
-                             gf_boolean_t bad_object, uint64_t *ctx_addr)
-{
-        int32_t ret = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
-
-        ctx = GF_CALLOC (1, sizeof (br_stub_inode_ctx_t),
-                         gf_br_stub_mt_inode_ctx_t);
-        if (!ctx)
-                goto error_return;
-
-        INIT_LIST_HEAD (&ctx->fd_list);
-        (markdirty) ? __br_stub_mark_inode_dirty (ctx)
-                : __br_stub_mark_inode_synced (ctx);
-        __br_stub_set_ongoing_version (ctx, version);
-
-        if (bad_object)
-                __br_stub_mark_object_bad (ctx);
-
-        if (fd) {
-                ret = br_stub_add_fd_to_inode (this, fd, ctx);
-                if (ret)
-                        goto free_ctx;
-        }
+br_stub_init_inode_versions(xlator_t *this, fd_t *fd, inode_t *inode,
+                            unsigned long version, gf_boolean_t markdirty,
+                            gf_boolean_t bad_object, uint64_t *ctx_addr)
+{
+    int32_t ret = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+
+    ctx = GF_CALLOC(1, sizeof(br_stub_inode_ctx_t), gf_br_stub_mt_inode_ctx_t);
+    if (!ctx)
+        goto error_return;
+
+    INIT_LIST_HEAD(&ctx->fd_list);
+    (markdirty) ? __br_stub_mark_inode_dirty(ctx)
+                : __br_stub_mark_inode_synced(ctx);
+    __br_stub_set_ongoing_version(ctx, version);
+
+    if (bad_object)
+        __br_stub_mark_object_bad(ctx);
 
-        ret = br_stub_set_inode_ctx (this, inode, ctx);
+    if (fd) {
+        ret = br_stub_add_fd_to_inode(this, fd, ctx);
         if (ret)
-                goto free_ctx;
+            goto free_ctx;
+    }
 
-        if (ctx_addr)
-                *ctx_addr = (uint64_t) ctx;
-        return 0;
+    ret = br_stub_set_inode_ctx(this, inode, ctx);
+    if (ret)
+        goto free_ctx;
+
+    if (ctx_addr)
+        *ctx_addr = (uint64_t)(uintptr_t)ctx;
+    return 0;
 
 free_ctx:
-        GF_FREE (ctx);
- error_return:
-        return -1;
+    GF_FREE(ctx);
+error_return:
+    return -1;
 }
 
 /**
  * modify the ongoing version of an inode.
  */
 static int
-br_stub_mod_inode_versions (xlator_t *this,
-                            fd_t *fd, inode_t *inode, unsigned long version)
+br_stub_mod_inode_versions(xlator_t *this, fd_t *fd, inode_t *inode,
+                           unsigned long version)
 {
-        int32_t ret = -1;
-        br_stub_inode_ctx_t *ctx = 0;
+    int32_t ret = -1;
+    br_stub_inode_ctx_t *ctx = 0;
 
-        LOCK (&inode->lock);
-        {
-                ctx = __br_stub_get_ongoing_version_ctx (this, inode, NULL);
-                if (ctx == NULL)
-                        goto unblock;
-                if (__br_stub_is_inode_dirty (ctx)) {
-                        __br_stub_set_ongoing_version (ctx, version);
-                        __br_stub_mark_inode_synced (ctx);
-                }
-
-                ret = 0;
+    LOCK(&inode->lock);
+    {
+        ctx = __br_stub_get_ongoing_version_ctx(this, inode, NULL);
+        if (ctx == NULL)
+            goto unblock;
+        if (__br_stub_is_inode_dirty(ctx)) {
+            __br_stub_set_ongoing_version(ctx, version);
+            __br_stub_mark_inode_synced(ctx);
         }
+
+        ret = 0;
+    }
 unblock:
-        UNLOCK (&inode->lock);
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
 static void
-br_stub_fill_local (br_stub_local_t *local,
-                    call_stub_t *stub, fd_t *fd, inode_t *inode, uuid_t gfid,
-                    int versioningtype, unsigned long memversion)
+br_stub_fill_local(br_stub_local_t *local, call_stub_t *stub, fd_t *fd,
+                   inode_t *inode, uuid_t gfid, int versioningtype,
+                   unsigned long memversion)
 {
-        local->fopstub = stub;
-        local->versioningtype = versioningtype;
-        local->u.context.version = memversion;
-        if (fd)
-                local->u.context.fd = fd_ref (fd);
-        if (inode)
-                local->u.context.inode = inode_ref (inode);
-        gf_uuid_copy (local->u.context.gfid, gfid);
+    local->fopstub = stub;
+    local->versioningtype = versioningtype;
+    local->u.context.version = memversion;
+    if (fd)
+        local->u.context.fd = fd_ref(fd);
+    if (inode)
+        local->u.context.inode = inode_ref(inode);
+    gf_uuid_copy(local->u.context.gfid, gfid);
 }
 
 static void
-br_stub_cleanup_local (br_stub_local_t *local)
+br_stub_cleanup_local(br_stub_local_t *local)
 {
-        if (!local)
-                return;
+    if (!local)
+        return;
 
-        local->fopstub = NULL;
-        local->versioningtype = 0;
-        local->u.context.version = 0;
-        if (local->u.context.fd) {
-                fd_unref (local->u.context.fd);
-                local->u.context.fd = NULL;
-        }
-        if (local->u.context.inode) {
-                inode_unref (local->u.context.inode);
-                local->u.context.inode = NULL;
-        }
-        memset (local->u.context.gfid, '\0', sizeof (uuid_t));
+    local->fopstub = NULL;
+    local->versioningtype = 0;
+    local->u.context.version = 0;
+    if (local->u.context.fd) {
+        fd_unref(local->u.context.fd);
+        local->u.context.fd = NULL;
+    }
+    if (local->u.context.inode) {
+        inode_unref(local->u.context.inode);
+        local->u.context.inode = NULL;
+    }
+    memset(local->u.context.gfid, '\0', sizeof(uuid_t));
 }
 
 static int
-br_stub_need_versioning (xlator_t *this,
-                         fd_t *fd, gf_boolean_t *versioning,
-                         gf_boolean_t *modified, br_stub_inode_ctx_t **ctx)
-{
-        int32_t ret      = -1;
-        uint64_t ctx_addr = 0;
-        br_stub_inode_ctx_t *c = NULL;
-        unsigned long        version  = BITROT_DEFAULT_CURRENT_VERSION;
-
-        *versioning = _gf_false;
-        *modified = _gf_false;
-
-        /* Bitrot stub inode context was initialized only in lookup, create
-         * and mknod cbk path. Object versioning was enabled by default
-         * irrespective of bitrot enbaled or not. But it's made optional now.
-         * As a consequence there could be cases where getting inode ctx would
-         * fail because it's not set yet.
-         * e.g., If versioning (with bitrot enable) is enabled while I/O is
-         * happening, it could directly get other fops like writev without
-         * lookup, where getting inode ctx would fail. Hence initialize the
-         * inode ctx on failure to get ctx. This is done in all places where
-         * applicable.
-         */
-        ret = br_stub_get_inode_ctx (this, fd->inode, &ctx_addr);
-        if (ret < 0) {
-                ret = br_stub_init_inode_versions (this, fd, fd->inode, version,
-                                                   _gf_true, _gf_false,
-                                                   &ctx_addr);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRS_MSG_GET_INODE_CONTEXT_FAILED, "failed to "
-                                " init the inode context for the inode %s",
-                                uuid_utoa (fd->inode->gfid));
-                        goto error_return;
-                }
+br_stub_need_versioning(xlator_t *this, fd_t *fd, gf_boolean_t *versioning,
+                        gf_boolean_t *modified, br_stub_inode_ctx_t **ctx)
+{
+    int32_t ret = -1;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *c = NULL;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+
+    *versioning = _gf_false;
+    *modified = _gf_false;
+
+    /* Bitrot stub inode context was initialized only in lookup, create
+     * and mknod cbk path. Object versioning was enabled by default
+     * irrespective of bitrot enabled or not. But it's made optional now.
+     * As a consequence there could be cases where getting inode ctx would
+     * fail because it's not set yet.
+     * e.g., If versioning (with bitrot enable) is enabled while I/O is
+     * happening, it could directly get other fops like writev without
+     * lookup, where getting inode ctx would fail. Hence initialize the
+     * inode ctx on failure to get ctx. This is done in all places where
+     * applicable.
+     */
+    ret = br_stub_get_inode_ctx(this, fd->inode, &ctx_addr);
+    if (ret < 0) {
+        ret = br_stub_init_inode_versions(this, fd, fd->inode, version,
+                                          _gf_true, _gf_false, &ctx_addr);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                    uuid_utoa(fd->inode->gfid), NULL);
+            goto error_return;
         }
+    }
 
-        c = (br_stub_inode_ctx_t *) (long) ctx_addr;
+    c = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
-        LOCK (&fd->inode->lock);
-        {
-                if (__br_stub_is_inode_dirty (c))
-                        *versioning = _gf_true;
-                if (__br_stub_is_inode_modified (c))
-                        *modified = _gf_true;
-        }
-        UNLOCK (&fd->inode->lock);
+    LOCK(&fd->inode->lock);
+    {
+        if (__br_stub_is_inode_dirty(c))
+            *versioning = _gf_true;
+        if (__br_stub_is_inode_modified(c))
+            *modified = _gf_true;
+    }
+    UNLOCK(&fd->inode->lock);
 
-        if (ctx)
-                *ctx = c;
-        return 0;
+    if (ctx)
+        *ctx = c;
+    return 0;
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 static int32_t
-br_stub_anon_fd_ctx (xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx)
-{
-        int32_t  ret = -1;
-        br_stub_fd_t *br_stub_fd = NULL;
-
-        br_stub_fd = br_stub_fd_ctx_get (this, fd);
-        if (!br_stub_fd) {
-                ret = br_stub_add_fd_to_inode (this, fd, ctx);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRS_MSG_ADD_FD_TO_INODE, "failed to add fd to "
-                                "the inode (gfid: %s)",
-                                uuid_utoa (fd->inode->gfid));
-                        goto out;
-                }
+br_stub_anon_fd_ctx(xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx)
+{
+    int32_t ret = -1;
+    br_stub_fd_t *br_stub_fd = NULL;
+
+    br_stub_fd = br_stub_fd_ctx_get(this, fd);
+    if (!br_stub_fd) {
+        ret = br_stub_add_fd_to_inode(this, fd, ctx);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ADD_FD_TO_INODE,
+                    "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-br_stub_versioning_prep (call_frame_t *frame,
-                         xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx)
+br_stub_versioning_prep(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        br_stub_inode_ctx_t *ctx)
 {
-        int32_t          ret = -1;
-        br_stub_local_t *local   = NULL;
+    int32_t ret = -1;
+    br_stub_local_t *local = NULL;
 
-        local = br_stub_alloc_local (this);
-        if (!local) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, BRS_MSG_NO_MEMORY,
-                        "local allocation failed (gfid: %s)",
-                        uuid_utoa (fd->inode->gfid));
-                goto error_return;
-        }
+    local = br_stub_alloc_local(this);
+    if (!local) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRS_MSG_NO_MEMORY, "gfid=%s",
+                uuid_utoa(fd->inode->gfid), NULL);
+        goto error_return;
+    }
 
-        if (fd_is_anonymous (fd)) {
-                ret = br_stub_anon_fd_ctx (this, fd, ctx);
-                if (ret)
-                        goto free_local;
-        }
+    if (fd_is_anonymous(fd)) {
+        ret = br_stub_anon_fd_ctx(this, fd, ctx);
+        if (ret)
+            goto free_local;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        return 0;
+    return 0;
 
- free_local:
-        br_stub_dealloc_local (local);
- error_return:
-        return -1;
+free_local:
+    br_stub_dealloc_local(local);
+error_return:
+    return -1;
 }
 
 static int
-br_stub_mark_inode_modified (xlator_t *this, br_stub_local_t *local)
-{
-        fd_t                *fd       = NULL;
-        int32_t              ret      = 0;
-        uint64_t             ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx      = NULL;
-        unsigned long        version  = BITROT_DEFAULT_CURRENT_VERSION;
-
-        fd = local->u.context.fd;
-
-        ret = br_stub_get_inode_ctx (this, fd->inode, &ctx_addr);
-        if (ret < 0) {
-                ret = br_stub_init_inode_versions (this, fd, fd->inode, version,
-                                                   _gf_true, _gf_false,
-                                                   &ctx_addr);
-                if (ret)
-                        goto error_return;
-        }
+br_stub_mark_inode_modified(xlator_t *this, br_stub_local_t *local)
+{
+    fd_t *fd = NULL;
+    int32_t ret = 0;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
 
-        ctx = (br_stub_inode_ctx_t *) (long) ctx_addr;
+    fd = local->u.context.fd;
 
-        LOCK (&fd->inode->lock);
-        {
-                __br_stub_set_inode_modified (ctx);
-        }
-        UNLOCK (&fd->inode->lock);
+    ret = br_stub_get_inode_ctx(this, fd->inode, &ctx_addr);
+    if (ret < 0) {
+        ret = br_stub_init_inode_versions(this, fd, fd->inode, version,
+                                          _gf_true, _gf_false, &ctx_addr);
+        if (ret)
+            goto error_return;
+    }
 
-        return 0;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
- error_return:
-        return -1;
+    LOCK(&fd->inode->lock);
+    {
+        __br_stub_set_inode_modified(ctx);
+    }
+    UNLOCK(&fd->inode->lock);
+
+    return 0;
+
+error_return:
+    return -1;
 }
 
 /**
@@ -641,71 +696,68 @@ br_stub_mark_inode_modified (xlator_t *this, br_stub_local_t *local)
  * and error is returned upwards.
  */
 static int
-br_stub_check_bad_object (xlator_t *this, inode_t *inode, int32_t *op_ret,
-                           int32_t *op_errno)
-{
-        int ret = -1;
-        unsigned long        version  = BITROT_DEFAULT_CURRENT_VERSION;
-
-        ret = br_stub_is_bad_object (this, inode);
-        if (ret == -2) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJECT_ACCESS,
-                        "%s is a bad object. Returning",
-                        uuid_utoa (inode->gfid));
-                *op_ret = -1;
-                *op_errno = EIO;
-        }
-
-        if (ret == -1) {
-                ret = br_stub_init_inode_versions (this, NULL, inode, version,
-                                                   _gf_true, _gf_false, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRS_MSG_GET_INODE_CONTEXT_FAILED,
-                                "failed to init inode context for %s",
-                                uuid_utoa (inode->gfid));
-                        *op_ret = -1;
-                        *op_errno = EINVAL;
-                }
+br_stub_check_bad_object(xlator_t *this, inode_t *inode, int32_t *op_ret,
+                         int32_t *op_errno)
+{
+    int ret = -1;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+
+    ret = br_stub_is_bad_object(this, inode);
+    if (ret == -2) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJECT_ACCESS,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        *op_ret = -1;
+        *op_errno = EIO;
+    }
+
+    if (ret == -1) {
+        ret = br_stub_init_inode_versions(this, NULL, inode, version, _gf_true,
+                                          _gf_false, NULL);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                    uuid_utoa(inode->gfid), NULL);
+            *op_ret = -1;
+            *op_errno = EINVAL;
         }
+    }
 
-        return ret;
+    return ret;
 }
 
 /**
  * callback for inode/fd versioning
  */
 int
-br_stub_fd_incversioning_cbk (call_frame_t *frame,
-                              void *cookie, xlator_t *this,
-                              int op_ret, int op_errno, dict_t *xdata)
-{
-        fd_t            *fd      = NULL;
-        inode_t         *inode   = NULL;
-        unsigned long    version = 0;
-        br_stub_local_t *local   = NULL;
-
-        local = (br_stub_local_t *)frame->local;
-        if (op_ret < 0)
-                goto done;
-        fd      = local->u.context.fd;
-        inode   = local->u.context.inode;
-        version = local->u.context.version;
-
-        op_ret = br_stub_mod_inode_versions (this, fd, inode, version);
-        if (op_ret < 0)
-                op_errno = EINVAL;
-
- done:
-        if (op_ret < 0) {
-                frame->local = NULL;
-                call_unwind_error (local->fopstub, -1, op_errno);
-                br_stub_cleanup_local (local);
-                br_stub_dealloc_local (local);
-        } else {
-                call_resume (local->fopstub);
-        }
-        return 0;
+br_stub_fd_incversioning_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, dict_t *xdata)
+{
+    fd_t *fd = NULL;
+    inode_t *inode = NULL;
+    unsigned long version = 0;
+    br_stub_local_t *local = NULL;
+
+    local = (br_stub_local_t *)frame->local;
+    if (op_ret < 0)
+        goto done;
+    fd = local->u.context.fd;
+    inode = local->u.context.inode;
+    version = local->u.context.version;
+
+    op_ret = br_stub_mod_inode_versions(this, fd, inode, version);
+    if (op_ret < 0)
+        op_errno = EINVAL;
+
+done:
+    if (op_ret < 0) {
+        frame->local = NULL;
+        call_unwind_error(local->fopstub, -1, op_errno);
+        br_stub_cleanup_local(local);
+        br_stub_dealloc_local(local);
+    } else {
+        call_resume(local->fopstub);
+    }
+    return 0;
 }
 
 /**
@@ -738,102 +790,101 @@ br_stub_fd_incversioning_cbk (call_frame_t *frame,
 /**
  * perform full or incremental versioning on an inode pointd by an
  * fd. incremental versioning is done when an inode is dirty and a
- * writeback is trigerred.
+ * writeback is triggered.
  */
 
 int
-br_stub_fd_versioning (xlator_t *this, call_frame_t *frame,
-                       call_stub_t *stub, dict_t *dict, fd_t *fd,
-                       br_stub_version_cbk *callback, unsigned long memversion,
-                       int versioningtype, int durable)
+br_stub_fd_versioning(xlator_t *this, call_frame_t *frame, call_stub_t *stub,
+                      dict_t *dict, fd_t *fd, br_stub_version_cbk *callback,
+                      unsigned long memversion, int versioningtype, int durable)
 {
-        int32_t          ret   = -1;
-        int              flags = 0;
-        dict_t          *xdata = NULL;
-        br_stub_local_t *local = NULL;
+    int32_t ret = -1;
+    int flags = 0;
+    dict_t *xdata = NULL;
+    br_stub_local_t *local = NULL;
 
-        xdata = dict_new ();
-        if (!xdata)
-                goto done;
+    xdata = dict_new();
+    if (!xdata)
+        goto done;
 
-        ret = dict_set_int32 (xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
-        if (ret)
-                goto dealloc_xdata;
+    ret = dict_set_int32(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+    if (ret)
+        goto dealloc_xdata;
 
-        if (durable) {
-                ret = dict_set_int32 (xdata, GLUSTERFS_DURABLE_OP, 0);
-                if (ret)
-                        goto dealloc_xdata;
-        }
+    if (durable) {
+        ret = dict_set_int32(xdata, GLUSTERFS_DURABLE_OP, 0);
+        if (ret)
+            goto dealloc_xdata;
+    }
 
-        local = frame->local;
+    local = frame->local;
 
-        br_stub_fill_local (local, stub, fd,
-                            fd->inode, fd->inode->gfid,
-                            versioningtype, memversion);
+    br_stub_fill_local(local, stub, fd, fd->inode, fd->inode->gfid,
+                       versioningtype, memversion);
 
-        STACK_WIND (frame, callback,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
+    STACK_WIND(frame, callback, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
 
-        ret = 0;
+    ret = 0;
 
- dealloc_xdata:
-        dict_unref (xdata);
- done:
-        return ret;
+dealloc_xdata:
+    dict_unref(xdata);
+done:
+    return ret;
 }
 
 static int
-br_stub_perform_incversioning (xlator_t *this,
-                               call_frame_t *frame, call_stub_t *stub,
-                               fd_t *fd, br_stub_inode_ctx_t *ctx)
-{
-        int32_t          ret               = -1;
-        dict_t          *dict              = NULL;
-        br_version_t    *obuf              = NULL;
-        unsigned long    writeback_version = 0;
-        int              op_errno          = 0;
-        br_stub_local_t *local             = NULL;
-
-        op_errno = EINVAL;
-        local = frame->local;
-
-        writeback_version = __br_stub_writeback_version (ctx);
-
-        op_errno = ENOMEM;
-        dict = dict_new ();
-        if (!dict)
-                goto done;
-        ret = br_stub_alloc_versions (&obuf, NULL, 0);
-        if (ret)
-                goto dealloc_dict;
-        ret = br_stub_prepare_version_request (this, dict,
-                                               obuf, writeback_version);
-        if (ret)
-                goto dealloc_versions;
-
-        ret = br_stub_fd_versioning
-                (this, frame, stub, dict,
-                 fd, br_stub_fd_incversioning_cbk, writeback_version,
-                 BR_STUB_INCREMENTAL_VERSIONING, !WRITEBACK_DURABLE);
-
- dealloc_versions:
-        br_stub_dealloc_versions (obuf);
- dealloc_dict:
-        dict_unref (dict);
- done:
-        if (ret) {
-                if (local)
-                        frame->local = NULL;
-                call_unwind_error (stub, -1, op_errno);
-                if (local) {
-                        br_stub_cleanup_local (local);
-                        br_stub_dealloc_local (local);
-                }
+br_stub_perform_incversioning(xlator_t *this, call_frame_t *frame,
+                              call_stub_t *stub, fd_t *fd,
+                              br_stub_inode_ctx_t *ctx)
+{
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    br_version_t *obuf = NULL;
+    unsigned long writeback_version = 0;
+    int op_errno = 0;
+    br_stub_local_t *local = NULL;
+
+    op_errno = EINVAL;
+    local = frame->local;
+
+    writeback_version = __br_stub_writeback_version(ctx);
+
+    op_errno = ENOMEM;
+    dict = dict_new();
+    if (!dict)
+        goto out;
+    ret = br_stub_alloc_versions(&obuf, NULL, 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ALLOC_MEM_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+    ret = br_stub_prepare_version_request(this, dict, obuf, writeback_version);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_VERSION_PREPARE_FAIL,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        br_stub_dealloc_versions(obuf);
+        goto out;
+    }
+
+    ret = br_stub_fd_versioning(
+        this, frame, stub, dict, fd, br_stub_fd_incversioning_cbk,
+        writeback_version, BR_STUB_INCREMENTAL_VERSIONING, !WRITEBACK_DURABLE);
+out:
+    if (dict)
+        dict_unref(dict);
+    if (ret) {
+        if (local)
+            frame->local = NULL;
+        call_unwind_error(stub, -1, op_errno);
+        if (local) {
+            br_stub_cleanup_local(local);
+            br_stub_dealloc_local(local);
         }
+    }
 
-        return ret;
+    return ret;
 }
 
 /** {{{ */
@@ -841,247 +892,271 @@ br_stub_perform_incversioning (xlator_t *this,
 /* fsetxattr() */
 
 int32_t
-br_stub_perform_objsign (call_frame_t *frame, xlator_t *this,
-                         fd_t *fd, dict_t *dict, int flags, dict_t *xdata)
+br_stub_perform_objsign(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        dict_t *dict, int flags, dict_t *xdata)
 {
-        STACK_WIND (frame, default_fsetxattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr, fd,
-                    dict, flags, xdata);
+    STACK_WIND(frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
 
-        dict_unref (xdata);
-        return 0;
+    dict_unref(xdata);
+    return 0;
 }
 
 void *
-br_stub_signth (void *arg)
+br_stub_signth(void *arg)
 {
-        xlator_t *this = arg;
-        br_stub_private_t *priv = this->private;
-        struct br_stub_signentry *sigstub = NULL;
-
-        THIS = this;
-        while (1) {
+    xlator_t *this = arg;
+    br_stub_private_t *priv = this->private;
+    struct br_stub_signentry *sigstub = NULL;
 
-                pthread_mutex_lock (&priv->lock);
-                {
-                        while (list_empty (&priv->squeue))
-                                pthread_cond_wait (&priv->cond, &priv->lock);
+    THIS = this;
+    while (1) {
+        /*
+         * Disabling bit-rot feature leads to this particular thread
+         * getting cleaned up by reconfigure via a call to the function
+         * gf_thread_cleanup_xint (which in turn calls pthread_cancel
+         * and pthread_join). But, if this thread had held the mutex
+         * &priv->lock at the time of cancellation, then it leads to
+         * deadlock in future when bit-rot feature is enabled (which
+         * again spawns this thread which cant hold the lock as the
+         * mutex is still held by the previous instance of the thread
+         * which got killed). Also, the br_stub_handle_object_signature
+         * function which is called whenever file has to be signed
+         * also gets blocked as it too attempts to acquire &priv->lock.
+         *
+         * So, arrange for the lock to be unlocked as part of the
+         * cleanup of this thread using pthread_cleanup_push and
+         * pthread_cleanup_pop.
+         */
+        pthread_cleanup_push(br_stub_lock_cleaner, &priv->lock);
+        pthread_mutex_lock(&priv->lock);
+        {
+            while (list_empty(&priv->squeue))
+                pthread_cond_wait(&priv->cond, &priv->lock);
 
-                        sigstub = list_first_entry
-                                (&priv->squeue, struct br_stub_signentry, list);
-                        list_del_init (&sigstub->list);
-                }
-                pthread_mutex_unlock (&priv->lock);
+            sigstub = list_first_entry(&priv->squeue, struct br_stub_signentry,
+                                       list);
+            list_del_init(&sigstub->list);
+        }
+        pthread_mutex_unlock(&priv->lock);
+        pthread_cleanup_pop(0);
 
-                call_resume (sigstub->stub);
+        call_resume(sigstub->stub);
 
-                GF_FREE (sigstub);
-        }
+        GF_FREE(sigstub);
+    }
 
-        return NULL;
+    return NULL;
 }
 
 static gf_boolean_t
-br_stub_internal_xattr (dict_t *dict) {
-
-        if (dict_get (dict, GLUSTERFS_SET_OBJECT_SIGNATURE) ||
-            dict_get (dict, GLUSTERFS_GET_OBJECT_SIGNATURE) ||
-            dict_get (dict, BR_REOPEN_SIGN_HINT_KEY) ||
-            dict_get (dict, BITROT_OBJECT_BAD_KEY) ||
-            dict_get (dict, BITROT_SIGNING_VERSION_KEY) ||
-            dict_get (dict, BITROT_CURRENT_VERSION_KEY))
-                return _gf_true;
+br_stub_internal_xattr(dict_t *dict)
+{
+    if (dict_get(dict, GLUSTERFS_SET_OBJECT_SIGNATURE) ||
+        dict_get(dict, GLUSTERFS_GET_OBJECT_SIGNATURE) ||
+        dict_get(dict, BR_REOPEN_SIGN_HINT_KEY) ||
+        dict_get(dict, BITROT_OBJECT_BAD_KEY) ||
+        dict_get(dict, BITROT_SIGNING_VERSION_KEY) ||
+        dict_get(dict, BITROT_CURRENT_VERSION_KEY))
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 int
-orderq (struct list_head *elem1, struct list_head *elem2)
+orderq(struct list_head *elem1, struct list_head *elem2)
 {
-        struct br_stub_signentry *s1 = NULL;
-        struct br_stub_signentry *s2 = NULL;
+    struct br_stub_signentry *s1 = NULL;
+    struct br_stub_signentry *s2 = NULL;
 
-        s1 = list_entry (elem1, struct br_stub_signentry, list);
-        s2 = list_entry (elem2, struct br_stub_signentry, list);
+    s1 = list_entry(elem1, struct br_stub_signentry, list);
+    s2 = list_entry(elem2, struct br_stub_signentry, list);
 
-        return (s1->v > s2->v);
+    return (s1->v > s2->v);
 }
 
 static int
-br_stub_compare_sign_version (xlator_t *this,
-                              inode_t *inode,
-                              br_signature_t *sbuf,
-                              dict_t *dict, int *fakesuccess)
-{
-        int32_t              ret     = -1;
-        uint64_t             tmp_ctx = 0;
-        gf_boolean_t         invalid = _gf_false;
-        br_stub_inode_ctx_t *ctx     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, sbuf, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-
-        ret = br_stub_get_inode_ctx (this, inode, &tmp_ctx);
-        if (ret) {
-                dict_del (dict, BITROT_SIGNING_VERSION_KEY);
-                goto out;
-        }
-
-        ctx = (br_stub_inode_ctx_t *)(long)tmp_ctx;
-
-        LOCK (&inode->lock);
-        {
-                if (ctx->currentversion < sbuf->signedversion) {
-                        invalid = _gf_true;
-                } else if (ctx->currentversion > sbuf->signedversion) {
-                        gf_msg_debug (this->name, 0, "\"Signing version\" "
-                                      "(%lu) lower than \"Current version \" "
-                                      "(%lu)", ctx->currentversion,
-                                      sbuf->signedversion);
-                        *fakesuccess = 1;
-                }
-        }
-        UNLOCK (&inode->lock);
-
-        if (invalid) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_SIGN_VERSION_ERROR, "Signing version exceeds "
-                        "current version [%lu > %lu]", sbuf->signedversion,
-                        ctx->currentversion);
-        }
+br_stub_compare_sign_version(xlator_t *this, inode_t *inode,
+                             br_signature_t *sbuf, dict_t *dict,
+                             int *fakesuccess)
+{
+    int32_t ret = -1;
+    uint64_t tmp_ctx = 0;
+    gf_boolean_t invalid = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, sbuf, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    ret = br_stub_get_inode_ctx(this, inode, &tmp_ctx);
+    if (ret) {
+        dict_del(dict, BITROT_SIGNING_VERSION_KEY);
+        goto out;
+    }
+
+    ctx = (br_stub_inode_ctx_t *)(long)tmp_ctx;
+
+    LOCK(&inode->lock);
+    {
+        if (ctx->currentversion < sbuf->signedversion) {
+            invalid = _gf_true;
+        } else if (ctx->currentversion > sbuf->signedversion) {
+            gf_msg_debug(this->name, 0,
+                         "\"Signing version\" "
+                         "(%lu) lower than \"Current version \" "
+                         "(%lu)",
+                         ctx->currentversion, sbuf->signedversion);
+            *fakesuccess = 1;
+        }
+    }
+    UNLOCK(&inode->lock);
+
+    if (invalid) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SIGN_VERSION_ERROR,
+                "Signing-ver=%lu", sbuf->signedversion, "current-ver=%lu",
+                ctx->currentversion, NULL);
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 static int
-br_stub_prepare_signature (xlator_t *this,
-                           dict_t *dict, inode_t *inode,
-                           br_isignature_t *sign, int *fakesuccess)
+br_stub_prepare_signature(xlator_t *this, dict_t *dict, inode_t *inode,
+                          br_isignature_t *sign, int *fakesuccess)
 {
-        int32_t ret = 0;
-        size_t signaturelen = 0;
-        br_signature_t *sbuf = NULL;
+    int32_t ret = -1;
+    size_t signaturelen = 0;
+    br_signature_t *sbuf = NULL;
 
-        if (!br_is_signature_type_valid (sign->signaturetype))
-                goto error_return;
+    if (!br_is_signature_type_valid(sign->signaturetype))
+        goto out;
 
-        signaturelen = sign->signaturelen;
-        ret = br_stub_alloc_versions (NULL, &sbuf, signaturelen);
-        if (ret)
-                goto error_return;
-        ret = br_stub_prepare_signing_request (dict, sbuf, sign, signaturelen);
-        if (ret)
-                goto dealloc_versions;
-
-        ret = br_stub_compare_sign_version (this, inode,
-                                            sbuf, dict, fakesuccess);
-        if (ret)
-                goto dealloc_versions;
-
-        return 0;
-
- dealloc_versions:
-        br_stub_dealloc_versions (sbuf);
- error_return:
-        return -1;
+    signaturelen = sign->signaturelen;
+    ret = br_stub_alloc_versions(NULL, &sbuf, signaturelen);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ALLOC_MEM_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        ret = -1;
+        goto out;
+    }
+    ret = br_stub_prepare_signing_request(dict, sbuf, sign, signaturelen);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_SIGN_PREPARE_FAIL,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        ret = -1;
+        br_stub_dealloc_versions(sbuf);
+        goto out;
+    }
+
+    /* At this point sbuf has been added to dict, so the memory will be freed
+     * when the data from the dict is destroyed
+     */
+    ret = br_stub_compare_sign_version(this, inode, sbuf, dict, fakesuccess);
+out:
+    return ret;
 }
 
 static void
-br_stub_handle_object_signature (call_frame_t *frame,
-                                 xlator_t *this, fd_t *fd, dict_t *dict,
-                                 br_isignature_t *sign, dict_t *xdata)
-{
-        int32_t                   ret         = -1;
-        int32_t                   op_ret      = -1;
-        int32_t                   op_errno    = EINVAL;
-        int                       fakesuccess = 0;
-        br_stub_private_t        *priv        = NULL;
-        struct br_stub_signentry *sigstub     = NULL;
-
-        priv = this->private;
-
-        if (frame->root->pid != GF_CLIENT_PID_BITD)
-                goto dofop;
-
-        ret = br_stub_prepare_signature (this, dict,
-                                         fd->inode, sign, &fakesuccess);
-        if (ret)
-                goto dofop;
-        if (fakesuccess) {
-                op_ret = op_errno = 0;
-                goto dofop;
-        }
-
-        dict_del (dict, GLUSTERFS_SET_OBJECT_SIGNATURE);
-
-        ret = -1;
-        if (!xdata) {
-                xdata = dict_new ();
-                if (!xdata)
-                        goto dofop;
-        } else {
-                dict_ref (xdata);
-        }
+br_stub_handle_object_signature(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                dict_t *dict, br_isignature_t *sign,
+                                dict_t *xdata)
+{
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    int fakesuccess = 0;
+    br_stub_private_t *priv = NULL;
+    struct br_stub_signentry *sigstub = NULL;
+
+    priv = this->private;
+
+    if (frame->root->pid != GF_CLIENT_PID_BITD) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, BRS_MSG_NON_BITD_PID,
+                "PID=%d", frame->root->pid, NULL);
+        goto dofop;
+    }
+
+    ret = br_stub_prepare_signature(this, dict, fd->inode, sign, &fakesuccess);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SIGN_PREPARE_FAIL,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto dofop;
+    }
+    if (fakesuccess) {
+        op_ret = op_errno = 0;
+        goto dofop;
+    }
+
+    dict_del(dict, GLUSTERFS_SET_OBJECT_SIGNATURE);
+
+    ret = -1;
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata)
+            goto dofop;
+    } else {
+        dict_ref(xdata);
+    }
 
-        ret = dict_set_int32 (xdata, GLUSTERFS_DURABLE_OP, 0);
-        if (ret)
-                goto unref_dict;
+    ret = dict_set_int32(xdata, GLUSTERFS_DURABLE_OP, 0);
+    if (ret)
+        goto unref_dict;
 
-        /* prepare dispatch stub to order object signing */
-        sigstub = GF_CALLOC (1, sizeof (*sigstub), gf_br_stub_mt_sigstub_t);
-        if (!sigstub)
-                goto unref_dict;
+    /* prepare dispatch stub to order object signing */
+    sigstub = GF_CALLOC(1, sizeof(*sigstub), gf_br_stub_mt_sigstub_t);
+    if (!sigstub)
+        goto unref_dict;
 
-        INIT_LIST_HEAD (&sigstub->list);
-        sigstub->v = ntohl (sign->signedversion);
-        sigstub->stub = fop_fsetxattr_stub (frame, br_stub_perform_objsign,
-                                            fd, dict, 0, xdata);
-        if (!sigstub->stub)
-                goto cleanup_stub;
+    INIT_LIST_HEAD(&sigstub->list);
+    sigstub->v = ntohl(sign->signedversion);
+    sigstub->stub = fop_fsetxattr_stub(frame, br_stub_perform_objsign, fd, dict,
+                                       0, xdata);
+    if (!sigstub->stub)
+        goto cleanup_stub;
 
-        pthread_mutex_lock (&priv->lock);
-        {
-                list_add_order (&sigstub->list, &priv->squeue, orderq);
-                pthread_cond_signal (&priv->cond);
-        }
-        pthread_mutex_unlock (&priv->lock);
+    pthread_mutex_lock(&priv->lock);
+    {
+        list_add_order(&sigstub->list, &priv->squeue, orderq);
+        pthread_cond_signal(&priv->cond);
+    }
+    pthread_mutex_unlock(&priv->lock);
 
-        return;
+    return;
 
- cleanup_stub:
-        GF_FREE (sigstub);
- unref_dict:
-        dict_unref (xdata);
- dofop:
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
+cleanup_stub:
+    GF_FREE(sigstub);
+unref_dict:
+    dict_unref(xdata);
+dofop:
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
 }
 
 int32_t
-br_stub_fsetxattr_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+br_stub_fsetxattr_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        int32_t ret = -1;
-        br_stub_local_t *local = NULL;
+    int32_t ret = -1;
+    br_stub_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
+    local = frame->local;
+    frame->local = NULL;
 
-        ret = br_stub_mark_inode_modified (this, local);
-        if (ret) {
-                op_ret = -1;
-                op_errno = EINVAL;
-        }
+    ret = br_stub_mark_inode_modified(this, local);
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
 
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
 
-        return 0;
+    return 0;
 }
 
 /**
@@ -1119,65 +1194,62 @@ br_stub_fsetxattr_resume (call_frame_t *frame, void *cookie, xlator_t *this,
  * }
  */
 static void
-br_stub_handle_object_reopen (call_frame_t *frame,
-                              xlator_t *this, fd_t *fd, uint32_t val)
-{
-        int32_t              ret         = -1;
-        int32_t              op_ret      = -1;
-        int32_t              op_errno    = EINVAL;
-        call_stub_t         *stub        = NULL;
-        gf_boolean_t         inc_version = _gf_false;
-        gf_boolean_t         modified    = _gf_false;
-        br_stub_inode_ctx_t *ctx         = NULL;
-        br_stub_local_t     *local       = NULL;
-        gf_boolean_t         goback      = _gf_true;
-
-        ret = br_stub_need_versioning (this, fd, &inc_version, &modified, &ctx);
-        if (ret)
-                goto unwind;
-
-        LOCK (&fd->inode->lock);
-        {
-                if ((val == BR_OBJECT_REOPEN) && inc_version)
-                        goback = _gf_false;
-                if (val == BR_OBJECT_RESIGN &&
-                    ctx->info_sign == BR_SIGN_NORMAL) {
-                        __br_stub_mark_inode_synced (ctx);
-                        __br_stub_set_inode_modified (ctx);
-                }
-                (void) __br_stub_inode_sign_state (ctx, GF_FOP_FSETXATTR, fd);
-        }
-        UNLOCK (&fd->inode->lock);
-
-        if (goback) {
-                op_ret = op_errno = 0;
-                goto unwind;
-        }
-
-        ret = br_stub_versioning_prep (frame, this, fd, ctx);
-        if (ret)
-                goto unwind;
-        local = frame->local;
-
-        stub = fop_fsetxattr_cbk_stub (frame, br_stub_fsetxattr_resume,
-                                       0, 0, NULL);
-        if (!stub) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
-                        "failed to allocate stub for fsetxattr fop (gfid: %s),"
-                        " unwinding", uuid_utoa (fd->inode->gfid));
-                goto cleanup_local;
-        }
-
-        (void) br_stub_perform_incversioning (this, frame, stub, fd, ctx);
-        return;
+br_stub_handle_object_reopen(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                             uint32_t val)
+{
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    call_stub_t *stub = NULL;
+    gf_boolean_t inc_version = _gf_false;
+    gf_boolean_t modified = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+    br_stub_local_t *local = NULL;
+    gf_boolean_t goback = _gf_true;
+
+    ret = br_stub_need_versioning(this, fd, &inc_version, &modified, &ctx);
+    if (ret)
+        goto unwind;
+
+    LOCK(&fd->inode->lock);
+    {
+        if ((val == BR_OBJECT_REOPEN) && inc_version)
+            goback = _gf_false;
+        if (val == BR_OBJECT_RESIGN && ctx->info_sign == BR_SIGN_NORMAL) {
+            __br_stub_mark_inode_synced(ctx);
+            __br_stub_set_inode_modified(ctx);
+        }
+        (void)__br_stub_inode_sign_state(ctx, GF_FOP_FSETXATTR, fd);
+    }
+    UNLOCK(&fd->inode->lock);
+
+    if (goback) {
+        op_ret = op_errno = 0;
+        goto unwind;
+    }
+
+    ret = br_stub_versioning_prep(frame, this, fd, ctx);
+    if (ret)
+        goto unwind;
+    local = frame->local;
+
+    stub = fop_fsetxattr_cbk_stub(frame, br_stub_fsetxattr_resume, 0, 0, NULL);
+    if (!stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
+                "fsetxattr gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto cleanup_local;
+    }
+
+    (void)br_stub_perform_incversioning(this, frame, stub, fd, ctx);
+    return;
+
+cleanup_local:
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
 
- cleanup_local:
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
-
- unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
+unwind:
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
 }
 
 /**
@@ -1188,89 +1260,83 @@ br_stub_handle_object_reopen (call_frame_t *frame,
  * to mark the object as bad.
  */
 int
-br_stub_fsetxattr_bad_object_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this, int32_t op_ret,
-                                  int32_t op_errno, dict_t *xdata)
-{
-        br_stub_local_t *local = NULL;
-        int32_t          ret   = -1;
-
-        local = frame->local;
-        frame->local = NULL;
-
-        if (op_ret < 0)
-                goto unwind;
-
-        /*
-         * What to do if marking the object as bad fails? (i.e. in memory
-         * marking within the inode context. If we are here means fsetxattr
-         * fop has succeeded on disk and the bad object xattr has been set).
-         * We can return failure to scruber, but there is nothing the scrubber
-         * can do with it (it might assume that the on disk setxattr itself has
-         * failed). The main purpose of this operation is to help identify the
-         * bad object by checking the inode context itself (thus avoiding the
-         * necessity of doing a getxattr fop on the disk).
-         *
-         * So as of now, success itself is being returned even though inode
-         * context set operation fails.
-         * In future if there is any change in the policy which can handle this,
-         * then appropriate response should be sent (i.e. success or error).
-         */
-        ret = br_stub_mark_object_bad (this, local->u.context.inode);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_MARK_FAIL,
-                        "failed to mark object %s as bad",
-                        uuid_utoa (local->u.context.inode->gfid));
-
-        ret = br_stub_add (this, local->u.context.inode->gfid);
+br_stub_fsetxattr_bad_object_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *xdata)
+{
+    br_stub_local_t *local = NULL;
+    int32_t ret = -1;
+
+    local = frame->local;
+    frame->local = NULL;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    /*
+     * What to do if marking the object as bad fails? (i.e. in memory
+     * marking within the inode context. If we are here means fsetxattr
+     * fop has succeeded on disk and the bad object xattr has been set).
+     * We can return failure to scruber, but there is nothing the scrubber
+     * can do with it (it might assume that the on disk setxattr itself has
+     * failed). The main purpose of this operation is to help identify the
+     * bad object by checking the inode context itself (thus avoiding the
+     * necessity of doing a getxattr fop on the disk).
+     *
+     * So as of now, success itself is being returned even though inode
+     * context set operation fails.
+     * In future if there is any change in the policy which can handle this,
+     * then appropriate response should be sent (i.e. success or error).
+     */
+    ret = br_stub_mark_object_bad(this, local->u.context.inode);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_MARK_FAIL,
+                "gfid=%s", uuid_utoa(local->u.context.inode->gfid), NULL);
+
+    ret = br_stub_add(this, local->u.context.inode->gfid);
 
 unwind:
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
-        return 0;
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+    return 0;
 }
 
 static int32_t
-br_stub_handle_bad_object_key (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                               dict_t *dict, int flags, dict_t *xdata)
-{
-        br_stub_local_t *local    = NULL;
-        int32_t          op_ret   = -1;
-        int32_t         op_errno = EINVAL;
-
-        if (frame->root->pid != GF_CLIENT_PID_SCRUB) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_NON_SCRUB_BAD_OBJ_MARK, "bad object marking "
-                        "on %s is not from the scrubber",
-                        uuid_utoa (fd->inode->gfid));
-                goto unwind;
-        }
-
-        local = br_stub_alloc_local (this);
-        if (!local) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_NO_MEMORY,
-                        "failed to allocate memory for fsetxattr on %s",
-                        uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+br_stub_handle_bad_object_key(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              dict_t *dict, int flags, dict_t *xdata)
+{
+    br_stub_local_t *local = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    if (frame->root->pid != GF_CLIENT_PID_SCRUB) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_NON_SCRUB_BAD_OBJ_MARK,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto unwind;
+    }
+
+    local = br_stub_alloc_local(this);
+    if (!local) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ALLOC_MEM_FAILED,
+                "fsetxattr gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-        br_stub_fill_local (local, NULL, fd, fd->inode,
-                            fd->inode->gfid, BR_STUB_NO_VERSIONING, 0);
-        frame->local = local;
+    br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                       BR_STUB_NO_VERSIONING, 0);
+    frame->local = local;
 
-        STACK_WIND (frame, br_stub_fsetxattr_bad_object_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetxattr, fd, dict, flags,
-                    xdata);
-        return 0;
+    STACK_WIND(frame, br_stub_fsetxattr_bad_object_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
-
 /**
  * As of now, versioning is done by the stub (though as a setxattr
  * operation) as part of inode modification operations such as writev,
@@ -1286,126 +1352,121 @@ unwind:
  *
  */
 static int32_t
-br_stub_handle_internal_xattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                               char *key)
+br_stub_handle_internal_xattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              char *key)
 {
-        int32_t          op_ret   = -1;
-        int32_t          op_errno = EINVAL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
 
-        gf_msg (this->name, GF_LOG_ERROR, 0,
-                BRS_MSG_SET_INTERNAL_XATTR, "setxattr called"
-                " on the internal xattr %s for inode %s", key,
-                uuid_utoa (fd->inode->gfid));
+    gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_SET_INTERNAL_XATTR,
+            "setxattr key=%s", key, "inode-gfid=%s", uuid_utoa(fd->inode->gfid),
+            NULL);
 
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
 static void
-br_stub_dump_xattr (xlator_t *this, dict_t *dict, int *op_errno)
-{
-        char                *format   = "(%s:%s)";
-        char                *dump     = NULL;
-
-        dump = GF_CALLOC (1, BR_STUB_DUMP_STR_SIZE, gf_br_stub_mt_misc);
-        if (!dump) {
-                *op_errno = ENOMEM;
-                goto out;
-        }
-        dict_dump_to_str (dict, dump, BR_STUB_DUMP_STR_SIZE, format);
-        gf_msg (this->name, GF_LOG_ERROR, 0,
-                BRS_MSG_SET_INTERNAL_XATTR, "fsetxattr called on "
-                "internal xattr %s", dump);
- out:
-        if (dump) {
-                GF_FREE (dump);
-        }
-        return;
+br_stub_dump_xattr(xlator_t *this, dict_t *dict, int *op_errno)
+{
+    char *format = "(%s:%s)";
+    char *dump = NULL;
+
+    dump = GF_CALLOC(1, BR_STUB_DUMP_STR_SIZE, gf_br_stub_mt_misc);
+    if (!dump) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
+    dict_dump_to_str(dict, dump, BR_STUB_DUMP_STR_SIZE, format);
+    gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_SET_INTERNAL_XATTR,
+            "fsetxattr dump=%s", dump, NULL);
+out:
+    if (dump) {
+        GF_FREE(dump);
+    }
+    return;
 }
 
 int
-br_stub_fsetxattr (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, dict_t *dict, int flags, dict_t *xdata)
-{
-        int32_t              ret      = 0;
-        uint32_t             val      = 0;
-        br_isignature_t     *sign     = NULL;
-        br_stub_private_t   *priv     = NULL;
-        int32_t              op_ret   = -1;
-        int32_t              op_errno = EINVAL;
-
-        priv = this->private;
-
-        if ((frame->root->pid != GF_CLIENT_PID_BITD &&
-            frame->root->pid != GF_CLIENT_PID_SCRUB) &&
-            br_stub_internal_xattr (dict)) {
-                br_stub_dump_xattr (this, dict, &op_errno);
-                goto unwind;
-        }
-
-        if (!priv->do_versioning)
-                goto wind;
-
-        if (!IA_ISREG (fd->inode->ia_type))
-                goto wind;
-
-        /* object signature request */
-        ret = dict_get_bin (dict, GLUSTERFS_SET_OBJECT_SIGNATURE,
-                            (void **) &sign);
-        if (!ret) {
-                br_stub_handle_object_signature (frame, this,
-                                                 fd, dict, sign, xdata);
-                goto done;
-        }
-
-        /* signing xattr */
-        if (dict_get(dict, BITROT_SIGNING_VERSION_KEY)) {
-                br_stub_handle_internal_xattr (frame, this, fd,
-                                               BITROT_SIGNING_VERSION_KEY);
-                goto done;
-        }
-
-        /* version xattr */
-        if (dict_get(dict, BITROT_CURRENT_VERSION_KEY)) {
-                br_stub_handle_internal_xattr (frame, this, fd,
-                                               BITROT_CURRENT_VERSION_KEY);
-                goto done;
-        }
-
-        if (dict_get (dict, GLUSTERFS_GET_OBJECT_SIGNATURE)) {
-                br_stub_handle_internal_xattr (frame, this, fd,
-                                               GLUSTERFS_GET_OBJECT_SIGNATURE);
-                goto done;
-        }
-
-        /* object reopen request */
-        ret = dict_get_uint32 (dict, BR_REOPEN_SIGN_HINT_KEY, &val);
-        if (!ret) {
-                br_stub_handle_object_reopen (frame, this, fd, val);
-                goto done;
-        }
-
-        /* handle bad object */
-        if (dict_get (dict, BITROT_OBJECT_BAD_KEY)) {
-                br_stub_handle_bad_object_key (frame, this, fd,
-                                               dict, flags, xdata);
-                goto done;
-        }
+br_stub_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                  int flags, dict_t *xdata)
+{
+    int32_t ret = 0;
+    uint32_t val = 0;
+    br_isignature_t *sign = NULL;
+    br_stub_private_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    priv = this->private;
+
+    if ((frame->root->pid != GF_CLIENT_PID_BITD &&
+         frame->root->pid != GF_CLIENT_PID_SCRUB) &&
+        br_stub_internal_xattr(dict)) {
+        br_stub_dump_xattr(this, dict, &op_errno);
+        goto unwind;
+    }
+
+    if (!priv->do_versioning)
+        goto wind;
+
+    if (!IA_ISREG(fd->inode->ia_type))
+        goto wind;
+
+    /* object signature request */
+    ret = dict_get_bin(dict, GLUSTERFS_SET_OBJECT_SIGNATURE, (void **)&sign);
+    if (!ret) {
+        gf_msg_debug(this->name, 0, "got SIGNATURE request on %s",
+                     uuid_utoa(fd->inode->gfid));
+        br_stub_handle_object_signature(frame, this, fd, dict, sign, xdata);
+        goto done;
+    }
+
+    /* signing xattr */
+    if (dict_get(dict, BITROT_SIGNING_VERSION_KEY)) {
+        br_stub_handle_internal_xattr(frame, this, fd,
+                                      BITROT_SIGNING_VERSION_KEY);
+        goto done;
+    }
+
+    /* version xattr */
+    if (dict_get(dict, BITROT_CURRENT_VERSION_KEY)) {
+        br_stub_handle_internal_xattr(frame, this, fd,
+                                      BITROT_CURRENT_VERSION_KEY);
+        goto done;
+    }
+
+    if (dict_get(dict, GLUSTERFS_GET_OBJECT_SIGNATURE)) {
+        br_stub_handle_internal_xattr(frame, this, fd,
+                                      GLUSTERFS_GET_OBJECT_SIGNATURE);
+        goto done;
+    }
+
+    /* object reopen request */
+    ret = dict_get_uint32(dict, BR_REOPEN_SIGN_HINT_KEY, &val);
+    if (!ret) {
+        br_stub_handle_object_reopen(frame, this, fd, val);
+        goto done;
+    }
+
+    /* handle bad object */
+    if (dict_get(dict, BITROT_OBJECT_BAD_KEY)) {
+        br_stub_handle_bad_object_key(frame, this, fd, dict, flags, xdata);
+        goto done;
+    }
 
 wind:
-        STACK_WIND (frame, default_fsetxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetxattr, fd, dict, flags,
-                    xdata);
-        return 0;
+    STACK_WIND(frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
 
 done:
-        return 0;
+    return 0;
 }
 
-
 /**
  * Currently BitD and scrubber are doing fsetxattr to either sign the object
  * or to mark it as bad. Hence setxattr on any of those keys is denied directly
@@ -1414,82 +1475,76 @@ done:
  * check has to be added below.
  */
 int
-br_stub_setxattr (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
+br_stub_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                 int flags, dict_t *xdata)
 {
-        int32_t  op_ret                    = -1;
-        int32_t  op_errno                  = EINVAL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
 
-        if (br_stub_internal_xattr (dict)) {
-                br_stub_dump_xattr (this, dict, &op_errno);
-                goto unwind;
-        }
+    if (br_stub_internal_xattr(dict)) {
+        br_stub_dump_xattr(this, dict, &op_errno);
+        goto unwind;
+    }
 
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD (this)->fops->setxattr, loc, dict, flags,
-                         xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
 /** }}} */
 
-
 /** {{{ */
 
 /* {f}removexattr() */
 
 int32_t
-br_stub_removexattr (call_frame_t *frame, xlator_t *this,
-                     loc_t *loc, const char *name, dict_t *xdata)
-{
-        int32_t op_ret    = -1;
-        int32_t op_errno  = EINVAL;
-
-        if (!strcmp (BITROT_OBJECT_BAD_KEY, name) ||
-            !strcmp (BITROT_SIGNING_VERSION_KEY, name) ||
-            !strcmp (BITROT_CURRENT_VERSION_KEY, name)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_REMOVE_INTERNAL_XATTR, "removexattr called"
-                        " on internal xattr %s for file %s", name, loc->path);
-                goto unwind;
-        }
-
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->removexattr,
-                         loc, name, xdata);
-        return 0;
+br_stub_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, dict_t *xdata)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    if (!strcmp(BITROT_OBJECT_BAD_KEY, name) ||
+        !strcmp(BITROT_SIGNING_VERSION_KEY, name) ||
+        !strcmp(BITROT_CURRENT_VERSION_KEY, name)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_REMOVE_INTERNAL_XATTR,
+                "name=%s", name, "file-path=%s", loc->path, NULL);
+        goto unwind;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(removexattr, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
 int32_t
-br_stub_fremovexattr (call_frame_t *frame, xlator_t *this,
-                      fd_t *fd, const char *name, dict_t *xdata)
-{
-        int32_t op_ret    = -1;
-        int32_t op_errno  = EINVAL;
-
-        if (!strcmp (BITROT_OBJECT_BAD_KEY, name) ||
-            !strcmp (BITROT_SIGNING_VERSION_KEY, name) ||
-            !strcmp (BITROT_CURRENT_VERSION_KEY, name)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_REMOVE_INTERNAL_XATTR, "removexattr called"
-                        " on internal xattr %s for inode %s", name,
-                        uuid_utoa (fd->inode->gfid));
-                goto unwind;
-        }
-
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fremovexattr,
-                         fd, name, xdata);
-        return 0;
+br_stub_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     const char *name, dict_t *xdata)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    if (!strcmp(BITROT_OBJECT_BAD_KEY, name) ||
+        !strcmp(BITROT_SIGNING_VERSION_KEY, name) ||
+        !strcmp(BITROT_CURRENT_VERSION_KEY, name)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_REMOVE_INTERNAL_XATTR,
+                "name=%s", name, "inode-gfid=%s", uuid_utoa(fd->inode->gfid),
+                NULL);
+        goto unwind;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fremovexattr, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
 /** }}} */
@@ -1499,17 +1554,17 @@ unwind:
 /* {f}getxattr() */
 
 int
-br_stub_listxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+br_stub_listxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        br_stub_remove_vxattrs (xattr);
+    br_stub_remove_vxattrs(xattr, _gf_true);
 
- unwind:
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, xattr, xdata);
-        return 0;
+unwind:
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, xattr, xdata);
+    return 0;
 }
 
 /**
@@ -1548,383 +1603,380 @@ br_stub_listxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
  * then treat the object as stale.
  */
 char
-br_stub_is_object_stale (xlator_t *this, call_frame_t *frame, inode_t *inode,
-                         br_version_t *obuf, br_signature_t *sbuf)
-{
-        uint64_t   ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
-        int32_t  ret = -1;
-        char stale = 0;
-
-        if (obuf->ongoingversion == sbuf->signedversion)
-                goto out;
-
-        if (frame->root->pid == GF_CLIENT_PID_SCRUB) {
-                stale = 1;
-                goto out;
-        }
-
-        ret = br_stub_get_inode_ctx (this, inode, &ctx_addr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_GET_INODE_CONTEXT_FAILED, "failed to get the "
-                        "inode context for %s", uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
-
-        LOCK (&inode->lock);
-        {
-                if ((!__br_stub_is_inode_dirty (ctx) &&
-                     ctx->info_sign != BR_SIGN_NORMAL) ||
-                    __br_stub_is_inode_dirty (ctx))
-                        stale = 1;
-        }
-        UNLOCK (&inode->lock);
+br_stub_is_object_stale(xlator_t *this, call_frame_t *frame, inode_t *inode,
+                        br_version_t *obuf, br_signature_t *sbuf)
+{
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    char stale = 0;
+
+    if (obuf->ongoingversion == sbuf->signedversion)
+        goto out;
+
+    if (frame->root->pid == GF_CLIENT_PID_SCRUB) {
+        stale = 1;
+        goto out;
+    }
+
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        goto out;
+    }
+
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+
+    LOCK(&inode->lock);
+    {
+        if ((!__br_stub_is_inode_dirty(ctx) &&
+             ctx->info_sign != BR_SIGN_NORMAL) ||
+            __br_stub_is_inode_dirty(ctx))
+            stale = 1;
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return stale;
+    return stale;
 }
 
 int
-br_stub_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
-{
-        int32_t              ret          = 0;
-        size_t               totallen     = 0;
-        size_t               signaturelen = 0;
-        br_stub_private_t   *priv         = NULL;
-        br_version_t        *obuf         = NULL;
-        br_signature_t      *sbuf         = NULL;
-        br_isignature_out_t *sign         = NULL;
-        br_vxattr_status_t   status;
-        br_stub_local_t     *local        = NULL;
-        inode_t             *inode        = NULL;
-        gf_boolean_t         bad_object   = _gf_false;
-        gf_boolean_t         ver_enabled  = _gf_false;
-
-        BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
-        priv = this->private;
-
-        if (op_ret < 0)
-                goto unwind;
-        BR_STUB_VER_COND_GOTO (priv, (!ver_enabled), delkeys);
-
-        if (cookie != (void *) BR_STUB_REQUEST_COOKIE)
-                goto unwind;
-
-        local = frame->local;
-        frame->local = NULL;
-        inode = local->u.context.inode;
-
-        op_ret   = -1;
-        status = br_version_xattr_state (xattr, &obuf, &sbuf, &bad_object);
-
-        op_errno = EIO;
-        if (bad_object)
-                goto delkeys;
-
-        op_errno = EINVAL;
-        if (status == BR_VXATTR_STATUS_INVALID)
-                goto delkeys;
-
-        op_errno = ENODATA;
-        if ((status == BR_VXATTR_STATUS_MISSING)
-            || (status == BR_VXATTR_STATUS_UNSIGNED))
-                goto delkeys;
-
-        /**
-         * okay.. we have enough information to satisfy the request,
-         * namely: version and signing extended attribute. what's
-         * pending is the signature length -- that's figured out
-         * indirectly via the size of the _whole_ xattr and the
-         * on-disk signing xattr header size.
-         */
+br_stub_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    int32_t ret = 0;
+    size_t totallen = 0;
+    size_t signaturelen = 0;
+    br_stub_private_t *priv = NULL;
+    br_version_t *obuf = NULL;
+    br_signature_t *sbuf = NULL;
+    br_isignature_out_t *sign = NULL;
+    br_vxattr_status_t status;
+    br_stub_local_t *local = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t bad_object = _gf_false;
+    gf_boolean_t ver_enabled = _gf_false;
+
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
+
+    if (op_ret < 0)
+        goto unwind;
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), delkeys);
+
+    if (cookie != (void *)BR_STUB_REQUEST_COOKIE)
+        goto unwind;
+
+    local = frame->local;
+    frame->local = NULL;
+    if (!local) {
+        op_ret = -1;
         op_errno = EINVAL;
-        ret = dict_get_uint32 (xattr, BITROT_SIGNING_XATTR_SIZE_KEY,
-                               (uint32_t *)&signaturelen);
-        if (ret)
-                goto delkeys;
-
-        signaturelen -= sizeof (br_signature_t);
-        totallen = sizeof (br_isignature_out_t) + signaturelen;
-
-        op_errno = ENOMEM;
-        sign = GF_CALLOC (1, totallen, gf_br_stub_mt_signature_t);
-        if (!sign)
-                goto delkeys;
-
-        sign->time[0] = obuf->timebuf[0];
-        sign->time[1] = obuf->timebuf[1];
-
-        /* Object's dirty state & current signed version */
-        sign->version = sbuf->signedversion;
-        sign->stale = br_stub_is_object_stale (this, frame, inode, obuf, sbuf);
-
-        /* Object's signature */
-        sign->signaturelen  = signaturelen;
-        sign->signaturetype = sbuf->signaturetype;
-        (void) memcpy (sign->signature, sbuf->signature, signaturelen);
-
-        op_errno = EINVAL;
-        ret = dict_set_bin (xattr, GLUSTERFS_GET_OBJECT_SIGNATURE,
-                            (void *)sign, totallen);
-        if (ret < 0) {
-                GF_FREE (sign);
-                goto delkeys;
-        }
-        op_errno = 0;
-        op_ret = totallen;
+        goto unwind;
+    }
+    inode = local->u.context.inode;
+
+    op_ret = -1;
+    status = br_version_xattr_state(xattr, &obuf, &sbuf, &bad_object);
+
+    op_errno = EIO;
+    if (bad_object)
+        goto delkeys;
+
+    op_errno = EINVAL;
+    if (status == BR_VXATTR_STATUS_INVALID)
+        goto delkeys;
+
+    op_errno = ENODATA;
+    if ((status == BR_VXATTR_STATUS_MISSING) ||
+        (status == BR_VXATTR_STATUS_UNSIGNED))
+        goto delkeys;
+
+    /**
+     * okay.. we have enough information to satisfy the request,
+     * namely: version and signing extended attribute. what's
+     * pending is the signature length -- that's figured out
+     * indirectly via the size of the _whole_ xattr and the
+     * on-disk signing xattr header size.
+     */
+    op_errno = EINVAL;
+    ret = dict_get_uint32(xattr, BITROT_SIGNING_XATTR_SIZE_KEY,
+                          (uint32_t *)&signaturelen);
+    if (ret)
+        goto delkeys;
+
+    signaturelen -= sizeof(br_signature_t);
+    totallen = sizeof(br_isignature_out_t) + signaturelen;
+
+    op_errno = ENOMEM;
+    sign = GF_CALLOC(1, totallen, gf_br_stub_mt_signature_t);
+    if (!sign)
+        goto delkeys;
+
+    sign->time[0] = obuf->timebuf[0];
+    sign->time[1] = obuf->timebuf[1];
+
+    /* Object's dirty state & current signed version */
+    sign->version = sbuf->signedversion;
+    sign->stale = br_stub_is_object_stale(this, frame, inode, obuf, sbuf);
+
+    /* Object's signature */
+    sign->signaturelen = signaturelen;
+    sign->signaturetype = sbuf->signaturetype;
+    (void)memcpy(sign->signature, sbuf->signature, signaturelen);
+
+    op_errno = EINVAL;
+    ret = dict_set_bin(xattr, GLUSTERFS_GET_OBJECT_SIGNATURE, (void *)sign,
+                       totallen);
+    if (ret < 0) {
+        GF_FREE(sign);
+        goto delkeys;
+    }
+    op_errno = 0;
+    op_ret = totallen;
+
+delkeys:
+    br_stub_remove_vxattrs(xattr, _gf_true);
 
- delkeys:
-        br_stub_remove_vxattrs (xattr);
-
- unwind:
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, xattr, xdata);
-        if (local) {
-                br_stub_cleanup_local (local);
-                br_stub_dealloc_local (local);
-        }
-        return 0;
+unwind:
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, xattr, xdata);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+    return 0;
 }
 
 static void
-br_stub_send_stub_init_time (call_frame_t *frame, xlator_t *this)
+br_stub_send_stub_init_time(call_frame_t *frame, xlator_t *this)
 {
-        int op_ret = 0;
-        int op_errno = 0;
-        dict_t *xattr = NULL;
-        br_stub_init_t stub = {{0,},};
-        br_stub_private_t *priv = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+    dict_t *xattr = NULL;
+    br_stub_init_t stub = {
+        {
+            0,
+        },
+    };
+    br_stub_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        xattr = dict_new ();
-        if (!xattr) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+    xattr = dict_new();
+    if (!xattr) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-        stub.timebuf[0] = priv->boot[0];
-        stub.timebuf[1] = priv->boot[1];
-        memcpy (stub.export, priv->export, strlen (priv->export) + 1);
+    stub.timebuf[0] = priv->boot[0];
+    stub.timebuf[1] = priv->boot[1];
+    memcpy(stub.export, priv->export, strlen(priv->export) + 1);
 
-        op_ret = dict_set_static_bin (xattr, GLUSTERFS_GET_BR_STUB_INIT_TIME,
-                                      (void *) &stub, sizeof (br_stub_init_t));
-        if (op_ret < 0) {
-                op_errno = EINVAL;
-                goto unwind;
-        }
+    op_ret = dict_set_static_bin(xattr, GLUSTERFS_GET_BR_STUB_INIT_TIME,
+                                 (void *)&stub, sizeof(br_stub_init_t));
+    if (op_ret < 0) {
+        op_errno = EINVAL;
+        goto unwind;
+    }
 
-        op_ret = sizeof (br_stub_init_t);
+    op_ret = sizeof(br_stub_init_t);
 
- unwind:
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, xattr, NULL);
+unwind:
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, xattr, NULL);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 }
 
 int
-br_stub_getxattr (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, const char *name, dict_t *xdata)
-{
-        void               *cookie   = NULL;
-        uuid_t              rootgfid = {0, };
-        fop_getxattr_cbk_t  cbk      = br_stub_getxattr_cbk;
-        int32_t             op_ret   = -1;
-        int32_t             op_errno = EINVAL;
-        br_stub_local_t    *local    = NULL;
-        br_stub_private_t  *priv     = NULL;
-
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, unwind);
-
-        rootgfid[15] = 1;
-
-        if (!name) {
-                cbk = br_stub_listxattr_cbk;
-                goto wind;
-        }
-
-        if (br_stub_is_internal_xattr (name))
-                goto unwind;
-
-        priv = this->private;
-        BR_STUB_VER_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        /**
-         * If xattr is node-uuid and the inode is marked bad, return EIO.
-         * Returning EIO would result in AFR to choose correct node-uuid
-         * coresponding to the subvolume * where the good copy of the
-         * file resides.
-         */
-        if (IA_ISREG (loc->inode->ia_type) && XATTR_IS_NODE_UUID (name) &&
-            br_stub_check_bad_object (this, loc->inode, &op_ret, &op_errno)) {
-                goto unwind;
-        }
-
-        /**
-         * this special extended attribute is allowed only on root
-         */
-        if (name
-            && (strncmp (name, GLUSTERFS_GET_BR_STUB_INIT_TIME,
-                          strlen (GLUSTERFS_GET_BR_STUB_INIT_TIME)) == 0)
-            && ((gf_uuid_compare (loc->gfid, rootgfid) == 0)
-                || (gf_uuid_compare (loc->inode->gfid, rootgfid) == 0))) {
-                BR_STUB_RESET_LOCAL_NULL (frame);
-                br_stub_send_stub_init_time (frame, this);
-                return 0;
+br_stub_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 const char *name, dict_t *xdata)
+{
+    void *cookie = NULL;
+    static uuid_t rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    fop_getxattr_cbk_t cbk = br_stub_getxattr_cbk;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_local_t *local = NULL;
+    br_stub_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
+
+    if (!name) {
+        cbk = br_stub_listxattr_cbk;
+        goto wind;
+    }
+
+    if (br_stub_is_internal_xattr(name))
+        goto unwind;
+
+    priv = this->private;
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    /**
+     * If xattr is node-uuid and the inode is marked bad, return EIO.
+     * Returning EIO would result in AFR to choose correct node-uuid
+     * corresponding to the subvolume * where the good copy of the
+     * file resides.
+     */
+    if (IA_ISREG(loc->inode->ia_type) && XATTR_IS_NODE_UUID(name) &&
+        br_stub_check_bad_object(this, loc->inode, &op_ret, &op_errno)) {
+        goto unwind;
+    }
+
+    /**
+     * this special extended attribute is allowed only on root
+     */
+    if (name &&
+        (strncmp(name, GLUSTERFS_GET_BR_STUB_INIT_TIME,
+                 sizeof(GLUSTERFS_GET_BR_STUB_INIT_TIME) - 1) == 0) &&
+        ((gf_uuid_compare(loc->gfid, rootgfid) == 0) ||
+         (gf_uuid_compare(loc->inode->gfid, rootgfid) == 0))) {
+        BR_STUB_RESET_LOCAL_NULL(frame);
+        br_stub_send_stub_init_time(frame, this);
+        return 0;
+    }
+
+    if (!IA_ISREG(loc->inode->ia_type))
+        goto wind;
+
+    if (name && (strncmp(name, GLUSTERFS_GET_OBJECT_SIGNATURE,
+                         sizeof(GLUSTERFS_GET_OBJECT_SIGNATURE) - 1) == 0)) {
+        cookie = (void *)BR_STUB_REQUEST_COOKIE;
+
+        local = br_stub_alloc_local(this);
+        if (!local) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
         }
 
-        if (!IA_ISREG (loc->inode->ia_type))
-                goto wind;
-
-        if (name && (strncmp (name, GLUSTERFS_GET_OBJECT_SIGNATURE,
-                              strlen (GLUSTERFS_GET_OBJECT_SIGNATURE)) == 0)) {
-                cookie = (void *) BR_STUB_REQUEST_COOKIE;
-
-                local = br_stub_alloc_local (this);
-                if (!local) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-                br_stub_fill_local (local, NULL, NULL, loc->inode,
-                                    loc->inode->gfid,
-                                    BR_STUB_NO_VERSIONING, 0);
-                frame->local = local;
-        }
+        br_stub_fill_local(local, NULL, NULL, loc->inode, loc->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        frame->local = local;
+    }
 
- wind:
-        STACK_WIND_COOKIE
-                      (frame, cbk, cookie, FIRST_CHILD (this),
-                       FIRST_CHILD (this)->fops->getxattr, loc, name, xdata);
-        return 0;
+wind:
+    STACK_WIND_COOKIE(frame, cbk, cookie, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 unwind:
-        BR_STUB_RESET_LOCAL_NULL (frame);
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, NULL, NULL);
-                return 0;
+    BR_STUB_RESET_LOCAL_NULL(frame);
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
 int
-br_stub_fgetxattr (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, const char *name, dict_t *xdata)
-{
-        void *cookie = NULL;
-        uuid_t rootgfid = {0, };
-        fop_fgetxattr_cbk_t cbk = br_stub_getxattr_cbk;
-        int32_t op_ret = -1;
-        int32_t op_errno = EINVAL;
-        br_stub_local_t *local = NULL;
-        br_stub_private_t  *priv = NULL;
-
-        rootgfid[15] = 1;
-        priv = this->private;
-
-        if (!name) {
-                cbk = br_stub_listxattr_cbk;
-                goto wind;
-        }
-
-        if (br_stub_is_internal_xattr (name))
-                goto unwind;
-
-        BR_STUB_VER_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        /**
-         * If xattr is node-uuid and the inode is marked bad, return EIO.
-         * Returning EIO would result in AFR to choose correct node-uuid
-         * coresponding to the subvolume * where the good copy of the
-         * file resides.
-         */
-        if (IA_ISREG (fd->inode->ia_type) && XATTR_IS_NODE_UUID (name) &&
-            br_stub_check_bad_object (this, fd->inode, &op_ret, &op_errno)) {
-                goto unwind;
-        }
-
-        /**
-         * this special extended attribute is allowed only on root
-         */
-        if (name
-            && (strncmp (name, GLUSTERFS_GET_BR_STUB_INIT_TIME,
-                         strlen (GLUSTERFS_GET_BR_STUB_INIT_TIME)) == 0)
-            && (gf_uuid_compare (fd->inode->gfid, rootgfid) == 0)) {
-                BR_STUB_RESET_LOCAL_NULL (frame);
-                br_stub_send_stub_init_time (frame, this);
-                return 0;
+br_stub_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
+{
+    void *cookie = NULL;
+    static uuid_t rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    fop_fgetxattr_cbk_t cbk = br_stub_getxattr_cbk;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_local_t *local = NULL;
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (!name) {
+        cbk = br_stub_listxattr_cbk;
+        goto wind;
+    }
+
+    if (br_stub_is_internal_xattr(name))
+        goto unwind;
+
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    /**
+     * If xattr is node-uuid and the inode is marked bad, return EIO.
+     * Returning EIO would result in AFR to choose correct node-uuid
+     * corresponding to the subvolume * where the good copy of the
+     * file resides.
+     */
+    if (IA_ISREG(fd->inode->ia_type) && XATTR_IS_NODE_UUID(name) &&
+        br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno)) {
+        goto unwind;
+    }
+
+    /**
+     * this special extended attribute is allowed only on root
+     */
+    if (name &&
+        (strncmp(name, GLUSTERFS_GET_BR_STUB_INIT_TIME,
+                 sizeof(GLUSTERFS_GET_BR_STUB_INIT_TIME) - 1) == 0) &&
+        (gf_uuid_compare(fd->inode->gfid, rootgfid) == 0)) {
+        BR_STUB_RESET_LOCAL_NULL(frame);
+        br_stub_send_stub_init_time(frame, this);
+        return 0;
+    }
+
+    if (!IA_ISREG(fd->inode->ia_type))
+        goto wind;
+
+    if (name && (strncmp(name, GLUSTERFS_GET_OBJECT_SIGNATURE,
+                         sizeof(GLUSTERFS_GET_OBJECT_SIGNATURE) - 1) == 0)) {
+        cookie = (void *)BR_STUB_REQUEST_COOKIE;
+
+        local = br_stub_alloc_local(this);
+        if (!local) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
         }
 
-        if (!IA_ISREG (fd->inode->ia_type))
-                goto wind;
-
-        if (name && (strncmp (name, GLUSTERFS_GET_OBJECT_SIGNATURE,
-                              strlen (GLUSTERFS_GET_OBJECT_SIGNATURE)) == 0)) {
-                cookie = (void *) BR_STUB_REQUEST_COOKIE;
-
-                local = br_stub_alloc_local (this);
-                if (!local) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-                br_stub_fill_local (local, NULL, fd, fd->inode,
-                                    fd->inode->gfid,
-                                    BR_STUB_NO_VERSIONING, 0);
-                frame->local = local;
-        }
+        br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        frame->local = local;
+    }
 
- wind:
-        STACK_WIND_COOKIE
-                      (frame, cbk, cookie, FIRST_CHILD (this),
-                       FIRST_CHILD (this)->fops->fgetxattr, fd, name, xdata);
-        return 0;
+wind:
+    STACK_WIND_COOKIE(frame, cbk, cookie, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 unwind:
-        BR_STUB_RESET_LOCAL_NULL (frame);
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, NULL, NULL);
-        return 0;
+    BR_STUB_RESET_LOCAL_NULL(frame);
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
 int32_t
-br_stub_readv (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+br_stub_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, uint32_t flags, dict_t *xdata)
 {
-        int32_t              op_ret   = -1;
-        int32_t              op_errno = EINVAL;
-        int32_t              ret      = -1;
-        br_stub_private_t   *priv     = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    int32_t ret = -1;
+    br_stub_private_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, unwind);
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, unwind);
 
-        priv = this->private;
-        if (!priv->do_versioning)
-                goto wind;
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto wind;
 
-        ret = br_stub_check_bad_object (this, fd->inode, &op_ret, &op_errno);
-        if (ret)
-                goto unwind;
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
 
 wind:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->readv, fd, size, offset,
-                         flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, NULL, 0, NULL,
-                             NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, NULL, 0, NULL, NULL,
+                        NULL);
+    return 0;
 }
 
 /**
@@ -1935,299 +1987,294 @@ unwind:
  * fds.
  */
 int32_t
-br_stub_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf, dict_t *xdata)
+br_stub_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-        int32_t            ret   = 0;
-        br_stub_local_t   *local = NULL;
+    int32_t ret = 0;
+    br_stub_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
+    local = frame->local;
+    frame->local = NULL;
 
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        ret = br_stub_mark_inode_modified (this, local);
-        if (ret) {
-                op_ret = -1;
-                op_errno = EINVAL;
-        }
+    ret = br_stub_mark_inode_modified(this, local);
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (writev, frame,
-                             op_ret, op_errno, prebuf, postbuf, xdata);
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-br_stub_writev_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                       struct iovec *vector, int32_t count, off_t offset,
-                       uint32_t flags, struct iobref *iobref, dict_t *xdata)
+br_stub_writev_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      struct iovec *vector, int32_t count, off_t offset,
+                      uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        STACK_WIND (frame, br_stub_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
-                    offset, flags, iobref, xdata);
-        return 0;
+    STACK_WIND(frame, br_stub_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 }
 
 /**
  * This is probably the most crucial part about the whole versioning thing.
  * There's absolutely no differentiation as such between an anonymous fd
  * and a regular fd except the fd context initialization. Object versioning
- * is perfomed when the inode is dirty. Parallel write operations are no
+ * is performed when the inode is dirty. Parallel write operations are no
  * special with each write performing object versioning followed by marking
  * the inode as non-dirty (synced). This is followed by the actual operation
  * (writev() in this case) which on a success marks the inode as modified.
  * This prevents signing of objects that have not been modified.
  */
 int32_t
-br_stub_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iovec *vector, int32_t count, off_t offset,
-                uint32_t flags, struct iobref *iobref, dict_t *xdata)
-{
-        call_stub_t         *stub        = NULL;
-        int32_t              op_ret      = -1;
-        int32_t              op_errno    = EINVAL;
-        gf_boolean_t         inc_version = _gf_false;
-        gf_boolean_t         modified    = _gf_false;
-        br_stub_inode_ctx_t *ctx         = NULL;
-        int32_t              ret         = -1;
-        fop_writev_cbk_t     cbk         = default_writev_cbk;
-        br_stub_local_t     *local       = NULL;
-        br_stub_private_t   *priv        = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-
-        priv = this->private;
-        if (!priv->do_versioning)
-                goto wind;
-
-        ret = br_stub_need_versioning (this, fd, &inc_version, &modified, &ctx);
-        if (ret)
-                goto unwind;
+br_stub_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t offset,
+               uint32_t flags, struct iobref *iobref, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t inc_version = _gf_false;
+    gf_boolean_t modified = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    fop_writev_cbk_t cbk = default_writev_cbk;
+    br_stub_local_t *local = NULL;
+    br_stub_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto wind;
+
+    ret = br_stub_need_versioning(this, fd, &inc_version, &modified, &ctx);
+    if (ret)
+        goto unwind;
+
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
+
+    /**
+     * The inode is not dirty and also witnessed at least one successful
+     * modification operation. Therefore, subsequent operations need not
+     * perform any special tracking.
+     */
+    if (!inc_version && modified)
+        goto wind;
+
+    /**
+     * okay.. so, either the inode needs versioning or the modification
+     * needs to be tracked. ->cbk is set to the appropriate callback
+     * routine for this.
+     * NOTE: ->local needs to be deallocated on failures from here on.
+     */
+    ret = br_stub_versioning_prep(frame, this, fd, ctx);
+    if (ret)
+        goto unwind;
+
+    local = frame->local;
+    if (!inc_version) {
+        br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        cbk = br_stub_writev_cbk;
+        goto wind;
+    }
+
+    stub = fop_writev_stub(frame, br_stub_writev_resume, fd, vector, count,
+                           offset, flags, iobref, xdata);
+
+    if (!stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
+                "write  gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto cleanup_local;
+    }
+
+    /* Perform Versioning */
+    return br_stub_perform_incversioning(this, frame, stub, fd, ctx);
 
-        ret = br_stub_check_bad_object (this, fd->inode, &op_ret, &op_errno);
-        if (ret)
-                goto unwind;
-
-        /**
-         * The inode is not dirty and also witnessed atleast one successful
-         * modification operation. Therefore, subsequent operations need not
-         * perform any special tracking.
-         */
-        if (!inc_version && modified)
-                goto wind;
-
-        /**
-         * okay.. so, either the inode needs versioning or the modification
-         * needs to be tracked. ->cbk is set to the appropriate callback
-         * routine for this.
-         * NOTE: ->local needs to be deallocated on failures from here on.
-         */
-        ret = br_stub_versioning_prep (frame, this, fd, ctx);
-        if (ret)
-                goto unwind;
-
-        local = frame->local;
-        if (!inc_version) {
-                br_stub_fill_local (local, NULL, fd, fd->inode,
-                                    fd->inode->gfid, BR_STUB_NO_VERSIONING, 0);
-                cbk = br_stub_writev_cbk;
-                goto wind;
-        }
-
-        stub = fop_writev_stub (frame, br_stub_writev_resume, fd, vector, count,
-                                offset, flags, iobref, xdata);
-
-        if (!stub) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
-                        "failed to allocate stub for write fop (gfid: %s), "
-                        "unwinding", uuid_utoa (fd->inode->gfid));
-                goto cleanup_local;
-        }
-
-        /* Perform Versioning */
-        return br_stub_perform_incversioning (this, frame, stub, fd, ctx);
-
- wind:
-        STACK_WIND (frame, cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref, xdata);
-        return 0;
+wind:
+    STACK_WIND(frame, cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
+               fd, vector, count, offset, flags, iobref, xdata);
+    return 0;
 
- cleanup_local:
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
+cleanup_local:
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
 
- unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, NULL, NULL,
-                             NULL);
+unwind:
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-br_stub_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
+br_stub_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        int32_t            ret   = -1;
-        br_stub_local_t   *local = NULL;
+    int32_t ret = -1;
+    br_stub_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
+    local = frame->local;
+    frame->local = NULL;
 
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        ret = br_stub_mark_inode_modified (this, local);
-        if (ret) {
-                op_ret = -1;
-                op_errno = EINVAL;
-        }
+    ret = br_stub_mark_inode_modified(this, local);
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (ftruncate, frame,
-                             op_ret, op_errno, prebuf, postbuf, xdata);
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-br_stub_ftruncate_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                          off_t offset, dict_t *xdata)
+br_stub_ftruncate_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                         off_t offset, dict_t *xdata)
 {
-        STACK_WIND (frame, br_stub_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
-        return 0;
+    STACK_WIND(frame, br_stub_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
 /* c.f. br_stub_writev() for explanation */
 int32_t
-br_stub_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   off_t offset, dict_t *xdata)
-{
-        br_stub_local_t     *local       = NULL;
-        call_stub_t         *stub        = NULL;
-        int32_t              op_ret      = -1;
-        int32_t              op_errno    = EINVAL;
-        gf_boolean_t         inc_version = _gf_false;
-        gf_boolean_t         modified    = _gf_false;
-        br_stub_inode_ctx_t *ctx         = NULL;
-        int32_t              ret         = -1;
-        fop_ftruncate_cbk_t  cbk         = default_ftruncate_cbk;
-        br_stub_private_t   *priv        = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-
-        priv = this->private;
-        if (!priv->do_versioning)
-                goto wind;
-
-        ret = br_stub_need_versioning (this, fd, &inc_version, &modified, &ctx);
-        if (ret)
-                goto unwind;
-
-        ret = br_stub_check_bad_object (this, fd->inode, &op_ret, &op_errno);
-        if (ret)
-                goto unwind;
-
-        if (!inc_version && modified)
-                goto wind;
-
-        ret = br_stub_versioning_prep (frame, this, fd, ctx);
-        if (ret)
-                goto unwind;
-
-        local = frame->local;
-        if (!inc_version) {
-                br_stub_fill_local (local, NULL, fd, fd->inode,
-                                    fd->inode->gfid, BR_STUB_NO_VERSIONING, 0);
-                cbk = br_stub_ftruncate_cbk;
-                goto wind;
-        }
-
-        stub = fop_ftruncate_stub (frame, br_stub_ftruncate_resume, fd, offset,
-                                   xdata);
-        if (!stub) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
-                        "failed to allocate stub for ftruncate fop (gfid: %s),"
-                        " unwinding", uuid_utoa (fd->inode->gfid));
-                goto cleanup_local;
-        }
+br_stub_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
+{
+    br_stub_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t inc_version = _gf_false;
+    gf_boolean_t modified = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    fop_ftruncate_cbk_t cbk = default_ftruncate_cbk;
+    br_stub_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto wind;
+
+    ret = br_stub_need_versioning(this, fd, &inc_version, &modified, &ctx);
+    if (ret)
+        goto unwind;
+
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
+
+    if (!inc_version && modified)
+        goto wind;
+
+    ret = br_stub_versioning_prep(frame, this, fd, ctx);
+    if (ret)
+        goto unwind;
+
+    local = frame->local;
+    if (!inc_version) {
+        br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        cbk = br_stub_ftruncate_cbk;
+        goto wind;
+    }
+
+    stub = fop_ftruncate_stub(frame, br_stub_ftruncate_resume, fd, offset,
+                              xdata);
+    if (!stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
+                "ftruncate gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto cleanup_local;
+    }
+
+    return br_stub_perform_incversioning(this, frame, stub, fd, ctx);
 
-        return br_stub_perform_incversioning (this, frame, stub, fd, ctx);
-
- wind:
-        STACK_WIND (frame, cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
-        return 0;
+wind:
+    STACK_WIND(frame, cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 
- cleanup_local:
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
+cleanup_local:
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
 
- unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, NULL, NULL,
-                             NULL);
+unwind:
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-br_stub_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+br_stub_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        br_stub_local_t *local = NULL;
+    int32_t ret = 0;
+    br_stub_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
+    local = frame->local;
+    frame->local = NULL;
 
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        ret = br_stub_mark_inode_modified (this, local);
-        if (ret) {
-                op_ret = -1;
-                op_errno = EINVAL;
-        }
+    ret = br_stub_mark_inode_modified(this, local);
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
-        return 0;
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+    return 0;
 }
 
 int32_t
-br_stub_truncate_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                          off_t offset, dict_t *xdata)
+br_stub_truncate_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        off_t offset, dict_t *xdata)
 {
-        br_stub_local_t *local = frame->local;
+    br_stub_local_t *local = frame->local;
 
-        fd_unref (local->u.context.fd);
-        STACK_WIND (frame, br_stub_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
-        return 0;
+    fd_unref(local->u.context.fd);
+    STACK_WIND(frame, br_stub_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
 /**
@@ -2245,97 +2292,92 @@ br_stub_truncate_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
  * c.f. br_writev_cbk() for explanation
  */
 int32_t
-br_stub_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  off_t offset, dict_t *xdata)
-{
-        br_stub_local_t     *local       = NULL;
-        call_stub_t         *stub        = NULL;
-        int32_t              op_ret      = -1;
-        int32_t              op_errno    = EINVAL;
-        gf_boolean_t         inc_version = _gf_false;
-        gf_boolean_t         modified    = _gf_false;
-        br_stub_inode_ctx_t *ctx         = NULL;
-        int32_t              ret         = -1;
-        fd_t                *fd          = NULL;
-        fop_truncate_cbk_t   cbk         = default_truncate_cbk;
-        br_stub_private_t   *priv        = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, unwind);
-
-        priv = this->private;
-        if (!priv->do_versioning)
-                goto wind;
-
-        fd = fd_anonymous (loc->inode);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_CREATE_ANONYMOUS_FD_FAILED, "failed to create "
-                        "anonymous fd for the inode %s",
-                        uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
-
-        ret = br_stub_need_versioning (this, fd, &inc_version, &modified, &ctx);
-        if (ret)
-                goto cleanup_fd;
-
-        ret = br_stub_check_bad_object (this, fd->inode, &op_ret, &op_errno);
-        if (ret)
-                goto unwind;
-
-        if (!inc_version && modified)
-                goto wind;
-
-        ret = br_stub_versioning_prep (frame, this, fd, ctx);
-        if (ret)
-                goto cleanup_fd;
-
-        local = frame->local;
-        if (!inc_version) {
-                br_stub_fill_local (local, NULL, fd, fd->inode,
-                                    fd->inode->gfid, BR_STUB_NO_VERSIONING, 0);
-                cbk = br_stub_truncate_cbk;
-                goto wind;
-        }
-
-        stub = fop_truncate_stub (frame, br_stub_truncate_resume, loc, offset,
-                                  xdata);
-        if (!stub) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
-                        "failed to allocate stub for truncate fop (gfid: %s), "
-                        "unwinding", uuid_utoa (fd->inode->gfid));
-                goto cleanup_local;
-        }
+br_stub_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
+{
+    br_stub_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t inc_version = _gf_false;
+    gf_boolean_t modified = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    fd_t *fd = NULL;
+    fop_truncate_cbk_t cbk = default_truncate_cbk;
+    br_stub_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
+
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto wind;
+
+    fd = fd_anonymous(loc->inode);
+    if (!fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_CREATE_ANONYMOUS_FD_FAILED,
+                "inode-gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+        goto unwind;
+    }
+
+    ret = br_stub_need_versioning(this, fd, &inc_version, &modified, &ctx);
+    if (ret)
+        goto cleanup_fd;
+
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
+
+    if (!inc_version && modified)
+        goto wind;
+
+    ret = br_stub_versioning_prep(frame, this, fd, ctx);
+    if (ret)
+        goto cleanup_fd;
+
+    local = frame->local;
+    if (!inc_version) {
+        br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        cbk = br_stub_truncate_cbk;
+        goto wind;
+    }
+
+    stub = fop_truncate_stub(frame, br_stub_truncate_resume, loc, offset,
+                             xdata);
+    if (!stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
+                "truncate gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto cleanup_local;
+    }
 
-        return br_stub_perform_incversioning (this, frame, stub, fd, ctx);
+    return br_stub_perform_incversioning(this, frame, stub, fd, ctx);
 
- wind:
-        STACK_WIND (frame, cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
-        if (fd)
-                fd_unref (fd);
-        return 0;
-
- cleanup_local:
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
- cleanup_fd:
-        fd_unref (fd);
- unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, NULL, NULL,
-                             NULL);
+wind:
+    STACK_WIND(frame, cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
+               loc, offset, xdata);
+    if (fd)
+        fd_unref(fd);
+    return 0;
+
+cleanup_local:
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+cleanup_fd:
+    fd_unref(fd);
+unwind:
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 /** }}} */
 
-
 /** {{{ */
 
 /* open() */
@@ -2355,76 +2397,70 @@ br_stub_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
  */
 
 int
-br_stub_open (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, int32_t flags, fd_t *fd, dict_t *xdata)
-{
-        int32_t              ret      = -1;
-        br_stub_inode_ctx_t *ctx      = NULL;
-        uint64_t             ctx_addr = 0;
-        int32_t              op_ret   = -1;
-        int32_t              op_errno = EINVAL;
-        br_stub_private_t   *priv     = NULL;
-        unsigned long        version  = BITROT_DEFAULT_CURRENT_VERSION;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, unwind);
-
-        priv = this->private;
-
-        if (!priv->do_versioning)
-                goto wind;
-
-        ret = br_stub_get_inode_ctx (this, fd->inode, &ctx_addr);
+br_stub_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             fd_t *fd, dict_t *xdata)
+{
+    int32_t ret = -1;
+    br_stub_inode_ctx_t *ctx = NULL;
+    uint64_t ctx_addr = 0;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_private_t *priv = NULL;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, unwind);
+
+    priv = this->private;
+
+    if (!priv->do_versioning)
+        goto wind;
+
+    ret = br_stub_get_inode_ctx(this, fd->inode, &ctx_addr);
+    if (ret) {
+        ret = br_stub_init_inode_versions(this, fd, fd->inode, version,
+                                          _gf_true, _gf_false, &ctx_addr);
         if (ret) {
-                ret = br_stub_init_inode_versions (this, fd, fd->inode, version,
-                                                   _gf_true, _gf_false,
-                                                   &ctx_addr);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRS_MSG_GET_INODE_CONTEXT_FAILED,
-                                "failed to init the inode context for "
-                                "the file %s (gfid: %s)", loc->path,
-                                uuid_utoa (fd->inode->gfid));
-                        goto unwind;
-                }
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_GET_INODE_CONTEXT_FAILED, "path=%s", loc->path,
+                    "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+            goto unwind;
         }
+    }
 
-        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
-        ret = br_stub_check_bad_object (this, fd->inode, &op_ret, &op_errno);
-        if (ret)
-                goto unwind;
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
 
-        if (frame->root->pid == GF_CLIENT_PID_SCRUB)
-                goto wind;
+    if (frame->root->pid == GF_CLIENT_PID_SCRUB)
+        goto wind;
 
-        if (flags == O_RDONLY)
-                goto wind;
+    if (flags == O_RDONLY)
+        goto wind;
 
-        ret = br_stub_add_fd_to_inode (this, fd, ctx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_ADD_FD_TO_LIST_FAILED,
-                        "failed add fd to the list (gfid: %s)",
-                        uuid_utoa (fd->inode->gfid));
-                goto unwind;
-        }
+    ret = br_stub_add_fd_to_inode(this, fd, ctx);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ADD_FD_TO_LIST_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto unwind;
+    }
 
 wind:
-        STACK_WIND (frame, default_open_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->open, loc, flags, fd, xdata);
-        return 0;
+    STACK_WIND(frame, default_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
 /** }}} */
 
-
 /** {{{ */
 
 /* creat() */
@@ -2434,143 +2470,137 @@ unwind:
  * fd to the inode context fd tracking list.
  */
 int32_t
-br_stub_add_fd_to_inode (xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx)
+br_stub_add_fd_to_inode(xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx)
 {
-        int32_t       ret        = -1;
-        br_stub_fd_t *br_stub_fd = NULL;
+    int32_t ret = -1;
+    br_stub_fd_t *br_stub_fd = NULL;
 
-        ret = br_stub_require_release_call (this, fd, &br_stub_fd);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_SET_FD_CONTEXT_FAILED, "failed to set the fd "
-                        "context for the file (gfid: %s)",
-                        uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
+    ret = br_stub_require_release_call(this, fd, &br_stub_fd);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_SET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
 
-        LOCK (&fd->inode->lock);
-        {
-                list_add_tail (&ctx->fd_list, &br_stub_fd->list);
-        }
-        UNLOCK (&fd->inode->lock);
+    LOCK(&fd->inode->lock);
+    {
+        list_add_tail(&ctx->fd_list, &br_stub_fd->list);
+    }
+    UNLOCK(&fd->inode->lock);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-br_stub_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno, fd_t *fd, inode_t *inode,
-                    struct iatt *stbuf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
-{
-        int32_t              ret      = 0;
-        uint64_t             ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx      = NULL;
-        unsigned long        version  = BITROT_DEFAULT_CURRENT_VERSION;
-        br_stub_private_t   *priv     = NULL;
-
-        priv = this->private;
-
-        if (op_ret < 0)
-                goto unwind;
-
-        if (!priv->do_versioning)
-                goto unwind;
-
-        ret = br_stub_get_inode_ctx (this, fd->inode, &ctx_addr);
-        if (ret < 0) {
-                ret = br_stub_init_inode_versions (this, fd, inode, version,
-                                                   _gf_true, _gf_false,
-                                                   &ctx_addr);
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = EINVAL;
-                }
-        } else {
-                ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
-                ret = br_stub_add_fd_to_inode (this, fd, ctx);
+br_stub_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, fd_t *fd, inode_t *inode,
+                   struct iatt *stbuf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+    int32_t ret = 0;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    if (!priv->do_versioning)
+        goto unwind;
+
+    ret = br_stub_get_inode_ctx(this, fd->inode, &ctx_addr);
+    if (ret < 0) {
+        ret = br_stub_init_inode_versions(this, fd, inode, version, _gf_true,
+                                          _gf_false, &ctx_addr);
+        if (ret) {
+            op_ret = -1;
+            op_errno = EINVAL;
         }
+    } else {
+        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+        ret = br_stub_add_fd_to_inode(this, fd, ctx);
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno,
-                             fd, inode, stbuf, preparent, postparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                        preparent, postparent, xdata);
+    return 0;
 }
 
 int
-br_stub_create (call_frame_t *frame,
-                xlator_t *this, loc_t *loc, int32_t flags,
-                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, unwind);
-
-        STACK_WIND (frame, br_stub_create_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+br_stub_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, unwind);
+
+    STACK_WIND(frame, br_stub_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (create, frame, -1, EINVAL, NULL, NULL, NULL, NULL,
-                             NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
 }
 
 int
-br_stub_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, inode_t *inode,
-                    struct iatt *stbuf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
-{
-        int32_t              ret      = -1;
-        unsigned long        version  = BITROT_DEFAULT_CURRENT_VERSION;
-        br_stub_private_t   *priv     = NULL;
-
-        priv = this->private;
-
-        if (op_ret < 0)
-                goto unwind;
-
-        if (!priv->do_versioning)
-                goto unwind;
-
-        ret = br_stub_init_inode_versions (this, NULL, inode, version,
-                                           _gf_true, _gf_false, NULL);
-        /**
-         * Like lookup, if init_inode_versions fail, return EINVAL
-         */
-        if (ret) {
-                op_ret = -1;
-                op_errno = EINVAL;
-        }
+br_stub_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, inode_t *inode, struct iatt *stbuf,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
+{
+    int32_t ret = -1;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    if (!priv->do_versioning)
+        goto unwind;
+
+    ret = br_stub_init_inode_versions(this, NULL, inode, version, _gf_true,
+                                      _gf_false, NULL);
+    /**
+     * Like lookup, if init_inode_versions fail, return EINVAL
+     */
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno,
-                             inode, stbuf, preparent, postparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(mknod, frame, op_ret, op_errno, inode, stbuf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
 int
-br_stub_mknod (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+br_stub_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t dev, mode_t umask, dict_t *xdata)
 {
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, unwind);
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
 
-        STACK_WIND (frame, br_stub_mknod_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->mknod,
-                    loc, mode, dev, umask, xdata);
-        return 0;
+    STACK_WIND(frame, br_stub_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, dev, umask, xdata);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (mknod, frame, -1, EINVAL, NULL, NULL, NULL,
-                             NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(mknod, frame, -1, EINVAL, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /** }}} */
@@ -2600,226 +2630,245 @@ unwind:
  * creattion failure as the lookup failure.
  */
 static int32_t
-br_stub_lookup_version (xlator_t *this,
-                        uuid_t gfid, inode_t *inode, dict_t *xattr)
-{
-        unsigned long       version = 0;
-        br_version_t       *obuf    = NULL;
-        br_signature_t     *sbuf    = NULL;
-        br_vxattr_status_t  status;
-        gf_boolean_t        bad_object = _gf_false;
-
-        /**
-         * versioning xattrs were requested from POSIX. if available, figure
-         * out the correct version to use in the inode context (start with
-         * the default version if unavailable). As of now versions are not
-         * persisted on-disk. The inode is marked dirty, so that the first
-         * operation (such as write(), etc..) triggers synchronization to
-         * disk.
-         */
-        status = br_version_xattr_state (xattr, &obuf, &sbuf, &bad_object);
-        version = ((status == BR_VXATTR_STATUS_FULL)
-                   || (status == BR_VXATTR_STATUS_UNSIGNED))
-                        ? obuf->ongoingversion : BITROT_DEFAULT_CURRENT_VERSION;
-
-        /**
-         * If signature is there, but version is not therem then that status is
-         * is treated as INVALID. So in that case, we should not initialize the
-         * inode context with wrong version names etc.
-         */
-        if (status == BR_VXATTR_STATUS_INVALID)
-                return -1;
+br_stub_lookup_version(xlator_t *this, uuid_t gfid, inode_t *inode,
+                       dict_t *xattr)
+{
+    unsigned long version = 0;
+    br_version_t *obuf = NULL;
+    br_signature_t *sbuf = NULL;
+    br_vxattr_status_t status;
+    gf_boolean_t bad_object = _gf_false;
+
+    /**
+     * versioning xattrs were requested from POSIX. if available, figure
+     * out the correct version to use in the inode context (start with
+     * the default version if unavailable). As of now versions are not
+     * persisted on-disk. The inode is marked dirty, so that the first
+     * operation (such as write(), etc..) triggers synchronization to
+     * disk.
+     */
+    status = br_version_xattr_state(xattr, &obuf, &sbuf, &bad_object);
+    version = ((status == BR_VXATTR_STATUS_FULL) ||
+               (status == BR_VXATTR_STATUS_UNSIGNED))
+                  ? obuf->ongoingversion
+                  : BITROT_DEFAULT_CURRENT_VERSION;
+
+    /**
+     * If signature is there, but version is not there then that status is
+     * is treated as INVALID. So in that case, we should not initialize the
+     * inode context with wrong version names etc.
+     */
+    if (status == BR_VXATTR_STATUS_INVALID)
+        return -1;
 
-        return br_stub_init_inode_versions (this, NULL, inode, version,
-                                            _gf_true, bad_object, NULL);
+    return br_stub_init_inode_versions(this, NULL, inode, version, _gf_true,
+                                       bad_object, NULL);
 }
 
-
 /** {{{ */
 
 int32_t
-br_stub_opendir (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, fd_t *fd, dict_t *xdata)
-{
-        br_stub_private_t    *priv = NULL;
-        br_stub_fd_t         *fd_ctx = NULL;
-        int32_t               op_ret = -1;
-        int32_t               op_errno = EINVAL;
-
-        priv = this->private;
-        if (gf_uuid_compare (fd->inode->gfid, priv->bad_object_dir_gfid))
-                goto normal;
-
-        fd_ctx = br_stub_fd_new ();
-        if (!fd_ctx) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+br_stub_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                dict_t *xdata)
+{
+    br_stub_private_t *priv = NULL;
+    br_stub_fd_t *fd_ctx = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
 
-        fd_ctx->bad_object.dir_eof = -1;
-        fd_ctx->bad_object.dir = sys_opendir (priv->stub_basepath);
-        if (!fd_ctx->bad_object.dir) {
-                op_errno = errno;
-                goto err_freectx;
-        }
+    priv = this->private;
+    if (gf_uuid_compare(fd->inode->gfid, priv->bad_object_dir_gfid))
+        goto normal;
+
+    fd_ctx = br_stub_fd_new();
+    if (!fd_ctx) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    fd_ctx->bad_object.dir_eof = -1;
+    fd_ctx->bad_object.dir = sys_opendir(priv->stub_basepath);
+    if (!fd_ctx->bad_object.dir) {
+        op_errno = errno;
+        goto err_freectx;
+    }
 
-        op_ret = br_stub_fd_ctx_set (this, fd, fd_ctx);
-        if (!op_ret)
-                goto unwind;
+    op_ret = br_stub_fd_ctx_set(this, fd, fd_ctx);
+    if (!op_ret)
+        goto unwind;
 
-        sys_closedir (fd_ctx->bad_object.dir);
+    sys_closedir(fd_ctx->bad_object.dir);
 
 err_freectx:
-        GF_FREE (fd_ctx);
+    GF_FREE(fd_ctx);
 unwind:
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, NULL);
+    return 0;
 
 normal:
-        STACK_WIND (frame, default_opendir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
-        return 0;
+    STACK_WIND(frame, default_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 }
 
 int32_t
-br_stub_readdir (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, size_t size, off_t off, dict_t *xdata)
-{
-        call_stub_t     *stub = NULL;
-        br_stub_private_t    *priv = NULL;
-
-        priv = this->private;
-        if (!priv->do_versioning)
-                goto out;
-
-        if (gf_uuid_compare (fd->inode->gfid, priv->bad_object_dir_gfid))
-                goto out;
-        stub = fop_readdir_stub (frame, br_stub_readdir_wrapper, fd, size, off,
-                                 xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
-        br_stub_worker_enqueue (this, stub);
-        return 0;
+br_stub_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t off, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto out;
+
+    if (gf_uuid_compare(fd->inode->gfid, priv->bad_object_dir_gfid))
+        goto out;
+    stub = fop_readdir_stub(frame, br_stub_readdir_wrapper, fd, size, off,
+                            xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(readdir, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+    }
+    br_stub_worker_enqueue(this, stub);
+    return 0;
 out:
-        STACK_WIND (frame, default_readdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
-        return 0;
+    STACK_WIND(frame, default_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
+    return 0;
 }
 
 int
-br_stub_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, gf_dirent_t *entries,
-                      dict_t *dict)
-{
-        int32_t            ret     = 0;
-        uint64_t           ctxaddr = 0;
-        gf_dirent_t       *entry   = NULL;
-        br_stub_private_t *priv    = NULL;
-        gf_boolean_t       ver_enabled = _gf_false;
-
-        BR_STUB_VER_ENABLED_IN_CALLPATH (frame, ver_enabled);
-        priv = this->private;
-        BR_STUB_VER_COND_GOTO (priv, (!ver_enabled), unwind);
-
-        if (op_ret < 0)
-                goto unwind;
-
-        list_for_each_entry (entry, &entries->list, list) {
-                if ((strcmp (entry->d_name, ".") == 0)
-                    || (strcmp (entry->d_name, "..") == 0))
-                        continue;
-
-                if (!IA_ISREG (entry->d_stat.ia_type))
-                        continue;
-
-                ret = br_stub_get_inode_ctx (this, entry->inode, &ctxaddr);
-                if (ret < 0)
-                        ctxaddr = 0;
-                if (ctxaddr) { /* already has the context */
-                        br_stub_remove_vxattrs (entry->dict);
-                        continue;
-                }
-
-                ret = br_stub_lookup_version
-                        (this, entry->inode->gfid, entry->inode, entry->dict);
-                br_stub_remove_vxattrs (entry->dict);
-                if (ret) {
-                        /**
-                         * there's no per-file granularity support in case of
-                         * failure. let's fail the entire request for now..
-                         */
-                        break;
-                }
+br_stub_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, gf_dirent_t *entries,
+                     dict_t *dict)
+{
+    int32_t ret = 0;
+    uint64_t ctxaddr = 0;
+    gf_dirent_t *entry = NULL;
+    br_stub_private_t *priv = NULL;
+    gf_boolean_t ver_enabled = _gf_false;
+
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), unwind);
+
+    if (op_ret < 0)
+        goto unwind;
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        if ((strcmp(entry->d_name, ".") == 0) ||
+            (strcmp(entry->d_name, "..") == 0))
+            continue;
+
+        if (!IA_ISREG(entry->d_stat.ia_type))
+            continue;
+
+        /*
+         * Readdirp for most part is a bulk lookup for all the entries
+         * present in the directory being read. Ideally, for each
+         * entry, the handling should be similar to that of a lookup
+         * callback. But for now, just keeping this as it has been
+         * until now (which means, this comment has been added much
+         * later as part of a change that wanted to send the flag
+         * of true/false to br_stub_remove_vxattrs to indicate whether
+         * the bad-object xattr should be removed from the entry->dict
+         * or not). Until this change, the function br_stub_remove_vxattrs
+         * was just removing all the xattrs associated with bit-rot-stub
+         * (like version, bad-object, signature etc). But, there are
+         * scenarios where we only want to send bad-object xattr and not
+         * others. So this comment is part of that change which also
+         * mentions about another possible change that might be needed
+         * in future.
+         * But for now, adding _gf_true means functionally its same as
+         * what this function was doing before. Just remove all the stub
+         * related xattrs.
+         */
+        ret = br_stub_get_inode_ctx(this, entry->inode, &ctxaddr);
+        if (ret < 0)
+            ctxaddr = 0;
+        if (ctxaddr) { /* already has the context */
+            br_stub_remove_vxattrs(entry->dict, _gf_true);
+            continue;
         }
 
+        ret = br_stub_lookup_version(this, entry->inode->gfid, entry->inode,
+                                     entry->dict);
+        br_stub_remove_vxattrs(entry->dict, _gf_true);
         if (ret) {
-                op_ret   = -1;
-                op_errno = EINVAL;
+            /**
+             * there's no per-file granularity support in case of
+             * failure. let's fail the entire request for now..
+             */
+            break;
         }
+    }
 
- unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, dict);
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
-        return 0;
+unwind:
+    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, dict);
+
+    return 0;
 }
 
 int
-br_stub_readdirp (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, size_t size, off_t offset, dict_t *dict)
+br_stub_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t offset, dict_t *dict)
 {
-        int32_t            ret      = -1;
-        int                op_errno = 0;
-        gf_boolean_t       xref     = _gf_false;
-        br_stub_private_t *priv     = NULL;
-
-        priv = this->private;
-        BR_STUB_VER_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        op_errno = ENOMEM;
-        if (!dict) {
-                dict = dict_new ();
-                if (!dict)
-                        goto unwind;
-        } else {
-                dict = dict_ref (dict);
-        }
+    int32_t ret = -1;
+    int op_errno = 0;
+    gf_boolean_t xref = _gf_false;
+    br_stub_private_t *priv = NULL;
 
-        xref = _gf_true;
+    priv = this->private;
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        op_errno = EINVAL;
-        ret = dict_set_uint32 (dict, BITROT_CURRENT_VERSION_KEY, 0);
-        if (ret)
-                goto unwind;
-        ret = dict_set_uint32 (dict, BITROT_SIGNING_VERSION_KEY, 0);
-        if (ret)
-                goto unwind;
-        ret = dict_set_uint32 (dict, BITROT_OBJECT_BAD_KEY, 0);
-        if (ret)
-                goto unwind;
+    op_errno = ENOMEM;
+    if (!dict) {
+        dict = dict_new();
+        if (!dict)
+            goto unwind;
+    } else {
+        dict = dict_ref(dict);
+    }
+
+    xref = _gf_true;
+
+    op_errno = EINVAL;
+    ret = dict_set_uint32(dict, BITROT_CURRENT_VERSION_KEY, 0);
+    if (ret)
+        goto unwind;
+    ret = dict_set_uint32(dict, BITROT_SIGNING_VERSION_KEY, 0);
+    if (ret)
+        goto unwind;
+    ret = dict_set_uint32(dict, BITROT_OBJECT_BAD_KEY, 0);
+    if (ret)
+        goto unwind;
 
 wind:
-        STACK_WIND (frame, br_stub_readdirp_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->readdirp, fd, size,
-                    offset, dict);
-        goto unref_dict;
+    STACK_WIND(frame, br_stub_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
+    goto unref_dict;
 
 unwind:
-        if (frame->local == (void *)0x1)
-                frame->local = NULL;
-        STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    if (frame->local == (void *)0x1)
+        frame->local = NULL;
+    STACK_UNWIND_STRICT(readdirp, frame, -1, op_errno, NULL, NULL);
+    return 0;
 
 unref_dict:
-        if (xref)
-                dict_unref (dict);
-        return 0;
+    if (xref)
+        dict_unref(dict);
+    return 0;
 }
 
 /** }}} */
 
-
 /** {{{ */
 
 /* lookup() */
@@ -2833,203 +2882,213 @@ unref_dict:
  * either forget () or lookup () will take care of removing the link.
  */
 void
-br_stub_handle_lookup_error (xlator_t *this, inode_t *inode, int32_t op_errno)
+br_stub_handle_lookup_error(xlator_t *this, inode_t *inode, int32_t op_errno)
 {
-        int32_t     ret = -1;
-        uint64_t   ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
 
-        if (op_errno != ENOENT)
-                goto out;
+    if (op_errno != ENOENT)
+        goto out;
 
-        if (!inode_is_linked (inode))
-                goto out;
+    if (!inode_is_linked(inode))
+        goto out;
 
-        ret = br_stub_get_inode_ctx (this, inode, &ctx_addr);
-        if (ret)
-                goto out;
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret)
+        goto out;
 
-        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
-        LOCK (&inode->lock);
-        {
-                if (__br_stub_is_bad_object (ctx))
-                        (void) br_stub_del (this, inode->gfid);
-        }
-        UNLOCK (&inode->lock);
-
-        if (__br_stub_is_bad_object (ctx)) {
-                /* File is not present, might be deleted for recovery,
-                 * del the bitrot inode context
-                 */
-                ctx_addr = 0;
-                inode_ctx_del (inode, this, &ctx_addr);
-                if (ctx_addr) {
-                        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
-                        GF_FREE (ctx);
-                }
+    LOCK(&inode->lock);
+    {
+        if (__br_stub_is_bad_object(ctx))
+            (void)br_stub_del(this, inode->gfid);
+    }
+    UNLOCK(&inode->lock);
+
+    if (__br_stub_is_bad_object(ctx)) {
+        /* File is not present, might be deleted for recovery,
+         * del the bitrot inode context
+         */
+        ctx_addr = 0;
+        inode_ctx_del(inode, this, &ctx_addr);
+        if (ctx_addr) {
+            ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+            GF_FREE(ctx);
         }
+    }
 
 out:
-        return;
+    return;
 }
 
 int
-br_stub_lookup_cbk (call_frame_t *frame, void *cookie,
-                    xlator_t *this, int op_ret, int op_errno, inode_t *inode,
-                    struct iatt *stbuf, dict_t *xattr, struct iatt *postparent)
+br_stub_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent)
 {
-        int32_t ret = 0;
-        br_stub_private_t *priv = NULL;
-        gf_boolean_t  ver_enabled = _gf_false;
-
-        BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
-        priv = this->private;
-
-        if (op_ret < 0) {
-                (void) br_stub_handle_lookup_error (this, inode, op_errno);
-                goto unwind;
-        }
+    int32_t ret = 0;
+    br_stub_private_t *priv = NULL;
+    gf_boolean_t ver_enabled = _gf_false;
+    gf_boolean_t remove_bad_file_marker = _gf_true;
 
-        BR_STUB_VER_COND_GOTO (priv, (!ver_enabled), delkey);
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
 
-        if (!IA_ISREG (stbuf->ia_type))
-                goto unwind;
+    if (op_ret < 0) {
+        (void)br_stub_handle_lookup_error(this, inode, op_errno);
 
-        /**
-         * If the object is bad, then "bad inode" marker has to be sent back
-         * in resoinse, for revalidated lookups as well. Some xlators such as
-         * quick-read might cache the data in revalidated lookup as fresh
-         * lookup would anyway have sent "bad inode" marker.
-         * In general send bad inode marker for every lookup operation on the
-         * bad object.
+        /*
+         * If the lookup error is not ENOENT, then it is better
+         * to send the bad file marker to the higher layer (if
+         * it has been set)
          */
-        if (cookie != (void *) BR_STUB_REQUEST_COOKIE) {
-                ret =  br_stub_mark_xdata_bad_object (this, inode, xattr);
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = EIO;
-                        goto unwind;
-                }
-                goto delkey;
-        }
-
-        ret = br_stub_lookup_version (this, stbuf->ia_gfid, inode, xattr);
-        if (ret < 0) {
-                op_ret   = -1;
-                op_errno = EINVAL;
-                goto delkey;
-        }
-
+        if (op_errno != ENOENT)
+            remove_bad_file_marker = _gf_false;
+        goto delkey;
+    }
+
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), delkey);
+
+    if (!IA_ISREG(stbuf->ia_type))
+        goto unwind;
+
+    /**
+     * If the object is bad, then "bad inode" marker has to be sent back
+     * in resoinse, for revalidated lookups as well. Some xlators such as
+     * quick-read might cache the data in revalidated lookup as fresh
+     * lookup would anyway have sent "bad inode" marker.
+     * In general send bad inode marker for every lookup operation on the
+     * bad object.
+     */
+    if (cookie != (void *)BR_STUB_REQUEST_COOKIE) {
+        ret = br_stub_mark_xdata_bad_object(this, inode, xattr);
+        if (ret) {
+            op_ret = -1;
+            op_errno = EIO;
+            /*
+             * This flag ensures that in the label @delkey below,
+             * bad file marker is not removed from the dictinary,
+             * but other virtual xattrs (such as version, signature)
+             * are removed.
+             */
+            remove_bad_file_marker = _gf_false;
+        }
+        goto delkey;
+    }
+
+    ret = br_stub_lookup_version(this, stbuf->ia_gfid, inode, xattr);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto delkey;
+    }
+
+    /**
+     * If the object is bad, send "bad inode" marker back in response
+     * for xlator(s) to act accordingly (such as quick-read, etc..)
+     */
+    ret = br_stub_mark_xdata_bad_object(this, inode, xattr);
+    if (ret) {
         /**
-         * If the object is bad, send "bad inode" marker back in response
-         * for xlator(s) to act accordingly (such as quick-read, etc..)
+         * aaha! bad object, but sorry we would not
+         * satisfy the request on allocation failures.
          */
-        ret = br_stub_mark_xdata_bad_object (this, inode, xattr);
-        if (ret) {
-                /**
-                 * aaha! bad object, but sorry we would not
-                 * satisfy the request on allocation failures.
-                 */
-                op_ret = -1;
-                op_errno = EIO;
-                goto unwind;
-        }
+        op_ret = -1;
+        op_errno = EIO;
+        goto delkey;
+    }
 
 delkey:
-        br_stub_remove_vxattrs (xattr);
+    br_stub_remove_vxattrs(xattr, remove_bad_file_marker);
 unwind:
-        STACK_UNWIND_STRICT (lookup, frame,
-                             op_ret, op_errno, inode, stbuf, xattr, postparent);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                        postparent);
 
-        return 0;
+    return 0;
 }
 
 int
-br_stub_lookup (call_frame_t *frame,
-                xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-        int32_t ret = 0;
-        int op_errno = 0;
-        void *cookie = NULL;
-        uint64_t ctx_addr = 0;
-        gf_boolean_t xref = _gf_false;
-        br_stub_private_t *priv = NULL;
-        call_stub_t       *stub = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot-stub", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, unwind);
-
-        priv = this->private;
-
-        BR_STUB_VER_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        if (!gf_uuid_compare (loc->gfid, priv->bad_object_dir_gfid) ||
-            !gf_uuid_compare (loc->pargfid, priv->bad_object_dir_gfid)) {
-
-                stub = fop_lookup_stub (frame, br_stub_lookup_wrapper, loc,
-                                        xdata);
-                if (!stub) {
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-                br_stub_worker_enqueue (this, stub);
-                return 0;
-        }
+br_stub_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    int32_t ret = 0;
+    int op_errno = 0;
+    void *cookie = NULL;
+    uint64_t ctx_addr = 0;
+    gf_boolean_t xref = _gf_false;
+    br_stub_private_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        ret = br_stub_get_inode_ctx (this, loc->inode, &ctx_addr);
-        if (ret < 0)
-                ctx_addr = 0;
-        if (ctx_addr != 0)
-                goto wind;
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
 
-        /**
-         * fresh lookup: request version keys from POSIX
-         */
-        op_errno = ENOMEM;
-        if (!xdata) {
-                xdata = dict_new ();
-                if (!xdata)
-                        goto unwind;
-        } else {
-                xdata = dict_ref (xdata);
+    priv = this->private;
+
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    if (!gf_uuid_compare(loc->gfid, priv->bad_object_dir_gfid) ||
+        !gf_uuid_compare(loc->pargfid, priv->bad_object_dir_gfid)) {
+        stub = fop_lookup_stub(frame, br_stub_lookup_wrapper, loc, xdata);
+        if (!stub) {
+            op_errno = ENOMEM;
+            goto unwind;
         }
+        br_stub_worker_enqueue(this, stub);
+        return 0;
+    }
 
-        xref = _gf_true;
+    ret = br_stub_get_inode_ctx(this, loc->inode, &ctx_addr);
+    if (ret < 0)
+        ctx_addr = 0;
+    if (ctx_addr != 0)
+        goto wind;
 
-        /**
-         * Requesting both xattrs provides a way of sanity checking the
-         * object. Anomaly checking is done in cbk by examining absence
-         * of either or both xattrs.
-         */
-        op_errno = EINVAL;
-        ret = dict_set_uint32 (xdata, BITROT_CURRENT_VERSION_KEY, 0);
-        if (ret)
-                goto unwind;
-        ret = dict_set_uint32 (xdata, BITROT_SIGNING_VERSION_KEY, 0);
-        if (ret)
-                goto unwind;
-        ret = dict_set_uint32 (xdata, BITROT_OBJECT_BAD_KEY, 0);
-        if (ret)
-                goto unwind;
-        cookie = (void *) BR_STUB_REQUEST_COOKIE;
+    /**
+     * fresh lookup: request version keys from POSIX
+     */
+    op_errno = ENOMEM;
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata)
+            goto unwind;
+    } else {
+        xdata = dict_ref(xdata);
+    }
+
+    xref = _gf_true;
+
+    /**
+     * Requesting both xattrs provides a way of sanity checking the
+     * object. Anomaly checking is done in cbk by examining absence
+     * of either or both xattrs.
+     */
+    op_errno = EINVAL;
+    ret = dict_set_uint32(xdata, BITROT_CURRENT_VERSION_KEY, 0);
+    if (ret)
+        goto unwind;
+    ret = dict_set_uint32(xdata, BITROT_SIGNING_VERSION_KEY, 0);
+    if (ret)
+        goto unwind;
+    ret = dict_set_uint32(xdata, BITROT_OBJECT_BAD_KEY, 0);
+    if (ret)
+        goto unwind;
+    cookie = (void *)BR_STUB_REQUEST_COOKIE;
 
- wind:
-        STACK_WIND_COOKIE (frame, br_stub_lookup_cbk, cookie,
-                           FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup,
-                           loc, xdata);
-        goto dealloc_dict;
+wind:
+    STACK_WIND_COOKIE(frame, br_stub_lookup_cbk, cookie, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    goto dealloc_dict;
 
- unwind:
-        if (frame->local == (void *) 0x1)
-                frame->local = NULL;
-        STACK_UNWIND_STRICT (lookup, frame,
-                             -1, op_errno, NULL, NULL, NULL, NULL);
- dealloc_dict:
-        if (xref)
-                dict_unref (xdata);
-        return 0;
+unwind:
+    if (frame->local == (void *)0x1)
+        frame->local = NULL;
+    STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+dealloc_dict:
+    if (xref)
+        dict_unref(xdata);
+    return 0;
 }
 
 /** }}} */
@@ -3038,64 +3097,64 @@ br_stub_lookup (call_frame_t *frame,
 
 /* stat */
 int
-br_stub_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+br_stub_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int32_t            ret      = 0;
-        int32_t            op_ret   = -1;
-        int32_t            op_errno = EINVAL;
-        br_stub_private_t *priv     = NULL;
+    int32_t ret = 0;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->do_versioning)
-                goto wind;
+    if (!priv->do_versioning)
+        goto wind;
 
-        if (!IA_ISREG (loc->inode->ia_type))
-                goto wind;
+    if (!IA_ISREG(loc->inode->ia_type))
+        goto wind;
 
-        ret = br_stub_check_bad_object (this, loc->inode, &op_ret, &op_errno);
-        if (ret)
-                goto unwind;
+    ret = br_stub_check_bad_object(this, loc->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
 
- wind:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->stat, loc, xdata);
-        return 0;
+wind:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->stat,
+                    loc, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
 /* fstat */
 int
-br_stub_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+br_stub_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int32_t            ret      = 0;
-        int32_t            op_ret   = -1;
-        int32_t            op_errno = EINVAL;
-        br_stub_private_t *priv     = NULL;
+    int32_t ret = 0;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->do_versioning)
-                goto wind;
+    if (!priv->do_versioning)
+        goto wind;
 
-        if (!IA_ISREG (fd->inode->ia_type))
-                goto wind;
+    if (!IA_ISREG(fd->inode->ia_type))
+        goto wind;
 
-        ret = br_stub_check_bad_object (this, fd->inode, &op_ret, &op_errno);
-        if (ret)
-                goto unwind;
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
 
- wind:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fstat, fd, xdata);
-        return 0;
+wind:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat,
+                    fd, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fstat, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
 /** }}} */
@@ -3105,114 +3164,114 @@ unwind:
 /* unlink() */
 
 int
-br_stub_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+br_stub_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
                    struct iatt *postparent, dict_t *xdata)
 {
-        br_stub_local_t     *local    = NULL;
-        inode_t             *inode    = NULL;
-        uint64_t             ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx      = NULL;
-        int32_t              ret      = -1;
-        br_stub_private_t   *priv     = NULL;
-        gf_boolean_t         ver_enabled = _gf_false;
-
-        BR_STUB_VER_ENABLED_IN_CALLPATH (frame, ver_enabled);
-        priv = this->private;
-        BR_STUB_VER_COND_GOTO (priv, (!ver_enabled), unwind);
-
-        local = frame->local;
-        frame->local = NULL;
-
-        if (op_ret < 0)
-                goto unwind;
-
-        inode = local->u.context.inode;
-        if (!IA_ISREG (inode->ia_type))
-           goto unwind;
-
-        ret = br_stub_get_inode_ctx (this, inode, &ctx_addr);
-        if (ret) {
-                /**
-                 * If the inode is bad AND context is not there, then there
-                 * is a possibility of the gfid of the object being listed
-                 * in the quarantine directory and will be shown in the
-                 * bad objects list. So continuing with the fop with a
-                 * warning log. The entry from the quarantine directory
-                 * has to be removed manually. Its not a good idea to fail
-                 * the fop, as the object has already been deleted.
-                 */
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_GET_INODE_CONTEXT_FAILED,
-                        "failed to get the context for the inode %s",
-                        uuid_utoa (inode->gfid));
-                goto unwind;
-        }
+    br_stub_local_t *local = NULL;
+    inode_t *inode = NULL;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    br_stub_private_t *priv = NULL;
+    gf_boolean_t ver_enabled = _gf_false;
+
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), unwind);
+
+    local = frame->local;
+    frame->local = NULL;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    if (!local) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_NULL_LOCAL, NULL);
+        goto unwind;
+    }
+    inode = local->u.context.inode;
+    if (!IA_ISREG(inode->ia_type))
+        goto unwind;
+
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret) {
+        /**
+         * If the inode is bad AND context is not there, then there
+         * is a possibility of the gfid of the object being listed
+         * in the quarantine directory and will be shown in the
+         * bad objects list. So continuing with the fop with a
+         * warning log. The entry from the quarantine directory
+         * has to be removed manually. Its not a good idea to fail
+         * the fop, as the object has already been deleted.
+         */
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+                "inode-gfid=%s", uuid_utoa(inode->gfid), NULL);
+        goto unwind;
+    }
 
-        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
-        LOCK (&inode->lock);
-        {
-                /**
-                 * Ignoring the return value of br_stub_del ().
-                 * There is not much that can be done if unlinking
-                 * of the entry in the quarantine directory fails.
-                 * The failure is logged.
-                 */
-                if (__br_stub_is_bad_object (ctx))
-                        (void) br_stub_del (this, inode->gfid);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        /**
+         * Ignoring the return value of br_stub_del ().
+         * There is not much that can be done if unlinking
+         * of the entry in the quarantine directory fails.
+         * The failure is logged.
+         */
+        if (__br_stub_is_bad_object(ctx))
+            (void)br_stub_del(this, inode->gfid);
+    }
+    UNLOCK(&inode->lock);
 
 unwind:
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
-        return 0;
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+    return 0;
 }
 
 int
-br_stub_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
-                dict_t *xdata)
+br_stub_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+               dict_t *xdata)
 {
-        br_stub_local_t   *local    = NULL;
-        int32_t            op_ret   = -1;
-        int32_t            op_errno = 0;
-        br_stub_private_t *priv     = NULL;
+    br_stub_local_t *local = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    br_stub_private_t *priv = NULL;
 
-        priv = this->private;
-        BR_STUB_VER_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        local = br_stub_alloc_local (this);
-        if (!local) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM, BRS_MSG_NO_MEMORY,
-                        "failed to allocate memory for local (path: %s, gfid: %s)",
-                        loc->path, uuid_utoa (loc->inode->gfid));
-                goto unwind;
-        }
+    local = br_stub_alloc_local(this);
+    if (!local) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRS_MSG_ALLOC_MEM_FAILED,
+                "local path=%s", loc->path, "gfid=%s",
+                uuid_utoa(loc->inode->gfid), NULL);
+        goto unwind;
+    }
 
-        br_stub_fill_local (local, NULL, NULL, loc->inode,
-                            loc->inode->gfid,
-                            BR_STUB_NO_VERSIONING, 0);
+    br_stub_fill_local(local, NULL, NULL, loc->inode, loc->inode->gfid,
+                       BR_STUB_NO_VERSIONING, 0);
 
-        frame->local = local;
+    frame->local = local;
 
 wind:
-        STACK_WIND (frame, br_stub_unlink_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->unlink, loc, flag, xdata);
-        return 0;
+    STACK_WIND(frame, br_stub_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, flag, xdata);
+    return 0;
 
 unwind:
-        if (frame->local == (void *)0x1)
-                frame->local = NULL;
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, NULL, NULL, NULL);
-        return 0;
+    if (frame->local == (void *)0x1)
+        frame->local = NULL;
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /** }}} */
 
 /** {{{ */
@@ -3220,20 +3279,20 @@ unwind:
 /* forget() */
 
 int
-br_stub_forget (xlator_t *this, inode_t *inode)
+br_stub_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
 
-        inode_ctx_del (inode, this, &ctx_addr);
-        if (!ctx_addr)
-                return 0;
+    inode_ctx_del(inode, this, &ctx_addr);
+    if (!ctx_addr)
+        return 0;
 
-        ctx = (br_stub_inode_ctx_t *) (long) ctx_addr;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
-        GF_FREE (ctx);
+    GF_FREE(ctx);
 
-        return 0;
+    return 0;
 }
 
 /** }}} */
@@ -3241,60 +3300,58 @@ br_stub_forget (xlator_t *this, inode_t *inode)
 /** {{{ */
 
 int32_t
-br_stub_noop (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+br_stub_noop(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
 {
-        STACK_DESTROY (frame->root);
-        return 0;
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
 static void
-br_stub_send_ipc_fop (xlator_t *this, fd_t *fd, unsigned long releaseversion,
-                      int sign_info)
-{
-        int32_t op = 0;
-        int32_t ret = 0;
-        dict_t *xdata = NULL;
-        call_frame_t *frame = NULL;
-        changelog_event_t ev = {0,};
-
-        ev.ev_type = CHANGELOG_OP_TYPE_BR_RELEASE;
-        ev.u.releasebr.version = releaseversion;
-        ev.u.releasebr.sign_info = sign_info;
-        gf_uuid_copy (ev.u.releasebr.gfid, fd->inode->gfid);
-
-        xdata = dict_new ();
-        if (!xdata) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM, BRS_MSG_NO_MEMORY,
-                        "dict allocation failed: cannot send IPC FOP "
-                        "to changelog");
-                goto out;
-        }
-
-        ret = dict_set_static_bin (xdata,
-                                   "RELEASE-EVENT", &ev, CHANGELOG_EV_SIZE);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, BRS_MSG_SET_EVENT_FAILED,
-                        "cannot set release event in dict");
-                goto dealloc_dict;
-        }
+br_stub_send_ipc_fop(xlator_t *this, fd_t *fd, unsigned long releaseversion,
+                     int sign_info)
+{
+    int32_t op = 0;
+    int32_t ret = 0;
+    dict_t *xdata = NULL;
+    call_frame_t *frame = NULL;
+    changelog_event_t ev = {
+        0,
+    };
+
+    ev.ev_type = CHANGELOG_OP_TYPE_BR_RELEASE;
+    ev.u.releasebr.version = releaseversion;
+    ev.u.releasebr.sign_info = sign_info;
+    gf_uuid_copy(ev.u.releasebr.gfid, fd->inode->gfid);
+
+    xdata = dict_new();
+    if (!xdata) {
+        gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, BRS_MSG_DICT_ALLOC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = dict_set_static_bin(xdata, "RELEASE-EVENT", &ev, CHANGELOG_EV_SIZE);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SET_EVENT_FAILED, NULL);
+        goto dealloc_dict;
+    }
 
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_CREATE_FRAME_FAILED,
-                        "create_frame() failure");
-                goto dealloc_dict;
-        }
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_CREATE_FRAME_FAILED,
+                NULL);
+        goto dealloc_dict;
+    }
 
-        op = GF_IPC_TARGET_CHANGELOG;
-        STACK_WIND (frame, br_stub_noop, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ipc, op, xdata);
+    op = GF_IPC_TARGET_CHANGELOG;
+    STACK_WIND(frame, br_stub_noop, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
 
- dealloc_dict:
-        dict_unref (xdata);
- out:
-        return;
+dealloc_dict:
+    dict_unref(xdata);
+out:
+    return;
 }
 
 /**
@@ -3305,7 +3362,7 @@ br_stub_send_ipc_fop (xlator_t *this, fd_t *fd, unsigned long releaseversion,
  * 3) BR_SIGN_QUICK => reopen has happened and this release should trigger sign
  * 2 events:
  * 1) GF_FOP_RELEASE
- * 2) GF_FOP_WRITE (actually a dummy write fro BitD)
+ * 2) GF_FOP_WRITE (actually a dummy write for BitD)
  *
  * This is how states are changed based on events:
  * EVENT: GF_FOP_RELEASE:
@@ -3318,115 +3375,113 @@ br_stub_send_ipc_fop (xlator_t *this, fd_t *fd, unsigned long releaseversion,
  *     set state = BR_SIGN_QUICK;
  */
 br_sign_state_t
-__br_stub_inode_sign_state (br_stub_inode_ctx_t *ctx,
-                            glusterfs_fop_t fop, fd_t *fd)
+__br_stub_inode_sign_state(br_stub_inode_ctx_t *ctx, glusterfs_fop_t fop,
+                           fd_t *fd)
 {
-        br_sign_state_t sign_info = BR_SIGN_INVALID;
-
-        switch (fop) {
+    br_sign_state_t sign_info = BR_SIGN_INVALID;
 
+    switch (fop) {
         case GF_FOP_FSETXATTR:
-                sign_info = ctx->info_sign = BR_SIGN_QUICK;
-                break;
+            sign_info = ctx->info_sign = BR_SIGN_QUICK;
+            break;
 
         case GF_FOP_RELEASE:
-                GF_ASSERT (ctx->info_sign != BR_SIGN_REOPEN_WAIT);
+            GF_ASSERT(ctx->info_sign != BR_SIGN_REOPEN_WAIT);
 
-                if (ctx->info_sign == BR_SIGN_NORMAL) {
-                        sign_info = ctx->info_sign = BR_SIGN_REOPEN_WAIT;
-                } else {
-                        sign_info = ctx->info_sign;
-                        ctx->info_sign = BR_SIGN_NORMAL;
-                }
+            if (ctx->info_sign == BR_SIGN_NORMAL) {
+                sign_info = ctx->info_sign = BR_SIGN_REOPEN_WAIT;
+            } else {
+                sign_info = ctx->info_sign;
+                ctx->info_sign = BR_SIGN_NORMAL;
+            }
 
-                break;
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        return sign_info;
+    return sign_info;
 }
 
 int32_t
-br_stub_release (xlator_t *this, fd_t *fd)
-{
-        int32_t              ret            = 0;
-        int32_t              flags          = 0;
-        inode_t             *inode          = NULL;
-        unsigned long        releaseversion = 0;
-        br_stub_inode_ctx_t *ctx            = NULL;
-        uint64_t             tmp            = 0;
-        br_stub_fd_t        *br_stub_fd     = NULL;
-        int32_t              signinfo       = 0;
-
-        inode = fd->inode;
-
-        LOCK (&inode->lock);
-        {
-                ctx = __br_stub_get_ongoing_version_ctx (this, inode, NULL);
-                if (ctx == NULL)
-                        goto unblock;
-                br_stub_fd = br_stub_fd_ctx_get (this, fd);
-                if (br_stub_fd) {
-                        list_del_init (&br_stub_fd->list);
-                }
-
-                ret = __br_stub_can_trigger_release
-                                    (inode, ctx, &releaseversion);
-                if (!ret)
-                        goto unblock;
-
-                signinfo = __br_stub_inode_sign_state (ctx, GF_FOP_RELEASE, fd);
-                signinfo = htonl (signinfo);
-
-                /* inode back to initital state: mark dirty */
-                if (ctx->info_sign == BR_SIGN_NORMAL) {
-                        __br_stub_mark_inode_dirty (ctx);
-                        __br_stub_unset_inode_modified (ctx);
-                }
-        }
- unblock:
-        UNLOCK (&inode->lock);
+br_stub_release(xlator_t *this, fd_t *fd)
+{
+    int32_t ret = 0;
+    int32_t flags = 0;
+    inode_t *inode = NULL;
+    unsigned long releaseversion = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    uint64_t tmp = 0;
+    br_stub_fd_t *br_stub_fd = NULL;
+    int32_t signinfo = 0;
+
+    inode = fd->inode;
+
+    LOCK(&inode->lock);
+    {
+        ctx = __br_stub_get_ongoing_version_ctx(this, inode, NULL);
+        if (ctx == NULL)
+            goto unblock;
+        br_stub_fd = br_stub_fd_ctx_get(this, fd);
+        if (br_stub_fd) {
+            list_del_init(&br_stub_fd->list);
+        }
+
+        ret = __br_stub_can_trigger_release(inode, ctx, &releaseversion);
+        if (!ret)
+            goto unblock;
+
+        signinfo = __br_stub_inode_sign_state(ctx, GF_FOP_RELEASE, fd);
+        signinfo = htonl(signinfo);
+
+        /* inode back to initital state: mark dirty */
+        if (ctx->info_sign == BR_SIGN_NORMAL) {
+            __br_stub_mark_inode_dirty(ctx);
+            __br_stub_unset_inode_modified(ctx);
+        }
+    }
+unblock:
+    UNLOCK(&inode->lock);
 
-        if (ret) {
-                gf_msg_debug (this->name, 0, "releaseversion: %lu | flags: %d "
-                              "| signinfo: %d",
-                              (unsigned long) ntohl (releaseversion), flags,
-                              ntohl(signinfo));
-                br_stub_send_ipc_fop (this, fd, releaseversion, signinfo);
-        }
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "releaseversion: %lu | flags: %d "
+                     "| signinfo: %d",
+                     (unsigned long)ntohl(releaseversion), flags,
+                     ntohl(signinfo));
+        br_stub_send_ipc_fop(this, fd, releaseversion, signinfo);
+    }
 
-        ret = fd_ctx_del (fd, this, &tmp);
-        br_stub_fd = (br_stub_fd_t *)(long)tmp;
+    ret = fd_ctx_del(fd, this, &tmp);
+    br_stub_fd = (br_stub_fd_t *)(long)tmp;
 
-        GF_FREE (br_stub_fd);
+    GF_FREE(br_stub_fd);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-br_stub_releasedir (xlator_t *this, fd_t *fd)
+br_stub_releasedir(xlator_t *this, fd_t *fd)
 {
-        br_stub_fd_t   *fctx = NULL;
-        uint64_t        ctx  = 0;
-        int             ret  = 0;
+    br_stub_fd_t *fctx = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
 
-        ret = fd_ctx_del (fd, this, &ctx);
-        if (ret < 0)
-                goto out;
-
-        fctx = (br_stub_fd_t *) (long) ctx;
-        if (fctx->bad_object.dir) {
-                ret = sys_closedir (fctx->bad_object.dir);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                BRS_MSG_BAD_OBJ_DIR_CLOSE_FAIL,
-                                "closedir error: %s", strerror (errno));
-        }
+    ret = fd_ctx_del(fd, this, &ctx);
+    if (ret < 0)
+        goto out;
 
-        GF_FREE (fctx);
+    fctx = (br_stub_fd_t *)(long)ctx;
+    if (fctx->bad_object.dir) {
+        ret = sys_closedir(fctx->bad_object.dir);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_DIR_CLOSE_FAIL,
+                    "error=%s", strerror(errno), NULL);
+    }
+
+    GF_FREE(fctx);
 out:
-        return 0;
+    return 0;
 }
 
 /** }}} */
@@ -3436,90 +3491,100 @@ out:
 /* ictxmerge */
 
 void
-br_stub_ictxmerge (xlator_t *this, fd_t *fd,
-                   inode_t *inode, inode_t *linked_inode)
-{
-        int32_t              ret        = 0;
-        uint64_t             ctxaddr    = 0;
-        uint64_t             lctxaddr   = 0;
-        br_stub_inode_ctx_t *ctx        = NULL;
-        br_stub_inode_ctx_t *lctx       = NULL;
-        br_stub_fd_t        *br_stub_fd = NULL;
-
-        ret = br_stub_get_inode_ctx (this, inode, &ctxaddr);
+br_stub_ictxmerge(xlator_t *this, fd_t *fd, inode_t *inode,
+                  inode_t *linked_inode)
+{
+    int32_t ret = 0;
+    uint64_t ctxaddr = 0;
+    uint64_t lctxaddr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    br_stub_inode_ctx_t *lctx = NULL;
+    br_stub_fd_t *br_stub_fd = NULL;
+
+    ret = br_stub_get_inode_ctx(this, inode, &ctxaddr);
+    if (ret < 0)
+        goto done;
+    ctx = (br_stub_inode_ctx_t *)(uintptr_t)ctxaddr;
+
+    LOCK(&linked_inode->lock);
+    {
+        ret = __br_stub_get_inode_ctx(this, linked_inode, &lctxaddr);
         if (ret < 0)
-                goto done;
-        ctx = (br_stub_inode_ctx_t *) ctxaddr;
+            goto unblock;
+        lctx = (br_stub_inode_ctx_t *)(uintptr_t)lctxaddr;
 
-        LOCK (&linked_inode->lock);
-        {
-                ret = __br_stub_get_inode_ctx (this, linked_inode, &lctxaddr);
-                if (ret < 0)
-                        goto unblock;
-                lctx = (br_stub_inode_ctx_t *) lctxaddr;
-
-                GF_ASSERT (list_is_singular (&ctx->fd_list));
-                br_stub_fd = list_first_entry (&ctx->fd_list, br_stub_fd_t,
-                                               list);
-                if (br_stub_fd) {
-                        GF_ASSERT (br_stub_fd->fd == fd);
-                        list_move_tail (&br_stub_fd->list, &lctx->fd_list);
-                }
+        GF_ASSERT(list_is_singular(&ctx->fd_list));
+        br_stub_fd = list_first_entry(&ctx->fd_list, br_stub_fd_t, list);
+        if (br_stub_fd) {
+            GF_ASSERT(br_stub_fd->fd == fd);
+            list_move_tail(&br_stub_fd->list, &lctx->fd_list);
         }
+    }
 unblock:
-        UNLOCK (&linked_inode->lock);
+    UNLOCK(&linked_inode->lock);
 
- done:
-        return;
+done:
+    return;
 }
 
 /** }}} */
 
-
 struct xlator_fops fops = {
-        .lookup    = br_stub_lookup,
-        .stat      = br_stub_stat,
-        .fstat     = br_stub_fstat,
-        .open      = br_stub_open,
-        .create    = br_stub_create,
-        .readdirp  = br_stub_readdirp,
-        .getxattr  = br_stub_getxattr,
-        .fgetxattr = br_stub_fgetxattr,
-        .fsetxattr = br_stub_fsetxattr,
-        .writev    = br_stub_writev,
-        .truncate  = br_stub_truncate,
-        .ftruncate = br_stub_ftruncate,
-        .mknod     = br_stub_mknod,
-        .readv     = br_stub_readv,
-        .removexattr = br_stub_removexattr,
-        .fremovexattr = br_stub_fremovexattr,
-        .setxattr  = br_stub_setxattr,
-        .opendir   = br_stub_opendir,
-        .readdir   = br_stub_readdir,
-        .unlink    = br_stub_unlink,
+    .lookup = br_stub_lookup,
+    .stat = br_stub_stat,
+    .fstat = br_stub_fstat,
+    .open = br_stub_open,
+    .create = br_stub_create,
+    .readdirp = br_stub_readdirp,
+    .getxattr = br_stub_getxattr,
+    .fgetxattr = br_stub_fgetxattr,
+    .fsetxattr = br_stub_fsetxattr,
+    .writev = br_stub_writev,
+    .truncate = br_stub_truncate,
+    .ftruncate = br_stub_ftruncate,
+    .mknod = br_stub_mknod,
+    .readv = br_stub_readv,
+    .removexattr = br_stub_removexattr,
+    .fremovexattr = br_stub_fremovexattr,
+    .setxattr = br_stub_setxattr,
+    .opendir = br_stub_opendir,
+    .readdir = br_stub_readdir,
+    .unlink = br_stub_unlink,
 };
 
 struct xlator_cbks cbks = {
-        .forget    = br_stub_forget,
-        .release   = br_stub_release,
-        .ictxmerge = br_stub_ictxmerge,
+    .forget = br_stub_forget,
+    .release = br_stub_release,
+    .ictxmerge = br_stub_ictxmerge,
 };
 
 struct volume_options options[] = {
-        { .key = {"bitrot"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .op_version  = {GD_OP_VERSION_3_7_0},
-          .flags       = OPT_FLAG_SETTABLE | OPT_FLAG_FORCE,
-          .tags        = {"bitrot"},
-          .description = "enable/disable bitrot stub"
-        },
-        { .key = {"export"},
-          .type = GF_OPTION_TYPE_PATH,
-          .op_version  = {GD_OP_VERSION_3_7_0},
-          .tags        = {"bitrot"},
-          .description = "brick path for versioning",
-          .default_value = "{{ brick.path }}"
-        },
-	{ .key  = {NULL} },
+    {.key = {"bitrot"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_FORCE,
+     .tags = {"bitrot"},
+     .description = "enable/disable bitrot stub"},
+    {.key = {"export"},
+     .type = GF_OPTION_TYPE_PATH,
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .tags = {"bitrot"},
+     .description = "brick path for versioning",
+     .default_value = "{{ brick.path }}"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "bitrot-stub",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub.h b/xlators/features/bit-rot/src/stub/bit-rot-stub.h
index ae4db0fd4f1..edd79a77e4f 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub.h
@@ -1,329 +1,325 @@
- /*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 #ifndef __BIT_ROT_STUB_H__
 #define __BIT_ROT_STUB_H__
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
-#include "call-stub.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/call-stub.h>
 #include "bit-rot-stub-mem-types.h"
-#include "syscall.h"
-#include "common-utils.h"
+#include <glusterfs/syscall.h>
+#include <glusterfs/common-utils.h>
 #include "bit-rot-common.h"
 #include "bit-rot-stub-messages.h"
 #include "glusterfs3-xdr.h"
-#include "syncop.h"
-#include "syncop-utils.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/syncop-utils.h>
 
-#define BAD_OBJECT_THREAD_STACK_SIZE   ((size_t)(1024*1024))
+#define BAD_OBJECT_THREAD_STACK_SIZE ((size_t)(1024 * 1024))
 #define BR_STUB_DUMP_STR_SIZE 65536
 
-#define BR_PATH_MAX_EXTRA (PATH_MAX+1024)
-#define BR_PATH_MAX_PLUS (PATH_MAX+2048)
+#define BR_PATH_MAX_EXTRA (PATH_MAX + 1024)
+#define BR_PATH_MAX_PLUS (PATH_MAX + 2048)
 
 /*
  * Oops. Spelling mistake. Correcting it
  */
-#define OLD_BR_STUB_QUARANTINE_DIR GF_HIDDEN_PATH"/quanrantine"
-#define BR_STUB_QUARANTINE_DIR GF_HIDDEN_PATH"/quarantine"
+#define OLD_BR_STUB_QUARANTINE_DIR GF_HIDDEN_PATH "/quanrantine"
+#define BR_STUB_QUARANTINE_DIR GF_HIDDEN_PATH "/quarantine"
 
 /* do not reference frame->local in cbk unless initialized.
  * Assigned 0x1 marks verisoning flag between call path and
  * cbk path.
  */
-#define BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, label) do {      \
-                if (priv->do_versioning)                               \
-                        frame->local = (void *)0x1;                    \
-                else                                                   \
-                        goto label;                                    \
-        } while (0)
-
-#define BR_STUB_VER_COND_GOTO(priv, cond, label) do {       \
-                if (!priv->do_versioning || cond)           \
-                        goto label;                         \
-        } while (0)
-
-#define BR_STUB_VER_ENABLED_IN_CALLPATH(frame, flag) do {   \
-                if (frame->local)                           \
-                        flag = _gf_true;                    \
-                if (frame->local == (void *) 0x1)           \
-                        frame->local = NULL;                \
-        } while (0)
-
-#define BR_STUB_RESET_LOCAL_NULL(frame) do {           \
-                if (frame->local == (void *)0x1)      \
-                        frame->local = NULL;           \
-        } while (0)
-
-typedef int (br_stub_version_cbk) (call_frame_t *, void *,
-                                   xlator_t *, int32_t, int32_t, dict_t *);
+#define BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, label)                   \
+    do {                                                                       \
+        if (priv->do_versioning)                                               \
+            frame->local = (void *)0x1;                                        \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define BR_STUB_VER_COND_GOTO(priv, cond, label)                               \
+    do {                                                                       \
+        if (!priv->do_versioning || cond)                                      \
+            goto label;                                                        \
+    } while (0)
+
+#define BR_STUB_VER_ENABLED_IN_CALLPATH(frame, flag)                           \
+    do {                                                                       \
+        if (frame->local)                                                      \
+            flag = _gf_true;                                                   \
+        if (frame->local == (void *)0x1)                                       \
+            frame->local = NULL;                                               \
+    } while (0)
+
+#define BR_STUB_RESET_LOCAL_NULL(frame)                                        \
+    do {                                                                       \
+        if (frame->local == (void *)0x1)                                       \
+            frame->local = NULL;                                               \
+    } while (0)
+
+typedef int(br_stub_version_cbk)(call_frame_t *, void *, xlator_t *, int32_t,
+                                 int32_t, dict_t *);
 
 typedef struct br_stub_inode_ctx {
-        int need_writeback;                     /* does the inode need
-                                                      a writeback to disk? */
-        unsigned long currentversion;           /* ongoing version */
-
-        int            info_sign;
-        struct list_head fd_list; /* list of open fds or fds participating in
-                                     write operations */
-        gf_boolean_t bad_object;
+    int need_writeback;           /* does the inode need
+                                        a writeback to disk? */
+    unsigned long currentversion; /* ongoing version */
+
+    int info_sign;
+    struct list_head fd_list; /* list of open fds or fds participating in
+                                 write operations */
+    gf_boolean_t bad_object;
 } br_stub_inode_ctx_t;
 
 typedef struct br_stub_fd {
-        fd_t *fd;
-        struct list_head list;
-        struct bad_object_dir {
-                DIR *dir;
-                off_t dir_eof;
-        } bad_object;
+    fd_t *fd;
+    struct list_head list;
+    struct bad_object_dir {
+        DIR *dir;
+        off_t dir_eof;
+    } bad_object;
 } br_stub_fd_t;
 
-#define I_DIRTY  (1<<0)        /* inode needs writeback */
-#define I_MODIFIED (1<<1)
-#define WRITEBACK_DURABLE 1    /* writeback is durable */
+#define I_DIRTY (1 << 0) /* inode needs writeback */
+#define I_MODIFIED (1 << 1)
+#define WRITEBACK_DURABLE 1 /* writeback is durable */
 
 /**
  * This could just have been a plain struct without unions and all,
  * but we may need additional things in the future.
  */
 typedef struct br_stub_local {
-        call_stub_t *fopstub;   /* stub for original fop */
-
-        int versioningtype;     /* not much used atm */
-
-        union {
-                struct br_stub_ctx {
-                        fd_t          *fd;
-                        uuid_t         gfid;
-                        inode_t       *inode;
-                        unsigned long  version;
-                } context;
-        } u;
+    call_stub_t *fopstub; /* stub for original fop */
+
+    int versioningtype; /* not much used atm */
+
+    union {
+        struct br_stub_ctx {
+            fd_t *fd;
+            uuid_t gfid;
+            inode_t *inode;
+            unsigned long version;
+        } context;
+    } u;
 } br_stub_local_t;
 
 #define BR_STUB_NO_VERSIONING (1 << 0)
 #define BR_STUB_INCREMENTAL_VERSIONING (1 << 1)
 
 typedef struct br_stub_private {
-        gf_boolean_t do_versioning;
+    gf_boolean_t do_versioning;
 
-        uint32_t boot[2];
-        char export[PATH_MAX];
+    uint32_t boot[2];
+    char export[PATH_MAX];
 
-        pthread_mutex_t lock;
-        pthread_cond_t  cond;
+    pthread_mutex_t lock;
+    pthread_cond_t cond;
 
-        struct list_head squeue;      /* ordered signing queue */
-        pthread_t signth;
-        struct bad_objects_container {
-                pthread_t thread;
-                pthread_mutex_t bad_lock;
-                pthread_cond_t  bad_cond;
-                struct list_head bad_queue;
-        } container;
-        struct mem_pool *local_pool;
+    struct list_head squeue; /* ordered signing queue */
+    pthread_t signth;
+    struct bad_objects_container {
+        pthread_t thread;
+        pthread_mutex_t bad_lock;
+        pthread_cond_t bad_cond;
+        struct list_head bad_queue;
+    } container;
+    struct mem_pool *local_pool;
 
-        char stub_basepath[BR_PATH_MAX_EXTRA];
+    char stub_basepath[BR_PATH_MAX_EXTRA];
 
-        uuid_t bad_object_dir_gfid;
+    uuid_t bad_object_dir_gfid;
 } br_stub_private_t;
 
 br_stub_fd_t *
-br_stub_fd_new (void);
-
+br_stub_fd_new(void);
 
 int
-__br_stub_fd_ctx_set (xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd);
+__br_stub_fd_ctx_set(xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd);
 
 br_stub_fd_t *
-__br_stub_fd_ctx_get (xlator_t *this, fd_t *fd);
+__br_stub_fd_ctx_get(xlator_t *this, fd_t *fd);
 
 br_stub_fd_t *
-br_stub_fd_ctx_get (xlator_t *this, fd_t *fd);
+br_stub_fd_ctx_get(xlator_t *this, fd_t *fd);
 
 int32_t
-br_stub_fd_ctx_set (xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd);
+br_stub_fd_ctx_set(xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd);
 
 static inline gf_boolean_t
-__br_stub_is_bad_object (br_stub_inode_ctx_t *ctx)
+__br_stub_is_bad_object(br_stub_inode_ctx_t *ctx)
 {
-        return ctx->bad_object;
+    return ctx->bad_object;
 }
 
 static inline void
-__br_stub_mark_object_bad (br_stub_inode_ctx_t *ctx)
+__br_stub_mark_object_bad(br_stub_inode_ctx_t *ctx)
 {
-        ctx->bad_object = _gf_true;
+    ctx->bad_object = _gf_true;
 }
 
 /* inode writeback helpers */
 static inline void
-__br_stub_mark_inode_dirty (br_stub_inode_ctx_t *ctx)
+__br_stub_mark_inode_dirty(br_stub_inode_ctx_t *ctx)
 {
-        ctx->need_writeback |= I_DIRTY;
+    ctx->need_writeback |= I_DIRTY;
 }
 
 static inline void
-__br_stub_mark_inode_synced (br_stub_inode_ctx_t *ctx)
+__br_stub_mark_inode_synced(br_stub_inode_ctx_t *ctx)
 {
-        ctx->need_writeback &= ~I_DIRTY;
+    ctx->need_writeback &= ~I_DIRTY;
 }
 
 static inline int
-__br_stub_is_inode_dirty (br_stub_inode_ctx_t *ctx)
+__br_stub_is_inode_dirty(br_stub_inode_ctx_t *ctx)
 {
-        return (ctx->need_writeback & I_DIRTY);
+    return (ctx->need_writeback & I_DIRTY);
 }
 
 /* inode mofification markers */
 static inline void
-__br_stub_set_inode_modified (br_stub_inode_ctx_t *ctx)
+__br_stub_set_inode_modified(br_stub_inode_ctx_t *ctx)
 {
-        ctx->need_writeback |= I_MODIFIED;
+    ctx->need_writeback |= I_MODIFIED;
 }
 
 static inline void
-__br_stub_unset_inode_modified (br_stub_inode_ctx_t *ctx)
+__br_stub_unset_inode_modified(br_stub_inode_ctx_t *ctx)
 {
-        ctx->need_writeback &= ~I_MODIFIED;
+    ctx->need_writeback &= ~I_MODIFIED;
 }
 
 static inline int
-__br_stub_is_inode_modified (br_stub_inode_ctx_t *ctx)
+__br_stub_is_inode_modified(br_stub_inode_ctx_t *ctx)
 {
-        return (ctx->need_writeback & I_MODIFIED);
+    return (ctx->need_writeback & I_MODIFIED);
 }
 
-
 static inline int
-br_stub_require_release_call (xlator_t *this, fd_t *fd, br_stub_fd_t **fd_ctx)
+br_stub_require_release_call(xlator_t *this, fd_t *fd, br_stub_fd_t **fd_ctx)
 {
-        int32_t ret = 0;
-        br_stub_fd_t *br_stub_fd = NULL;
+    int32_t ret = 0;
+    br_stub_fd_t *br_stub_fd = NULL;
 
-        br_stub_fd = br_stub_fd_new ();
-        if (!br_stub_fd)
-                return -1;
+    br_stub_fd = br_stub_fd_new();
+    if (!br_stub_fd)
+        return -1;
 
-        br_stub_fd->fd = fd;
-        INIT_LIST_HEAD (&br_stub_fd->list);
+    br_stub_fd->fd = fd;
+    INIT_LIST_HEAD(&br_stub_fd->list);
 
-        ret = br_stub_fd_ctx_set (this, fd, br_stub_fd);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        BRS_MSG_SET_CONTEXT_FAILED,
-                        "could not set fd context (for release callback");
-        else
-                *fd_ctx = br_stub_fd;
+    ret = br_stub_fd_ctx_set(this, fd, br_stub_fd);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SET_CONTEXT_FAILED,
+                NULL);
+    else
+        *fd_ctx = br_stub_fd;
 
-        return ret;
+    return ret;
 }
 
 /* get/set inode context helpers */
 
 static inline int
-__br_stub_get_inode_ctx (xlator_t *this,
-                         inode_t *inode, uint64_t *ctx)
+__br_stub_get_inode_ctx(xlator_t *this, inode_t *inode, uint64_t *ctx)
 {
-        return __inode_ctx_get (inode, this, ctx);
+    return __inode_ctx_get(inode, this, ctx);
 }
 
 static inline int
-br_stub_get_inode_ctx (xlator_t *this,
-                       inode_t *inode, uint64_t *ctx)
+br_stub_get_inode_ctx(xlator_t *this, inode_t *inode, uint64_t *ctx)
 {
-        int ret = -1;
+    int ret = -1;
 
-        LOCK (&inode->lock);
-        {
-                ret = __br_stub_get_inode_ctx (this, inode, ctx);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __br_stub_get_inode_ctx(this, inode, ctx);
+    }
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
 static inline int
-br_stub_set_inode_ctx (xlator_t *this,
-                       inode_t *inode, br_stub_inode_ctx_t *ctx)
+br_stub_set_inode_ctx(xlator_t *this, inode_t *inode, br_stub_inode_ctx_t *ctx)
 {
-        uint64_t ctx_addr = (uint64_t) ctx;
-        return inode_ctx_set (inode, this, &ctx_addr);
+    uint64_t ctx_addr = (uint64_t)(uintptr_t)ctx;
+    return inode_ctx_set(inode, this, &ctx_addr);
 }
 
 /* version get/set helpers */
 
 static inline unsigned long
-__br_stub_writeback_version (br_stub_inode_ctx_t *ctx)
+__br_stub_writeback_version(br_stub_inode_ctx_t *ctx)
 {
-        return (ctx->currentversion + 1);
+    return (ctx->currentversion + 1);
 }
 
 static inline void
-__br_stub_set_ongoing_version (br_stub_inode_ctx_t *ctx, unsigned long version)
+__br_stub_set_ongoing_version(br_stub_inode_ctx_t *ctx, unsigned long version)
 {
-        if (ctx->currentversion < version)
-                ctx->currentversion = version;
-        else
-                gf_msg ("bit-rot-stub", GF_LOG_WARNING, 0,
-                        BRS_MSG_CHANGE_VERSION_FAILED, "current version: %lu"
-                        "new version: %lu", ctx->currentversion, version);
+    if (ctx->currentversion < version)
+        ctx->currentversion = version;
+    else
+        gf_smsg("bit-rot-stub", GF_LOG_WARNING, 0,
+                BRS_MSG_CHANGE_VERSION_FAILED, "current version=%lu",
+                ctx->currentversion, "new version=%lu", version, NULL);
 }
 
 static inline int
-__br_stub_can_trigger_release (inode_t *inode,
-                               br_stub_inode_ctx_t *ctx, unsigned long *version)
+__br_stub_can_trigger_release(inode_t *inode, br_stub_inode_ctx_t *ctx,
+                              unsigned long *version)
 {
-        /**
-         * If the inode is modified, then it has to be dirty. An inode is
-         * marked dirty once version is increased. Its marked as modified
-         * when the modification call (write/truncate) which triggered
-         * the versioning is successful.
-         */
-        if (__br_stub_is_inode_modified (ctx)
-            && list_empty (&ctx->fd_list)
-            && (ctx->info_sign != BR_SIGN_REOPEN_WAIT)) {
-
-                GF_ASSERT (__br_stub_is_inode_dirty (ctx) == 0);
+    /**
+     * If the inode is modified, then it has to be dirty. An inode is
+     * marked dirty once version is increased. Its marked as modified
+     * when the modification call (write/truncate) which triggered
+     * the versioning is successful.
+     */
+    if (__br_stub_is_inode_modified(ctx) && list_empty(&ctx->fd_list) &&
+        (ctx->info_sign != BR_SIGN_REOPEN_WAIT)) {
+        GF_ASSERT(__br_stub_is_inode_dirty(ctx) == 0);
 
-                if (version)
-                        *version = htonl (ctx->currentversion);
-                return 1;
-        }
+        if (version)
+            *version = htonl(ctx->currentversion);
+        return 1;
+    }
 
-        return 0;
+    return 0;
 }
 
 static inline int32_t
-br_stub_get_ongoing_version (xlator_t *this,
-                             inode_t *inode, unsigned long *version)
+br_stub_get_ongoing_version(xlator_t *this, inode_t *inode,
+                            unsigned long *version)
 {
-        int32_t ret = 0;
-        uint64_t ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
-
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx_addr);
-                if (ret < 0)
-                        goto unblock;
-                ctx = (br_stub_inode_ctx_t *) (long) ctx_addr;
-                *version = ctx->currentversion;
-        }
- unblock:
-        UNLOCK (&inode->lock);
-
-        return ret;
+    int32_t ret = 0;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctx_addr);
+        if (ret < 0)
+            goto unblock;
+        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+        *version = ctx->currentversion;
+    }
+unblock:
+    UNLOCK(&inode->lock);
+
+    return ret;
 }
 
 /**
@@ -332,45 +328,52 @@ br_stub_get_ongoing_version (xlator_t *this,
  * *needs* to be valid in the caller.
  */
 static inline br_stub_inode_ctx_t *
-__br_stub_get_ongoing_version_ctx (xlator_t *this,
-                                   inode_t *inode, unsigned long *version)
+__br_stub_get_ongoing_version_ctx(xlator_t *this, inode_t *inode,
+                                  unsigned long *version)
 {
-        int32_t ret = 0;
-        uint64_t ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
-
-        ret = __inode_ctx_get (inode, this, &ctx_addr);
-        if (ret < 0)
-                return NULL;
-        ctx = (br_stub_inode_ctx_t *) (long) ctx_addr;
-        if (version)
-                *version = ctx->currentversion;
-
-        return ctx;
+    int32_t ret = 0;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+
+    ret = __inode_ctx_get(inode, this, &ctx_addr);
+    if (ret < 0)
+        return NULL;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+    if (version)
+        *version = ctx->currentversion;
+
+    return ctx;
 }
 
 /* filter for xattr fetch */
 static inline int
-br_stub_is_internal_xattr (const char *name)
+br_stub_is_internal_xattr(const char *name)
 {
-        if (name
-            && ((strncmp (name, BITROT_CURRENT_VERSION_KEY,
-                          strlen (BITROT_CURRENT_VERSION_KEY)) == 0)
-                || (strncmp (name, BITROT_SIGNING_VERSION_KEY,
-                             strlen (BITROT_SIGNING_VERSION_KEY)) == 0)))
-                return 1;
-        return 0;
+    if (name && ((strncmp(name, BITROT_CURRENT_VERSION_KEY,
+                          SLEN(BITROT_CURRENT_VERSION_KEY)) == 0) ||
+                 (strncmp(name, BITROT_SIGNING_VERSION_KEY,
+                          SLEN(BITROT_SIGNING_VERSION_KEY)) == 0)))
+        return 1;
+    return 0;
 }
 
 static inline void
-br_stub_remove_vxattrs (dict_t *xattr)
+br_stub_remove_vxattrs(dict_t *xattr, gf_boolean_t remove_bad_marker)
 {
-        if (xattr) {
-                dict_del (xattr, BITROT_OBJECT_BAD_KEY);
-                dict_del (xattr, BITROT_CURRENT_VERSION_KEY);
-                dict_del (xattr, BITROT_SIGNING_VERSION_KEY);
-                dict_del (xattr, BITROT_SIGNING_XATTR_SIZE_KEY);
-        }
+    if (xattr) {
+        /*
+         * When a file is corrupted, bad-object should be
+         * set in the dict. But, other info such as version,
+         * signature etc should not be set. Hence the flag
+         * remove_bad_marker. The consumer should know whether
+         * to send the bad-object info in the dict or not.
+         */
+        if (remove_bad_marker)
+            dict_del(xattr, BITROT_OBJECT_BAD_KEY);
+        dict_del(xattr, BITROT_CURRENT_VERSION_KEY);
+        dict_del(xattr, BITROT_SIGNING_VERSION_KEY);
+        dict_del(xattr, BITROT_SIGNING_XATTR_SIZE_KEY);
+    }
 }
 
 /**
@@ -384,64 +387,60 @@ br_stub_remove_vxattrs (dict_t *xattr)
  * errors can be made into enums.
  */
 static inline int
-br_stub_is_bad_object (xlator_t *this, inode_t *inode)
+br_stub_is_bad_object(xlator_t *this, inode_t *inode)
 {
-        int                  bad_object = 0;
-        gf_boolean_t         tmp        = _gf_false;
-        uint64_t             ctx_addr   = 0;
-        br_stub_inode_ctx_t *ctx        = NULL;
-        int32_t              ret        = -1;
-
-        ret = br_stub_get_inode_ctx (this, inode, &ctx_addr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_GET_INODE_CONTEXT_FAILED,
-                        "failed to get the inode context for the inode %s",
-                        uuid_utoa (inode->gfid));
-                bad_object = -1;
-                goto out;
-        }
-
-        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
-
-        LOCK (&inode->lock);
-        {
-                tmp = __br_stub_is_bad_object (ctx);
-                if (tmp)
-                        bad_object = -2;
-        }
-        UNLOCK (&inode->lock);
+    int bad_object = 0;
+    gf_boolean_t tmp = _gf_false;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+                "inode-gfid=%s", uuid_utoa(inode->gfid), NULL);
+        bad_object = -1;
+        goto out;
+    }
+
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+
+    LOCK(&inode->lock);
+    {
+        tmp = __br_stub_is_bad_object(ctx);
+        if (tmp)
+            bad_object = -2;
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return bad_object;
+    return bad_object;
 }
 
 static inline int32_t
-br_stub_mark_object_bad (xlator_t *this, inode_t *inode)
+br_stub_mark_object_bad(xlator_t *this, inode_t *inode)
 {
-        int32_t  ret = -1;
-        uint64_t ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
-
-        ret = br_stub_get_inode_ctx (this, inode, &ctx_addr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        BRS_MSG_GET_INODE_CONTEXT_FAILED, "failed to get the "
-                        "inode context for the inode %s",
-                        uuid_utoa (inode->gfid));
-               goto out;
-        }
+    int32_t ret = -1;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
 
-        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+                "inode-gfid=%s", uuid_utoa(inode->gfid), NULL);
+        goto out;
+    }
+
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
-        LOCK (&inode->lock);
-        {
-                __br_stub_mark_object_bad (ctx);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        __br_stub_mark_object_bad(ctx);
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -449,68 +448,68 @@ out:
  * given to the caller and the caller has to decide what to do.
  */
 static inline int32_t
-br_stub_mark_xdata_bad_object (xlator_t *this, inode_t *inode, dict_t *xdata)
+br_stub_mark_xdata_bad_object(xlator_t *this, inode_t *inode, dict_t *xdata)
 {
-        int32_t    ret = 0;
+    int32_t ret = 0;
 
-        if (br_stub_is_bad_object (this, inode) == -2)
-                ret = dict_set_int32 (xdata, GLUSTERFS_BAD_INODE, 1);
+    if (br_stub_is_bad_object(this, inode) == -2)
+        ret = dict_set_int32(xdata, GLUSTERFS_BAD_INODE, 1);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-br_stub_add_fd_to_inode (xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx);
+br_stub_add_fd_to_inode(xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx);
 
 br_sign_state_t
-__br_stub_inode_sign_state (br_stub_inode_ctx_t *ctx, glusterfs_fop_t fop,
-                            fd_t *fd);
+__br_stub_inode_sign_state(br_stub_inode_ctx_t *ctx, glusterfs_fop_t fop,
+                           fd_t *fd);
 
 int
-br_stub_dir_create (xlator_t *this, br_stub_private_t *priv);
+br_stub_dir_create(xlator_t *this, br_stub_private_t *priv);
 
 int
-br_stub_add (xlator_t *this, uuid_t gfid);
+br_stub_add(xlator_t *this, uuid_t gfid);
 
 int32_t
-br_stub_create_stub_gfid (xlator_t *this, char *stub_gfid_path, uuid_t gfid);
+br_stub_create_stub_gfid(xlator_t *this, char *stub_gfid_path, uuid_t gfid);
 
 int
-br_stub_dir_create (xlator_t *this, br_stub_private_t *priv);
+br_stub_dir_create(xlator_t *this, br_stub_private_t *priv);
 
 call_stub_t *
-__br_stub_dequeue (struct list_head *callstubs);
+__br_stub_dequeue(struct list_head *callstubs);
 
 void
-__br_stub_enqueue (struct list_head *callstubs, call_stub_t *stub);
+__br_stub_enqueue(struct list_head *callstubs, call_stub_t *stub);
 
 void
-br_stub_worker_enqueue (xlator_t *this, call_stub_t *stub);
+br_stub_worker_enqueue(xlator_t *this, call_stub_t *stub);
 
 void *
-br_stub_worker (void *data);
+br_stub_worker(void *data);
 
 int32_t
-br_stub_lookup_wrapper (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, dict_t *xattr_req);
+br_stub_lookup_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xattr_req);
 
 int32_t
-br_stub_readdir_wrapper (call_frame_t *frame, xlator_t *this,
-                         fd_t *fd, size_t size, off_t off, dict_t *xdata);
+br_stub_readdir_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        size_t size, off_t off, dict_t *xdata);
 
 int
-br_stub_del (xlator_t *this, uuid_t gfid);
+br_stub_del(xlator_t *this, uuid_t gfid);
 
 int
-br_stub_bad_objects_path (xlator_t *this, fd_t *fd, gf_dirent_t *entries,
-                          dict_t **dict);
+br_stub_bad_objects_path(xlator_t *this, fd_t *fd, gf_dirent_t *entries,
+                         dict_t **dict);
 
 void
-br_stub_entry_xattr_fill (xlator_t *this, char *hpath, gf_dirent_t *entry,
-                          dict_t *dict);
+br_stub_entry_xattr_fill(xlator_t *this, char *hpath, gf_dirent_t *entry,
+                         dict_t *dict);
 
 int
-br_stub_get_path_of_gfid (xlator_t *this, inode_t *parent, inode_t *inode,
-                          uuid_t gfid, char **path);
+br_stub_get_path_of_gfid(xlator_t *this, inode_t *parent, inode_t *inode,
+                         uuid_t gfid, char **path);
 
 #endif /* __BIT_ROT_STUB_H__ */
diff --git a/xlators/features/changelog/lib/examples/c/get-changes-multi.c b/xlators/features/changelog/lib/examples/c/get-changes-multi.c
index 3741bdf6edc..5ea5bbb6630 100644
--- a/xlators/features/changelog/lib/examples/c/get-changes-multi.c
+++ b/xlators/features/changelog/lib/examples/c/get-changes-multi.c
@@ -25,64 +25,66 @@
 
 #include "changelog.h"
 
-void *brick_init (void *xl, struct gf_brick_spec *brick)
+void *
+brick_init(void *xl, struct gf_brick_spec *brick)
 {
-        return brick;
+    return brick;
 }
 
-void brick_fini (void *xl, char *brick, void *data)
+void
+brick_fini(void *xl, char *brick, void *data)
 {
-        return;
+    return;
 }
 
-void brick_callback (void *xl, char *brick,
-                    void *data, changelog_event_t *ev)
+void
+brick_callback(void *xl, char *brick, void *data, changelog_event_t *ev)
 {
-        printf ("->callback: (brick,type) [%s:%d]\n", brick, ev->ev_type);
+    printf("->callback: (brick,type) [%s:%d]\n", brick, ev->ev_type);
 }
 
-void fill_brick_spec (struct gf_brick_spec *brick, char *path)
+void
+fill_brick_spec(struct gf_brick_spec *brick, char *path)
 {
-        brick->brick_path = strdup (path);
-        brick->filter = CHANGELOG_OP_TYPE_BR_RELEASE;
-
-        brick->init         = brick_init;
-        brick->fini         = brick_fini;
-        brick->callback     = brick_callback;
-        brick->connected    = NULL;
-        brick->disconnected = NULL;
+    brick->brick_path = strdup(path);
+    brick->filter = CHANGELOG_OP_TYPE_BR_RELEASE;
+
+    brick->init = brick_init;
+    brick->fini = brick_fini;
+    brick->callback = brick_callback;
+    brick->connected = NULL;
+    brick->disconnected = NULL;
 }
 
 int
-main (int argc, char **argv)
+main(int argc, char **argv)
 {
-        int ret = 0;
-        void *bricks = NULL;
-        struct gf_brick_spec *brick = NULL;
+    int ret = 0;
+    void *bricks = NULL;
+    struct gf_brick_spec *brick = NULL;
 
-        bricks = calloc (2, sizeof (struct gf_brick_spec));
-        if (!bricks)
-                goto error_return;
+    bricks = calloc(2, sizeof(struct gf_brick_spec));
+    if (!bricks)
+        goto error_return;
 
-        brick = (struct gf_brick_spec *)bricks;
-        fill_brick_spec (brick, "/export/z1/zwoop");
+    brick = (struct gf_brick_spec *)bricks;
+    fill_brick_spec(brick, "/export/z1/zwoop");
 
-        brick++;
-        fill_brick_spec (brick, "/export/z2/zwoop");
+    brick++;
+    fill_brick_spec(brick, "/export/z2/zwoop");
 
-        ret = gf_changelog_init (NULL);
-        if (ret)
-                goto error_return;
+    ret = gf_changelog_init(NULL);
+    if (ret)
+        goto error_return;
 
-        ret = gf_changelog_register_generic ((struct gf_brick_spec *)bricks, 2,
-                                             0, "/tmp/multi-changes.log", 9,
-                                             NULL);
-        if (ret)
-                goto error_return;
+    ret = gf_changelog_register_generic((struct gf_brick_spec *)bricks, 2, 0,
+                                        "/tmp/multi-changes.log", 9, NULL);
+    if (ret)
+        goto error_return;
 
-        /* let callbacks do the job */
-        select (0, NULL, NULL, NULL, NULL);
+    /* let callbacks do the job */
+    select(0, NULL, NULL, NULL, NULL);
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
diff --git a/xlators/features/changelog/lib/examples/c/get-changes.c b/xlators/features/changelog/lib/examples/c/get-changes.c
index ef766c566b6..8bc651c24a4 100644
--- a/xlators/features/changelog/lib/examples/c/get-changes.c
+++ b/xlators/features/changelog/lib/examples/c/get-changes.c
@@ -27,67 +27,67 @@
 
 #include "changelog.h"
 
-#define handle_error(fn)                                \
-        printf ("%s (reason: %s)\n", fn, strerror (errno))
+#define handle_error(fn) printf("%s (reason: %s)\n", fn, strerror(errno))
 
 int
-main (int argc, char ** argv)
+main(int argc, char **argv)
 {
-        int     i           = 0;
-        int     ret         = 0;
-        ssize_t nr_changes  = 0;
-        ssize_t changes     = 0;
-        char fbuf[PATH_MAX] = {0,};
-
-        ret = gf_changelog_init (NULL);
-        if (ret) {
-                handle_error ("Init failed");
-                goto out;
+    int i = 0;
+    int ret = 0;
+    ssize_t nr_changes = 0;
+    ssize_t changes = 0;
+    char fbuf[PATH_MAX] = {
+        0,
+    };
+
+    ret = gf_changelog_init(NULL);
+    if (ret) {
+        handle_error("Init failed");
+        goto out;
+    }
+
+    /* get changes for brick "/home/vshankar/export/yow/yow-1" */
+    ret = gf_changelog_register("/export/z1/zwoop", "/tmp/scratch",
+                                "/tmp/change.log", 9, 5);
+    if (ret) {
+        handle_error("register failed");
+        goto out;
+    }
+
+    while (1) {
+        i = 0;
+        nr_changes = gf_changelog_scan();
+        if (nr_changes < 0) {
+            handle_error("scan(): ");
+            break;
         }
 
-        /* get changes for brick "/home/vshankar/export/yow/yow-1" */
-        ret = gf_changelog_register ("/export/z1/zwoop",
-                                     "/tmp/scratch", "/tmp/change.log", 9, 5);
-        if (ret) {
-                handle_error ("register failed");
-                goto out;
-        }
-
-        while (1) {
-                i = 0;
-                nr_changes = gf_changelog_scan ();
-                if (nr_changes < 0) {
-                        handle_error ("scan(): ");
-                        break;
-                }
-
-                if (nr_changes == 0)
-                        goto next;
+        if (nr_changes == 0)
+            goto next;
 
-                printf ("Got %ld changelog files\n", nr_changes);
+        printf("Got %ld changelog files\n", nr_changes);
 
-                while ( (changes =
-                         gf_changelog_next_change (fbuf, PATH_MAX)) > 0) {
-                        printf ("changelog file [%d]: %s\n", ++i, fbuf);
+        while ((changes = gf_changelog_next_change(fbuf, PATH_MAX)) > 0) {
+            printf("changelog file [%d]: %s\n", ++i, fbuf);
 
-                        /* process changelog */
-                        /* ... */
-                        /* ... */
-                        /* ... */
-                        /* done processing */
+            /* process changelog */
+            /* ... */
+            /* ... */
+            /* ... */
+            /* done processing */
 
-                        ret = gf_changelog_done (fbuf);
-                        if (ret)
-                                handle_error ("gf_changelog_done");
-                }
+            ret = gf_changelog_done(fbuf);
+            if (ret)
+                handle_error("gf_changelog_done");
+        }
 
-                if (changes == -1)
-                        handle_error ("gf_changelog_next_change");
+        if (changes == -1)
+            handle_error("gf_changelog_next_change");
 
-        next:
-                sleep (10);
-        }
+    next:
+        sleep(10);
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
diff --git a/xlators/features/changelog/lib/examples/c/get-history.c b/xlators/features/changelog/lib/examples/c/get-history.c
index ee3ec0ad100..3e888d75ca6 100644
--- a/xlators/features/changelog/lib/examples/c/get-history.c
+++ b/xlators/features/changelog/lib/examples/c/get-history.c
@@ -27,90 +27,90 @@
 
 #include "changelog.h"
 
-#define handle_error(fn)                                \
-        printf ("%s (reason: %s)\n", fn, strerror (errno))
+#define handle_error(fn) printf("%s (reason: %s)\n", fn, strerror(errno))
 
 int
-main (int argc, char ** argv)
+main(int argc, char **argv)
 {
-        int     i            = 0;
-        int     ret          = 0;
-        ssize_t nr_changes   = 0;
-        ssize_t changes      = 0;
-        char fbuf[PATH_MAX]  = {0,};
-        unsigned long end_ts = 0;
-
-        ret = gf_changelog_init (NULL);
-        if (ret) {
-                handle_error ("init failed");
-                goto out;
+    int i = 0;
+    int ret = 0;
+    ssize_t nr_changes = 0;
+    ssize_t changes = 0;
+    char fbuf[PATH_MAX] = {
+        0,
+    };
+    unsigned long end_ts = 0;
+
+    ret = gf_changelog_init(NULL);
+    if (ret) {
+        handle_error("init failed");
+        goto out;
+    }
+
+    ret = gf_changelog_register("/export/z1/zwoop", "/tmp/scratch_v1",
+                                "/tmp/changes.log", 9, 5);
+    if (ret) {
+        handle_error("register failed");
+        goto out;
+    }
+
+    int a, b;
+    printf("give the two numbers start and end\t");
+    scanf("%d%d", &a, &b);
+    ret = gf_history_changelog("/export/z1/zwoop/.glusterfs/changelogs", a, b,
+                               3, &end_ts);
+    if (ret == -1) {
+        printf("history failed");
+        goto out;
+    }
+
+    printf("end time till when changelog available : %d , ret(%d) \t", end_ts,
+           ret);
+    fflush(stdout);
+
+    while (1) {
+        nr_changes = gf_history_changelog_scan();
+        printf("scanned, nr_changes : %d\n", nr_changes);
+        if (nr_changes < 0) {
+            handle_error("scan(): ");
+            break;
         }
 
-        ret = gf_changelog_register ("/export/z1/zwoop",
-                                     "/tmp/scratch_v1", "/tmp/changes.log",
-                                     9, 5);
-        if (ret) {
-                handle_error ("register failed");
-                goto out;
+        if (nr_changes == 0) {
+            printf("done scanning \n");
+            goto out;
         }
 
-        int a, b;
-        printf ("give the two numbers start and end\t");
-        scanf ("%d%d", &a, &b);
-        ret = gf_history_changelog ("/export/z1/zwoop/.glusterfs/changelogs",
-                                    a, b, 3, &end_ts);
-        if (ret == -1) {
-                printf ("history failed");
-                goto out;
-        }
+        printf("Got %ld changelog files\n", nr_changes);
+
+        while ((changes = gf_history_changelog_next_change(fbuf, PATH_MAX)) >
+               0) {
+            printf("changelog file [%d]: %s\n", ++i, fbuf);
 
-        printf ("end time till when changelog available : %d , ret(%d) \t", end_ts, ret);
-        fflush(stdout);
-
-        while (1) {
-                nr_changes = gf_history_changelog_scan ();
-                printf ("scanned, nr_changes : %d\n",nr_changes);
-                if (nr_changes < 0) {
-                        handle_error ("scan(): ");
-                        break;
-                }
-
-                if (nr_changes == 0) {
-                        printf ("done scanning \n");
-                        goto out;
-                }
-
-                printf ("Got %ld changelog files\n", nr_changes);
-
-                while ( (changes =
-                         gf_history_changelog_next_change (fbuf, PATH_MAX)) > 0) {
-                        printf ("changelog file [%d]: %s\n", ++i, fbuf);
-
-                        /* process changelog */
-                        /* ... */
-                        /* ... */
-                        /* ... */
-                        /* done processing */
-
-                        ret = gf_history_changelog_done (fbuf);
-                        if (ret)
-                                handle_error ("gf_changelog_done");
-                }
-                /*
-                if (changes == -1)
-                        handle_error ("gf_changelog_next_change");
-                if (nr_changes ==1){
-                        printf("continue scanning\n");
-                }
-
-                if(nr_changes == 0){
-                        printf("done scanning \n");
-                        goto out;
-                }
-                */
+            /* process changelog */
+            /* ... */
+            /* ... */
+            /* ... */
+            /* done processing */
+
+            ret = gf_history_changelog_done(fbuf);
+            if (ret)
+                handle_error("gf_changelog_done");
+        }
+        /*
+        if (changes == -1)
+                handle_error ("gf_changelog_next_change");
+        if (nr_changes ==1){
+                printf("continue scanning\n");
         }
 
+        if(nr_changes == 0){
+                printf("done scanning \n");
+                goto out;
+        }
+        */
+    }
 
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/changelog/lib/examples/python/changes.py b/xlators/features/changelog/lib/examples/python/changes.py
index 077c1108cce..c410d3b000d 100644..100755
--- a/xlators/features/changelog/lib/examples/python/changes.py
+++ b/xlators/features/changelog/lib/examples/python/changes.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 
 from __future__ import print_function
 import os
diff --git a/xlators/features/changelog/lib/examples/python/libgfchangelog.py b/xlators/features/changelog/lib/examples/python/libgfchangelog.py
index 2cdbf1152b9..2da9f2d2a8c 100644
--- a/xlators/features/changelog/lib/examples/python/libgfchangelog.py
+++ b/xlators/features/changelog/lib/examples/python/libgfchangelog.py
@@ -1,8 +1,10 @@
 import os
 from ctypes import *
+from ctypes.util import find_library
 
 class Changes(object):
-    libgfc = CDLL("libgfchangelog.so", mode=RTLD_GLOBAL, use_errno=True)
+    libgfc = CDLL(find_library("gfchangelog"), mode=RTLD_GLOBAL,
+                  use_errno=True)
 
     @classmethod
     def geterrno(cls):
diff --git a/xlators/features/changelog/lib/src/Makefile.am b/xlators/features/changelog/lib/src/Makefile.am
index 92ee26827da..c933ec53ed2 100644
--- a/xlators/features/changelog/lib/src/Makefile.am
+++ b/xlators/features/changelog/lib/src/Makefile.am
@@ -1,7 +1,7 @@
 libgfchangelog_la_CFLAGS = -Wall $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS) \
 	-DDATADIR=\"$(localstatedir)\"
 
-libgfchangelog_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 -fpic \
+libgfchangelog_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 -D__USE_LARGEFILE64 -fpic \
 	-I../../../src/ -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/xlators/features/changelog/src \
 	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
@@ -13,10 +13,10 @@ libgfchangelog_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
 
-libgfchangelog_la_LDFLAGS = $(GF_LDFLAGS) -version-info \
-	$(LIBGFCHANGELOG_LT_VERSION)
+libgfchangelog_la_LDFLAGS = $(GF_LDFLAGS) \
+        -version-info $(LIBGFCHANGELOG_LT_VERSION) \
+        $(GF_NO_UNDEFINED)
 
-libgfchangelogdir = $(includedir)/glusterfs/gfchangelog
 lib_LTLIBRARIES = libgfchangelog.la
 
 CONTRIB_BUILDDIR = $(top_builddir)/contrib
diff --git a/xlators/features/changelog/lib/src/changelog-lib-messages.h b/xlators/features/changelog/lib/src/changelog-lib-messages.h
index 2061217b801..d7fe7274353 100644
--- a/xlators/features/changelog/lib/src/changelog-lib-messages.h
+++ b/xlators/features/changelog/lib/src/changelog-lib-messages.h
@@ -11,7 +11,7 @@
 #ifndef _CHANGELOG_LIB_MESSAGES_H_
 #define _CHANGELOG_LIB_MESSAGES_H_
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
 
 /* To add new message IDs, append new identifiers at the end of the list.
  *
@@ -23,39 +23,52 @@
  * glfs-message-id.h.
  */
 
-GLFS_MSGID(CHANGELOG_LIB,
-        CHANGELOG_LIB_MSG_OPEN_FAILED,
-        CHANGELOG_LIB_MSG_FAILED_TO_RMDIR,
-        CHANGELOG_LIB_MSG_SCRATCH_DIR_ENTRIES_CREATION_ERROR,
-        CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED,
-        CHANGELOG_LIB_MSG_OPENDIR_ERROR,
-        CHANGELOG_LIB_MSG_RENAME_FAILED,
-        CHANGELOG_LIB_MSG_READ_ERROR,
-        CHANGELOG_LIB_MSG_HTIME_ERROR,
-        CHANGELOG_LIB_MSG_GET_TIME_ERROR,
-        CHANGELOG_LIB_MSG_WRITE_FAILED,
-        CHANGELOG_LIB_MSG_PTHREAD_ERROR,
-        CHANGELOG_LIB_MSG_MMAP_FAILED,
-        CHANGELOG_LIB_MSG_MUNMAP_FAILED,
-        CHANGELOG_LIB_MSG_ASCII_ERROR,
-        CHANGELOG_LIB_MSG_STAT_FAILED,
-        CHANGELOG_LIB_MSG_GET_XATTR_FAILED,
-        CHANGELOG_LIB_MSG_PUBLISH_ERROR,
-        CHANGELOG_LIB_MSG_PARSE_ERROR,
-        CHANGELOG_LIB_MSG_TOTAL_LOG_INFO,
-        CHANGELOG_LIB_MSG_CLEANUP_ERROR,
-        CHANGELOG_LIB_MSG_UNLINK_FAILED,
-        CHANGELOG_LIB_MSG_NOTIFY_REGISTER_FAILED,
-        CHANGELOG_LIB_MSG_INVOKE_RPC_FAILED,
-        CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO,
-        CHANGELOG_LIB_MSG_CLEANING_BRICK_ENTRY_INFO,
-        CHANGELOG_LIB_MSG_FREEING_ENTRY_INFO,
-        CHANGELOG_LIB_MSG_XDR_DECODING_FAILED,
-        CHANGELOG_LIB_MSG_NOTIFY_REGISTER_INFO,
-        CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
-        CHANGELOG_LIB_MSG_COPY_FROM_BUFFER_FAILED,
-        CHANGELOG_LIB_MSG_PTHREAD_JOIN_FAILED,
-        CHANGELOG_LIB_MSG_HIST_FAILED
-);
+GLFS_MSGID(
+    CHANGELOG_LIB, CHANGELOG_LIB_MSG_OPEN_FAILED,
+    CHANGELOG_LIB_MSG_FAILED_TO_RMDIR,
+    CHANGELOG_LIB_MSG_SCRATCH_DIR_ENTRIES_CREATION_ERROR,
+    CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED, CHANGELOG_LIB_MSG_OPENDIR_ERROR,
+    CHANGELOG_LIB_MSG_RENAME_FAILED, CHANGELOG_LIB_MSG_READ_ERROR,
+    CHANGELOG_LIB_MSG_HTIME_ERROR, CHANGELOG_LIB_MSG_GET_TIME_ERROR,
+    CHANGELOG_LIB_MSG_WRITE_FAILED, CHANGELOG_LIB_MSG_PTHREAD_ERROR,
+    CHANGELOG_LIB_MSG_MMAP_FAILED, CHANGELOG_LIB_MSG_MUNMAP_FAILED,
+    CHANGELOG_LIB_MSG_ASCII_ERROR, CHANGELOG_LIB_MSG_STAT_FAILED,
+    CHANGELOG_LIB_MSG_GET_XATTR_FAILED, CHANGELOG_LIB_MSG_PUBLISH_ERROR,
+    CHANGELOG_LIB_MSG_PARSE_ERROR, CHANGELOG_LIB_MSG_MIN_MAX_INFO,
+    CHANGELOG_LIB_MSG_CLEANUP_ERROR, CHANGELOG_LIB_MSG_UNLINK_FAILED,
+    CHANGELOG_LIB_MSG_NOTIFY_REGISTER_FAILED,
+    CHANGELOG_LIB_MSG_INVOKE_RPC_FAILED, CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO,
+    CHANGELOG_LIB_MSG_CLEANING_BRICK_ENTRY_INFO,
+    CHANGELOG_LIB_MSG_FREEING_ENTRY_INFO, CHANGELOG_LIB_MSG_XDR_DECODING_FAILED,
+    CHANGELOG_LIB_MSG_NOTIFY_REGISTER_INFO,
+    CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
+    CHANGELOG_LIB_MSG_COPY_FROM_BUFFER_FAILED,
+    CHANGELOG_LIB_MSG_PTHREAD_JOIN_FAILED, CHANGELOG_LIB_MSG_HIST_FAILED,
+    CHANGELOG_LIB_MSG_DRAINED_EVENT_INFO, CHANGELOG_LIB_MSG_PARSE_ERROR_CEASED,
+    CHANGELOG_LIB_MSG_REQUESTING_INFO, CHANGELOG_LIB_MSG_FINAL_INFO);
+
+#define CHANGELOG_LIB_MSG_NOTIFY_REGISTER_INFO_STR "Registering brick"
+#define CHANGELOG_LIB_MSG_RENAME_FAILED_STR "error moving changelog file"
+#define CHANGELOG_LIB_MSG_OPEN_FAILED_STR "cannot open changelog file"
+#define CHANGELOG_LIB_MSG_UNLINK_FAILED_STR "failed to unlink"
+#define CHANGELOG_LIB_MSG_FAILED_TO_RMDIR_STR "failed to rmdir"
+#define CHANGELOG_LIB_MSG_STAT_FAILED_STR "stat failed on changelog file"
+#define CHANGELOG_LIB_MSG_PARSE_ERROR_STR "could not parse changelog"
+#define CHANGELOG_LIB_MSG_PARSE_ERROR_CEASED_STR                               \
+    "parsing error, ceased publishing..."
+#define CHANGELOG_LIB_MSG_HTIME_ERROR_STR "fop failed on htime file"
+#define CHANGELOG_LIB_MSG_GET_XATTR_FAILED_STR                                 \
+    "error extracting max timstamp from htime file"
+#define CHANGELOG_LIB_MSG_MIN_MAX_INFO_STR "changelogs min max"
+#define CHANGELOG_LIB_MSG_REQUESTING_INFO_STR "Requesting historical changelogs"
+#define CHANGELOG_LIB_MSG_FINAL_INFO_STR "FINAL"
+#define CHANGELOG_LIB_MSG_HIST_FAILED_STR                                      \
+    "Requested changelog range is not available"
+#define CHANGELOG_LIB_MSG_GET_TIME_ERROR_STR "wrong result"
+#define CHANGELOG_LIB_MSG_CLEANING_BRICK_ENTRY_INFO_STR                        \
+    "Cleaning brick entry for brick"
+#define CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO_STR "Draining event"
+#define CHANGELOG_LIB_MSG_DRAINED_EVENT_INFO_STR "Drained event"
+#define CHANGELOG_LIB_MSG_FREEING_ENTRY_INFO_STR "freeing entry"
 
 #endif /* !_CHANGELOG_MESSAGES_H_ */
diff --git a/xlators/features/changelog/lib/src/gf-changelog-api.c b/xlators/features/changelog/lib/src/gf-changelog-api.c
index 372550c7acf..81a5cbfec10 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-api.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-api.c
@@ -8,10 +8,10 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "compat-uuid.h"
-#include "globals.h"
-#include "glusterfs.h"
-#include "syscall.h"
+#include <glusterfs/compat-uuid.h>
+#include <glusterfs/globals.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/syscall.h>
 
 #include "gf-changelog-helpers.h"
 #include "gf-changelog-journal.h"
@@ -19,57 +19,54 @@
 #include "changelog-lib-messages.h"
 
 int
-gf_changelog_done (char *file)
+gf_changelog_done(char *file)
 {
-        int                     ret    = -1;
-        char                   *buffer = NULL;
-        xlator_t               *this   = NULL;
-        gf_changelog_journal_t *jnl    = NULL;
-        char to_path[PATH_MAX]         = {0,};
-
-        errno = EINVAL;
-
-        this = THIS;
-        if (!this)
-                goto out;
-
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
-
-        if (!file || !strlen (file))
-                goto out;
-
-        /* make sure 'file' is inside ->jnl_working_dir */
-        buffer = realpath (file, NULL);
-        if (!buffer)
-                goto out;
-
-        if (strncmp (jnl->jnl_working_dir,
-                     buffer, strlen (jnl->jnl_working_dir)))
-                goto out;
-
-        (void) snprintf (to_path, PATH_MAX, "%s%s",
-                         jnl->jnl_processed_dir, basename (buffer));
-        gf_msg_debug (this->name, 0,
-                      "moving %s to processed directory", file);
-        ret = sys_rename (buffer, to_path);
-        if (ret) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_RENAME_FAILED,
-                         "cannot move changelog file",
-                         "from=%s", file,
-                         "to=%s", to_path,
-                         NULL);
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    char *buffer = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    char to_path[PATH_MAX] = {
+        0,
+    };
+
+    errno = EINVAL;
+
+    this = THIS;
+    if (!this)
+        goto out;
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
+
+    if (!file || !strlen(file))
+        goto out;
+
+    /* make sure 'file' is inside ->jnl_working_dir */
+    buffer = realpath(file, NULL);
+    if (!buffer)
+        goto out;
+
+    if (strncmp(jnl->jnl_working_dir, buffer, strlen(jnl->jnl_working_dir)))
+        goto out;
+
+    (void)snprintf(to_path, PATH_MAX, "%s%s", jnl->jnl_processed_dir,
+                   basename(buffer));
+    gf_msg_debug(this->name, 0, "moving %s to processed directory", file);
+    ret = sys_rename(buffer, to_path);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_RENAME_FAILED, "from=%s", file, "to=%s",
+                to_path, NULL);
+        goto out;
+    }
+
+    ret = 0;
 
- out:
-        if (buffer)
-                free (buffer); /* allocated by realpath() */
-        return ret;
+out:
+    if (buffer)
+        free(buffer); /* allocated by realpath() */
+    return ret;
 }
 
 /**
@@ -77,28 +74,28 @@ gf_changelog_done (char *file)
  *  for a set of changelogs, start from the beginning
  */
 int
-gf_changelog_start_fresh ()
+gf_changelog_start_fresh()
 {
-        xlator_t *this = NULL;
-        gf_changelog_journal_t *jnl = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
 
-        this = THIS;
-        if (!this)
-                goto out;
+    this = THIS;
+    if (!this)
+        goto out;
 
-        errno = EINVAL;
+    errno = EINVAL;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
 
-        if (gf_ftruncate (jnl->jnl_fd, 0))
-                goto out;
+    if (gf_ftruncate(jnl->jnl_fd, 0))
+        goto out;
 
-        return 0;
+    return 0;
 
- out:
-        return -1;
+out:
+    return -1;
 }
 
 /**
@@ -107,40 +104,42 @@ gf_changelog_start_fresh ()
  * consumed.
  */
 ssize_t
-gf_changelog_next_change (char *bufptr, size_t maxlen)
+gf_changelog_next_change(char *bufptr, size_t maxlen)
 {
-        ssize_t         size       = -1;
-        int             tracker_fd = 0;
-        xlator_t       *this       = NULL;
-        gf_changelog_journal_t *jnl = NULL;
-        char buffer[PATH_MAX]      = {0,};
+    ssize_t size = -1;
+    int tracker_fd = 0;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    char buffer[PATH_MAX] = {
+        0,
+    };
 
-        errno = EINVAL;
+    errno = EINVAL;
 
-        this = THIS;
-        if (!this)
-                goto out;
+    this = THIS;
+    if (!this)
+        goto out;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
 
-        tracker_fd = jnl->jnl_fd;
+    tracker_fd = jnl->jnl_fd;
 
-        size = gf_readline (tracker_fd, buffer, maxlen);
-        if (size < 0) {
-                size = -1;
-                goto out;
-        }
+    size = gf_readline(tracker_fd, buffer, maxlen);
+    if (size < 0) {
+        size = -1;
+        goto out;
+    }
 
-        if (size == 0)
-                goto out;
+    if (size == 0)
+        goto out;
 
-        memcpy (bufptr, buffer, size - 1);
-        bufptr[size - 1] = '\0';
+    memcpy(bufptr, buffer, size - 1);
+    bufptr[size - 1] = '\0';
 
 out:
-        return size;
+    return size;
 }
 
 /**
@@ -152,70 +151,74 @@ out:
  * This call also acts as a cancellation point for the consumer.
  */
 ssize_t
-gf_changelog_scan ()
+gf_changelog_scan()
 {
-        int             tracker_fd  = 0;
-        size_t          off         = 0;
-        xlator_t       *this        = NULL;
-        size_t          nr_entries  = 0;
-        gf_changelog_journal_t *jnl = NULL;
-        struct dirent  *entry       = NULL;
-        struct dirent   scratch[2]  = {{0,},};
-        char            buffer[PATH_MAX] = {0,};
-
-        this = THIS;
-        if (!this)
-                goto out;
-
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
-        if (JNL_IS_API_DISCONNECTED (jnl)) {
-                errno = ENOTCONN;
-                goto out;
+    int tracker_fd = 0;
+    size_t off = 0;
+    xlator_t *this = NULL;
+    size_t nr_entries = 0;
+    gf_changelog_journal_t *jnl = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char buffer[PATH_MAX] = {
+        0,
+    };
+
+    this = THIS;
+    if (!this)
+        goto out;
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
+    if (JNL_IS_API_DISCONNECTED(jnl)) {
+        errno = ENOTCONN;
+        goto out;
+    }
+
+    errno = EINVAL;
+
+    tracker_fd = jnl->jnl_fd;
+    if (gf_ftruncate(tracker_fd, 0))
+        goto out;
+
+    rewinddir(jnl->jnl_dir);
+
+    for (;;) {
+        errno = 0;
+        entry = sys_readdir(jnl->jnl_dir, scratch);
+        if (!entry || errno != 0)
+            break;
+
+        if (!strcmp(basename(entry->d_name), ".") ||
+            !strcmp(basename(entry->d_name), ".."))
+            continue;
+
+        nr_entries++;
+
+        GF_CHANGELOG_FILL_BUFFER(jnl->jnl_processing_dir, buffer, off,
+                                 strlen(jnl->jnl_processing_dir));
+        GF_CHANGELOG_FILL_BUFFER(entry->d_name, buffer, off,
+                                 strlen(entry->d_name));
+        GF_CHANGELOG_FILL_BUFFER("\n", buffer, off, 1);
+
+        if (gf_changelog_write(tracker_fd, buffer, off) != off) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_WRITE_FAILED,
+                   "error writing changelog filename"
+                   " to tracker file");
+            break;
         }
+        off = 0;
+    }
 
-        errno = EINVAL;
-
-        tracker_fd = jnl->jnl_fd;
-        if (gf_ftruncate (tracker_fd, 0))
-                goto out;
-
-        rewinddir (jnl->jnl_dir);
-
-        for (;;) {
-                errno = 0;
-                entry = sys_readdir (jnl->jnl_dir, scratch);
-                if (!entry || errno != 0)
-                        break;
-
-                if (!strcmp (basename (entry->d_name), ".")
-                     || !strcmp (basename (entry->d_name), ".."))
-                        continue;
-
-                nr_entries++;
-
-                GF_CHANGELOG_FILL_BUFFER (jnl->jnl_processing_dir,
-                                          buffer, off,
-                                          strlen (jnl->jnl_processing_dir));
-                GF_CHANGELOG_FILL_BUFFER (entry->d_name, buffer,
-                                          off, strlen (entry->d_name));
-                GF_CHANGELOG_FILL_BUFFER ("\n", buffer, off, 1);
-
-                if (gf_changelog_write (tracker_fd, buffer, off) != off) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_LIB_MSG_WRITE_FAILED,
-                                "error writing changelog filename"
-                                " to tracker file");
-                        break;
-                }
-                off = 0;
-        }
-
-        if (!entry) {
-                if (gf_lseek (tracker_fd, 0, SEEK_SET) != -1)
-                        return nr_entries;
-        }
- out:
-        return -1;
+    if (!entry) {
+        if (gf_lseek(tracker_fd, 0, SEEK_SET) != -1)
+            return nr_entries;
+    }
+out:
+    return -1;
 }
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.c b/xlators/features/changelog/lib/src/gf-changelog-helpers.c
index 45cf55778e4..75f8a6dfc08 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-helpers.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.c
@@ -11,40 +11,36 @@
 #include "changelog-mem-types.h"
 #include "gf-changelog-helpers.h"
 #include "changelog-lib-messages.h"
-#include "syscall.h"
-
-ssize_t gf_changelog_read_path (int fd, char *buffer, size_t bufsize)
-{
-        return sys_read (fd, buffer, bufsize);
-}
+#include <glusterfs/syscall.h>
 
 size_t
-gf_changelog_write (int fd, char *buffer, size_t len)
+gf_changelog_write(int fd, char *buffer, size_t len)
 {
-        ssize_t size = 0;
-        size_t written = 0;
+    ssize_t size = 0;
+    size_t written = 0;
 
-        while (written < len) {
-                size = sys_write (fd, buffer + written, len - written);
-                if (size <= 0)
-                        break;
+    while (written < len) {
+        size = sys_write(fd, buffer + written, len - written);
+        if (size <= 0)
+            break;
 
-                written += size;
-        }
+        written += size;
+    }
 
-        return written;
+    return written;
 }
 
 void
-gf_rfc3986_encode_space_newline (unsigned char *s, char *enc, char *estr)
+gf_rfc3986_encode_space_newline(unsigned char *s, char *enc, char *estr)
 {
-        for (; *s; s++) {
-                if (estr[*s])
-                        sprintf(enc, "%c", estr[*s]);
-                else
-                        sprintf(enc, "%%%02X", *s);
-                while (*++enc);
-        }
+    for (; *s; s++) {
+        if (estr[*s])
+            sprintf(enc, "%c", estr[*s]);
+        else
+            sprintf(enc, "%%%02X", *s);
+        while (*++enc)
+            ;
+    }
 }
 
 /**
@@ -57,163 +53,118 @@ gf_rfc3986_encode_space_newline (unsigned char *s, char *enc, char *estr)
  * that can be done via @fflush(fp), @ftruncate(fd) and @fseek(fp),
  * but this involves mixing POSIX file descriptors and stream FILE *).
  *
- * NOTE: This implmentation still does work with more than one fd's
+ * NOTE: This implementation still does work with more than one fd's
  *       used to perform gf_readline(). For this very reason it's not
  *       made a part of libglusterfs.
  */
 
-static pthread_key_t rl_key;
-static pthread_once_t rl_once = PTHREAD_ONCE_INIT;
-
-static void
-readline_destructor (void *ptr)
-{
-        GF_FREE (ptr);
-}
-
-static void
-readline_once (void)
-{
-        pthread_key_create (&rl_key, readline_destructor);
-}
+static __thread read_line_t thread_tsd = {};
 
 static ssize_t
-my_read (read_line_t *tsd, int fd, char *ptr)
-{
-        if (tsd->rl_cnt <= 0) {
-                tsd->rl_cnt = sys_read (fd, tsd->rl_buf, MAXLINE);
-
-                if (tsd->rl_cnt < 0)
-                        return -1;
-                else if (tsd->rl_cnt == 0)
-                        return 0;
-                tsd->rl_bufptr = tsd->rl_buf;
-        }
-
-        tsd->rl_cnt--;
-        *ptr = *tsd->rl_bufptr++;
-        return 1;
-}
-
-static int
-gf_readline_init_once (read_line_t **tsd)
+my_read(read_line_t *tsd, int fd, char *ptr)
 {
-        if (pthread_once (&rl_once, readline_once) != 0)
-                return -1;
+    if (tsd->rl_cnt <= 0) {
+        tsd->rl_cnt = sys_read(fd, tsd->rl_buf, MAXLINE);
 
-        *tsd = pthread_getspecific (rl_key);
-        if (*tsd)
-                goto out;
-
-        *tsd = GF_CALLOC (1, sizeof (**tsd),
-                          gf_changelog_mt_libgfchangelog_rl_t);
-        if (!*tsd)
-                return -1;
-
-        if (pthread_setspecific (rl_key, *tsd) != 0)
-                return -1;
+        if (tsd->rl_cnt < 0)
+            return -1;
+        else if (tsd->rl_cnt == 0)
+            return 0;
+        tsd->rl_bufptr = tsd->rl_buf;
+    }
 
- out:
-        return 0;
+    tsd->rl_cnt--;
+    *ptr = *tsd->rl_bufptr++;
+    return 1;
 }
 
 ssize_t
-gf_readline (int fd, void *vptr, size_t maxlen)
+gf_readline(int fd, void *vptr, size_t maxlen)
 {
-        size_t       n   = 0;
-        size_t       rc  = 0;
-        char         c   = ' ';
-        char        *ptr = NULL;
-        read_line_t *tsd = NULL;
-
-        if (gf_readline_init_once (&tsd))
-                return -1;
-
-        ptr = vptr;
-        for (n = 1; n < maxlen; n++) {
-                if ( (rc = my_read (tsd, fd, &c)) == 1 ) {
-                        *ptr++ = c;
-                        if (c == '\n')
-                                break;
-                } else if (rc == 0) {
-                        *ptr = '\0';
-                        return (n - 1);
-                } else
-                        return -1;
-        }
-
-        *ptr = '\0';
-        return n;
-
+    size_t n = 0;
+    size_t rc = 0;
+    char c = ' ';
+    char *ptr = NULL;
+    read_line_t *tsd = &thread_tsd;
+
+    ptr = vptr;
+    for (n = 1; n < maxlen; n++) {
+        if ((rc = my_read(tsd, fd, &c)) == 1) {
+            *ptr++ = c;
+            if (c == '\n')
+                break;
+        } else if (rc == 0) {
+            *ptr = '\0';
+            return (n - 1);
+        } else
+            return -1;
+    }
+
+    *ptr = '\0';
+    return n;
 }
 
 off_t
-gf_lseek (int fd, off_t offset, int whence)
+gf_lseek(int fd, off_t offset, int whence)
 {
-        off_t        off = 0;
-        read_line_t *tsd = NULL;
+    off_t off = 0;
+    read_line_t *tsd = &thread_tsd;
 
-        if (gf_readline_init_once (&tsd))
-                return -1;
+    off = sys_lseek(fd, offset, whence);
+    if (off == -1)
+        return -1;
 
-        off = sys_lseek (fd, offset, whence);
-        if (off == -1)
-                return -1;
+    tsd->rl_cnt = 0;
+    tsd->rl_bufptr = tsd->rl_buf;
 
-        tsd->rl_cnt = 0;
-        tsd->rl_bufptr = tsd->rl_buf;
-
-        return off;
+    return off;
 }
 
 int
-gf_ftruncate (int fd, off_t length)
+gf_ftruncate(int fd, off_t length)
 {
-        read_line_t *tsd = NULL;
+    read_line_t *tsd = &thread_tsd;
 
-        if (gf_readline_init_once (&tsd))
-                return -1;
+    if (sys_ftruncate(fd, 0))
+        return -1;
 
-        if (sys_ftruncate (fd, 0))
-                return -1;
+    tsd->rl_cnt = 0;
+    tsd->rl_bufptr = tsd->rl_buf;
 
-        tsd->rl_cnt = 0;
-        tsd->rl_bufptr = tsd->rl_buf;
-
-        return 0;
+    return 0;
 }
 
 int
-gf_thread_cleanup (xlator_t *this, pthread_t thread)
+gf_thread_cleanup(xlator_t *this, pthread_t thread)
 {
-        int ret = 0;
-        void *res = NULL;
-
-        ret = pthread_cancel (thread);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
-                        "Failed to send cancellation to thread");
-                goto error_return;
-        }
-
-        ret = pthread_join (thread, &res);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
-                        "failed to join thread");
-                goto error_return;
-        }
-
-        if (res != PTHREAD_CANCELED) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
-                        "Thread could not be cleaned up");
-                goto error_return;
-        }
-
-        return 0;
-
- error_return:
-        return -1;
+    int ret = 0;
+    void *res = NULL;
+
+    ret = pthread_cancel(thread);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0,
+               CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
+               "Failed to send cancellation to thread");
+        goto error_return;
+    }
+
+    ret = pthread_join(thread, &res);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0,
+               CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
+               "failed to join thread");
+        goto error_return;
+    }
+
+    if (res != PTHREAD_CANCELED) {
+        gf_msg(this->name, GF_LOG_WARNING, 0,
+               CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
+               "Thread could not be cleaned up");
+        goto error_return;
+    }
+
+    return 0;
+
+error_return:
+    return -1;
 }
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.h b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
index b05628ee70d..9c609d33172 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-helpers.h
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
@@ -14,36 +14,37 @@
 #include <unistd.h>
 #include <dirent.h>
 #include <limits.h>
-#include "locking.h"
+#include <glusterfs/locking.h>
 
-#include <xlator.h>
+#include <glusterfs/xlator.h>
 
 #include "changelog.h"
 
 #include "changelog-rpc-common.h"
 #include "gf-changelog-journal.h"
 
-#define GF_CHANGELOG_TRACKER  "tracker"
+#define GF_CHANGELOG_TRACKER "tracker"
 
-#define GF_CHANGELOG_CURRENT_DIR    ".current"
-#define GF_CHANGELOG_PROCESSED_DIR  ".processed"
+#define GF_CHANGELOG_CURRENT_DIR ".current"
+#define GF_CHANGELOG_PROCESSED_DIR ".processed"
 #define GF_CHANGELOG_PROCESSING_DIR ".processing"
-#define GF_CHANGELOG_HISTORY_DIR    ".history"
+#define GF_CHANGELOG_HISTORY_DIR ".history"
 #define TIMESTAMP_LENGTH 10
 
 #ifndef MAXLINE
 #define MAXLINE 4096
 #endif
 
-#define GF_CHANGELOG_FILL_BUFFER(ptr, ascii, off, len) do {     \
-                memcpy (ascii + off, ptr, len);                 \
-                off += len;                                     \
-        } while (0)
+#define GF_CHANGELOG_FILL_BUFFER(ptr, ascii, off, len)                         \
+    do {                                                                       \
+        memcpy(ascii + off, ptr, len);                                         \
+        off += len;                                                            \
+    } while (0)
 
 typedef struct read_line {
-        int rl_cnt;
-        char *rl_bufptr;
-        char rl_buf[MAXLINE];
+    int rl_cnt;
+    char *rl_bufptr;
+    char rl_buf[MAXLINE];
 } read_line_t;
 
 struct gf_changelog;
@@ -55,51 +56,50 @@ struct gf_event;
  * ->next_seq holds the next _expected_ sequence number.
  */
 struct gf_event_list {
-        pthread_mutex_t lock;               /* protects this structure */
-        pthread_cond_t  cond;
+    pthread_mutex_t lock; /* protects this structure */
+    pthread_cond_t cond;
 
-        pthread_t invoker;
+    pthread_t invoker;
 
-        unsigned long next_seq;             /* next sequence number expected:
-                                               zero during bootstrap */
+    unsigned long next_seq; /* next sequence number expected:
+                               zero during bootstrap */
 
-        struct gf_changelog *entry;         /* backpointer to it's brick
-                                               encapsulator (entry) */
-        struct list_head events;            /* list of events */
+    struct gf_changelog *entry; /* backpointer to it's brick
+                                   encapsulator (entry) */
+    struct list_head events;    /* list of events */
 };
 
 /**
  * include a refcount if it's of use by additional layers
  */
 struct gf_event {
-        int count;
+    int count;
 
-        unsigned long seq;
+    unsigned long seq;
 
-        struct list_head list;
+    struct list_head list;
 
-        struct iovec iov[0];
+    struct iovec iov[0];
 };
-#define GF_EVENT_CALLOC_SIZE(cnt, len)                                  \
-        (sizeof (struct gf_event) + (cnt * sizeof (struct iovec)) + len)
+#define GF_EVENT_CALLOC_SIZE(cnt, len)                                         \
+    (sizeof(struct gf_event) + (cnt * sizeof(struct iovec)) + len)
 
 /**
  * assign the base address of the IO vector to the correct memory
 o * area and set it's addressable length.
  */
-#define GF_EVENT_ASSIGN_IOVEC(vec, event, len, pos)                     \
-        do {                                                            \
-                vec->iov_base = ((char *)event) +                       \
-                        sizeof (struct gf_event) +                      \
-                        (event->count * sizeof (struct iovec)) + pos;   \
-                vec->iov_len = len;                                     \
-                pos += len;                                             \
-        } while (0)
+#define GF_EVENT_ASSIGN_IOVEC(vec, event, len, pos)                            \
+    do {                                                                       \
+        vec->iov_base = ((char *)event) + sizeof(struct gf_event) +            \
+                        (event->count * sizeof(struct iovec)) + pos;           \
+        vec->iov_len = len;                                                    \
+        pos += len;                                                            \
+    } while (0)
 
 typedef enum gf_changelog_conn_state {
-        GF_CHANGELOG_CONN_STATE_PENDING = 0,
-        GF_CHANGELOG_CONN_STATE_ACCEPTED,
-        GF_CHANGELOG_CONN_STATE_DISCONNECTED,
+    GF_CHANGELOG_CONN_STATE_PENDING = 0,
+    GF_CHANGELOG_CONN_STATE_ACCEPTED,
+    GF_CHANGELOG_CONN_STATE_DISCONNECTED,
 } gf_changelog_conn_state_t;
 
 /**
@@ -107,153 +107,149 @@ typedef enum gf_changelog_conn_state {
  * notifications are streamed.
  */
 typedef struct gf_changelog {
-        gf_lock_t statelock;
-        gf_changelog_conn_state_t connstate;
+    gf_lock_t statelock;
+    gf_changelog_conn_state_t connstate;
 
-        xlator_t *this;
+    xlator_t *this;
 
-        struct list_head list;              /* list of instances */
+    struct list_head list; /* list of instances */
 
-        char brick[PATH_MAX];               /* brick path for this end-point */
+    char brick[PATH_MAX]; /* brick path for this end-point */
 
-        changelog_rpc_t grpc;               /* rpc{-clnt,svc} for this brick */
-#define RPC_PROBER(ent)  ent->grpc.rpc
-#define RPC_REBORP(ent)  ent->grpc.svc
-#define RPC_SOCK(ent)    ent->grpc.sock
+    changelog_rpc_t grpc; /* rpc{-clnt,svc} for this brick */
+#define RPC_PROBER(ent) ent->grpc.rpc
+#define RPC_REBORP(ent) ent->grpc.svc
+#define RPC_SOCK(ent) ent->grpc.sock
 
-        unsigned int notify;                /* notification flag(s) */
+    unsigned int notify; /* notification flag(s) */
 
-        FINI       *fini;                   /* destructor callback */
-        CALLBACK   *callback;               /* event callback dispatcher */
-        CONNECT    *connected;              /* connect callback */
-        DISCONNECT *disconnected;           /* disconnection callback */
+    FINI *fini;               /* destructor callback */
+    CALLBACK *callback;       /* event callback dispatcher */
+    CONNECT *connected;       /* connect callback */
+    DISCONNECT *disconnected; /* disconnection callback */
 
-        void *ptr;                          /* owner specific private data */
-        xlator_t *invokerxl;                /* consumers _this_, if valid,
-                                               assigned to THIS before cbk is
-                                               invoked */
+    void *ptr;           /* owner specific private data */
+    xlator_t *invokerxl; /* consumers _this_, if valid,
+                            assigned to THIS before cbk is
+                            invoked */
 
-        gf_boolean_t ordered;
+    gf_boolean_t ordered;
 
-        void (*queueevent) (struct gf_event_list *, struct gf_event *);
-        void (*pickevent) (struct gf_event_list *, struct gf_event **);
+    void (*queueevent)(struct gf_event_list *, struct gf_event *);
+    void (*pickevent)(struct gf_event_list *, struct gf_event **);
 
-        struct gf_event_list event;
+    struct gf_event_list event;
 } gf_changelog_t;
 
 static inline int
-gf_changelog_filter_check (gf_changelog_t *entry, changelog_event_t *event)
+gf_changelog_filter_check(gf_changelog_t *entry, changelog_event_t *event)
 {
-        if (event->ev_type & entry->notify)
-                return 1;
-        return 0;
+    if (event->ev_type & entry->notify)
+        return 1;
+    return 0;
 }
 
-#define GF_NEED_ORDERED_EVENTS(ent)  (ent->ordered == _gf_true)
+#define GF_NEED_ORDERED_EVENTS(ent) (ent->ordered == _gf_true)
 
 /** private structure */
 typedef struct gf_private {
-        pthread_mutex_t lock;            /* protects ->connections, cleanups */
-        pthread_cond_t  cond;
+    pthread_mutex_t lock; /* protects ->connections, cleanups */
+    pthread_cond_t cond;
 
-        void *api;                       /* pointer for API access */
+    void *api; /* pointer for API access */
 
-        pthread_t poller;                /* event poller thread */
-        pthread_t connectionjanitor;     /* connection cleaner */
+    pthread_t poller;            /* event poller thread */
+    pthread_t connectionjanitor; /* connection cleaner */
 
-        struct list_head connections;    /* list of connections */
-        struct list_head cleanups;       /* list of connection to be
-                                            cleaned up */
+    struct list_head connections; /* list of connections */
+    struct list_head cleanups;    /* list of connection to be
+                                     cleaned up */
 } gf_private_t;
 
-#define GF_CHANGELOG_GET_API_PTR(this) (((gf_private_t *) this->private)->api)
+#define GF_CHANGELOG_GET_API_PTR(this) (((gf_private_t *)this->private)->api)
 
 /**
  * upcall: invoke callback with _correct_ THIS
  */
-#define GF_CHANGELOG_INVOKE_CBK(this, cbk, brick, args ...)             \
-        do {                                                            \
-                xlator_t *old_this = NULL;                              \
-                xlator_t *invokerxl = NULL;                             \
-                                                                        \
-                invokerxl = entry->invokerxl;                           \
-                old_this = this;                                        \
-                                                                        \
-                if (invokerxl) {                                        \
-                        THIS = invokerxl;                               \
-                }                                                       \
-                                                                        \
-                cbk (invokerxl, brick, args);                           \
-                THIS = old_this;                                        \
-                                                                        \
-        } while (0)
-
-#define SAVE_THIS(xl)                           \
-        do {                                    \
-                old_this = xl;                  \
-                THIS = master;                  \
-        } while (0)
-
-#define RESTORE_THIS()                          \
-        do {                                    \
-                if (old_this)                   \
-                        THIS = old_this;        \
-        } while (0)
+#define GF_CHANGELOG_INVOKE_CBK(this, cbk, brick, args...)                     \
+    do {                                                                       \
+        xlator_t *old_this = NULL;                                             \
+        xlator_t *invokerxl = NULL;                                            \
+                                                                               \
+        invokerxl = entry->invokerxl;                                          \
+        old_this = this;                                                       \
+                                                                               \
+        if (invokerxl) {                                                       \
+            THIS = invokerxl;                                                  \
+        }                                                                      \
+                                                                               \
+        cbk(invokerxl, brick, args);                                           \
+        THIS = old_this;                                                       \
+                                                                               \
+    } while (0)
+
+#define SAVE_THIS(xl)                                                          \
+    do {                                                                       \
+        old_this = xl;                                                         \
+        THIS = master;                                                         \
+    } while (0)
+
+#define RESTORE_THIS()                                                         \
+    do {                                                                       \
+        if (old_this)                                                          \
+            THIS = old_this;                                                   \
+    } while (0)
 
 /** APIs and the rest */
 
 void *
-gf_changelog_process (void *data);
-
-ssize_t
-gf_changelog_read_path (int fd, char *buffer, size_t bufsize);
+gf_changelog_process(void *data);
 
 void
-gf_rfc3986_encode_space_newline (unsigned char *s, char *enc, char *estr);
+gf_rfc3986_encode_space_newline(unsigned char *s, char *enc, char *estr);
 
 size_t
-gf_changelog_write (int fd, char *buffer, size_t len);
+gf_changelog_write(int fd, char *buffer, size_t len);
 
 ssize_t
-gf_readline (int fd, void *vptr, size_t maxlen);
+gf_readline(int fd, void *vptr, size_t maxlen);
 
 int
-gf_ftruncate (int fd, off_t length);
+gf_ftruncate(int fd, off_t length);
 
 off_t
-gf_lseek (int fd, off_t offset, int whence);
+gf_lseek(int fd, off_t offset, int whence);
 
 int
-gf_changelog_consume (xlator_t *this,
-                      gf_changelog_journal_t *jnl,
-                      char *from_path, gf_boolean_t no_publish);
+gf_changelog_consume(xlator_t *this, gf_changelog_journal_t *jnl,
+                     char *from_path, gf_boolean_t no_publish);
 int
-gf_changelog_publish (xlator_t *this,
-                      gf_changelog_journal_t *jnl, char *from_path);
+gf_changelog_publish(xlator_t *this, gf_changelog_journal_t *jnl,
+                     char *from_path);
 int
-gf_thread_cleanup (xlator_t *this, pthread_t thread);
+gf_thread_cleanup(xlator_t *this, pthread_t thread);
 void *
-gf_changelog_callback_invoker (void *arg);
+gf_changelog_callback_invoker(void *arg);
 
 int
-gf_cleanup_event (xlator_t *, struct gf_event_list *);
+gf_cleanup_event(xlator_t *, struct gf_event_list *);
 
 /* (un)ordered event queueing */
 void
-queue_ordered_event (struct gf_event_list *, struct gf_event *);
+queue_ordered_event(struct gf_event_list *, struct gf_event *);
 
 void
-queue_unordered_event (struct gf_event_list *, struct gf_event *);
+queue_unordered_event(struct gf_event_list *, struct gf_event *);
 
 /* (un)ordered event picking */
 void
-pick_event_ordered (struct gf_event_list *, struct gf_event **);
+pick_event_ordered(struct gf_event_list *, struct gf_event **);
 
 void
-pick_event_unordered (struct gf_event_list *, struct gf_event **);
+pick_event_unordered(struct gf_event_list *, struct gf_event **);
 
 /* connection janitor thread */
 void *
-gf_changelog_connection_janitor (void *);
+gf_changelog_connection_janitor(void *);
 
 #endif
diff --git a/xlators/features/changelog/lib/src/gf-changelog-journal-handler.c b/xlators/features/changelog/lib/src/gf-changelog-journal-handler.c
index 07de76e7cda..7f6e2329e71 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-journal-handler.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-journal-handler.c
@@ -8,11 +8,11 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "compat-uuid.h"
-#include "globals.h"
-#include "glusterfs.h"
-#include "syscall.h"
-#include "compat-errno.h"
+#include <glusterfs/compat-uuid.h>
+#include <glusterfs/globals.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/compat-errno.h>
 
 #include "gf-changelog-helpers.h"
 
@@ -25,112 +25,107 @@
 
 extern int byebye;
 
-enum changelog_versions {
-    VERSION_1_1 = 0,
-    VERSION_1_2 = 1
-};
+enum changelog_versions { VERSION_1_1 = 0, VERSION_1_2 = 1 };
 
 /**
  * number of gfid records after fop number
  */
-int nr_gfids[2][GF_FOP_MAXVALUE] = {
-    {
-        [GF_FOP_MKNOD]   = 1,
-        [GF_FOP_MKDIR]   = 1,
-        [GF_FOP_UNLINK]  = 1,
-        [GF_FOP_RMDIR]   = 1,
-        [GF_FOP_SYMLINK] = 1,
-        [GF_FOP_RENAME]  = 2,
-        [GF_FOP_LINK]    = 1,
-        [GF_FOP_CREATE]  = 1,
-    },
-    {
-        [GF_FOP_MKNOD]   = 1,
-        [GF_FOP_MKDIR]   = 1,
-        [GF_FOP_UNLINK]  = 2,
-        [GF_FOP_RMDIR]   = 2,
-        [GF_FOP_SYMLINK] = 1,
-        [GF_FOP_RENAME]  = 2,
-        [GF_FOP_LINK]    = 1,
-        [GF_FOP_CREATE]  = 1,
-    }
-};
-
-int nr_extra_recs[2][GF_FOP_MAXVALUE] = {
-    {
-        [GF_FOP_MKNOD]   = 3,
-        [GF_FOP_MKDIR]   = 3,
-        [GF_FOP_UNLINK]  = 0,
-        [GF_FOP_RMDIR]   = 0,
-        [GF_FOP_SYMLINK] = 0,
-        [GF_FOP_RENAME]  = 0,
-        [GF_FOP_LINK]    = 0,
-        [GF_FOP_CREATE]  = 3,
-    },
-    {
-        [GF_FOP_MKNOD]   = 3,
-        [GF_FOP_MKDIR]   = 3,
-        [GF_FOP_UNLINK]  = 0,
-        [GF_FOP_RMDIR]   = 0,
-        [GF_FOP_SYMLINK] = 0,
-        [GF_FOP_RENAME]  = 0,
-        [GF_FOP_LINK]    = 0,
-        [GF_FOP_CREATE]  = 3,
-    }
-};
+int nr_gfids[2][GF_FOP_MAXVALUE] = {{
+                                        [GF_FOP_MKNOD] = 1,
+                                        [GF_FOP_MKDIR] = 1,
+                                        [GF_FOP_UNLINK] = 1,
+                                        [GF_FOP_RMDIR] = 1,
+                                        [GF_FOP_SYMLINK] = 1,
+                                        [GF_FOP_RENAME] = 2,
+                                        [GF_FOP_LINK] = 1,
+                                        [GF_FOP_CREATE] = 1,
+                                    },
+                                    {
+                                        [GF_FOP_MKNOD] = 1,
+                                        [GF_FOP_MKDIR] = 1,
+                                        [GF_FOP_UNLINK] = 2,
+                                        [GF_FOP_RMDIR] = 2,
+                                        [GF_FOP_SYMLINK] = 1,
+                                        [GF_FOP_RENAME] = 2,
+                                        [GF_FOP_LINK] = 1,
+                                        [GF_FOP_CREATE] = 1,
+                                    }};
+
+int nr_extra_recs[2][GF_FOP_MAXVALUE] = {{
+                                             [GF_FOP_MKNOD] = 3,
+                                             [GF_FOP_MKDIR] = 3,
+                                             [GF_FOP_UNLINK] = 0,
+                                             [GF_FOP_RMDIR] = 0,
+                                             [GF_FOP_SYMLINK] = 0,
+                                             [GF_FOP_RENAME] = 0,
+                                             [GF_FOP_LINK] = 0,
+                                             [GF_FOP_CREATE] = 3,
+                                         },
+                                         {
+                                             [GF_FOP_MKNOD] = 3,
+                                             [GF_FOP_MKDIR] = 3,
+                                             [GF_FOP_UNLINK] = 0,
+                                             [GF_FOP_RMDIR] = 0,
+                                             [GF_FOP_SYMLINK] = 0,
+                                             [GF_FOP_RENAME] = 0,
+                                             [GF_FOP_LINK] = 0,
+                                             [GF_FOP_CREATE] = 3,
+                                         }};
 
 static char *
-binary_to_ascii (uuid_t uuid)
+binary_to_ascii(uuid_t uuid)
 {
-        return uuid_utoa (uuid);
+    return uuid_utoa(uuid);
 }
 
 static char *
-conv_noop (char *ptr) { return ptr; }
-
-#define VERIFY_SEPARATOR(ptr, plen, perr)       \
-        {                                       \
-                if (*(ptr + plen) != '\0') {    \
-                        perr = 1;               \
-                        break;                  \
-                }                               \
-        }
+conv_noop(char *ptr)
+{
+    return ptr;
+}
 
-#define MOVER_MOVE(mover, nleft, bytes)         \
-        {                                       \
-                mover += bytes;                 \
-                nleft -= bytes;                 \
-        }                                       \
-
-#define PARSE_GFID(mov, ptr, le, fn, perr)                      \
-        {                                                       \
-                VERIFY_SEPARATOR (mov, le, perr);               \
-                ptr = fn (mov);                                 \
-                if (!ptr) {                                     \
-                        perr = 1;                               \
-                        break;                                  \
-                }                                               \
-        }
+#define VERIFY_SEPARATOR(ptr, plen, perr)                                      \
+    {                                                                          \
+        if (*(ptr + plen) != '\0') {                                           \
+            perr = 1;                                                          \
+            break;                                                             \
+        }                                                                      \
+    }
 
-#define FILL_AND_MOVE(pt, buf, of, mo, nl, le)                          \
-        {                                                               \
-                GF_CHANGELOG_FILL_BUFFER (pt, buf, of, strlen (pt));    \
-                MOVER_MOVE (mo, nl, le);                                \
-        }
+#define MOVER_MOVE(mover, nleft, bytes)                                        \
+    {                                                                          \
+        mover += bytes;                                                        \
+        nleft -= bytes;                                                        \
+    }
+
+#define PARSE_GFID(mov, ptr, le, fn, perr)                                     \
+    {                                                                          \
+        VERIFY_SEPARATOR(mov, le, perr);                                       \
+        ptr = fn(mov);                                                         \
+        if (!ptr) {                                                            \
+            perr = 1;                                                          \
+            break;                                                             \
+        }                                                                      \
+    }
 
+#define FILL_AND_MOVE(pt, buf, of, mo, nl, le)                                 \
+    {                                                                          \
+        GF_CHANGELOG_FILL_BUFFER(pt, buf, of, strlen(pt));                     \
+        MOVER_MOVE(mo, nl, le);                                                \
+    }
 
-#define PARSE_GFID_MOVE(ptr, uuid, mover, nleft, perr)          \
-        {                                                       \
-                memcpy (uuid, mover, sizeof (uuid_t));          \
-                ptr = binary_to_ascii (uuid);                   \
-                if (!ptr) {                                     \
-                        perr = 1;                               \
-                        break;                                  \
-                }                                               \
-                MOVER_MOVE (mover, nleft, sizeof (uuid_t));     \
-        }                                                       \
+#define PARSE_GFID_MOVE(ptr, uuid, mover, nleft, perr)                         \
+    {                                                                          \
+        memcpy(uuid, mover, sizeof(uuid_t));                                   \
+        ptr = binary_to_ascii(uuid);                                           \
+        if (!ptr) {                                                            \
+            perr = 1;                                                          \
+            break;                                                             \
+        }                                                                      \
+        MOVER_MOVE(mover, nleft, sizeof(uuid_t));                              \
+    }
 
-#define LINE_BUFSIZE  (3*PATH_MAX) /* enough buffer for extra chars too */
+#define LINE_BUFSIZE (3 * PATH_MAX) /* enough buffer for extra chars too */
 
 /**
  * using mmap() makes parsing easy. fgets() cannot be used here as
@@ -145,111 +140,107 @@ conv_noop (char *ptr) { return ptr; }
  */
 
 static int
-gf_changelog_parse_binary (xlator_t *this,
-                           gf_changelog_journal_t *jnl,
-                           int from_fd, int to_fd,
-                           size_t start_offset, struct stat *stbuf,
-                           int version_idx)
+gf_changelog_parse_binary(xlator_t *this, gf_changelog_journal_t *jnl,
+                          int from_fd, int to_fd, size_t start_offset,
+                          struct stat *stbuf, int version_idx)
 
 {
-        int     ret              = -1;
-        off_t   off              = 0;
-        off_t   nleft            = 0;
-        uuid_t  uuid             = {0,};
-        char   *ptr              = NULL;
-        char   *bname_start      = NULL;
-        char   *bname_end        = NULL;
-        char   *mover            = NULL;
-        void   *start            = NULL;
-        char    current_mover    = ' ';
-        size_t  blen             = 0;
-        int     parse_err        = 0;
-        char   *ascii            = NULL;
-
-        ascii = GF_CALLOC (LINE_BUFSIZE, sizeof(char), gf_common_mt_char);
-
-        nleft = stbuf->st_size;
-
-        start = mmap (NULL, nleft, PROT_READ, MAP_PRIVATE, from_fd, 0);
-        if (start == MAP_FAILED) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_MMAP_FAILED,
-                        "mmap() error");
-                goto out;
-        }
-
-        mover = start;
+    int ret = -1;
+    off_t off = 0;
+    off_t nleft = 0;
+    uuid_t uuid = {
+        0,
+    };
+    char *ptr = NULL;
+    char *bname_start = NULL;
+    char *bname_end = NULL;
+    char *mover = NULL;
+    void *start = NULL;
+    char current_mover = ' ';
+    size_t blen = 0;
+    int parse_err = 0;
+    char *ascii = NULL;
+
+    ascii = GF_CALLOC(LINE_BUFSIZE, sizeof(char), gf_common_mt_char);
+
+    nleft = stbuf->st_size;
+
+    start = mmap(NULL, nleft, PROT_READ, MAP_PRIVATE, from_fd, 0);
+    if (start == MAP_FAILED) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_MMAP_FAILED,
+               "mmap() error");
+        goto out;
+    }
 
-        MOVER_MOVE (mover, nleft, start_offset);
+    mover = start;
 
-        while (nleft > 0) {
+    MOVER_MOVE(mover, nleft, start_offset);
 
-                off = blen = 0;
-                ptr = bname_start = bname_end = NULL;
+    while (nleft > 0) {
+        off = blen = 0;
+        ptr = bname_start = bname_end = NULL;
 
-                current_mover = *mover;
+        current_mover = *mover;
 
-                switch (current_mover) {
-                case 'D':
-                case 'M':
-                        MOVER_MOVE (mover, nleft, 1);
-                        PARSE_GFID_MOVE (ptr, uuid, mover, nleft, parse_err);
+        switch (current_mover) {
+            case 'D':
+            case 'M':
+                MOVER_MOVE(mover, nleft, 1);
+                PARSE_GFID_MOVE(ptr, uuid, mover, nleft, parse_err);
 
-                        break;
+                break;
 
-                case 'E':
-                        MOVER_MOVE (mover, nleft, 1);
-                        PARSE_GFID_MOVE (ptr, uuid, mover, nleft, parse_err);
+            case 'E':
+                MOVER_MOVE(mover, nleft, 1);
+                PARSE_GFID_MOVE(ptr, uuid, mover, nleft, parse_err);
 
-                        bname_start = mover;
-                        bname_end = strchr (mover, '\n');
-                        if (bname_end == NULL) {
-                                parse_err = 1;
-                                break;
-                        }
+                bname_start = mover;
+                bname_end = strchr(mover, '\n');
+                if (bname_end == NULL) {
+                    parse_err = 1;
+                    break;
+                }
 
-                        blen = bname_end - bname_start;
-                        MOVER_MOVE (mover, nleft, blen);
+                blen = bname_end - bname_start;
+                MOVER_MOVE(mover, nleft, blen);
 
-                        break;
+                break;
 
-                default:
-                        parse_err = 1;
-                }
+            default:
+                parse_err = 1;
+        }
 
-                if (parse_err)
-                        break;
+        if (parse_err)
+            break;
+
+        GF_CHANGELOG_FILL_BUFFER(&current_mover, ascii, off, 1);
+        GF_CHANGELOG_FILL_BUFFER(" ", ascii, off, 1);
+        GF_CHANGELOG_FILL_BUFFER(ptr, ascii, off, strlen(ptr));
+        if (blen)
+            GF_CHANGELOG_FILL_BUFFER(bname_start, ascii, off, blen);
+        GF_CHANGELOG_FILL_BUFFER("\n", ascii, off, 1);
+
+        if (gf_changelog_write(to_fd, ascii, off) != off) {
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   CHANGELOG_LIB_MSG_ASCII_ERROR,
+                   "processing binary changelog failed due to "
+                   " error in writing ascii change");
+            break;
+        }
 
-                GF_CHANGELOG_FILL_BUFFER (&current_mover, ascii, off, 1);
-                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
-                GF_CHANGELOG_FILL_BUFFER (ptr, ascii, off, strlen (ptr));
-                if (blen)
-                        GF_CHANGELOG_FILL_BUFFER (bname_start,
-                                                  ascii, off, blen);
-                GF_CHANGELOG_FILL_BUFFER ("\n", ascii, off, 1);
-
-                if (gf_changelog_write (to_fd, ascii, off) != off) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_LIB_MSG_ASCII_ERROR,
-                                "processing binary changelog failed due to "
-                                " error in writing ascii change");
-                        break;
-                }
+        MOVER_MOVE(mover, nleft, 1);
+    }
 
-                MOVER_MOVE (mover, nleft, 1);
-        }
+    if ((nleft == 0) && (!parse_err))
+        ret = 0;
 
-        if ((nleft == 0) && (!parse_err))
-                ret = 0;
-
-        if (munmap (start, stbuf->st_size))
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_MUNMAP_FAILED,
-                        "munmap() error");
- out:
-        if (ascii)
-                GF_FREE (ascii);
-        return ret;
+    if (munmap(start, stbuf->st_size))
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_MUNMAP_FAILED,
+               "munmap() error");
+out:
+    if (ascii)
+        GF_FREE(ascii);
+    return ret;
 }
 
 /**
@@ -258,798 +249,781 @@ gf_changelog_parse_binary (xlator_t *this,
  *  - use fop name rather than fop number
  */
 static int
-gf_changelog_parse_ascii (xlator_t *this,
-                          gf_changelog_journal_t *jnl,
-                          int from_fd, int to_fd,
-                          size_t start_offset, struct stat *stbuf,
-                          int version_idx)
+gf_changelog_parse_ascii(xlator_t *this, gf_changelog_journal_t *jnl,
+                         int from_fd, int to_fd, size_t start_offset,
+                         struct stat *stbuf, int version_idx)
 {
-        int           ng            = 0;
-        int           ret           = -1;
-        int           fop           = 0;
-        int           len           = 0;
-        off_t         off           = 0;
-        off_t         nleft         = 0;
-        char         *ptr           = NULL;
-        char         *eptr          = NULL;
-        void         *start         = NULL;
-        char         *mover         = NULL;
-        int           parse_err     = 0;
-        char          current_mover = ' ';
-        char         *ascii         = NULL;
-        const char   *fopname       = NULL;
-
-        ascii = GF_CALLOC (LINE_BUFSIZE, sizeof(char), gf_common_mt_char);
-
-        nleft = stbuf->st_size;
-
-        start = mmap (NULL, nleft, PROT_READ, MAP_PRIVATE, from_fd, 0);
-        if (start == MAP_FAILED) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_MMAP_FAILED,
-                        "mmap() error");
-                goto out;
-        }
+    int ng = 0;
+    int ret = -1;
+    int fop = 0;
+    int len = 0;
+    off_t off = 0;
+    off_t nleft = 0;
+    char *ptr = NULL;
+    char *eptr = NULL;
+    void *start = NULL;
+    char *mover = NULL;
+    int parse_err = 0;
+    char current_mover = ' ';
+    char *ascii = NULL;
+    const char *fopname = NULL;
+
+    ascii = GF_CALLOC(LINE_BUFSIZE, sizeof(char), gf_common_mt_char);
+
+    nleft = stbuf->st_size;
+
+    start = mmap(NULL, nleft, PROT_READ, MAP_PRIVATE, from_fd, 0);
+    if (start == MAP_FAILED) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_MMAP_FAILED,
+               "mmap() error");
+        goto out;
+    }
 
-        mover = start;
+    mover = start;
 
-        MOVER_MOVE (mover, nleft, start_offset);
+    MOVER_MOVE(mover, nleft, start_offset);
 
-        while (nleft > 0) {
-                off = 0;
-                current_mover = *mover;
+    while (nleft > 0) {
+        off = 0;
+        current_mover = *mover;
 
-                GF_CHANGELOG_FILL_BUFFER (&current_mover, ascii, off, 1);
-                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+        GF_CHANGELOG_FILL_BUFFER(&current_mover, ascii, off, 1);
+        GF_CHANGELOG_FILL_BUFFER(" ", ascii, off, 1);
 
-                switch (current_mover) {
-                case 'D':
-                        MOVER_MOVE (mover, nleft, 1);
+        switch (current_mover) {
+            case 'D':
+                MOVER_MOVE(mover, nleft, 1);
 
-                        /* target gfid */
-                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
-                                    conv_noop, parse_err);
-                        FILL_AND_MOVE(ptr, ascii, off,
-                                      mover, nleft, UUID_CANONICAL_FORM_LEN);
-                        break;
-                case 'M':
-                        MOVER_MOVE (mover, nleft, 1);
+                /* target gfid */
+                PARSE_GFID(mover, ptr, UUID_CANONICAL_FORM_LEN, conv_noop,
+                           parse_err);
+                FILL_AND_MOVE(ptr, ascii, off, mover, nleft,
+                              UUID_CANONICAL_FORM_LEN);
+                break;
+            case 'M':
+                MOVER_MOVE(mover, nleft, 1);
+
+                /* target gfid */
+                PARSE_GFID(mover, ptr, UUID_CANONICAL_FORM_LEN, conv_noop,
+                           parse_err);
+                FILL_AND_MOVE(ptr, ascii, off, mover, nleft,
+                              UUID_CANONICAL_FORM_LEN);
+                FILL_AND_MOVE(" ", ascii, off, mover, nleft, 1);
+
+                /* fop */
+                len = strlen(mover);
+                VERIFY_SEPARATOR(mover, len, parse_err);
+
+                fop = atoi(mover);
+                fopname = gf_fop_list[fop];
+                if (fopname == NULL) {
+                    parse_err = 1;
+                    break;
+                }
 
-                        /* target gfid */
-                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
-                                    conv_noop, parse_err);
-                        FILL_AND_MOVE (ptr, ascii, off,
-                                       mover, nleft, UUID_CANONICAL_FORM_LEN);
-                        FILL_AND_MOVE (" ", ascii, off, mover, nleft, 1);
+                MOVER_MOVE(mover, nleft, len);
 
-                        /* fop */
-                        len = strlen (mover);
-                        VERIFY_SEPARATOR (mover, len, parse_err);
+                len = strlen(fopname);
+                GF_CHANGELOG_FILL_BUFFER(fopname, ascii, off, len);
 
-                        fop = atoi (mover);
-                        fopname = gf_fop_list[fop];
-                        if (fopname == NULL) {
-                                parse_err = 1;
-                                break;
-                        }
+                break;
 
-                        MOVER_MOVE (mover, nleft, len);
+            case 'E':
+                MOVER_MOVE(mover, nleft, 1);
+
+                /* target gfid */
+                PARSE_GFID(mover, ptr, UUID_CANONICAL_FORM_LEN, conv_noop,
+                           parse_err);
+                FILL_AND_MOVE(ptr, ascii, off, mover, nleft,
+                              UUID_CANONICAL_FORM_LEN);
+                FILL_AND_MOVE(" ", ascii, off, mover, nleft, 1);
+
+                /* fop */
+                len = strlen(mover);
+                VERIFY_SEPARATOR(mover, len, parse_err);
+
+                fop = atoi(mover);
+                fopname = gf_fop_list[fop];
+                if (fopname == NULL) {
+                    parse_err = 1;
+                    break;
+                }
 
-                        len = strlen (fopname);
-                        GF_CHANGELOG_FILL_BUFFER (fopname, ascii, off, len);
+                MOVER_MOVE(mover, nleft, len);
 
-                        break;
+                len = strlen(fopname);
+                GF_CHANGELOG_FILL_BUFFER(fopname, ascii, off, len);
 
-                case 'E':
-                        MOVER_MOVE (mover, nleft, 1);
-
-                        /* target gfid */
-                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
-                                    conv_noop, parse_err);
-                        FILL_AND_MOVE (ptr, ascii, off,
-                                       mover, nleft, UUID_CANONICAL_FORM_LEN);
-                        FILL_AND_MOVE (" ", ascii, off,
-                                       mover, nleft, 1);
-
-                        /* fop */
-                        len = strlen (mover);
-                        VERIFY_SEPARATOR (mover, len, parse_err);
-
-                        fop = atoi (mover);
-                        fopname = gf_fop_list[fop];
-                        if (fopname == NULL) {
-                                parse_err = 1;
-                                break;
-                        }
-
-                        MOVER_MOVE (mover, nleft, len);
-
-                        len = strlen (fopname);
-                        GF_CHANGELOG_FILL_BUFFER (fopname, ascii, off, len);
-
-                        ng = nr_extra_recs[version_idx][fop];
-                        for (; ng > 0; ng--) {
-                                MOVER_MOVE (mover, nleft, 1);
-                                len = strlen (mover);
-                                VERIFY_SEPARATOR (mover, len, parse_err);
-
-                                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
-                                FILL_AND_MOVE (mover, ascii,
-                                               off, mover, nleft, len);
-                        }
-
-                        /* pargfid + bname */
-                        ng = nr_gfids[version_idx][fop];
-                        while (ng-- > 0) {
-                                MOVER_MOVE (mover, nleft, 1);
-                                len = strlen (mover);
-                                if (!len) {
-                                        MOVER_MOVE (mover, nleft, 1);
-                                        continue;
-                                }
-
-                                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
-
-                                PARSE_GFID (mover, ptr, len,
-                                            conv_noop, parse_err);
-                                eptr = calloc (3, strlen (ptr));
-                                if (!eptr) {
-                                        parse_err = 1;
-                                        break;
-                                }
-
-                                gf_rfc3986_encode_space_newline (
-                                                   (unsigned char *) ptr,
-                                                   eptr,
-                                                   jnl->rfc3986_space_newline);
-                                FILL_AND_MOVE (eptr, ascii, off,
-                                               mover, nleft, len);
-                                free (eptr);
-                        }
+                ng = nr_extra_recs[version_idx][fop];
+                for (; ng > 0; ng--) {
+                    MOVER_MOVE(mover, nleft, 1);
+                    len = strlen(mover);
+                    VERIFY_SEPARATOR(mover, len, parse_err);
 
-                        break;
-                default:
-                        parse_err = 1;
+                    GF_CHANGELOG_FILL_BUFFER(" ", ascii, off, 1);
+                    FILL_AND_MOVE(mover, ascii, off, mover, nleft, len);
                 }
 
-                if (parse_err)
+                /* pargfid + bname */
+                ng = nr_gfids[version_idx][fop];
+                while (ng-- > 0) {
+                    MOVER_MOVE(mover, nleft, 1);
+                    len = strlen(mover);
+                    if (!len) {
+                        MOVER_MOVE(mover, nleft, 1);
+                        continue;
+                    }
+
+                    GF_CHANGELOG_FILL_BUFFER(" ", ascii, off, 1);
+
+                    PARSE_GFID(mover, ptr, len, conv_noop, parse_err);
+                    eptr = calloc(3, strlen(ptr));
+                    if (!eptr) {
+                        parse_err = 1;
                         break;
+                    }
 
-                GF_CHANGELOG_FILL_BUFFER ("\n", ascii, off, 1);
-
-                if (gf_changelog_write (to_fd, ascii, off) != off) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_LIB_MSG_ASCII_ERROR,
-                                "processing ascii changelog failed due to "
-                                " error in writing change");
-                        break;
+                    gf_rfc3986_encode_space_newline((unsigned char *)ptr, eptr,
+                                                    jnl->rfc3986_space_newline);
+                    FILL_AND_MOVE(eptr, ascii, off, mover, nleft, len);
+                    free(eptr);
                 }
 
-                MOVER_MOVE (mover, nleft, 1);
+                break;
+            default:
+                parse_err = 1;
+        }
+
+        if (parse_err)
+            break;
+
+        GF_CHANGELOG_FILL_BUFFER("\n", ascii, off, 1);
 
+        if (gf_changelog_write(to_fd, ascii, off) != off) {
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   CHANGELOG_LIB_MSG_ASCII_ERROR,
+                   "processing ascii changelog failed due to "
+                   " error in writing change");
+            break;
         }
 
-        if ((nleft == 0) && (!parse_err))
-                ret = 0;
+        MOVER_MOVE(mover, nleft, 1);
+    }
+
+    if ((nleft == 0) && (!parse_err))
+        ret = 0;
 
-        if (munmap (start, stbuf->st_size))
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_MUNMAP_FAILED,
-                        "munmap() error");
+    if (munmap(start, stbuf->st_size))
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_MUNMAP_FAILED,
+               "munmap() error");
 
- out:
-        if (ascii)
-                GF_FREE (ascii);
+out:
+    if (ascii)
+        GF_FREE(ascii);
 
-        return ret;
+    return ret;
 }
 
 static int
-gf_changelog_decode (xlator_t *this, gf_changelog_journal_t *jnl,
-                     int from_fd, int to_fd, struct stat *stbuf, int *zerob)
+gf_changelog_decode(xlator_t *this, gf_changelog_journal_t *jnl, int from_fd,
+                    int to_fd, struct stat *stbuf, int *zerob)
 {
-        int    ret        = -1;
-        int    encoding   = -1;
-        int major_version = -1;
-        int minor_version = -1;
-        int version_idx   = -1;
-        size_t elen       = 0;
-        char buffer[1024] = {0,};
-
-        CHANGELOG_GET_HEADER_INFO (from_fd, buffer, 1024, encoding,
-                                   major_version, minor_version, elen);
-        if (encoding == -1) /* unknown encoding */
-                goto out;
-
-        if (major_version == -1) /* unknown major version */
-                goto out;
-
-        if (minor_version == -1) /* unknown minor version */
-                goto out;
-
-        if (!CHANGELOG_VALID_ENCODING (encoding))
-                goto out;
-
-        if (elen == stbuf->st_size) {
-                *zerob = 1;
-                goto out;
-        }
+    int ret = -1;
+    int encoding = -1;
+    int major_version = -1;
+    int minor_version = -1;
+    int version_idx = -1;
+    size_t elen = 0;
+    char buffer[1024] = {
+        0,
+    };
+
+    CHANGELOG_GET_HEADER_INFO(from_fd, buffer, sizeof(buffer), encoding,
+                              major_version, minor_version, elen);
+    if (encoding == -1) /* unknown encoding */
+        goto out;
+
+    if (major_version == -1) /* unknown major version */
+        goto out;
+
+    if (minor_version == -1) /* unknown minor version */
+        goto out;
+
+    if (!CHANGELOG_VALID_ENCODING(encoding))
+        goto out;
+
+    if (elen == stbuf->st_size) {
+        *zerob = 1;
+        goto out;
+    }
 
-        if (major_version == 1 && minor_version == 1) {
-                version_idx = VERSION_1_1;
-        } else if (major_version == 1 && minor_version == 2) {
-                version_idx = VERSION_1_2;
-        }
+    if (major_version == 1 && minor_version == 1) {
+        version_idx = VERSION_1_1;
+    } else if (major_version == 1 && minor_version == 2) {
+        version_idx = VERSION_1_2;
+    }
 
-        if (version_idx == -1) /* unknown version number */
-                goto out;
+    if (version_idx == -1) /* unknown version number */
+        goto out;
 
-        /**
-         * start processing after the header
-         */
-        if (sys_lseek (from_fd, elen, SEEK_SET) < 0) {
-                goto out;
-        }
-        switch (encoding) {
+    /**
+     * start processing after the header
+     */
+    if (sys_lseek(from_fd, elen, SEEK_SET) < 0) {
+        goto out;
+    }
+    switch (encoding) {
         case CHANGELOG_ENCODE_BINARY:
-                /**
-                 * this ideally should have been a part of changelog-encoders.c
-                 * (ie. part of the changelog translator).
-                 */
-                ret = gf_changelog_parse_binary (this, jnl, from_fd,
-                                                 to_fd, elen, stbuf,
-                                                 version_idx);
-                break;
+            /**
+             * this ideally should have been a part of changelog-encoders.c
+             * (ie. part of the changelog translator).
+             */
+            ret = gf_changelog_parse_binary(this, jnl, from_fd, to_fd, elen,
+                                            stbuf, version_idx);
+            break;
 
         case CHANGELOG_ENCODE_ASCII:
-                ret = gf_changelog_parse_ascii (this, jnl, from_fd,
-                                                to_fd, elen, stbuf,
-                                                version_idx);
-                break;
-        }
+            ret = gf_changelog_parse_ascii(this, jnl, from_fd, to_fd, elen,
+                                           stbuf, version_idx);
+            break;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-gf_changelog_publish (xlator_t *this,
-                      gf_changelog_journal_t *jnl, char *from_path)
+gf_changelog_publish(xlator_t *this, gf_changelog_journal_t *jnl,
+                     char *from_path)
 {
-        int         ret        = 0;
-        char dest[PATH_MAX]    = {0,};
-        char to_path[PATH_MAX] = {0,};
-        struct stat stbuf      = {0,};
-
-        (void) snprintf (to_path, PATH_MAX, "%s%s",
-                         jnl->jnl_current_dir, basename (from_path));
-
-        /* handle zerob file that wont exist in current */
-        ret = sys_stat (to_path, &stbuf);
-        if (ret) {
-                if (errno == ENOENT)
-                        ret = 0;
-                goto out;
-        }
+    int ret = 0;
+    char dest[PATH_MAX] = {
+        0,
+    };
+    char to_path[PATH_MAX] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+
+    if (snprintf(to_path, PATH_MAX, "%s%s", jnl->jnl_current_dir,
+                 basename(from_path)) >= PATH_MAX)
+        return -1;
 
-        (void) snprintf (dest, PATH_MAX, "%s%s",
-                         jnl->jnl_processing_dir, basename (from_path));
+    /* handle zerob file that won't exist in current */
+    ret = sys_stat(to_path, &stbuf);
+    if (ret) {
+        if (errno == ENOENT)
+            ret = 0;
+        goto out;
+    }
 
-        ret = sys_rename (to_path, dest);
-        if (ret) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_RENAME_FAILED,
-                         "error moving changelog to processing dir",
-                         "path=%s", to_path,
-                         NULL);
-        }
+    if (snprintf(dest, PATH_MAX, "%s%s", jnl->jnl_processing_dir,
+                 basename(from_path)) >= PATH_MAX)
+        return -1;
+
+    ret = sys_rename(to_path, dest);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_RENAME_FAILED, "from=%s", to_path, "to=%s",
+                dest, NULL);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-gf_changelog_consume (xlator_t *this,
-                      gf_changelog_journal_t *jnl,
-                      char *from_path, gf_boolean_t no_publish)
+gf_changelog_consume(xlator_t *this, gf_changelog_journal_t *jnl,
+                     char *from_path, gf_boolean_t no_publish)
 {
-        int         ret        = -1;
-        int         fd1        = 0;
-        int         fd2        = 0;
-        int         zerob      = 0;
-        struct stat stbuf      = {0,};
-        char dest[PATH_MAX]    = {0,};
-        char to_path[PATH_MAX] = {0,};
-
-        ret = sys_stat (from_path, &stbuf);
-        if (ret || !S_ISREG(stbuf.st_mode)) {
-                ret = -1;
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_STAT_FAILED,
-                         "stat failed on changelog file",
-                         "path=%s", from_path,
-                         NULL);
-                goto out;
-        }
+    int ret = -1;
+    int fd1 = 0;
+    int fd2 = 0;
+    int zerob = 0;
+    struct stat stbuf = {
+        0,
+    };
+    char dest[PATH_MAX] = {
+        0,
+    };
+    char to_path[PATH_MAX] = {
+        0,
+    };
+
+    if (snprintf(to_path, PATH_MAX, "%s%s", jnl->jnl_current_dir,
+                 basename(from_path)) >= PATH_MAX)
+        goto out;
+    if (snprintf(dest, PATH_MAX, "%s%s", jnl->jnl_processing_dir,
+                 basename(from_path)) >= PATH_MAX)
+        goto out;
+
+    ret = sys_stat(from_path, &stbuf);
+    if (ret || !S_ISREG(stbuf.st_mode)) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_STAT_FAILED,
+                "path=%s", from_path, NULL);
+        goto out;
+    }
 
-        fd1 = open (from_path, O_RDONLY);
-        if (fd1 < 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_OPEN_FAILED,
-                         "cannot open changelog file",
-                         "path=%s", from_path,
-                         NULL);
-                goto out;
-        }
+    fd1 = open(from_path, O_RDONLY);
+    if (fd1 < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_OPEN_FAILED,
+                "path=%s", from_path, NULL);
+        goto out;
+    }
 
-        (void) snprintf (to_path, PATH_MAX, "%s%s",
-                         jnl->jnl_current_dir, basename (from_path));
-        (void) snprintf (dest, PATH_MAX, "%s%s",
-                         jnl->jnl_processing_dir, basename (from_path));
-
-        fd2 = open (to_path, O_CREAT | O_TRUNC | O_RDWR,
-                    S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (fd2 < 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_OPEN_FAILED,
-                         "cannot create ascii changelog file",
-                         "path=%s", to_path,
-                         NULL);
+    fd2 = open(to_path, O_CREAT | O_TRUNC | O_RDWR,
+               S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (fd2 < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_OPEN_FAILED,
+                "path=%s", to_path, NULL);
+        goto close_fd;
+    } else {
+        ret = gf_changelog_decode(this, jnl, fd1, fd2, &stbuf, &zerob);
+
+        sys_close(fd2);
+
+        if (!ret) {
+            /* move it to processing on a successful
+               decode */
+            if (no_publish == _gf_true)
                 goto close_fd;
-        } else {
-                ret = gf_changelog_decode (this, jnl, fd1,
-                                           fd2, &stbuf, &zerob);
-
-                sys_close (fd2);
-
-                if (!ret) {
-                        /* move it to processing on a successful
-                           decode */
-                        if (no_publish == _gf_true)
-                                goto close_fd;
-                        ret = sys_rename (to_path, dest);
-                        if (ret)
-                                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                                         CHANGELOG_LIB_MSG_RENAME_FAILED,
-                                         "error moving changelog to processing dir",
-                                         "path=%s", to_path,
-                                         NULL);
-                }
+            ret = sys_rename(to_path, dest);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        CHANGELOG_LIB_MSG_RENAME_FAILED, "from=%s", to_path,
+                        "to=%s", dest, NULL);
+        }
 
-                /* remove it from .current if it's an empty file */
-                if (zerob) {
-                        /* zerob changelogs must be unlinked */
-                        ret = sys_unlink (to_path);
-                        if (ret)
-                                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                                         CHANGELOG_LIB_MSG_UNLINK_FAILED,
-                                         "could not unlink empty changelog",
-                                         "path=%s", to_path,
-                                         NULL);
-                }
+        /* remove it from .current if it's an empty file */
+        if (zerob) {
+            /* zerob changelogs must be unlinked */
+            ret = sys_unlink(to_path);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        CHANGELOG_LIB_MSG_UNLINK_FAILED, "name=empty changelog",
+                        "path=%s", to_path, NULL);
         }
+    }
 
- close_fd:
-        sys_close (fd1);
+close_fd:
+    sys_close(fd1);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 void *
-gf_changelog_process (void *data)
+gf_changelog_process(void *data)
 {
-        xlator_t *this = NULL;
-        gf_changelog_journal_t *jnl = NULL;
-        gf_changelog_entry_t *entry = NULL;
-        gf_changelog_processor_t *jnl_proc = NULL;
-
-        jnl = data;
-        jnl_proc = jnl->jnl_proc;
-        THIS = jnl->this;
-        this = jnl->this;
-
-        while (1) {
-                pthread_mutex_lock (&jnl_proc->lock);
-                {
-                        while (list_empty (&jnl_proc->entries)) {
-                                jnl_proc->waiting = _gf_true;
-                                pthread_cond_wait
-                                        (&jnl_proc->cond, &jnl_proc->lock);
-                        }
-
-                        entry = list_first_entry (&jnl_proc->entries,
-                                                  gf_changelog_entry_t, list);
-                        if (entry)
-                                list_del (&entry->list);
-
-                        jnl_proc->waiting = _gf_false;
-                }
-                pthread_mutex_unlock (&jnl_proc->lock);
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_entry_t *entry = NULL;
+    gf_changelog_processor_t *jnl_proc = NULL;
+
+    jnl = data;
+    jnl_proc = jnl->jnl_proc;
+    THIS = jnl->this;
+    this = jnl->this;
+
+    while (1) {
+        pthread_mutex_lock(&jnl_proc->lock);
+        {
+            while (list_empty(&jnl_proc->entries)) {
+                jnl_proc->waiting = _gf_true;
+                pthread_cond_wait(&jnl_proc->cond, &jnl_proc->lock);
+            }
 
-                if (entry) {
-                        (void) gf_changelog_consume (this, jnl,
-                                                     entry->path, _gf_false);
-                        GF_FREE (entry);
-                }
+            entry = list_first_entry(&jnl_proc->entries, gf_changelog_entry_t,
+                                     list);
+            if (entry)
+                list_del(&entry->list);
+
+            jnl_proc->waiting = _gf_false;
         }
+        pthread_mutex_unlock(&jnl_proc->lock);
 
-        return NULL;
+        if (entry) {
+            (void)gf_changelog_consume(this, jnl, entry->path, _gf_false);
+            GF_FREE(entry);
+        }
+    }
+
+    return NULL;
 }
 
 void
-gf_changelog_queue_journal (gf_changelog_processor_t *jnl_proc,
-                            changelog_event_t *event)
+gf_changelog_queue_journal(gf_changelog_processor_t *jnl_proc,
+                           changelog_event_t *event)
 {
-        size_t len = 0;
-        gf_changelog_entry_t *entry = NULL;
+    size_t len = 0;
+    gf_changelog_entry_t *entry = NULL;
 
-        entry = GF_CALLOC (1, sizeof (gf_changelog_entry_t),
-                           gf_changelog_mt_libgfchangelog_entry_t);
-        if (!entry)
-                return;
-        INIT_LIST_HEAD (&entry->list);
+    entry = GF_CALLOC(1, sizeof(gf_changelog_entry_t),
+                      gf_changelog_mt_libgfchangelog_entry_t);
+    if (!entry)
+        return;
+    INIT_LIST_HEAD(&entry->list);
 
-        len = strlen (event->u.journal.path);
-        (void)memcpy (entry->path, event->u.journal.path, len+1);
-        entry->path[len] = '\0';
+    len = strlen(event->u.journal.path);
+    (void)memcpy(entry->path, event->u.journal.path, len + 1);
+    entry->path[len] = '\0';
 
-        pthread_mutex_lock (&jnl_proc->lock);
-        {
-                list_add_tail (&entry->list, &jnl_proc->entries);
-                if (jnl_proc->waiting)
-                        pthread_cond_signal (&jnl_proc->cond);
-        }
-        pthread_mutex_unlock (&jnl_proc->lock);
+    pthread_mutex_lock(&jnl_proc->lock);
+    {
+        list_add_tail(&entry->list, &jnl_proc->entries);
+        if (jnl_proc->waiting)
+            pthread_cond_signal(&jnl_proc->cond);
+    }
+    pthread_mutex_unlock(&jnl_proc->lock);
 
-        return;
+    return;
 }
 
 void
-gf_changelog_handle_journal (void *xl, char *brick,
-                             void *cbkdata, changelog_event_t *event)
+gf_changelog_handle_journal(void *xl, char *brick, void *cbkdata,
+                            changelog_event_t *event)
 {
-        gf_changelog_journal_t   *jnl      = NULL;
-        gf_changelog_processor_t *jnl_proc = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_processor_t *jnl_proc = NULL;
 
-        jnl      = cbkdata;
-        jnl_proc = jnl->jnl_proc;
+    jnl = cbkdata;
+    jnl_proc = jnl->jnl_proc;
 
-        gf_changelog_queue_journal (jnl_proc, event);
+    gf_changelog_queue_journal(jnl_proc, event);
 }
 
 void
-gf_changelog_journal_disconnect (void *xl, char *brick, void *data)
+gf_changelog_journal_disconnect(void *xl, char *brick, void *data)
 {
-        gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *jnl = NULL;
 
-        jnl = data;
+    jnl = data;
 
-        pthread_spin_lock (&jnl->lock);
-        {
-                JNL_SET_API_STATE (jnl, JNL_API_DISCONNECTED);
-        };
-        pthread_spin_unlock (&jnl->lock);
+    pthread_spin_lock(&jnl->lock);
+    {
+        JNL_SET_API_STATE(jnl, JNL_API_DISCONNECTED);
+    };
+    pthread_spin_unlock(&jnl->lock);
 }
 
 void
-gf_changelog_journal_connect (void *xl, char *brick, void *data)
+gf_changelog_journal_connect(void *xl, char *brick, void *data)
 {
-        gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *jnl = NULL;
 
-        jnl = data;
+    jnl = data;
 
-        pthread_spin_lock (&jnl->lock);
-        {
-                JNL_SET_API_STATE (jnl, JNL_API_CONNECTED);
-        };
-        pthread_spin_unlock (&jnl->lock);
+    pthread_spin_lock(&jnl->lock);
+    {
+        JNL_SET_API_STATE(jnl, JNL_API_CONNECTED);
+    };
+    pthread_spin_unlock(&jnl->lock);
 
-        return;
+    return;
 }
 
 void
-gf_changelog_cleanup_processor (gf_changelog_journal_t *jnl)
+gf_changelog_cleanup_processor(gf_changelog_journal_t *jnl)
 {
-        int ret = 0;
-        xlator_t *this = NULL;
-        gf_changelog_processor_t *jnl_proc = NULL;
-
-        this = THIS;
-        if (!this || !jnl || !jnl->jnl_proc)
-                goto error_return;
-
-        jnl_proc = jnl->jnl_proc;
-
-        ret = gf_thread_cleanup (this, jnl_proc->processor);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_LIB_MSG_CLEANUP_ERROR,
-                        "failed to cleanup processor thread");
-                goto error_return;
-        }
+    int ret = 0;
+    xlator_t *this = NULL;
+    gf_changelog_processor_t *jnl_proc = NULL;
 
-        (void)pthread_mutex_destroy (&jnl_proc->lock);
-        (void)pthread_cond_destroy (&jnl_proc->cond);
+    this = THIS;
+    if (!this || !jnl || !jnl->jnl_proc)
+        goto error_return;
 
-        GF_FREE (jnl_proc);
+    jnl_proc = jnl->jnl_proc;
 
- error_return:
-        return;
+    ret = gf_thread_cleanup(this, jnl_proc->processor);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_CLEANUP_ERROR,
+               "failed to cleanup processor thread");
+        goto error_return;
+    }
+
+    (void)pthread_mutex_destroy(&jnl_proc->lock);
+    (void)pthread_cond_destroy(&jnl_proc->cond);
+
+    GF_FREE(jnl_proc);
+
+error_return:
+    return;
 }
 
 int
-gf_changelog_init_processor (gf_changelog_journal_t *jnl)
+gf_changelog_init_processor(gf_changelog_journal_t *jnl)
 {
-        int ret = -1;
-        gf_changelog_processor_t *jnl_proc = NULL;
+    int ret = -1;
+    gf_changelog_processor_t *jnl_proc = NULL;
 
-        jnl_proc = GF_CALLOC (1, sizeof (gf_changelog_processor_t),
-                              gf_changelog_mt_libgfchangelog_t);
-        if (!jnl_proc)
-                goto error_return;
-
-        ret = pthread_mutex_init (&jnl_proc->lock, NULL);
-        if (ret != 0)
-                goto free_jnl_proc;
-        ret = pthread_cond_init (&jnl_proc->cond, NULL);
-        if (ret != 0)
-                goto cleanup_mutex;
-
-        INIT_LIST_HEAD (&jnl_proc->entries);
-        jnl_proc->waiting = _gf_false;
-        jnl->jnl_proc = jnl_proc;
-
-        ret = gf_thread_create (&jnl_proc->processor,
-                                NULL, gf_changelog_process, jnl, "clogproc");
-        if (ret != 0) {
-                jnl->jnl_proc = NULL;
-                goto cleanup_cond;
-        }
+    jnl_proc = GF_CALLOC(1, sizeof(gf_changelog_processor_t),
+                         gf_changelog_mt_libgfchangelog_t);
+    if (!jnl_proc)
+        goto error_return;
+
+    ret = pthread_mutex_init(&jnl_proc->lock, NULL);
+    if (ret != 0)
+        goto free_jnl_proc;
+    ret = pthread_cond_init(&jnl_proc->cond, NULL);
+    if (ret != 0)
+        goto cleanup_mutex;
+
+    INIT_LIST_HEAD(&jnl_proc->entries);
+    jnl_proc->waiting = _gf_false;
+    jnl->jnl_proc = jnl_proc;
+
+    ret = gf_thread_create(&jnl_proc->processor, NULL, gf_changelog_process,
+                           jnl, "clogproc");
+    if (ret != 0) {
+        jnl->jnl_proc = NULL;
+        goto cleanup_cond;
+    }
 
-        return 0;
+    return 0;
 
- cleanup_cond:
-        (void) pthread_cond_destroy (&jnl_proc->cond);
- cleanup_mutex:
-        (void) pthread_mutex_destroy (&jnl_proc->lock);
- free_jnl_proc:
-        GF_FREE (jnl_proc);
- error_return:
-        return -1;
+cleanup_cond:
+    (void)pthread_cond_destroy(&jnl_proc->cond);
+cleanup_mutex:
+    (void)pthread_mutex_destroy(&jnl_proc->lock);
+free_jnl_proc:
+    GF_FREE(jnl_proc);
+error_return:
+    return -1;
 }
 
 static void
-gf_changelog_cleanup_fds (gf_changelog_journal_t *jnl)
+gf_changelog_cleanup_fds(gf_changelog_journal_t *jnl)
 {
-        /* tracker fd */
-        if (jnl->jnl_fd != -1)
-                sys_close (jnl->jnl_fd);
-        /* processing dir */
-        if (jnl->jnl_dir)
-                sys_closedir (jnl->jnl_dir);
-
-        if (jnl->jnl_working_dir)
-                free (jnl->jnl_working_dir); /* allocated by realpath */
+    /* tracker fd */
+    if (jnl->jnl_fd != -1)
+        sys_close(jnl->jnl_fd);
+    /* processing dir */
+    if (jnl->jnl_dir)
+        sys_closedir(jnl->jnl_dir);
+
+    if (jnl->jnl_working_dir)
+        free(jnl->jnl_working_dir); /* allocated by realpath */
 }
 
 static int
-gf_changelog_open_dirs (xlator_t *this, gf_changelog_journal_t *jnl)
+gf_changelog_open_dirs(xlator_t *this, gf_changelog_journal_t *jnl)
 {
-        int  ret                    = -1;
-        DIR *dir                    = NULL;
-        int  tracker_fd             = 0;
-        char tracker_path[PATH_MAX] = {0,};
-
-        /* .current */
-        (void) snprintf (jnl->jnl_current_dir, PATH_MAX,
-                         "%s/"GF_CHANGELOG_CURRENT_DIR"/",
-                         jnl->jnl_working_dir);
-        ret = recursive_rmdir (jnl->jnl_current_dir);
-        if (ret) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_FAILED_TO_RMDIR,
-                         "Failed to rmdir",
-                         "path=%s", jnl->jnl_current_dir,
-                         NULL);
-                goto out;
-        }
-        ret = mkdir_p (jnl->jnl_current_dir, 0600, _gf_false);
-        if (ret)
-                goto out;
+    int ret = -1;
+    DIR *dir = NULL;
+    int tracker_fd = 0;
+    char tracker_path[PATH_MAX] = {
+        0,
+    };
+
+    /* .current */
+    (void)snprintf(jnl->jnl_current_dir, PATH_MAX,
+                   "%s/" GF_CHANGELOG_CURRENT_DIR "/", jnl->jnl_working_dir);
+    ret = recursive_rmdir(jnl->jnl_current_dir);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_FAILED_TO_RMDIR, "path=%s",
+                jnl->jnl_current_dir, NULL);
+        goto out;
+    }
+    ret = mkdir_p(jnl->jnl_current_dir, 0600, _gf_false);
+    if (ret)
+        goto out;
+
+    /* .processed */
+    (void)snprintf(jnl->jnl_processed_dir, PATH_MAX,
+                   "%s/" GF_CHANGELOG_PROCESSED_DIR "/", jnl->jnl_working_dir);
+    ret = mkdir_p(jnl->jnl_processed_dir, 0600, _gf_false);
+    if (ret)
+        goto out;
+
+    /* .processing */
+    (void)snprintf(jnl->jnl_processing_dir, PATH_MAX,
+                   "%s/" GF_CHANGELOG_PROCESSING_DIR "/", jnl->jnl_working_dir);
+    ret = recursive_rmdir(jnl->jnl_processing_dir);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_FAILED_TO_RMDIR, "path=%s",
+                jnl->jnl_processing_dir, NULL);
+        goto out;
+    }
 
-        /* .processed */
-        (void) snprintf (jnl->jnl_processed_dir, PATH_MAX,
-                         "%s/"GF_CHANGELOG_PROCESSED_DIR"/",
-                         jnl->jnl_working_dir);
-        ret = mkdir_p (jnl->jnl_processed_dir, 0600, _gf_false);
-        if (ret)
-                goto out;
-
-        /* .processing */
-        (void) snprintf (jnl->jnl_processing_dir, PATH_MAX,
-                         "%s/"GF_CHANGELOG_PROCESSING_DIR"/",
-                         jnl->jnl_working_dir);
-        ret = recursive_rmdir (jnl->jnl_processing_dir);
-        if (ret) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_FAILED_TO_RMDIR,
-                         "Failed to rmdir",
-                         "path=%s", jnl->jnl_processing_dir,
-                         NULL);
-                goto out;
-        }
+    ret = mkdir_p(jnl->jnl_processing_dir, 0600, _gf_false);
+    if (ret)
+        goto out;
 
-        ret = mkdir_p (jnl->jnl_processing_dir, 0600, _gf_false);
-        if (ret)
-                goto out;
-
-        dir = sys_opendir (jnl->jnl_processing_dir);
-        if (!dir) {
-                gf_msg ("", GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_OPENDIR_ERROR,
-                        "opendir() error");
-                goto out;
-        }
+    dir = sys_opendir(jnl->jnl_processing_dir);
+    if (!dir) {
+        gf_msg("", GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_OPENDIR_ERROR,
+               "opendir() error");
+        goto out;
+    }
 
-        jnl->jnl_dir = dir;
+    jnl->jnl_dir = dir;
 
-        (void) snprintf (tracker_path, PATH_MAX,
-                         "%s/"GF_CHANGELOG_TRACKER, jnl->jnl_working_dir);
+    (void)snprintf(tracker_path, PATH_MAX, "%s/" GF_CHANGELOG_TRACKER,
+                   jnl->jnl_working_dir);
 
-        tracker_fd = open (tracker_path, O_CREAT | O_APPEND | O_RDWR,
-                           S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (tracker_fd < 0) {
-                sys_closedir (jnl->jnl_dir);
-                ret = -1;
-                goto out;
-        }
+    tracker_fd = open(tracker_path, O_CREAT | O_APPEND | O_RDWR,
+                      S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (tracker_fd < 0) {
+        sys_closedir(jnl->jnl_dir);
+        ret = -1;
+        goto out;
+    }
 
-        jnl->jnl_fd = tracker_fd;
-        ret = 0;
- out:
-        return ret;
+    jnl->jnl_fd = tracker_fd;
+    ret = 0;
+out:
+    return ret;
 }
 
 int
-gf_changelog_init_history (xlator_t *this,
-                           gf_changelog_journal_t *jnl,
-                           char *brick_path)
+gf_changelog_init_history(xlator_t *this, gf_changelog_journal_t *jnl,
+                          char *brick_path)
 {
-        int i   = 0;
-        int ret = 0;
-        char hist_scratch_dir[PATH_MAX] = {0,};
+    int i = 0;
+    int ret = 0;
+    char hist_scratch_dir[PATH_MAX] = {
+        0,
+    };
 
-        jnl->hist_jnl = GF_CALLOC (1, sizeof (*jnl),
-                         gf_changelog_mt_libgfchangelog_t);
-        if (!jnl->hist_jnl)
-                goto error_return;
+    jnl->hist_jnl = GF_CALLOC(1, sizeof(*jnl),
+                              gf_changelog_mt_libgfchangelog_t);
+    if (!jnl->hist_jnl)
+        goto error_return;
 
-        jnl->hist_jnl->jnl_dir = NULL;
-        jnl->hist_jnl->jnl_fd =  -1;
+    jnl->hist_jnl->jnl_dir = NULL;
+    jnl->hist_jnl->jnl_fd = -1;
 
-        (void) snprintf (hist_scratch_dir, PATH_MAX,
-                         "%s/"GF_CHANGELOG_HISTORY_DIR"/",
-                         jnl->jnl_working_dir);
+    (void)snprintf(hist_scratch_dir, PATH_MAX,
+                   "%s/" GF_CHANGELOG_HISTORY_DIR "/", jnl->jnl_working_dir);
 
-        ret = mkdir_p (hist_scratch_dir, 0600, _gf_false);
-        if (ret)
-                goto dealloc_hist;
-
-        jnl->hist_jnl->jnl_working_dir = realpath (hist_scratch_dir, NULL);
-        if (!jnl->hist_jnl->jnl_working_dir)
-                goto dealloc_hist;
-
-        ret = gf_changelog_open_dirs (this, jnl->hist_jnl);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_LIB_MSG_OPENDIR_ERROR,
-                        "could not create entries in history scratch dir");
-                goto dealloc_hist;
-        }
+    ret = mkdir_p(hist_scratch_dir, 0600, _gf_false);
+    if (ret)
+        goto dealloc_hist;
 
-        (void) strncpy (jnl->hist_jnl->jnl_brickpath, brick_path, PATH_MAX-1);
-        jnl->hist_jnl->jnl_brickpath[PATH_MAX-1] = 0;
+    jnl->hist_jnl->jnl_working_dir = realpath(hist_scratch_dir, NULL);
+    if (!jnl->hist_jnl->jnl_working_dir)
+        goto dealloc_hist;
 
-        for (i = 0; i < 256; i++) {
-                jnl->hist_jnl->rfc3986_space_newline[i] =
-                        (i == ' ' || i == '\n' || i == '%') ? 0 : i;
-        }
+    ret = gf_changelog_open_dirs(this, jnl->hist_jnl);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_OPENDIR_ERROR,
+               "could not create entries in history scratch dir");
+        goto dealloc_hist;
+    }
 
-        return 0;
+    if (snprintf(jnl->hist_jnl->jnl_brickpath, PATH_MAX, "%s", brick_path) >=
+        PATH_MAX)
+        goto dealloc_hist;
 
- dealloc_hist:
-        GF_FREE (jnl->hist_jnl);
-        jnl->hist_jnl = NULL;
- error_return:
-        return -1;
+    for (i = 0; i < 256; i++) {
+        jnl->hist_jnl->rfc3986_space_newline[i] = (i == ' ' || i == '\n' ||
+                                                   i == '%')
+                                                      ? 0
+                                                      : i;
+    }
+
+    return 0;
+
+dealloc_hist:
+    GF_FREE(jnl->hist_jnl);
+    jnl->hist_jnl = NULL;
+error_return:
+    return -1;
 }
 
 void
-gf_changelog_journal_fini (void *xl, char *brick, void *data)
+gf_changelog_journal_fini(void *xl, char *brick, void *data)
 {
-        gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *jnl = NULL;
 
-        jnl = data;
+    jnl = data;
 
-        gf_changelog_cleanup_processor (jnl);
+    gf_changelog_cleanup_processor(jnl);
 
-        gf_changelog_cleanup_fds (jnl);
-        if (jnl->hist_jnl)
-                gf_changelog_cleanup_fds (jnl->hist_jnl);
+    gf_changelog_cleanup_fds(jnl);
+    if (jnl->hist_jnl)
+        gf_changelog_cleanup_fds(jnl->hist_jnl);
 
-        GF_FREE (jnl);
+    GF_FREE(jnl);
 }
 
 void *
-gf_changelog_journal_init (void *xl, struct gf_brick_spec *brick)
+gf_changelog_journal_init(void *xl, struct gf_brick_spec *brick)
 {
-        int                     i           = 0;
-        int                     ret         = 0;
-        xlator_t               *this        = NULL;
-        struct stat             buf         = {0,};
-        char                   *scratch_dir = NULL;
-        gf_changelog_journal_t *jnl         = NULL;
-
-        this = xl;
-        scratch_dir = (char *) brick->ptr;
-
-        jnl = GF_CALLOC (1, sizeof (gf_changelog_journal_t),
-                         gf_changelog_mt_libgfchangelog_t);
-        if (!jnl)
-                goto error_return;
-
-        if (sys_stat (scratch_dir, &buf) && errno == ENOENT) {
-                ret = mkdir_p (scratch_dir, 0600, _gf_true);
-                if (ret)
-                        goto dealloc_private;
-        }
-
-        jnl->jnl_working_dir = realpath (scratch_dir, NULL);
-        if (!jnl->jnl_working_dir)
-                goto dealloc_private;
-
-        ret = gf_changelog_open_dirs (this, jnl);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_LIB_MSG_OPENDIR_ERROR,
-                        "could not create entries in scratch dir");
-                goto dealloc_private;
-        }
+    int i = 0;
+    int ret = 0;
+    xlator_t *this = NULL;
+    struct stat buf = {
+        0,
+    };
+    char *scratch_dir = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+
+    this = xl;
+    scratch_dir = (char *)brick->ptr;
+
+    jnl = GF_CALLOC(1, sizeof(gf_changelog_journal_t),
+                    gf_changelog_mt_libgfchangelog_t);
+    if (!jnl)
+        goto error_return;
+
+    if (snprintf(jnl->jnl_brickpath, PATH_MAX, "%s", brick->brick_path) >=
+        PATH_MAX)
+        goto dealloc_private;
+
+    if (sys_stat(scratch_dir, &buf) && errno == ENOENT) {
+        ret = mkdir_p(scratch_dir, 0600, _gf_true);
+        if (ret)
+            goto dealloc_private;
+    }
 
-        (void) strncpy (jnl->jnl_brickpath, brick->brick_path, PATH_MAX-1);
-        jnl->jnl_brickpath[PATH_MAX-1] = 0;
+    jnl->jnl_working_dir = realpath(scratch_dir, NULL);
+    if (!jnl->jnl_working_dir)
+        goto dealloc_private;
 
-        /* RFC 3986 {de,en}coding */
-        for (i = 0; i < 256; i++) {
-                jnl->rfc3986_space_newline[i] =
-                        (i == ' ' || i == '\n' || i == '%') ? 0 : i;
-        }
+    ret = gf_changelog_open_dirs(this, jnl);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_OPENDIR_ERROR,
+               "could not create entries in scratch dir");
+        goto dealloc_private;
+    }
 
-        ret = gf_changelog_init_history (this, jnl, brick->brick_path);
-        if (ret)
-                goto cleanup_fds;
+    /* RFC 3986 {de,en}coding */
+    for (i = 0; i < 256; i++) {
+        jnl->rfc3986_space_newline[i] = (i == ' ' || i == '\n' || i == '%') ? 0
+                                                                            : i;
+    }
 
-        /* initialize journal processor */
-        jnl->this = this;
-        ret = gf_changelog_init_processor (jnl);
-        if (ret)
-                goto cleanup_fds;
-
-        JNL_SET_API_STATE (jnl, JNL_API_CONN_INPROGESS);
-        ret = pthread_spin_init (&jnl->lock, 0);
-        if (ret != 0)
-                goto cleanup_processor;
-        return jnl;
-
- cleanup_processor:
-        gf_changelog_cleanup_processor (jnl);
- cleanup_fds:
-        gf_changelog_cleanup_fds (jnl);
-        if (jnl->hist_jnl)
-                gf_changelog_cleanup_fds (jnl->hist_jnl);
- dealloc_private:
-        GF_FREE (jnl);
- error_return:
-        return NULL;
+    ret = gf_changelog_init_history(this, jnl, brick->brick_path);
+    if (ret)
+        goto cleanup_fds;
+
+    /* initialize journal processor */
+    jnl->this = this;
+    ret = gf_changelog_init_processor(jnl);
+    if (ret)
+        goto cleanup_fds;
+
+    JNL_SET_API_STATE(jnl, JNL_API_CONN_INPROGESS);
+    ret = pthread_spin_init(&jnl->lock, 0);
+    if (ret != 0)
+        goto cleanup_processor;
+    return jnl;
+
+cleanup_processor:
+    gf_changelog_cleanup_processor(jnl);
+cleanup_fds:
+    gf_changelog_cleanup_fds(jnl);
+    if (jnl->hist_jnl)
+        gf_changelog_cleanup_fds(jnl->hist_jnl);
+dealloc_private:
+    GF_FREE(jnl);
+error_return:
+    return NULL;
 }
diff --git a/xlators/features/changelog/lib/src/gf-changelog-journal.h b/xlators/features/changelog/lib/src/gf-changelog-journal.h
index 46d50f159d9..ba5b9bf827e 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-journal.h
+++ b/xlators/features/changelog/lib/src/gf-changelog-journal.h
@@ -17,91 +17,91 @@
 #include "changelog.h"
 
 enum api_conn {
-        JNL_API_CONNECTED,
-        JNL_API_CONN_INPROGESS,
-        JNL_API_DISCONNECTED,
+    JNL_API_CONNECTED,
+    JNL_API_CONN_INPROGESS,
+    JNL_API_DISCONNECTED,
 };
 
 typedef struct gf_changelog_entry {
-        char path[PATH_MAX];
+    char path[PATH_MAX];
 
-        struct list_head list;
+    struct list_head list;
 } gf_changelog_entry_t;
 
 typedef struct gf_changelog_processor {
-        pthread_mutex_t  lock;     /* protects ->entries */
-        pthread_cond_t   cond;     /* waiter during empty list */
-        gf_boolean_t     waiting;
+    pthread_mutex_t lock; /* protects ->entries */
+    pthread_cond_t cond;  /* waiter during empty list */
+    gf_boolean_t waiting;
 
-        pthread_t processor;       /* thread-id of journal processing thread */
+    pthread_t processor; /* thread-id of journal processing thread */
 
-        struct list_head entries;
+    struct list_head entries;
 } gf_changelog_processor_t;
 
 typedef struct gf_changelog_journal {
-        DIR *jnl_dir;                       /* 'processing' directory stream */
+    DIR *jnl_dir; /* 'processing' directory stream */
 
-        int jnl_fd;                         /* fd to the tracker file */
+    int jnl_fd; /* fd to the tracker file */
 
-        char jnl_brickpath[PATH_MAX];       /* brick path for this end-point */
+    char jnl_brickpath[PATH_MAX]; /* brick path for this end-point */
 
-        gf_changelog_processor_t *jnl_proc;
+    gf_changelog_processor_t *jnl_proc;
 
-        char *jnl_working_dir;              /* scratch directory */
+    char *jnl_working_dir; /* scratch directory */
 
-        char jnl_current_dir[PATH_MAX];
-        char jnl_processed_dir[PATH_MAX];
-        char jnl_processing_dir[PATH_MAX];
+    char jnl_current_dir[PATH_MAX];
+    char jnl_processed_dir[PATH_MAX];
+    char jnl_processing_dir[PATH_MAX];
 
-        char rfc3986_space_newline[256];    /* RFC 3986 string encoding */
+    char rfc3986_space_newline[256]; /* RFC 3986 string encoding */
 
-        struct gf_changelog_journal *hist_jnl;
-        int hist_done;                      /* holds 0 done scanning,
-                                               1 keep scanning and -1 error */
+    struct gf_changelog_journal *hist_jnl;
+    int hist_done; /* holds 0 done scanning,
+                      1 keep scanning and -1 error */
 
-        pthread_spinlock_t lock;
-        int connected;
-        xlator_t *this;
+    pthread_spinlock_t lock;
+    int connected;
+    xlator_t *this;
 } gf_changelog_journal_t;
 
-#define JNL_SET_API_STATE(jnl, state)  (jnl->connected = state)
-#define JNL_IS_API_DISCONNECTED(jnl)  (jnl->connected == JNL_API_DISCONNECTED)
+#define JNL_SET_API_STATE(jnl, state) (jnl->connected = state)
+#define JNL_IS_API_DISCONNECTED(jnl) (jnl->connected == JNL_API_DISCONNECTED)
 
 /* History API */
 typedef struct gf_changelog_history_data {
-        int           len;
+    int len;
 
-        int           htime_fd;
+    int htime_fd;
 
-        /* parallelism count */
-        int           n_parallel;
+    /* parallelism count */
+    int n_parallel;
 
-        /* history from, to indexes */
-        unsigned long from;
-        unsigned long to;
-        xlator_t      *this;
+    /* history from, to indexes */
+    unsigned long from;
+    unsigned long to;
+    xlator_t *this;
 } gf_changelog_history_data_t;
 
 typedef struct gf_changelog_consume_data {
-        /** set of inputs */
+    /** set of inputs */
 
-        /* fd to read from */
-        int             fd;
+    /* fd to read from */
+    int fd;
 
-        /* from @offset */
-        off_t           offset;
+    /* from @offset */
+    off_t offset;
 
-        xlator_t       *this;
+    xlator_t *this;
 
-        gf_changelog_journal_t *jnl;
+    gf_changelog_journal_t *jnl;
 
-        /** set of outputs */
+    /** set of outputs */
 
-        /* return value */
-        int retval;
+    /* return value */
+    int retval;
 
-        /* journal processed */
-        char changelog[PATH_MAX];
+    /* journal processed */
+    char changelog[PATH_MAX];
 } gf_changelog_consume_data_t;
 
 /* event handler */
diff --git a/xlators/features/changelog/lib/src/gf-changelog-reborp.c b/xlators/features/changelog/lib/src/gf-changelog-reborp.c
index d1b3059a60c..56b11cbb705 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-reborp.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-reborp.c
@@ -15,147 +15,130 @@
 #include "changelog-rpc-common.h"
 #include "changelog-lib-messages.h"
 
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 
 /**
  * Reverse socket: actual data transfer handler. Connection
  * initiator is PROBER, data transfer is REBORP.
  */
 
-struct rpcsvc_program *gf_changelog_reborp_programs[];
+static struct rpcsvc_program *gf_changelog_reborp_programs[];
 
 void *
-gf_changelog_connection_janitor (void *arg)
+gf_changelog_connection_janitor(void *arg)
 {
-        int32_t ret = 0;
-        xlator_t *this = NULL;
-        gf_private_t *priv = NULL;
-        gf_changelog_t *entry = NULL;
-        struct gf_event *event = NULL;
-        struct gf_event_list *ev = NULL;
-        unsigned long drained = 0;
-
-        this = arg;
-        THIS = this;
-
-        priv = this->private;
-
-        while (1) {
-                pthread_mutex_lock (&priv->lock);
-                {
-                        while (list_empty (&priv->cleanups))
-                                pthread_cond_wait (&priv->cond, &priv->lock);
-
-                        entry = list_first_entry (&priv->cleanups,
-                                                  gf_changelog_t, list);
-                        list_del_init (&entry->list);
-                }
-                pthread_mutex_unlock (&priv->lock);
-
-                drained = 0;
-                ev = &entry->event;
-
-                gf_smsg (this->name, GF_LOG_INFO, 0,
-                         CHANGELOG_LIB_MSG_CLEANING_BRICK_ENTRY_INFO,
-                         "Cleaning brick entry for brick",
-                         "brick=%s", entry->brick,
-                         NULL);
-
-                /* 0x0: disbale rpc-clnt */
-                rpc_clnt_disable (RPC_PROBER (entry));
-
-                /* 0x1: cleanup callback invoker thread */
-                ret = gf_cleanup_event (this, ev);
-                if (ret)
-                        continue;
-
-                /* 0x2: drain pending events */
-                while (!list_empty (&ev->events)) {
-                        event = list_first_entry (&ev->events,
-                                                  struct gf_event, list);
-                        gf_smsg (this->name, GF_LOG_INFO, 0,
-                                 CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO,
-                                 "Draining event",
-                                 "seq=%lu", event->seq,
-                                 "payload=%d", event->count,
-                                 NULL);
-
-                        GF_FREE (event);
-                        drained++;
-                }
-
-                gf_smsg (this->name, GF_LOG_INFO, 0,
-                         CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO,
-                         "Drained events",
-                         "num=%lu", drained,
-                         NULL);
-
-                /* 0x3: freeup brick entry */
-                gf_smsg (this->name, GF_LOG_INFO, 0,
-                         CHANGELOG_LIB_MSG_FREEING_ENTRY_INFO,
-                         "freeing entry",
-                         "entry=%p", entry,
-                         NULL);
-                LOCK_DESTROY (&entry->statelock);
-                GF_FREE (entry);
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    gf_private_t *priv = NULL;
+    gf_changelog_t *entry = NULL;
+    struct gf_event *event = NULL;
+    struct gf_event_list *ev = NULL;
+    unsigned long drained = 0;
+
+    this = arg;
+    THIS = this;
+
+    priv = this->private;
+
+    while (1) {
+        pthread_mutex_lock(&priv->lock);
+        {
+            while (list_empty(&priv->cleanups))
+                pthread_cond_wait(&priv->cond, &priv->lock);
+
+            entry = list_first_entry(&priv->cleanups, gf_changelog_t, list);
+            list_del_init(&entry->list);
+        }
+        pthread_mutex_unlock(&priv->lock);
+
+        drained = 0;
+        ev = &entry->event;
+
+        gf_smsg(this->name, GF_LOG_INFO, 0,
+                CHANGELOG_LIB_MSG_CLEANING_BRICK_ENTRY_INFO, "brick=%s",
+                entry->brick, NULL);
+
+        /* 0x0: disable rpc-clnt */
+        rpc_clnt_disable(RPC_PROBER(entry));
+
+        /* 0x1: cleanup callback invoker thread */
+        ret = gf_cleanup_event(this, ev);
+        if (ret)
+            continue;
+
+        /* 0x2: drain pending events */
+        while (!list_empty(&ev->events)) {
+            event = list_first_entry(&ev->events, struct gf_event, list);
+            gf_smsg(this->name, GF_LOG_INFO, 0,
+                    CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO, "seq=%lu",
+                    event->seq, "payload=%d", event->count, NULL);
+
+            GF_FREE(event);
+            drained++;
         }
 
-        return NULL;
+        gf_smsg(this->name, GF_LOG_INFO, 0,
+                CHANGELOG_LIB_MSG_DRAINED_EVENT_INFO, "num=%lu", drained, NULL);
+
+        /* 0x3: freeup brick entry */
+        gf_smsg(this->name, GF_LOG_INFO, 0,
+                CHANGELOG_LIB_MSG_FREEING_ENTRY_INFO, "entry=%p", entry, NULL);
+        LOCK_DESTROY(&entry->statelock);
+        GF_FREE(entry);
+    }
+
+    return NULL;
 }
 
 int
-gf_changelog_reborp_rpcsvc_notify (rpcsvc_t *rpc, void *mydata,
-                                   rpcsvc_event_t event, void *data)
+gf_changelog_reborp_rpcsvc_notify(rpcsvc_t *rpc, void *mydata,
+                                  rpcsvc_event_t event, void *data)
 {
-        int             ret      = 0;
-        xlator_t       *this     = NULL;
-        gf_changelog_t *entry    = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+    gf_changelog_t *entry = NULL;
 
-        if (!(event == RPCSVC_EVENT_ACCEPT ||
-              event == RPCSVC_EVENT_DISCONNECT))
-                return 0;
+    if (!(event == RPCSVC_EVENT_ACCEPT || event == RPCSVC_EVENT_DISCONNECT))
+        return 0;
 
-        entry = mydata;
-        this = entry->this;
+    entry = mydata;
+    this = entry->this;
 
-        switch (event) {
+    switch (event) {
         case RPCSVC_EVENT_ACCEPT:
-                ret = sys_unlink (RPC_SOCK(entry));
-                if (ret != 0)
-                        gf_smsg (this->name, GF_LOG_WARNING, errno,
-                                 CHANGELOG_LIB_MSG_UNLINK_FAILED,
-                                 "failed to unlink "
-                                 "reverse socket",
-                                 "path=%s", RPC_SOCK (entry),
-                                 NULL);
-                if (entry->connected)
-                        GF_CHANGELOG_INVOKE_CBK (this, entry->connected,
-                                                 entry->brick, entry->ptr);
-                break;
+            ret = sys_unlink(RPC_SOCK(entry));
+            if (ret != 0)
+                gf_smsg(this->name, GF_LOG_WARNING, errno,
+                        CHANGELOG_LIB_MSG_UNLINK_FAILED, "name=reverse socket",
+                        "path=%s", RPC_SOCK(entry), NULL);
+            if (entry->connected)
+                GF_CHANGELOG_INVOKE_CBK(this, entry->connected, entry->brick,
+                                        entry->ptr);
+            break;
         case RPCSVC_EVENT_DISCONNECT:
-                if (entry->disconnected)
-                        GF_CHANGELOG_INVOKE_CBK (this, entry->disconnected,
-                                                 entry->brick, entry->ptr);
-                /* passthrough */
+            if (entry->disconnected)
+                GF_CHANGELOG_INVOKE_CBK(this, entry->disconnected, entry->brick,
+                                        entry->ptr);
+            /* passthrough */
         default:
-                break;
-        }
+            break;
+    }
 
-        return 0;
+    return 0;
 }
 
 rpcsvc_t *
-gf_changelog_reborp_init_rpc_listner (xlator_t *this,
-                                      char *path, char *sock, void *cbkdata)
+gf_changelog_reborp_init_rpc_listner(xlator_t *this, char *path, char *sock,
+                                     void *cbkdata)
 {
-        CHANGELOG_MAKE_TMP_SOCKET_PATH (path, sock, UNIX_PATH_MAX);
-        return changelog_rpc_server_init (this, sock, cbkdata,
-                                          gf_changelog_reborp_rpcsvc_notify,
-                                          gf_changelog_reborp_programs);
+    CHANGELOG_MAKE_TMP_SOCKET_PATH(path, sock, UNIX_PATH_MAX);
+    return changelog_rpc_server_init(this, sock, cbkdata,
+                                     gf_changelog_reborp_rpcsvc_notify,
+                                     gf_changelog_reborp_programs);
 }
 
 /**
- * This is dirty and painful as of now untill there is event filtering in the
+ * This is dirty and painful as of now until there is event filtering in the
  * server. The entire event buffer is scanned and interested events are picked,
  * whereas we _should_ be notified with the events we were interested in
  * (selected at the time of probe). As of now this is complete BS and needs
@@ -164,29 +147,27 @@ gf_changelog_reborp_init_rpc_listner (xlator_t *this,
  * @FIXME: cleanup this bugger once server filters events.
  */
 void
-gf_changelog_invoke_callback (gf_changelog_t *entry,
-                              struct iovec **vec, int payloadcnt)
+gf_changelog_invoke_callback(gf_changelog_t *entry, struct iovec **vec,
+                             int payloadcnt)
 {
-        int i = 0;
-        int evsize = 0;
-        xlator_t *this = NULL;
-        changelog_event_t *event = NULL;
-
-        this = entry->this;
-
-        for (; i < payloadcnt; i++) {
-                event = (changelog_event_t *)vec[i]->iov_base;
-                evsize = vec[i]->iov_len / CHANGELOG_EV_SIZE;
-
-                for (; evsize > 0; evsize--, event++) {
-                        if (gf_changelog_filter_check (entry, event)) {
-                                GF_CHANGELOG_INVOKE_CBK (this,
-                                                         entry->callback,
-                                                         entry->brick,
-                                                         entry->ptr, event);
-                        }
-                }
+    int i = 0;
+    int evsize = 0;
+    xlator_t *this = NULL;
+    changelog_event_t *event = NULL;
+
+    this = entry->this;
+
+    for (; i < payloadcnt; i++) {
+        event = (changelog_event_t *)vec[i]->iov_base;
+        evsize = vec[i]->iov_len / CHANGELOG_EV_SIZE;
+
+        for (; evsize > 0; evsize--, event++) {
+            if (gf_changelog_filter_check(entry, event)) {
+                GF_CHANGELOG_INVOKE_CBK(this, entry->callback, entry->brick,
+                                        entry->ptr, event);
+            }
         }
+    }
 }
 
 /**
@@ -197,218 +178,218 @@ gf_changelog_invoke_callback (gf_changelog_t *entry,
  */
 
 int
-__is_expected_sequence (struct gf_event_list *ev, struct gf_event *event)
+__is_expected_sequence(struct gf_event_list *ev, struct gf_event *event)
 {
-        return (ev->next_seq == event->seq);
+    return (ev->next_seq == event->seq);
 }
 
 int
-__can_process_event (struct gf_event_list *ev, struct gf_event **event)
+__can_process_event(struct gf_event_list *ev, struct gf_event **event)
 {
-        *event = list_first_entry (&ev->events, struct gf_event, list);
+    *event = list_first_entry(&ev->events, struct gf_event, list);
 
-        if (__is_expected_sequence (ev, *event)) {
-                list_del (&(*event)->list);
-                ev->next_seq++;
-                return 1;
-        }
+    if (__is_expected_sequence(ev, *event)) {
+        list_del(&(*event)->list);
+        ev->next_seq++;
+        return 1;
+    }
 
-        return 0;
+    return 0;
 }
 
 void
-pick_event_ordered (struct gf_event_list *ev, struct gf_event **event)
+pick_event_ordered(struct gf_event_list *ev, struct gf_event **event)
 {
-        pthread_mutex_lock (&ev->lock);
-        {
-                while (list_empty (&ev->events)
-                       || !__can_process_event (ev, event))
-                        pthread_cond_wait (&ev->cond, &ev->lock);
-        }
-        pthread_mutex_unlock (&ev->lock);
+    pthread_mutex_lock(&ev->lock);
+    {
+        while (list_empty(&ev->events) || !__can_process_event(ev, event))
+            pthread_cond_wait(&ev->cond, &ev->lock);
+    }
+    pthread_mutex_unlock(&ev->lock);
 }
 
 void
-pick_event_unordered (struct gf_event_list *ev, struct gf_event **event)
+pick_event_unordered(struct gf_event_list *ev, struct gf_event **event)
 {
-        pthread_mutex_lock (&ev->lock);
-        {
-                while (list_empty (&ev->events))
-                        pthread_cond_wait (&ev->cond, &ev->lock);
-                *event = list_first_entry (&ev->events, struct gf_event, list);
-                list_del (&(*event)->list);
-        }
-        pthread_mutex_unlock (&ev->lock);
+    pthread_mutex_lock(&ev->lock);
+    {
+        while (list_empty(&ev->events))
+            pthread_cond_wait(&ev->cond, &ev->lock);
+        *event = list_first_entry(&ev->events, struct gf_event, list);
+        list_del(&(*event)->list);
+    }
+    pthread_mutex_unlock(&ev->lock);
 }
 
 void *
-gf_changelog_callback_invoker (void *arg)
+gf_changelog_callback_invoker(void *arg)
 {
-        xlator_t             *this   = NULL;
-        gf_changelog_t       *entry  = NULL;
-        struct iovec         *vec    = NULL;
-        struct gf_event      *event  = NULL;
-        struct gf_event_list *ev     = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_t *entry = NULL;
+    struct iovec *vec = NULL;
+    struct gf_event *event = NULL;
+    struct gf_event_list *ev = NULL;
 
-        ev    = arg;
-        entry = ev->entry;
-        THIS = this = entry->this;
+    ev = arg;
+    entry = ev->entry;
+    THIS = this = entry->this;
 
-        while (1) {
-                entry->pickevent (ev, &event);
+    while (1) {
+        entry->pickevent(ev, &event);
 
-                vec = (struct iovec *) &event->iov;
-                gf_changelog_invoke_callback (entry, &vec, event->count);
+        vec = (struct iovec *)&event->iov;
+        gf_changelog_invoke_callback(entry, &vec, event->count);
 
-                GF_FREE (event);
-        }
+        GF_FREE(event);
+    }
 
-        return NULL;
+    return NULL;
 }
 
 static int
-orderfn (struct list_head *pos1, struct list_head *pos2)
+orderfn(struct list_head *pos1, struct list_head *pos2)
 {
-        struct gf_event *event1 = NULL;
-        struct gf_event *event2 = NULL;
+    struct gf_event *event1 = NULL;
+    struct gf_event *event2 = NULL;
 
-        event1 = list_entry (pos1, struct gf_event, list);
-        event2 = list_entry (pos2, struct gf_event, list);
+    event1 = list_entry(pos1, struct gf_event, list);
+    event2 = list_entry(pos2, struct gf_event, list);
 
-        if  (event1->seq > event2->seq)
-                return 1;
-        return -1;
+    if (event1->seq > event2->seq)
+        return 1;
+    return -1;
 }
 
 void
-queue_ordered_event (struct gf_event_list *ev, struct gf_event *event)
+queue_ordered_event(struct gf_event_list *ev, struct gf_event *event)
 {
-        /* add event to the ordered event list and wake up listner(s) */
-        pthread_mutex_lock (&ev->lock);
-        {
-                list_add_order (&event->list, &ev->events, orderfn);
-                if (!ev->next_seq)
-                        ev->next_seq = event->seq;
-                if (ev->next_seq == event->seq)
-                        pthread_cond_signal (&ev->cond);
-        }
-        pthread_mutex_unlock (&ev->lock);
+    /* add event to the ordered event list and wake up listener(s) */
+    pthread_mutex_lock(&ev->lock);
+    {
+        list_add_order(&event->list, &ev->events, orderfn);
+        if (!ev->next_seq)
+            ev->next_seq = event->seq;
+        if (ev->next_seq == event->seq)
+            pthread_cond_signal(&ev->cond);
+    }
+    pthread_mutex_unlock(&ev->lock);
 }
 
 void
-queue_unordered_event (struct gf_event_list *ev, struct gf_event *event)
+queue_unordered_event(struct gf_event_list *ev, struct gf_event *event)
 {
-        /* add event to the tail of the queue and wake up listener(s) */
-        pthread_mutex_lock (&ev->lock);
-        {
-                list_add_tail (&event->list, &ev->events);
-                pthread_cond_signal (&ev->cond);
-        }
-        pthread_mutex_unlock (&ev->lock);
+    /* add event to the tail of the queue and wake up listener(s) */
+    pthread_mutex_lock(&ev->lock);
+    {
+        list_add_tail(&event->list, &ev->events);
+        pthread_cond_signal(&ev->cond);
+    }
+    pthread_mutex_unlock(&ev->lock);
 }
 
 int
-gf_changelog_event_handler (rpcsvc_request_t *req,
-                            xlator_t *this, gf_changelog_t *entry)
+gf_changelog_event_handler(rpcsvc_request_t *req, xlator_t *this,
+                           gf_changelog_t *entry)
 {
-        int                   i          = 0;
-        size_t                payloadlen = 0;
-        ssize_t               len        = 0;
-        int                   payloadcnt = 0;
-        changelog_event_req   rpc_req    = {0,};
-        changelog_event_rsp   rpc_rsp    = {0,};
-        struct iovec         *vec        = NULL;
-        struct gf_event      *event      = NULL;
-        struct gf_event_list *ev         = NULL;
-
-        ev = &entry->event;
-
-        len = xdr_to_generic (req->msg[0],
-                              &rpc_req, (xdrproc_t)xdr_changelog_event_req);
-        if (len < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_LIB_MSG_XDR_DECODING_FAILED,
-                        "xdr decoding failed");
-                req->rpc_err = GARBAGE_ARGS;
-                goto handle_xdr_error;
-        }
-
-        if (len < req->msg[0].iov_len) {
-                payloadcnt = 1;
-                payloadlen = (req->msg[0].iov_len - len);
-        }
-        for (i = 1; i < req->count; i++) {
-                payloadcnt++;
-                payloadlen += req->msg[i].iov_len;
-        }
-
-        event = GF_CALLOC (1, GF_EVENT_CALLOC_SIZE (payloadcnt, payloadlen),
-                           gf_changelog_mt_libgfchangelog_event_t);
-        if (!event)
-                goto handle_xdr_error;
-        INIT_LIST_HEAD (&event->list);
-
-        payloadlen   = 0;
-        event->seq   = rpc_req.seq;
-        event->count = payloadcnt;
-
-        /* deep copy IO vectors */
-        vec = &event->iov[0];
-        GF_EVENT_ASSIGN_IOVEC (vec, event,
-                               (req->msg[0].iov_len - len), payloadlen);
-        (void) memcpy (vec->iov_base,
-                       req->msg[0].iov_base + len, vec->iov_len);
-
-        for (i = 1; i < req->count; i++) {
-                vec = &event->iov[i];
-                GF_EVENT_ASSIGN_IOVEC (vec, event,
-                                       req->msg[i].iov_len, payloadlen);
-                (void) memcpy (event->iov[i].iov_base,
-                               req->msg[i].iov_base, req->msg[i].iov_len);
-        }
-
-        gf_msg_debug (this->name, 0,
-                      "seq: %lu [%s] (time: %lu.%lu), (vec: %d, len: %zd)",
-                      rpc_req.seq, entry->brick, rpc_req.tv_sec,
-                      rpc_req.tv_usec, payloadcnt, payloadlen);
-
-        /* dispatch event */
-        entry->queueevent (ev, event);
-
-        /* ack sequence number */
-        rpc_rsp.op_ret = 0;
-        rpc_rsp.seq    = rpc_req.seq;
-
-        goto submit_rpc;
-
- handle_xdr_error:
-        rpc_rsp.op_ret = -1;
-        rpc_rsp.seq    = 0;     /* invalid */
- submit_rpc:
-        return changelog_rpc_sumbit_reply (req, &rpc_rsp, NULL, 0, NULL,
-                                           (xdrproc_t)xdr_changelog_event_rsp);
+    int i = 0;
+    size_t payloadlen = 0;
+    ssize_t len = 0;
+    int payloadcnt = 0;
+    changelog_event_req rpc_req = {
+        0,
+    };
+    changelog_event_rsp rpc_rsp = {
+        0,
+    };
+    struct iovec *vec = NULL;
+    struct gf_event *event = NULL;
+    struct gf_event_list *ev = NULL;
+
+    ev = &entry->event;
+
+    len = xdr_to_generic(req->msg[0], &rpc_req,
+                         (xdrproc_t)xdr_changelog_event_req);
+    if (len < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               CHANGELOG_LIB_MSG_XDR_DECODING_FAILED, "xdr decoding failed");
+        req->rpc_err = GARBAGE_ARGS;
+        goto handle_xdr_error;
+    }
+
+    if (len < req->msg[0].iov_len) {
+        payloadcnt = 1;
+        payloadlen = (req->msg[0].iov_len - len);
+    }
+    for (i = 1; i < req->count; i++) {
+        payloadcnt++;
+        payloadlen += req->msg[i].iov_len;
+    }
+
+    event = GF_CALLOC(1, GF_EVENT_CALLOC_SIZE(payloadcnt, payloadlen),
+                      gf_changelog_mt_libgfchangelog_event_t);
+    if (!event)
+        goto handle_xdr_error;
+    INIT_LIST_HEAD(&event->list);
+
+    payloadlen = 0;
+    event->seq = rpc_req.seq;
+    event->count = payloadcnt;
+
+    /* deep copy IO vectors */
+    vec = &event->iov[0];
+    GF_EVENT_ASSIGN_IOVEC(vec, event, (req->msg[0].iov_len - len), payloadlen);
+    (void)memcpy(vec->iov_base, req->msg[0].iov_base + len, vec->iov_len);
+
+    for (i = 1; i < req->count; i++) {
+        vec = &event->iov[i];
+        GF_EVENT_ASSIGN_IOVEC(vec, event, req->msg[i].iov_len, payloadlen);
+        (void)memcpy(event->iov[i].iov_base, req->msg[i].iov_base,
+                     req->msg[i].iov_len);
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "seq: %" PRIu64 " [%s] (time: %" PRIu64 ".%" PRIu64
+                 "), "
+                 "(vec: %d, len: %zd)",
+                 rpc_req.seq, entry->brick, rpc_req.tv_sec, rpc_req.tv_usec,
+                 payloadcnt, payloadlen);
+
+    /* dispatch event */
+    entry->queueevent(ev, event);
+
+    /* ack sequence number */
+    rpc_rsp.op_ret = 0;
+    rpc_rsp.seq = rpc_req.seq;
+
+    goto submit_rpc;
+
+handle_xdr_error:
+    rpc_rsp.op_ret = -1;
+    rpc_rsp.seq = 0; /* invalid */
+submit_rpc:
+    return changelog_rpc_sumbit_reply(req, &rpc_rsp, NULL, 0, NULL,
+                                      (xdrproc_t)xdr_changelog_event_rsp);
 }
 
 int
-gf_changelog_reborp_handle_event (rpcsvc_request_t *req)
+gf_changelog_reborp_handle_event(rpcsvc_request_t *req)
 {
-        xlator_t       *this  = NULL;
-        rpcsvc_t       *svc   = NULL;
-        gf_changelog_t *entry = NULL;
+    xlator_t *this = NULL;
+    rpcsvc_t *svc = NULL;
+    gf_changelog_t *entry = NULL;
 
-        svc = rpcsvc_request_service (req);
-        entry = svc->mydata;
+    svc = rpcsvc_request_service(req);
+    entry = svc->mydata;
 
-        this = THIS = entry->this;
+    this = THIS = entry->this;
 
-        return gf_changelog_event_handler (req, this, entry);
+    return gf_changelog_event_handler(req, this, entry);
 }
 
-rpcsvc_actor_t gf_changelog_reborp_actors[CHANGELOG_REV_PROC_MAX] = {
-        [CHANGELOG_REV_PROC_EVENT] = {
-                "CHANGELOG EVENT HANDLER", CHANGELOG_REV_PROC_EVENT,
-                gf_changelog_reborp_handle_event, NULL, 0, DRC_NA
-        },
+static rpcsvc_actor_t gf_changelog_reborp_actors[CHANGELOG_REV_PROC_MAX] = {
+    [CHANGELOG_REV_PROC_EVENT] = {"CHANGELOG EVENT HANDLER",
+                                  gf_changelog_reborp_handle_event, NULL,
+                                  CHANGELOG_REV_PROC_EVENT, DRC_NA, 0},
 };
 
 /**
@@ -417,16 +398,16 @@ rpcsvc_actor_t gf_changelog_reborp_actors[CHANGELOG_REV_PROC_MAX] = {
  * and that's required to invoke the callback with the appropriate
  * brick path and it's private data.
  */
-struct rpcsvc_program gf_changelog_reborp_prog = {
-        .progname  = "LIBGFCHANGELOG REBORP",
-        .prognum   = CHANGELOG_REV_RPC_PROCNUM,
-        .progver   = CHANGELOG_REV_RPC_PROCVER,
-        .numactors = CHANGELOG_REV_PROC_MAX,
-        .actors    = gf_changelog_reborp_actors,
-        .synctask  = _gf_false,
+static struct rpcsvc_program gf_changelog_reborp_prog = {
+    .progname = "LIBGFCHANGELOG REBORP",
+    .prognum = CHANGELOG_REV_RPC_PROCNUM,
+    .progver = CHANGELOG_REV_RPC_PROCVER,
+    .numactors = CHANGELOG_REV_PROC_MAX,
+    .actors = gf_changelog_reborp_actors,
+    .synctask = _gf_false,
 };
 
-struct rpcsvc_program *gf_changelog_reborp_programs[] = {
-        &gf_changelog_reborp_prog,
-        NULL,
+static struct rpcsvc_program *gf_changelog_reborp_programs[] = {
+    &gf_changelog_reborp_prog,
+    NULL,
 };
diff --git a/xlators/features/changelog/lib/src/gf-changelog-rpc.c b/xlators/features/changelog/lib/src/gf-changelog-rpc.c
index c1139423d6d..8ec6ffbcebc 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-rpc.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-rpc.c
@@ -16,32 +16,32 @@ struct rpc_clnt_program gf_changelog_clnt;
 
 /* TODO: piggyback reconnect to called (upcall) */
 int
-gf_changelog_rpc_notify (struct rpc_clnt *rpc,
-                         void *mydata, rpc_clnt_event_t event, void *data)
+gf_changelog_rpc_notify(struct rpc_clnt *rpc, void *mydata,
+                        rpc_clnt_event_t event, void *data)
 {
-        switch (event) {
+    switch (event) {
         case RPC_CLNT_CONNECT:
-                rpc_clnt_set_connected (&rpc->conn);
-                break;
+            break;
         case RPC_CLNT_DISCONNECT:
         case RPC_CLNT_MSG:
         case RPC_CLNT_DESTROY:
         case RPC_CLNT_PING:
-                break;
-        }
+            break;
+    }
 
-        return 0;
+    return 0;
 }
 
 struct rpc_clnt *
-gf_changelog_rpc_init (xlator_t *this, gf_changelog_t *entry)
+gf_changelog_rpc_init(xlator_t *this, gf_changelog_t *entry)
 {
-        char sockfile[UNIX_PATH_MAX] = {0,};
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
 
-        CHANGELOG_MAKE_SOCKET_PATH (entry->brick,
-                                    sockfile, UNIX_PATH_MAX);
-        return changelog_rpc_client_init (this, entry,
-                                          sockfile, gf_changelog_rpc_notify);
+    CHANGELOG_MAKE_SOCKET_PATH(entry->brick, sockfile, UNIX_PATH_MAX);
+    return changelog_rpc_client_init(this, entry, sockfile,
+                                     gf_changelog_rpc_notify);
 }
 
 /**
@@ -49,51 +49,50 @@ gf_changelog_rpc_init (xlator_t *this, gf_changelog_t *entry)
  */
 
 int
-gf_probe_changelog_cbk (struct rpc_req *req,
-                        struct iovec *iovec, int count, void *myframe)
+gf_probe_changelog_cbk(struct rpc_req *req, struct iovec *iovec, int count,
+                       void *myframe)
 {
-        return 0;
+    return 0;
 }
 
 int
-gf_probe_changelog_filter (call_frame_t *frame, xlator_t *this, void *data)
+gf_probe_changelog_filter(call_frame_t *frame, xlator_t *this, void *data)
 {
-        char *sock = NULL;
-        gf_changelog_t *entry = NULL;
-        changelog_probe_req req = {0,};
-
-        entry = data;
-        sock = RPC_SOCK (entry);
-
-        (void) memcpy (&req.sock, sock, strlen (sock));
-        req.filter = entry->notify;
-
-        /* invoke RPC */
-        return changelog_rpc_sumbit_req (RPC_PROBER (entry), (void *) &req,
-                                         frame, &gf_changelog_clnt,
-                                         CHANGELOG_RPC_PROBE_FILTER, NULL, 0,
-                                         NULL, this, gf_probe_changelog_cbk,
-                                         (xdrproc_t) xdr_changelog_probe_req);
+    char *sock = NULL;
+    gf_changelog_t *entry = NULL;
+    changelog_probe_req req = {
+        0,
+    };
+
+    entry = data;
+    sock = RPC_SOCK(entry);
+
+    (void)memcpy(&req.sock, sock, strlen(sock));
+    req.filter = entry->notify;
+
+    /* invoke RPC */
+    return changelog_rpc_sumbit_req(
+        RPC_PROBER(entry), (void *)&req, frame, &gf_changelog_clnt,
+        CHANGELOG_RPC_PROBE_FILTER, NULL, 0, NULL, this, gf_probe_changelog_cbk,
+        (xdrproc_t)xdr_changelog_probe_req);
 }
 
 int
-gf_changelog_invoke_rpc (xlator_t *this, gf_changelog_t *entry, int procidx)
+gf_changelog_invoke_rpc(xlator_t *this, gf_changelog_t *entry, int procidx)
 {
-        return changelog_invoke_rpc (this, RPC_PROBER (entry),
-                                     &gf_changelog_clnt, procidx, entry);
+    return changelog_invoke_rpc(this, RPC_PROBER(entry), &gf_changelog_clnt,
+                                procidx, entry);
 }
 
 struct rpc_clnt_procedure gf_changelog_procs[CHANGELOG_RPC_PROC_MAX] = {
-        [CHANGELOG_RPC_PROC_NULL] = {"NULL", NULL},
-        [CHANGELOG_RPC_PROBE_FILTER] = {
-                "PROBE FILTER", gf_probe_changelog_filter
-        },
+    [CHANGELOG_RPC_PROC_NULL] = {"NULL", NULL},
+    [CHANGELOG_RPC_PROBE_FILTER] = {"PROBE FILTER", gf_probe_changelog_filter},
 };
 
 struct rpc_clnt_program gf_changelog_clnt = {
-        .progname  = "LIBGFCHANGELOG",
-        .prognum   = CHANGELOG_RPC_PROGNUM,
-        .progver   = CHANGELOG_RPC_PROGVER,
-        .numproc   = CHANGELOG_RPC_PROC_MAX,
-        .proctable = gf_changelog_procs,
+    .progname = "LIBGFCHANGELOG",
+    .prognum = CHANGELOG_RPC_PROGNUM,
+    .progver = CHANGELOG_RPC_PROGVER,
+    .numproc = CHANGELOG_RPC_PROC_MAX,
+    .proctable = gf_changelog_procs,
 };
diff --git a/xlators/features/changelog/lib/src/gf-changelog-rpc.h b/xlators/features/changelog/lib/src/gf-changelog-rpc.h
index 1c982eef809..5c82d6f1c08 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-rpc.h
+++ b/xlators/features/changelog/lib/src/gf-changelog-rpc.h
@@ -11,16 +11,18 @@
 #ifndef __GF_CHANGELOG_RPC_H
 #define __GF_CHANGELOG_RPC_H
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
 #include "gf-changelog-helpers.h"
 #include "changelog-rpc-common.h"
 
-struct rpc_clnt *gf_changelog_rpc_init (xlator_t *, gf_changelog_t *);
+struct rpc_clnt *
+gf_changelog_rpc_init(xlator_t *, gf_changelog_t *);
 
-int gf_changelog_invoke_rpc (xlator_t *, gf_changelog_t *, int);
+int
+gf_changelog_invoke_rpc(xlator_t *, gf_changelog_t *, int);
 
 rpcsvc_t *
-gf_changelog_reborp_init_rpc_listner (xlator_t *, char *, char *, void *);
+gf_changelog_reborp_init_rpc_listner(xlator_t *, char *, char *, void *);
 
 #endif
diff --git a/xlators/features/changelog/lib/src/gf-changelog.c b/xlators/features/changelog/lib/src/gf-changelog.c
index 6ad6c63740d..57c3d39ef76 100644
--- a/xlators/features/changelog/lib/src/gf-changelog.c
+++ b/xlators/features/changelog/lib/src/gf-changelog.c
@@ -22,11 +22,11 @@
 #endif
 #include <string.h>
 
-#include "globals.h"
-#include "glusterfs.h"
-#include "logging.h"
-#include "defaults.h"
-#include "syncop.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/syncop.h>
 
 #include "gf-changelog-rpc.h"
 #include "gf-changelog-helpers.h"
@@ -45,291 +45,315 @@
  */
 xlator_t *master = NULL;
 
-static inline
-gf_private_t *gf_changelog_alloc_priv ()
+static inline gf_private_t *
+gf_changelog_alloc_priv()
 {
-        int ret = 0;
-        gf_private_t *priv = NULL;
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_changelog_mt_priv_t);
-        if (!priv)
-                goto error_return;
-        INIT_LIST_HEAD (&priv->connections);
-        INIT_LIST_HEAD (&priv->cleanups);
-
-        ret = pthread_mutex_init (&priv->lock, NULL);
-        if (ret != 0)
-                goto free_priv;
-        ret = pthread_cond_init (&priv->cond, NULL);
-        if (ret != 0)
-                goto cleanup_mutex;
-
-        priv->api = NULL;
-        return priv;
-
- cleanup_mutex:
-        (void) pthread_mutex_destroy (&priv->lock);
- free_priv:
-        GF_FREE (priv);
- error_return:
-        return NULL;
+    int ret = 0;
+    gf_private_t *priv = NULL;
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_changelog_mt_priv_t);
+    if (!priv)
+        goto error_return;
+    INIT_LIST_HEAD(&priv->connections);
+    INIT_LIST_HEAD(&priv->cleanups);
+
+    ret = pthread_mutex_init(&priv->lock, NULL);
+    if (ret != 0)
+        goto free_priv;
+    ret = pthread_cond_init(&priv->cond, NULL);
+    if (ret != 0)
+        goto cleanup_mutex;
+
+    priv->api = NULL;
+    return priv;
+
+cleanup_mutex:
+    (void)pthread_mutex_destroy(&priv->lock);
+free_priv:
+    GF_FREE(priv);
+error_return:
+    return NULL;
 }
 
-#define GF_CHANGELOG_EVENT_POOL_SIZE   16384
+#define GF_CHANGELOG_EVENT_POOL_SIZE 16384
 #define GF_CHANGELOG_EVENT_THREAD_COUNT 4
 
 static int
-gf_changelog_ctx_defaults_init (glusterfs_ctx_t *ctx)
+gf_changelog_ctx_defaults_init(glusterfs_ctx_t *ctx)
 {
-        cmd_args_t    *cmd_args = NULL;
-        struct rlimit  lim = {0, };
-        call_pool_t   *pool = NULL;
-        int            ret         = -1;
+    cmd_args_t *cmd_args = NULL;
+    struct rlimit lim = {
+        0,
+    };
+    call_pool_t *pool = NULL;
+    int ret = -1;
+
+    ret = xlator_mem_acct_init(THIS, gf_changelog_mt_end);
+    if (ret != 0)
+        return -1;
 
-        ret = xlator_mem_acct_init (THIS, gf_changelog_mt_end);
-        if (ret != 0)
-                return -1;
+    ctx->process_uuid = generate_glusterfs_ctx_id();
+    if (!ctx->process_uuid)
+        return -1;
 
-        ctx->process_uuid = generate_glusterfs_ctx_id ();
-        if (!ctx->process_uuid)
-                return -1;
+    ctx->page_size = 128 * GF_UNIT_KB;
 
-        ctx->page_size  = 128 * GF_UNIT_KB;
+    ctx->iobuf_pool = iobuf_pool_new();
+    if (!ctx->iobuf_pool)
+        goto free_pool;
 
-        ctx->iobuf_pool = iobuf_pool_new ();
-        if (!ctx->iobuf_pool)
-                return -1;
+    ctx->event_pool = gf_event_pool_new(GF_CHANGELOG_EVENT_POOL_SIZE,
+                                        GF_CHANGELOG_EVENT_THREAD_COUNT);
+    if (!ctx->event_pool)
+        goto free_pool;
 
-        ctx->event_pool = event_pool_new (GF_CHANGELOG_EVENT_POOL_SIZE,
-                                          GF_CHANGELOG_EVENT_THREAD_COUNT);
-        if (!ctx->event_pool)
-                return -1;
+    pool = GF_CALLOC(1, sizeof(call_pool_t),
+                     gf_changelog_mt_libgfchangelog_call_pool_t);
+    if (!pool)
+        goto free_pool;
 
-        pool = GF_CALLOC (1, sizeof (call_pool_t),
-                          gf_changelog_mt_libgfchangelog_call_pool_t);
-        if (!pool)
-                return -1;
+    /* frame_mem_pool size 112 * 64 */
+    pool->frame_mem_pool = mem_pool_new(call_frame_t, 32);
+    if (!pool->frame_mem_pool)
+        goto free_pool;
 
-        /* frame_mem_pool size 112 * 64 */
-        pool->frame_mem_pool = mem_pool_new (call_frame_t, 32);
-        if (!pool->frame_mem_pool)
-                return -1;
+    /* stack_mem_pool size 256 * 128 */
+    pool->stack_mem_pool = mem_pool_new(call_stack_t, 16);
 
-        /* stack_mem_pool size 256 * 128 */
-        pool->stack_mem_pool = mem_pool_new (call_stack_t, 16);
+    if (!pool->stack_mem_pool)
+        goto free_pool;
 
-        if (!pool->stack_mem_pool)
-                return -1;
+    ctx->stub_mem_pool = mem_pool_new(call_stub_t, 16);
+    if (!ctx->stub_mem_pool)
+        goto free_pool;
 
-        ctx->stub_mem_pool = mem_pool_new (call_stub_t, 16);
-        if (!ctx->stub_mem_pool)
-                return -1;
+    ctx->dict_pool = mem_pool_new(dict_t, 32);
+    if (!ctx->dict_pool)
+        goto free_pool;
 
-        ctx->dict_pool = mem_pool_new (dict_t, 32);
-        if (!ctx->dict_pool)
-                return -1;
+    ctx->dict_pair_pool = mem_pool_new(data_pair_t, 512);
+    if (!ctx->dict_pair_pool)
+        goto free_pool;
 
-        ctx->dict_pair_pool = mem_pool_new (data_pair_t, 512);
-        if (!ctx->dict_pair_pool)
-                return -1;
+    ctx->dict_data_pool = mem_pool_new(data_t, 512);
+    if (!ctx->dict_data_pool)
+        goto free_pool;
 
-        ctx->dict_data_pool = mem_pool_new (data_t, 512);
-        if (!ctx->dict_data_pool)
-                return -1;
+    ctx->logbuf_pool = mem_pool_new(log_buf_t, 256);
+    if (!ctx->logbuf_pool)
+        goto free_pool;
 
-        ctx->logbuf_pool = mem_pool_new (log_buf_t, 256);
-        if (!ctx->logbuf_pool)
-                return -1;
+    INIT_LIST_HEAD(&pool->all_frames);
+    LOCK_INIT(&pool->lock);
+    ctx->pool = pool;
 
-        INIT_LIST_HEAD (&pool->all_frames);
-        LOCK_INIT (&pool->lock);
-        ctx->pool = pool;
+    LOCK_INIT(&ctx->lock);
 
-        LOCK_INIT (&ctx->lock);
+    cmd_args = &ctx->cmd_args;
 
-        cmd_args = &ctx->cmd_args;
+    INIT_LIST_HEAD(&cmd_args->xlator_options);
 
-        INIT_LIST_HEAD (&cmd_args->xlator_options);
+    lim.rlim_cur = RLIM_INFINITY;
+    lim.rlim_max = RLIM_INFINITY;
+    setrlimit(RLIMIT_CORE, &lim);
 
-        lim.rlim_cur = RLIM_INFINITY;
-        lim.rlim_max = RLIM_INFINITY;
-        setrlimit (RLIMIT_CORE, &lim);
+    return 0;
 
-        return 0;
+free_pool:
+    if (pool) {
+        GF_FREE(pool->frame_mem_pool);
+
+        GF_FREE(pool->stack_mem_pool);
+
+        GF_FREE(pool);
+    }
+
+    GF_FREE(ctx->stub_mem_pool);
+
+    GF_FREE(ctx->dict_pool);
+
+    GF_FREE(ctx->dict_pair_pool);
+
+    GF_FREE(ctx->dict_data_pool);
+
+    GF_FREE(ctx->logbuf_pool);
+
+    GF_FREE(ctx->iobuf_pool);
+
+    GF_FREE(ctx->event_pool);
+
+    return -1;
 }
 
 /* TODO: cleanup ctx defaults */
 void
-gf_changelog_cleanup_this (xlator_t *this)
+gf_changelog_cleanup_this(xlator_t *this)
 {
-        glusterfs_ctx_t *ctx = NULL;
+    glusterfs_ctx_t *ctx = NULL;
 
-        if (!this)
-                return;
+    if (!this)
+        return;
 
-        ctx = this->ctx;
-        syncenv_destroy (ctx->env);
-        free (ctx);
+    ctx = this->ctx;
+    syncenv_destroy(ctx->env);
+    free(ctx);
 
-        this->private = NULL;
-        this->ctx = NULL;
+    this->private = NULL;
+    this->ctx = NULL;
 
-        mem_pools_fini ();
+    mem_pools_fini();
 }
 
 static int
-gf_changelog_init_context ()
+gf_changelog_init_context()
 {
-        glusterfs_ctx_t *ctx = NULL;
+    glusterfs_ctx_t *ctx = NULL;
 
-        ctx = glusterfs_ctx_new ();
-        if (!ctx)
-                goto error_return;
+    ctx = glusterfs_ctx_new();
+    if (!ctx)
+        goto error_return;
 
-        if (glusterfs_globals_init (ctx))
-                goto free_ctx;
+    if (glusterfs_globals_init(ctx))
+        goto free_ctx;
 
-        THIS->ctx = ctx;
-        if (gf_changelog_ctx_defaults_init (ctx))
-                goto free_ctx;
+    THIS->ctx = ctx;
+    if (gf_changelog_ctx_defaults_init(ctx))
+        goto free_ctx;
 
-        ctx->env = syncenv_new (0, 0, 0);
-        if (!ctx->env)
-                goto free_ctx;
-        return 0;
+    ctx->env = syncenv_new(0, 0, 0);
+    if (!ctx->env)
+        goto free_ctx;
+    return 0;
 
- free_ctx:
-        free (ctx);
-        THIS->ctx = NULL;
- error_return:
-        return -1;
+free_ctx:
+    free(ctx);
+    THIS->ctx = NULL;
+error_return:
+    return -1;
 }
 
 static int
-gf_changelog_init_master ()
+gf_changelog_init_master()
 {
-        int              ret = 0;
+    int ret = 0;
 
-        mem_pools_init_early ();
-        ret = gf_changelog_init_context ();
-        mem_pools_init_late ();
+    ret = gf_changelog_init_context();
+    mem_pools_init();
 
-        return ret;
+    return ret;
 }
 
 /* TODO: cleanup clnt/svc on failure */
 int
-gf_changelog_setup_rpc (xlator_t *this,
-                        gf_changelog_t *entry, int proc)
+gf_changelog_setup_rpc(xlator_t *this, gf_changelog_t *entry, int proc)
 {
-        int              ret = 0;
-        rpcsvc_t        *svc = NULL;
-        struct rpc_clnt *rpc = NULL;
-
-        /**
-         * Initialize a connect back socket. A probe() RPC call to the server
-         * triggers a reverse connect.
-         */
-        svc = gf_changelog_reborp_init_rpc_listner (this, entry->brick,
-                                                    RPC_SOCK (entry), entry);
-        if (!svc)
-                goto error_return;
-        RPC_REBORP (entry) = svc;
-
-        /* Initialize an RPC client */
-        rpc = gf_changelog_rpc_init (this, entry);
-        if (!rpc)
-                goto error_return;
-        RPC_PROBER (entry) = rpc;
-
-        /**
-         * @FIXME
-         * till we have connection state machine, let's delay the RPC call
-         * for now..
-         */
-        sleep (2);
-
-        /**
-         * Probe changelog translator for reverse connection. After a successful
-         * call, there's less use of the client and can be disconnected, but
-         * let's leave the connection active for any future RPC calls.
-         */
-        ret = gf_changelog_invoke_rpc (this, entry, proc);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_LIB_MSG_INVOKE_RPC_FAILED,
-                        "Could not initiate probe RPC, bailing out!!!");
-                goto error_return;
-        }
-
-        return 0;
-
- error_return:
-        return -1;
+    int ret = 0;
+    rpcsvc_t *svc = NULL;
+    struct rpc_clnt *rpc = NULL;
+
+    /**
+     * Initialize a connect back socket. A probe() RPC call to the server
+     * triggers a reverse connect.
+     */
+    svc = gf_changelog_reborp_init_rpc_listner(this, entry->brick,
+                                               RPC_SOCK(entry), entry);
+    if (!svc)
+        goto error_return;
+    RPC_REBORP(entry) = svc;
+
+    /* Initialize an RPC client */
+    rpc = gf_changelog_rpc_init(this, entry);
+    if (!rpc)
+        goto error_return;
+    RPC_PROBER(entry) = rpc;
+
+    /**
+     * @FIXME
+     * till we have connection state machine, let's delay the RPC call
+     * for now..
+     */
+    sleep(2);
+
+    /**
+     * Probe changelog translator for reverse connection. After a successful
+     * call, there's less use of the client and can be disconnected, but
+     * let's leave the connection active for any future RPC calls.
+     */
+    ret = gf_changelog_invoke_rpc(this, entry, proc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_INVOKE_RPC_FAILED,
+               "Could not initiate probe RPC, bailing out!!!");
+        goto error_return;
+    }
+
+    return 0;
+
+error_return:
+    return -1;
 }
 
 int
-gf_cleanup_event (xlator_t *this, struct gf_event_list *ev)
+gf_cleanup_event(xlator_t *this, struct gf_event_list *ev)
 {
-        int ret = 0;
-
-        ret = gf_thread_cleanup (this, ev->invoker);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        CHANGELOG_LIB_MSG_CLEANUP_ERROR,
-                        "cannot cleanup callback invoker thread."
-                        " Not freeing resources");
-                return -1;
-        }
+    int ret = 0;
+
+    ret = gf_thread_cleanup(this, ev->invoker);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret,
+               CHANGELOG_LIB_MSG_CLEANUP_ERROR,
+               "cannot cleanup callback invoker thread."
+               " Not freeing resources");
+        return -1;
+    }
 
-        ev->entry = NULL;
+    ev->entry = NULL;
 
-        return 0;
+    return 0;
 }
 
 static int
-gf_init_event (gf_changelog_t *entry)
+gf_init_event(gf_changelog_t *entry)
 {
-        int ret = 0;
-        struct gf_event_list *ev = NULL;
-
-        ev = &entry->event;
-        ev->entry = entry;
-
-        ret = pthread_mutex_init (&ev->lock, NULL);
-        if (ret != 0)
-                goto error_return;
-        ret = pthread_cond_init (&ev->cond, NULL);
-        if (ret != 0)
-                goto cleanup_mutex;
-        INIT_LIST_HEAD (&ev->events);
-
-        ev->next_seq = 0;  /* bootstrap sequencing */
-
-        if (GF_NEED_ORDERED_EVENTS (entry)) {
-                entry->pickevent  = pick_event_ordered;
-                entry->queueevent = queue_ordered_event;
-        } else {
-                entry->pickevent  = pick_event_unordered;
-                entry->queueevent = queue_unordered_event;
-        }
-
-        ret = gf_thread_create (&ev->invoker, NULL,
-                                gf_changelog_callback_invoker, ev, "clogcbki");
-        if (ret != 0) {
-                entry->pickevent = NULL;
-                entry->queueevent = NULL;
-                goto cleanup_cond;
-        }
-
-        return 0;
-
- cleanup_cond:
-        (void) pthread_cond_destroy (&ev->cond);
- cleanup_mutex:
-        (void) pthread_mutex_destroy (&ev->lock);
- error_return:
-        return -1;
+    int ret = 0;
+    struct gf_event_list *ev = NULL;
+
+    ev = &entry->event;
+    ev->entry = entry;
+
+    ret = pthread_mutex_init(&ev->lock, NULL);
+    if (ret != 0)
+        goto error_return;
+    ret = pthread_cond_init(&ev->cond, NULL);
+    if (ret != 0)
+        goto cleanup_mutex;
+    INIT_LIST_HEAD(&ev->events);
+
+    ev->next_seq = 0; /* bootstrap sequencing */
+
+    if (GF_NEED_ORDERED_EVENTS(entry)) {
+        entry->pickevent = pick_event_ordered;
+        entry->queueevent = queue_ordered_event;
+    } else {
+        entry->pickevent = pick_event_unordered;
+        entry->queueevent = queue_unordered_event;
+    }
+
+    ret = gf_thread_create(&ev->invoker, NULL, gf_changelog_callback_invoker,
+                           ev, "clogcbki");
+    if (ret != 0) {
+        entry->pickevent = NULL;
+        entry->queueevent = NULL;
+        goto cleanup_cond;
+    }
+
+    return 0;
+
+cleanup_cond:
+    (void)pthread_cond_destroy(&ev->cond);
+cleanup_mutex:
+    (void)pthread_mutex_destroy(&ev->lock);
+error_return:
+    return -1;
 }
 
 /**
@@ -339,250 +363,241 @@ gf_init_event (gf_changelog_t *entry)
  *  - destroy rpc{-clnt, svc}
  */
 int
-gf_cleanup_brick_connection (xlator_t *this, gf_changelog_t *entry)
+gf_cleanup_brick_connection(xlator_t *this, gf_changelog_t *entry)
 {
-        return 0;
+    return 0;
 }
 
 int
-gf_cleanup_connections (xlator_t *this)
+gf_cleanup_connections(xlator_t *this)
 {
-        return 0;
+    return 0;
 }
 
 static int
-gf_setup_brick_connection (xlator_t *this,
-                           struct gf_brick_spec *brick,
-                           gf_boolean_t ordered, void *xl)
+gf_setup_brick_connection(xlator_t *this, struct gf_brick_spec *brick,
+                          gf_boolean_t ordered, void *xl)
 {
-        int ret = 0;
-        gf_private_t *priv = NULL;
-        gf_changelog_t *entry = NULL;
-
-        priv = this->private;
-
-        if (!brick->callback || !brick->init || !brick->fini)
-                goto error_return;
-
-        entry = GF_CALLOC (1, sizeof (*entry),
-                           gf_changelog_mt_libgfchangelog_t);
-        if (!entry)
-                goto error_return;
-        INIT_LIST_HEAD (&entry->list);
-
-        LOCK_INIT (&entry->statelock);
-        entry->connstate = GF_CHANGELOG_CONN_STATE_PENDING;
-
-        entry->notify = brick->filter;
-        (void) strncpy (entry->brick, brick->brick_path, PATH_MAX-1);
-        entry->brick[PATH_MAX-1] = 0;
-
-        entry->this = this;
-        entry->invokerxl = xl;
-
-        entry->ordered = ordered;
-        ret = gf_init_event (entry);
-        if (ret)
-                goto free_entry;
-
-        entry->fini         = brick->fini;
-        entry->callback     = brick->callback;
-        entry->connected    = brick->connected;
-        entry->disconnected = brick->disconnected;
-
-        entry->ptr = brick->init (this, brick);
-        if (!entry->ptr)
-                goto cleanup_event;
-        priv->api = entry->ptr;  /* pointer to API, if required */
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                list_add_tail (&entry->list, &priv->connections);
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        ret = gf_changelog_setup_rpc (this, entry, CHANGELOG_RPC_PROBE_FILTER);
-        if (ret)
-                goto cleanup_event;
-        return 0;
-
- cleanup_event:
-        (void) gf_cleanup_event (this, &entry->event);
- free_entry:
-        gf_msg_debug (this->name, 0, "freeing entry %p", entry);
-        list_del (&entry->list); /* FIXME: kludge for now */
-        GF_FREE (entry);
- error_return:
-        return -1;
+    int ret = 0;
+    gf_private_t *priv = NULL;
+    gf_changelog_t *entry = NULL;
+
+    priv = this->private;
+
+    if (!brick->callback || !brick->init || !brick->fini)
+        goto error_return;
+
+    entry = GF_CALLOC(1, sizeof(*entry), gf_changelog_mt_libgfchangelog_t);
+    if (!entry)
+        goto error_return;
+    INIT_LIST_HEAD(&entry->list);
+
+    LOCK_INIT(&entry->statelock);
+    entry->connstate = GF_CHANGELOG_CONN_STATE_PENDING;
+
+    entry->notify = brick->filter;
+    if (snprintf(entry->brick, PATH_MAX, "%s", brick->brick_path) >= PATH_MAX)
+        goto free_entry;
+
+    entry->this = this;
+    entry->invokerxl = xl;
+
+    entry->ordered = ordered;
+    ret = gf_init_event(entry);
+    if (ret)
+        goto free_entry;
+
+    entry->fini = brick->fini;
+    entry->callback = brick->callback;
+    entry->connected = brick->connected;
+    entry->disconnected = brick->disconnected;
+
+    entry->ptr = brick->init(this, brick);
+    if (!entry->ptr)
+        goto cleanup_event;
+    priv->api = entry->ptr; /* pointer to API, if required */
+
+    pthread_mutex_lock(&priv->lock);
+    {
+        list_add_tail(&entry->list, &priv->connections);
+    }
+    pthread_mutex_unlock(&priv->lock);
+
+    ret = gf_changelog_setup_rpc(this, entry, CHANGELOG_RPC_PROBE_FILTER);
+    if (ret)
+        goto cleanup_event;
+    return 0;
+
+cleanup_event:
+    (void)gf_cleanup_event(this, &entry->event);
+free_entry:
+    gf_msg_debug(this->name, 0, "freeing entry %p", entry);
+    list_del(&entry->list); /* FIXME: kludge for now */
+    GF_FREE(entry);
+error_return:
+    return -1;
 }
 
 int
-gf_changelog_register_brick (xlator_t *this,
-                             struct gf_brick_spec *brick,
-                             gf_boolean_t ordered, void *xl)
+gf_changelog_register_brick(xlator_t *this, struct gf_brick_spec *brick,
+                            gf_boolean_t ordered, void *xl)
 {
-        return gf_setup_brick_connection (this, brick, ordered, xl);
+    return gf_setup_brick_connection(this, brick, ordered, xl);
 }
 
 static int
-gf_changelog_setup_logging (xlator_t *this, char *logfile, int loglevel)
+gf_changelog_setup_logging(xlator_t *this, char *logfile, int loglevel)
 {
-        /* passing ident as NULL means to use default ident for syslog */
-        if (gf_log_init (this->ctx, logfile, NULL))
-                return -1;
+    /* passing ident as NULL means to use default ident for syslog */
+    if (gf_log_init(this->ctx, logfile, NULL))
+        return -1;
 
-        gf_log_set_loglevel (this->ctx, (loglevel == -1) ? GF_LOG_INFO :
-                             loglevel);
-        return 0;
+    gf_log_set_loglevel(this->ctx, (loglevel == -1) ? GF_LOG_INFO : loglevel);
+    return 0;
 }
 
 static int
-gf_changelog_set_master (xlator_t *master, void *xl)
+gf_changelog_set_master(xlator_t *master, void *xl)
 {
-        int32_t ret = 0;
-        xlator_t *this = NULL;
-        xlator_t *old_this = NULL;
-        gf_private_t *priv = NULL;
-
-        this = xl;
-        if (!this || !this->ctx) {
-                ret = gf_changelog_init_master ();
-                if (ret)
-                        return -1;
-                this = THIS;
-        }
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    xlator_t *old_this = NULL;
+    gf_private_t *priv = NULL;
+
+    this = xl;
+    if (!this || !this->ctx) {
+        ret = gf_changelog_init_master();
+        if (ret)
+            return -1;
+        this = THIS;
+    }
 
-        master->ctx = this->ctx;
+    master->ctx = this->ctx;
 
-        INIT_LIST_HEAD (&master->volume_options);
-        SAVE_THIS (THIS);
+    INIT_LIST_HEAD(&master->volume_options);
+    SAVE_THIS(THIS);
 
-        ret = xlator_mem_acct_init (THIS, gf_changelog_mt_end);
-        if (ret != 0)
-                goto restore_this;
+    ret = xlator_mem_acct_init(THIS, gf_changelog_mt_end);
+    if (ret != 0)
+        goto restore_this;
 
-        priv = gf_changelog_alloc_priv ();
-        if (!priv) {
-                ret = -1;
-                goto restore_this;
-        }
+    priv = gf_changelog_alloc_priv();
+    if (!priv) {
+        ret = -1;
+        goto restore_this;
+    }
 
-        if (!xl) {
-                /* poller thread */
-                ret = gf_thread_create (&priv->poller,
-                                        NULL, changelog_rpc_poller, THIS,
-                                        "clogpoll");
-                if (ret != 0) {
-                        GF_FREE (priv);
-                        gf_msg (master->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED,
-                                "failed to spawn poller thread");
-                        goto restore_this;
-                }
+    if (!xl) {
+        /* poller thread */
+        ret = gf_thread_create(&priv->poller, NULL, changelog_rpc_poller, THIS,
+                               "clogpoll");
+        if (ret != 0) {
+            GF_FREE(priv);
+            gf_msg(master->name, GF_LOG_ERROR, 0,
+                   CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED,
+                   "failed to spawn poller thread");
+            goto restore_this;
         }
+    }
 
-        master->private = priv;
+    master->private = priv;
 
- restore_this:
-        RESTORE_THIS ();
+restore_this:
+    RESTORE_THIS();
 
-        return ret;
+    return ret;
 }
 
 int
-gf_changelog_init (void *xl)
+gf_changelog_init(void *xl)
 {
-        int ret = 0;
-        gf_private_t *priv = NULL;
-
-        if (master)
-                return 0;
-
-        master = calloc (1, sizeof (*master));
-        if (!master)
-                goto error_return;
-
-        master->name = strdup ("gfchangelog");
-        if (!master->name)
-                goto dealloc_master;
-
-        ret = gf_changelog_set_master (master, xl);
-        if (ret)
-                goto dealloc_name;
-
-        priv = master->private;
-        ret = gf_thread_create (&priv->connectionjanitor, NULL,
-                                gf_changelog_connection_janitor, master,
-                                "clogjan");
-        if (ret != 0) {
-                /* TODO: cleanup priv, mutex (poller thread for !xl) */
-                goto dealloc_name;
-        }
+    int ret = 0;
+    gf_private_t *priv = NULL;
 
+    if (master)
         return 0;
 
- dealloc_name:
-        free (master->name);
- dealloc_master:
-        free (master);
-        master = NULL;
- error_return:
-        return -1;
+    master = calloc(1, sizeof(*master));
+    if (!master)
+        goto error_return;
+
+    master->name = strdup("gfchangelog");
+    if (!master->name)
+        goto dealloc_master;
+
+    ret = gf_changelog_set_master(master, xl);
+    if (ret)
+        goto dealloc_name;
+
+    priv = master->private;
+    ret = gf_thread_create(&priv->connectionjanitor, NULL,
+                           gf_changelog_connection_janitor, master, "clogjan");
+    if (ret != 0) {
+        /* TODO: cleanup priv, mutex (poller thread for !xl) */
+        goto dealloc_name;
+    }
+
+    return 0;
+
+dealloc_name:
+    free(master->name);
+dealloc_master:
+    free(master);
+    master = NULL;
+error_return:
+    return -1;
 }
 
 int
-gf_changelog_register_generic (struct gf_brick_spec *bricks, int count,
-                               int ordered, char *logfile, int lvl, void *xl)
+gf_changelog_register_generic(struct gf_brick_spec *bricks, int count,
+                              int ordered, char *logfile, int lvl, void *xl)
 {
-        int                   ret        = 0;
-        xlator_t             *this       = NULL;
-        xlator_t             *old_this   = NULL;
-        struct gf_brick_spec *brick      = NULL;
-        gf_boolean_t          need_order = _gf_false;
+    int ret = 0;
+    xlator_t *this = NULL;
+    xlator_t *old_this = NULL;
+    struct gf_brick_spec *brick = NULL;
+    gf_boolean_t need_order = _gf_false;
 
-        SAVE_THIS (xl);
+    SAVE_THIS(xl);
 
-        this = THIS;
-        if (!this)
-                goto error_return;
+    this = THIS;
+    if (!this)
+        goto error_return;
 
-        ret = gf_changelog_setup_logging (this, logfile, lvl);
-        if (ret)
-                goto error_return;
-
-        need_order = (ordered) ? _gf_true : _gf_false;
-
-        brick = bricks;
-        while (count--) {
-                gf_smsg (this->name, GF_LOG_INFO, 0,
-                         CHANGELOG_LIB_MSG_NOTIFY_REGISTER_INFO,
-                         "Registering brick",
-                         "brick=%s", brick->brick_path,
-                         "notify_filter=%d", brick->filter,
-                         NULL);
-
-                ret = gf_changelog_register_brick (this, brick, need_order, xl);
-                if (ret != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_LIB_MSG_NOTIFY_REGISTER_FAILED,
-                                "Error registering with changelog xlator");
-                        break;
-                }
-
-                brick++;
+    ret = gf_changelog_setup_logging(this, logfile, lvl);
+    if (ret)
+        goto error_return;
+
+    need_order = (ordered) ? _gf_true : _gf_false;
+
+    brick = bricks;
+    while (count--) {
+        gf_smsg(this->name, GF_LOG_INFO, 0,
+                CHANGELOG_LIB_MSG_NOTIFY_REGISTER_INFO, "brick=%s",
+                brick->brick_path, "notify_filter=%d", brick->filter, NULL);
+
+        ret = gf_changelog_register_brick(this, brick, need_order, xl);
+        if (ret != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   CHANGELOG_LIB_MSG_NOTIFY_REGISTER_FAILED,
+                   "Error registering with changelog xlator");
+            break;
         }
 
-        if (ret != 0)
-                goto cleanup_inited_bricks;
+        brick++;
+    }
 
-        RESTORE_THIS();
-        return 0;
+    if (ret != 0)
+        goto cleanup_inited_bricks;
 
- cleanup_inited_bricks:
-        gf_cleanup_connections (this);
- error_return:
-        RESTORE_THIS();
-        return -1;
+    RESTORE_THIS();
+    return 0;
+
+cleanup_inited_bricks:
+    gf_cleanup_connections(this);
+error_return:
+    RESTORE_THIS();
+    return -1;
 }
 
 /**
@@ -609,27 +624,29 @@ gf_changelog_register_generic (struct gf_brick_spec *bricks, int count,
  * For generic API, refer gf_changelog_register_generic().
  */
 int
-gf_changelog_register (char *brick_path, char *scratch_dir,
-                       char *log_file, int log_level, int max_reconnects)
+gf_changelog_register(char *brick_path, char *scratch_dir, char *log_file,
+                      int log_level, int max_reconnects)
 {
-        struct gf_brick_spec brick = {0,};
+    struct gf_brick_spec brick = {
+        0,
+    };
 
-        if (master)
-                THIS = master;
-        else
-                return -1;
+    if (master)
+        THIS = master;
+    else
+        return -1;
 
-        brick.brick_path = brick_path;
-        brick.filter     = CHANGELOG_OP_TYPE_JOURNAL;
+    brick.brick_path = brick_path;
+    brick.filter = CHANGELOG_OP_TYPE_JOURNAL;
 
-        brick.init         = gf_changelog_journal_init;
-        brick.fini         = gf_changelog_journal_fini;
-        brick.callback     = gf_changelog_handle_journal;
-        brick.connected    = gf_changelog_journal_connect;
-        brick.disconnected = gf_changelog_journal_disconnect;
+    brick.init = gf_changelog_journal_init;
+    brick.fini = gf_changelog_journal_fini;
+    brick.callback = gf_changelog_handle_journal;
+    brick.connected = gf_changelog_journal_connect;
+    brick.disconnected = gf_changelog_journal_disconnect;
 
-        brick.ptr = scratch_dir;
+    brick.ptr = scratch_dir;
 
-        return gf_changelog_register_generic (&brick, 1, 1,
-                                              log_file, log_level, NULL);
+    return gf_changelog_register_generic(&brick, 1, 1, log_file, log_level,
+                                         NULL);
 }
diff --git a/xlators/features/changelog/lib/src/gf-history-changelog.c b/xlators/features/changelog/lib/src/gf-history-changelog.c
index 2f245a725f6..a16219f3664 100644
--- a/xlators/features/changelog/lib/src/gf-history-changelog.c
+++ b/xlators/features/changelog/lib/src/gf-history-changelog.c
@@ -8,10 +8,10 @@
 #endif
 #include <string.h>
 
-#include "globals.h"
-#include "glusterfs.h"
-#include "logging.h"
-#include "syscall.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/syscall.h>
 
 #include "gf-changelog-helpers.h"
 #include "gf-changelog-journal.h"
@@ -36,62 +36,60 @@
  *   -1: On error.
  */
 int
-gf_history_changelog_done (char *file)
+gf_history_changelog_done(char *file)
 {
-        int                     ret               = -1;
-        char                   *buffer            = NULL;
-        xlator_t               *this              = NULL;
-        gf_changelog_journal_t *jnl               = NULL;
-        gf_changelog_journal_t *hist_jnl          = NULL;
-        char                    to_path[PATH_MAX] = {0,};
+    int ret = -1;
+    char *buffer = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    char to_path[PATH_MAX] = {
+        0,
+    };
+
+    errno = EINVAL;
+
+    this = THIS;
+    if (!this)
+        goto out;
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
+
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl)
+        goto out;
+
+    if (!file || !strlen(file))
+        goto out;
+
+    /* make sure 'file' is inside ->jnl_working_dir */
+    buffer = realpath(file, NULL);
+    if (!buffer)
+        goto out;
+
+    if (strncmp(hist_jnl->jnl_working_dir, buffer,
+                strlen(hist_jnl->jnl_working_dir)))
+        goto out;
+
+    (void)snprintf(to_path, PATH_MAX, "%s%s", hist_jnl->jnl_processed_dir,
+                   basename(buffer));
+    gf_msg_debug(this->name, 0, "moving %s to processed directory", file);
+    ret = sys_rename(buffer, to_path);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_RENAME_FAILED, "from=%s", file, "to=%s",
+                to_path, NULL);
+        goto out;
+    }
+
+    ret = 0;
 
-        errno = EINVAL;
-
-        this = THIS;
-        if (!this)
-                goto out;
-
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
-
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl)
-                goto out;
-
-        if (!file || !strlen (file))
-                goto out;
-
-        /* make sure 'file' is inside ->jnl_working_dir */
-        buffer = realpath (file, NULL);
-        if (!buffer)
-                goto out;
-
-        if (strncmp (hist_jnl->jnl_working_dir,
-                     buffer, strlen (hist_jnl->jnl_working_dir)))
-                goto out;
-
-        (void) snprintf (to_path, PATH_MAX, "%s%s",
-                         hist_jnl->jnl_processed_dir, basename (buffer));
-        gf_msg_debug (this->name, 0,
-                      "moving %s to processed directory", file);
-        ret = sys_rename (buffer, to_path);
-        if (ret) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_RENAME_FAILED,
-                         "cannot move changelog file",
-                         "from=%s", file,
-                         "to=%s", to_path,
-                         NULL);
-                goto out;
-        }
-
-        ret = 0;
-
- out:
-        if (buffer)
-                free (buffer); /* allocated by realpath() */
-        return ret;
+out:
+    if (buffer)
+        free(buffer); /* allocated by realpath() */
+    return ret;
 }
 
 /**
@@ -105,33 +103,33 @@ gf_history_changelog_done (char *file)
  *    -1: On error.
  */
 int
-gf_history_changelog_start_fresh ()
+gf_history_changelog_start_fresh()
 {
-        xlator_t               *this     = NULL;
-        gf_changelog_journal_t *jnl      = NULL;
-        gf_changelog_journal_t *hist_jnl = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
 
-        this = THIS;
-        if (!this)
-                goto out;
+    this = THIS;
+    if (!this)
+        goto out;
 
-        errno = EINVAL;
+    errno = EINVAL;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
 
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl)
-                goto out;
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl)
+        goto out;
 
-        if (gf_ftruncate (hist_jnl->jnl_fd, 0))
-                goto out;
+    if (gf_ftruncate(hist_jnl->jnl_fd, 0))
+        goto out;
 
-        return 0;
+    return 0;
 
- out:
-        return -1;
+out:
+    return -1;
 }
 
 /**
@@ -150,50 +148,52 @@ gf_history_changelog_start_fresh ()
  *     -1  : On error.
  */
 ssize_t
-gf_history_changelog_next_change (char *bufptr, size_t maxlen)
+gf_history_changelog_next_change(char *bufptr, size_t maxlen)
 {
-        ssize_t                 size             = -1;
-        int                     tracker_fd       = 0;
-        xlator_t               *this             = NULL;
-        gf_changelog_journal_t *jnl              = NULL;
-        gf_changelog_journal_t *hist_jnl         = NULL;
-        char                    buffer[PATH_MAX] = {0,};
-
-        if (maxlen > PATH_MAX) {
-                errno = ENAMETOOLONG;
-                goto out;
-        }
+    ssize_t size = -1;
+    int tracker_fd = 0;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    char buffer[PATH_MAX] = {
+        0,
+    };
 
-        errno = EINVAL;
+    if (maxlen > PATH_MAX) {
+        errno = ENAMETOOLONG;
+        goto out;
+    }
 
-        this = THIS;
-        if (!this)
-                goto out;
+    errno = EINVAL;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
+    this = THIS;
+    if (!this)
+        goto out;
 
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl)
-                goto out;
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
 
-        tracker_fd = hist_jnl->jnl_fd;
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl)
+        goto out;
 
-        size = gf_readline (tracker_fd, buffer, maxlen);
-        if (size < 0) {
-                size = -1;
-                goto out;
-        }
+    tracker_fd = hist_jnl->jnl_fd;
 
-        if (size == 0)
-                goto out;
+    size = gf_readline(tracker_fd, buffer, maxlen);
+    if (size < 0) {
+        size = -1;
+        goto out;
+    }
 
-        memcpy (bufptr, buffer, size - 1);
-        bufptr[size - 1] = '\0';
+    if (size == 0)
+        goto out;
+
+    memcpy(bufptr, buffer, size - 1);
+    bufptr[size - 1] = '\0';
 
 out:
-        return size;
+    return size;
 }
 
 /**
@@ -214,97 +214,100 @@ out:
  *
  */
 ssize_t
-gf_history_changelog_scan ()
+gf_history_changelog_scan()
 {
-        int                     tracker_fd   = 0;
-        size_t                  off          = 0;
-        xlator_t               *this         = NULL;
-        size_t                  nr_entries   = 0;
-        gf_changelog_journal_t *jnl          = NULL;
-        gf_changelog_journal_t *hist_jnl     = NULL;
-        struct dirent          *entry        = NULL;
-        struct dirent           scratch[2]   = {{0,},};
-        char buffer[PATH_MAX]                = {0,};
-        static int              is_last_scan;
-
-        this = THIS;
-        if (!this)
-                goto out;
+    int tracker_fd = 0;
+    size_t off = 0;
+    xlator_t *this = NULL;
+    size_t nr_entries = 0;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char buffer[PATH_MAX] = {
+        0,
+    };
+    static int is_last_scan;
+
+    this = THIS;
+    if (!this)
+        goto out;
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
+    if (JNL_IS_API_DISCONNECTED(jnl)) {
+        errno = ENOTCONN;
+        goto out;
+    }
+
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl)
+        goto out;
+
+retry:
+    if (is_last_scan == 1)
+        return 0;
+    if (hist_jnl->hist_done == 0)
+        is_last_scan = 1;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
-        if (JNL_IS_API_DISCONNECTED (jnl)) {
-                errno = ENOTCONN;
-                goto out;
-        }
+    errno = EINVAL;
+    if (hist_jnl->hist_done == -1)
+        goto out;
 
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl)
-                goto out;
+    tracker_fd = hist_jnl->jnl_fd;
 
- retry:
-        if (is_last_scan == 1)
-                return 0;
-        if (hist_jnl->hist_done == 0)
-                is_last_scan = 1;
+    if (gf_ftruncate(tracker_fd, 0))
+        goto out;
 
-        errno = EINVAL;
-        if (hist_jnl->hist_done == -1)
-                goto out;
+    rewinddir(hist_jnl->jnl_dir);
 
-        tracker_fd = hist_jnl->jnl_fd;
+    for (;;) {
+        errno = 0;
+        entry = sys_readdir(hist_jnl->jnl_dir, scratch);
+        if (!entry || errno != 0)
+            break;
 
-        if (gf_ftruncate (tracker_fd, 0))
-                goto out;
+        if (strcmp(basename(entry->d_name), ".") == 0 ||
+            strcmp(basename(entry->d_name), "..") == 0)
+            continue;
+
+        nr_entries++;
 
-        rewinddir (hist_jnl->jnl_dir);
-
-        for (;;) {
-                errno = 0;
-                entry = sys_readdir (hist_jnl->jnl_dir, scratch);
-                if (!entry || errno != 0)
-                        break;
-
-                if (strcmp (basename (entry->d_name), ".") == 0 ||
-                    strcmp (basename (entry->d_name), "..") == 0)
-                        continue;
-
-                nr_entries++;
-
-                GF_CHANGELOG_FILL_BUFFER (hist_jnl->jnl_processing_dir,
-                                          buffer, off,
-                                          strlen (hist_jnl->jnl_processing_dir));
-                GF_CHANGELOG_FILL_BUFFER (entry->d_name, buffer,
-                                          off, strlen (entry->d_name));
-                GF_CHANGELOG_FILL_BUFFER ("\n", buffer, off, 1);
-
-                if (gf_changelog_write (tracker_fd, buffer, off) != off) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_LIB_MSG_WRITE_FAILED,
-                                "error writing changelog filename"
-                                " to tracker file");
-                        break;
-                }
-                off = 0;
+        GF_CHANGELOG_FILL_BUFFER(hist_jnl->jnl_processing_dir, buffer, off,
+                                 strlen(hist_jnl->jnl_processing_dir));
+        GF_CHANGELOG_FILL_BUFFER(entry->d_name, buffer, off,
+                                 strlen(entry->d_name));
+        GF_CHANGELOG_FILL_BUFFER("\n", buffer, off, 1);
+
+        if (gf_changelog_write(tracker_fd, buffer, off) != off) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_WRITE_FAILED,
+                   "error writing changelog filename"
+                   " to tracker file");
+            break;
         }
+        off = 0;
+    }
+
+    gf_msg_debug(this->name, 0, "hist_done %d, is_last_scan: %d",
+                 hist_jnl->hist_done, is_last_scan);
 
-        gf_msg_debug (this->name, 0,
-                      "hist_done %d, is_last_scan: %d",
-                      hist_jnl->hist_done, is_last_scan);
-
-        if (!entry) {
-                if (gf_lseek (tracker_fd, 0, SEEK_SET) != -1) {
-                        if (nr_entries > 0)
-                                return nr_entries;
-                        else {
-                                sleep(1);
-                                goto retry;
-                        }
-                }
+    if (!entry) {
+        if (gf_lseek(tracker_fd, 0, SEEK_SET) != -1) {
+            if (nr_entries > 0)
+                return nr_entries;
+            else {
+                sleep(1);
+                goto retry;
+            }
         }
- out:
-        return -1;
+    }
+out:
+    return -1;
 }
 
 /*
@@ -312,36 +315,36 @@ gf_history_changelog_scan ()
  * Returns 0 on success(updates given time-stamp), -1 on failure.
  */
 int
-gf_history_get_timestamp (int fd, int index, int len,
-                          unsigned long *ts)
+gf_history_get_timestamp(int fd, int index, int len, unsigned long *ts)
 {
-        xlator_t        *this             = NULL;
-        int             n_read            = -1;
-        char            path_buf[PATH_MAX]= {0,};
-        char            *iter             = path_buf;
-        size_t          offset            = index * (len+1);
-        unsigned long   value             = 0;
-        int             ret               = 0;
-
-        this = THIS;
-        if (!this) {
-                return -1;
-        }
-
-        n_read = sys_pread (fd, path_buf, len, offset);
-        if (n_read < 0 ) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_READ_ERROR,
-                         "could not read from htime file");
-                goto out;
-        }
-        iter+= len - TIMESTAMP_LENGTH;
-        sscanf (iter, "%lu",&value);
+    xlator_t *this = NULL;
+    int n_read = -1;
+    char path_buf[PATH_MAX] = {
+        0,
+    };
+    char *iter = path_buf;
+    size_t offset = index * (len + 1);
+    unsigned long value = 0;
+    int ret = 0;
+
+    this = THIS;
+    if (!this) {
+        return -1;
+    }
+
+    n_read = sys_pread(fd, path_buf, len, offset);
+    if (n_read < 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_READ_ERROR,
+               "could not read from htime file");
+        goto out;
+    }
+    iter += len - TIMESTAMP_LENGTH;
+    sscanf(iter, "%lu", &value);
 out:
-        if(ret == 0)
-                *ts = value;
-        return ret;
+    if (ret == 0)
+        *ts = value;
+    return ret;
 }
 
 /*
@@ -349,38 +352,37 @@ out:
  * Checks whether @value is there next to @target_index or not
  */
 int
-gf_history_check ( int fd, int target_index, unsigned long value, int len)
+gf_history_check(int fd, int target_index, unsigned long value, int len)
 {
-        int             ret = 0;
-        unsigned long   ts1 = 0;
-        unsigned long   ts2 = 0;
-
-        if (target_index == 0) {
-                ret = gf_history_get_timestamp (fd, target_index, len, &ts1);
-                if (ret == -1)
-                        goto out;
-                if (value <= ts1)
-                        goto out;
-                else {
-                        ret = -1;
-                        goto out;
-                }
-        }
+    int ret = 0;
+    unsigned long ts1 = 0;
+    unsigned long ts2 = 0;
 
-        ret = gf_history_get_timestamp (fd, target_index, len, &ts1);
-        if (ret ==-1)
-                goto out;
-        ret = gf_history_get_timestamp (fd, target_index -1, len, &ts2);
-        if (ret ==-1)
-                goto out;
-
-        if ( (value <= ts1) && (value > ts2) ) {
-                goto out;
-        }
-        else
-                ret = -1;
+    if (target_index == 0) {
+        ret = gf_history_get_timestamp(fd, target_index, len, &ts1);
+        if (ret == -1)
+            goto out;
+        if (value <= ts1)
+            goto out;
+        else {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = gf_history_get_timestamp(fd, target_index, len, &ts1);
+    if (ret == -1)
+        goto out;
+    ret = gf_history_get_timestamp(fd, target_index - 1, len, &ts2);
+    if (ret == -1)
+        goto out;
+
+    if ((value <= ts1) && (value > ts2)) {
+        goto out;
+    } else
+        ret = -1;
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -400,83 +402,74 @@ out:
  */
 
 int
-gf_history_b_search (int fd, unsigned long value,
-                     unsigned long from, unsigned long to, int len)
+gf_history_b_search(int fd, unsigned long value, unsigned long from,
+                    unsigned long to, int len)
 {
-        int             m_index   = -1;
-        unsigned long   cur_value = 0;
-        unsigned long   ts1       = 0;
-        int             ret       = 0;
-
-        m_index = (from + to)/2;
-
-        if ( (to - from) <=1 ) {
-                /* either one or 2 changelogs left */
-                if ( to != from ) {
-                        /* check if value is less or greater than to
-                         * return accordingly
-                         */
-                        ret = gf_history_get_timestamp (fd, from, len, &ts1);
-                        if (ret ==-1)
-                                goto out;
-                        if ( ts1 >= value) {
-                                /* actually compatision should be
-                                 * exactly == but considering
-                                 *
-                                 * case of only 2 changelogs in htime file
-                                 */
-                                return from;
-                        }
-                        else
-                                return to;
-                }
-                else
-                        return to;
-        }
-
-        ret = gf_history_get_timestamp (fd, m_index, len, &cur_value);
+    int m_index = -1;
+    unsigned long cur_value = 0;
+    unsigned long ts1 = 0;
+    int ret = 0;
+
+    m_index = (from + to) / 2;
+
+    if ((to - from) <= 1) {
+        /* either one or 2 changelogs left */
+        if (to != from) {
+            /* check if value is less or greater than to
+             * return accordingly
+             */
+            ret = gf_history_get_timestamp(fd, from, len, &ts1);
+            if (ret == -1)
+                goto out;
+            if (ts1 >= value) {
+                /* actually compatision should be
+                 * exactly == but considering
+                 *
+                 * case of only 2 changelogs in htime file
+                 */
+                return from;
+            } else
+                return to;
+        } else
+            return to;
+    }
+
+    ret = gf_history_get_timestamp(fd, m_index, len, &cur_value);
+    if (ret == -1)
+        goto out;
+    if (cur_value == value) {
+        return m_index;
+    } else if (value > cur_value) {
+        ret = gf_history_get_timestamp(fd, m_index + 1, len, &cur_value);
         if (ret == -1)
-                goto out;
-        if (cur_value == value) {
+            goto out;
+        if (value < cur_value)
+            return m_index + 1;
+        else
+            return gf_history_b_search(fd, value, m_index + 1, to, len);
+    } else {
+        if (m_index == 0) {
+            /*  we are sure that values exists
+             *  in this htime file
+             */
+            return 0;
+        } else {
+            ret = gf_history_get_timestamp(fd, m_index - 1, len, &cur_value);
+            if (ret == -1)
+                goto out;
+            if (value > cur_value) {
                 return m_index;
+            } else
+                return gf_history_b_search(fd, value, from, m_index - 1, len);
         }
-        else if (value > cur_value) {
-                ret = gf_history_get_timestamp (fd, m_index+1, len, &cur_value);
-                if (ret == -1)
-                        goto out;
-                if (value < cur_value)
-                        return m_index + 1;
-                else
-                        return gf_history_b_search (fd, value,
-                                                    m_index+1, to, len);
-        }
-        else {
-                if (m_index ==0) {
-                       /*  we are sure that values exists
-                        *  in this htime file
-                        */
-                        return 0;
-                }
-                else {
-                        ret = gf_history_get_timestamp (fd, m_index-1, len,
-                                                        &cur_value);
-                        if (ret == -1)
-                                goto out;
-                        if (value > cur_value) {
-                                return m_index;
-                        }
-                        else
-                                return gf_history_b_search (fd, value, from,
-                                                            m_index-1, len);
-                }
-        }
+    }
 out:
-        return -1;
+    return -1;
 }
 
 /*
  * Description: Checks if the changelog path is usable or not,
- *              which is differenciated by checking for "changelog"
+ *              which is differentiated by checking for "changelog"
  *              in the path and not "CHANGELOG".
  *
  * Returns:
@@ -484,65 +477,59 @@ out:
  * 0 : No, Not usable ( contains, "changelog")
  */
 int
-gf_is_changelog_usable (char *cl_path)
+gf_is_changelog_usable(char *cl_path)
 {
-        int             ret             = -1;
-        const char      low_c[]         = "changelog";
-        char            *str_ret        = NULL;
-        char            *bname          = NULL;
+    int ret = -1;
+    const char low_c[] = "changelog";
+    char *str_ret = NULL;
+    char *bname = NULL;
 
-        bname = basename (cl_path);
+    bname = basename(cl_path);
 
-        str_ret = strstr (bname, low_c);
+    str_ret = strstr(bname, low_c);
 
-        if (str_ret != NULL)
-                ret = 0;
-        else
-                ret = 1;
-
-        return ret;
+    if (str_ret != NULL)
+        ret = 0;
+    else
+        ret = 1;
 
+    return ret;
 }
 
 void *
-gf_changelog_consume_wrap (void* data)
+gf_changelog_consume_wrap(void *data)
 {
-        int                          ret   = -1;
-        ssize_t                      nread = 0;
-        xlator_t                    *this  = NULL;
-        gf_changelog_consume_data_t *ccd   = NULL;
-
-        ccd = (gf_changelog_consume_data_t *) data;
-        this = ccd->this;
-
-        ccd->retval = -1;
-
-        nread = sys_pread (ccd->fd, ccd->changelog, PATH_MAX-1, ccd->offset);
-        if (nread < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_READ_ERROR,
-                        "cannot read from history metadata file");
-                goto out;
-        }
-
-        /* TODO: handle short reads and EOF. */
-        if (gf_is_changelog_usable (ccd->changelog) == 1) {
-
-                ret = gf_changelog_consume (ccd->this,
-                                            ccd->jnl, ccd->changelog, _gf_true);
-                if (ret) {
-                        gf_smsg (this->name, GF_LOG_ERROR,
-                                 0, CHANGELOG_LIB_MSG_PARSE_ERROR,
-                                 "could not parse changelog",
-                                 "name=%s", ccd->changelog,
-                                 NULL);
-                        goto out;
-                }
+    int ret = -1;
+    ssize_t nread = 0;
+    xlator_t *this = NULL;
+    gf_changelog_consume_data_t *ccd = NULL;
+
+    ccd = (gf_changelog_consume_data_t *)data;
+    this = ccd->this;
+
+    ccd->retval = -1;
+
+    nread = sys_pread(ccd->fd, ccd->changelog, PATH_MAX - 1, ccd->offset);
+    if (nread < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_READ_ERROR,
+               "cannot read from history metadata file");
+        goto out;
+    }
+
+    /* TODO: handle short reads and EOF. */
+    if (gf_is_changelog_usable(ccd->changelog) == 1) {
+        ret = gf_changelog_consume(ccd->this, ccd->jnl, ccd->changelog,
+                                   _gf_true);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_PARSE_ERROR,
+                    "name=%s", ccd->changelog, NULL);
+            goto out;
         }
-        ccd->retval = 0;
+    }
+    ccd->retval = 0;
 
- out:
-        return NULL;
+out:
+    return NULL;
 }
 
 /**
@@ -551,133 +538,132 @@ gf_changelog_consume_wrap (void* data)
  * to index "to" in open htime file whose fd is "fd".
  */
 
-#define MAX_PARALLELS  10
+#define MAX_PARALLELS 10
 
 void *
-gf_history_consume (void * data)
+gf_history_consume(void *data)
 {
-        xlator_t                    *this              = NULL;
-        gf_changelog_journal_t              *jnl               = NULL;
-        gf_changelog_journal_t              *hist_jnl          = NULL;
-        int                          ret               = 0;
-        int                          iter              = 0;
-        int                          fd                = -1;
-        int                          from              = -1;
-        int                          to                = -1;
-        int                          len               = -1;
-        int                          n_parallel        = 0;
-        int                          n_envoked         = 0;
-        gf_boolean_t                 publish           = _gf_true;
-        pthread_t th_id[MAX_PARALLELS]                 = {0,};
-        gf_changelog_history_data_t *hist_data         = NULL;
-        gf_changelog_consume_data_t ccd[MAX_PARALLELS] = {{0},};
-        gf_changelog_consume_data_t *curr              = NULL;
-        char thread_name[GF_THREAD_NAMEMAX]            = {0,};
-
-        hist_data = (gf_changelog_history_data_t *) data;
-        if (hist_data == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        fd         = hist_data->htime_fd;
-        from       = hist_data->from;
-        to         = hist_data->to;
-        len        = hist_data->len;
-        n_parallel = hist_data->n_parallel;
-
-        THIS = hist_data->this;
-        this = hist_data->this;
-        if (!this) {
-                ret = -1;
-                goto out;
-        }
-
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl) {
-                ret = -1;
-                goto out;
-        }
-
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl) {
-                ret = -1;
-                goto out;
-        }
-
-        while (from <= to) {
-                n_envoked = 0;
-
-                for (iter = 0 ; (iter < n_parallel) && (from <= to); iter++) {
-                        curr = &ccd[iter];
-
-                        curr->this   = this;
-                        curr->jnl    = hist_jnl;
-                        curr->fd     = fd;
-                        curr->offset = from * (len + 1);
-
-                        curr->retval = 0;
-                        memset (curr->changelog, '\0', PATH_MAX);
-                        snprintf (thread_name, sizeof(thread_name), "%s%d",
-                                  "clogc", iter + 1);
-
-                        ret = gf_thread_create (&th_id[iter], NULL,
-                                                gf_changelog_consume_wrap, curr,
-                                                thread_name);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, ret,
-                                        CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED
-                                        , "could not create consume-thread");
-                                goto sync;
-                        } else
-                                n_envoked++;
-
-                        from++;
-                }
-
-        sync:
-                for (iter = 0; iter < n_envoked; iter++) {
-                        ret = pthread_join (th_id[iter], NULL);
-                        if (ret) {
-                                publish = _gf_false;
-                                gf_msg (this->name, GF_LOG_ERROR, ret,
-                                        CHANGELOG_LIB_MSG_PTHREAD_JOIN_FAILED,
-                                        "pthread_join() error");
-                                /* try to join the rest */
-                                continue;
-                        }
-
-                        if (publish == _gf_false)
-                                continue;
-
-                        curr = &ccd[iter];
-                        if (ccd->retval) {
-                                publish = _gf_false;
-                                gf_msg (this->name, GF_LOG_ERROR,
-                                        0, CHANGELOG_LIB_MSG_PARSE_ERROR,
-                                        "parsing error, ceased publishing...");
-                                continue;
-                        }
-
-                        ret = gf_changelog_publish (curr->this,
-                                                    curr->jnl, curr->changelog);
-                        if (ret) {
-                                publish = _gf_false;
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_LIB_MSG_PUBLISH_ERROR,
-                                        "publish error, ceased publishing...");
-                        }
-                }
-        }
-
-       /* informing "parsing done". */
-        hist_jnl->hist_done = (publish == _gf_true) ? 0 : -1;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    int ret = 0;
+    int iter = 0;
+    int fd = -1;
+    int from = -1;
+    int to = -1;
+    int len = -1;
+    int n_parallel = 0;
+    int n_envoked = 0;
+    gf_boolean_t publish = _gf_true;
+    pthread_t th_id[MAX_PARALLELS] = {
+        0,
+    };
+    gf_changelog_history_data_t *hist_data = NULL;
+    gf_changelog_consume_data_t ccd[MAX_PARALLELS] = {
+        {0},
+    };
+    gf_changelog_consume_data_t *curr = NULL;
+
+    hist_data = (gf_changelog_history_data_t *)data;
+    if (hist_data == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    fd = hist_data->htime_fd;
+    from = hist_data->from;
+    to = hist_data->to;
+    len = hist_data->len;
+    n_parallel = hist_data->n_parallel;
+
+    THIS = hist_data->this;
+    this = hist_data->this;
+    if (!this) {
+        ret = -1;
+        goto out;
+    }
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl) {
+        ret = -1;
+        goto out;
+    }
+
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl) {
+        ret = -1;
+        goto out;
+    }
+
+    while (from <= to) {
+        n_envoked = 0;
+
+        for (iter = 0; (iter < n_parallel) && (from <= to); iter++) {
+            curr = &ccd[iter];
+
+            curr->this = this;
+            curr->jnl = hist_jnl;
+            curr->fd = fd;
+            curr->offset = from * (len + 1);
+
+            curr->retval = 0;
+            memset(curr->changelog, '\0', PATH_MAX);
+
+            ret = gf_thread_create(&th_id[iter], NULL,
+                                   gf_changelog_consume_wrap, curr,
+                                   "clogc%03hx", (iter + 1) & 0x3ff);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ret,
+                       CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED,
+                       "could not create consume-thread");
+                goto sync;
+            } else
+                n_envoked++;
+
+            from++;
+        }
+
+    sync:
+        for (iter = 0; iter < n_envoked; iter++) {
+            ret = pthread_join(th_id[iter], NULL);
+            if (ret) {
+                publish = _gf_false;
+                gf_msg(this->name, GF_LOG_ERROR, ret,
+                       CHANGELOG_LIB_MSG_PTHREAD_JOIN_FAILED,
+                       "pthread_join() error");
+                /* try to join the rest */
+                continue;
+            }
+
+            if (publish == _gf_false)
+                continue;
+
+            curr = &ccd[iter];
+            if (ccd->retval) {
+                publish = _gf_false;
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_LIB_MSG_PARSE_ERROR_CEASED, NULL);
+                continue;
+            }
+
+            ret = gf_changelog_publish(curr->this, curr->jnl, curr->changelog);
+            if (ret) {
+                publish = _gf_false;
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       CHANGELOG_LIB_MSG_PUBLISH_ERROR,
+                       "publish error, ceased publishing...");
+            }
+        }
+    }
+
+    /* informing "parsing done". */
+    hist_jnl->hist_done = (publish == _gf_true) ? 0 : -1;
 
 out:
-        if (fd != -1)
-                (void) sys_close (fd);
-        GF_FREE (hist_data);
-        return NULL;
+    if (fd != -1)
+        (void)sys_close(fd);
+    GF_FREE(hist_data);
+    return NULL;
 }
 
 /**
@@ -707,317 +693,328 @@ out:
  *   -2 : Ignore this metadata file and process next
  */
 int
-gf_changelog_extract_min_max (const char *dname, const char *htime_dir,
-                              int *fd, unsigned long *total,
-                              unsigned long *min_ts, unsigned long *max_ts)
+gf_changelog_extract_min_max(const char *dname, const char *htime_dir, int *fd,
+                             unsigned long *total, unsigned long *min_ts,
+                             unsigned long *max_ts)
 {
-        int          ret          = -1;
-        xlator_t    *this         = NULL;
-        char htime_file[PATH_MAX] = {0,};
-        struct stat  stbuf        = {0,};
-        char        *iter         = NULL;
-        char x_value[30]          = {0,};
-
-        this = THIS;
-
-        snprintf (htime_file, PATH_MAX, "%s/%s", htime_dir, dname);
+    int ret = -1;
+    xlator_t *this = NULL;
+    char htime_file[PATH_MAX] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+    char *iter = NULL;
+    char x_value[30] = {
+        0,
+    };
+
+    this = THIS;
+
+    snprintf(htime_file, PATH_MAX, "%s/%s", htime_dir, dname);
+
+    iter = (htime_file + strlen(htime_file) - TIMESTAMP_LENGTH);
+    sscanf(iter, "%lu", min_ts);
+
+    ret = sys_stat(htime_file, &stbuf);
+    if (ret) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_HTIME_ERROR,
+                "op=stat", "path=%s", htime_file, NULL);
+        goto out;
+    }
+
+    /* ignore everything except regular files */
+    if (!S_ISREG(stbuf.st_mode)) {
+        ret = -2;
+        goto out;
+    }
+
+    *fd = open(htime_file, O_RDONLY);
+    if (*fd < 0) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_HTIME_ERROR,
+                "op=open", "path=%s", htime_file, NULL);
+        goto out;
+    }
+
+    /* Looks good, extract max timestamp */
+    ret = sys_fgetxattr(*fd, HTIME_KEY, x_value, sizeof(x_value));
+    if (ret < 0) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_GET_XATTR_FAILED, "path=%s", htime_file,
+                NULL);
+        goto out;
+    }
+
+    sscanf(x_value, "%lu:%lu", max_ts, total);
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_LIB_MSG_MIN_MAX_INFO,
+            "min=%lu", *min_ts, "max=%lu", *max_ts, "total_changelogs=%lu",
+            *total, NULL);
+
+    ret = 0;
 
-        iter = (htime_file + strlen (htime_file) - TIMESTAMP_LENGTH);
-        sscanf (iter ,"%lu",min_ts);
+out:
+    return ret;
+}
 
-        ret = sys_stat (htime_file, &stbuf);
+/* gf_history_changelog returns actual_end and spawns threads to
+ * parse historical changelogs. The return values are as follows.
+ *     0 : On success
+ *     1 : Successful, but partial historical changelogs available,
+ *         end time falls into different htime file or future time
+ *    -2 : Error, requested historical changelog not available, not
+ *         even partial
+ *    -1 : On any error
+ */
+int
+gf_history_changelog(char *changelog_dir, unsigned long start,
+                     unsigned long end, int n_parallel,
+                     unsigned long *actual_end)
+{
+    int ret = 0;
+    int len = -1;
+    int fd = -1;
+    int n_read = -1;
+    unsigned long min_ts = 0;
+    unsigned long max_ts = 0;
+    unsigned long end2 = 0;
+    unsigned long ts1 = 0;
+    unsigned long ts2 = 0;
+    unsigned long to = 0;
+    unsigned long from = 0;
+    unsigned long total_changelog = 0;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    gf_changelog_history_data_t *hist_data = NULL;
+    DIR *dirp = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    pthread_t consume_th = 0;
+    char htime_dir[PATH_MAX] = {
+        0,
+    };
+    char buffer[PATH_MAX] = {
+        0,
+    };
+    gf_boolean_t partial_history = _gf_false;
+
+    pthread_attr_t attr;
+
+    this = THIS;
+    if (!this) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = pthread_attr_init(&attr);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_PTHREAD_ERROR,
+               "Pthread init failed");
+        return -1;
+    }
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl) {
+        ret = -1;
+        goto out;
+    }
+
+    hist_jnl = (gf_changelog_journal_t *)jnl->hist_jnl;
+    if (!hist_jnl) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_LIB_MSG_REQUESTING_INFO,
+            "start=%lu", start, "end=%lu", end, NULL);
+
+    /* basic sanity check */
+    if (start > end || n_parallel <= 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_HIST_FAILED,
+                "start=%lu", start, "end=%lu", end, "thread_count=%d",
+                n_parallel, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    /* cap parallelism count */
+    if (n_parallel > MAX_PARALLELS)
+        n_parallel = MAX_PARALLELS;
+
+    CHANGELOG_FILL_HTIME_DIR(changelog_dir, htime_dir);
+
+    dirp = sys_opendir(htime_dir);
+    if (dirp == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_HTIME_ERROR,
+                "op=opendir", "path=%s", htime_dir, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    for (;;) {
+        errno = 0;
+
+        entry = sys_readdir(dirp, scratch);
+
+        if (!entry || errno != 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_LIB_MSG_HIST_FAILED, "start=%lu", start,
+                    "end=%lu", end, NULL);
+            ret = -2;
+            break;
+        }
+
+        ret = gf_changelog_extract_min_max(entry->d_name, htime_dir, &fd,
+                                           &total_changelog, &min_ts, &max_ts);
         if (ret) {
-                ret = -1;
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_HTIME_ERROR,
-                         "stat() failed on htime file",
-                         "path=%s", htime_file,
-                         NULL);
-                goto out;
-        }
-
-        /* ignore everything except regular files */
-        if (!S_ISREG (stbuf.st_mode)) {
-                ret = -2;
-                goto out;
+            if (-2 == ret)
+                continue;
+            goto out;
         }
 
-        *fd = open (htime_file, O_RDONLY);
-        if (*fd < 0) {
+        if (start >= min_ts && start < max_ts) {
+            /**
+             * TODO: handle short reads later...
+             */
+            n_read = sys_read(fd, buffer, PATH_MAX);
+            if (n_read < 0) {
                 ret = -1;
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_HTIME_ERROR,
-                         "open() failed for htime file",
-                         "path=%s", htime_file,
-                         NULL);
+                gf_msg(this->name, GF_LOG_ERROR, errno,
+                       CHANGELOG_LIB_MSG_READ_ERROR,
+                       "unable to read htime file");
                 goto out;
-        }
+            }
 
-        /* Looks good, extract max timestamp */
-        ret = sys_fgetxattr (*fd, HTIME_KEY, x_value, sizeof (x_value));
-        if (ret < 0) {
-                ret = -1;
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_GET_XATTR_FAILED,
-                         "error extracting max timstamp from htime file"
-                         "path=%s", htime_file,
-                         NULL);
-                goto out;
-        }
-
-        sscanf (x_value, "%lu:%lu", max_ts, total);
-        gf_smsg (this->name, GF_LOG_INFO, 0,
-                 CHANGELOG_LIB_MSG_TOTAL_LOG_INFO,
-                 "changelogs min max",
-                 "min=%lu", *min_ts,
-                 "max=%lu", *max_ts,
-                 "total_changelogs=%lu", *total,
-                 NULL);
+            len = strlen(buffer);
 
-        ret = 0;
+            /**
+             * search @start in the htime file returning it's index
+             * (@from)
+             */
+            from = gf_history_b_search(fd, start, 0, total_changelog - 1, len);
 
- out:
-        return ret;
-}
-
-int
-gf_history_changelog (char* changelog_dir, unsigned long start,
-                      unsigned long end, int n_parallel,
-                      unsigned long *actual_end)
-{
-        int                             ret                     = 0;
-        int                             len                     = -1;
-        int                             fd                      = -1;
-        int                             n_read                  = -1;
-        unsigned long                   min_ts                  = 0;
-        unsigned long                   max_ts                  = 0;
-        unsigned long                   end2                    = 0;
-        unsigned long                   ts1                     = 0;
-        unsigned long                   ts2                     = 0;
-        unsigned long                   to                      = 0;
-        unsigned long                   from                    = 0;
-        unsigned long                   total_changelog         = 0;
-        xlator_t                       *this                    = NULL;
-        gf_changelog_journal_t         *jnl                     = NULL;
-        gf_changelog_journal_t         *hist_jnl                = NULL;
-        gf_changelog_history_data_t    *hist_data               = NULL;
-        DIR                            *dirp                    = NULL;
-        struct dirent                  *entry                   = NULL;
-        struct dirent                   scratch[2]              = {{0,},};
-        pthread_t                       consume_th              = 0;
-        char                            htime_dir[PATH_MAX]     = {0,};
-        char                            buffer[PATH_MAX]        = {0,};
-
-        pthread_attr_t attr;
-
-        this = THIS;
-        if (!this) {
+            /* ensuring correctness of gf_b_search */
+            if (gf_history_check(fd, from, start, len) != 0) {
+                ret = -1;
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_LIB_MSG_GET_TIME_ERROR, "for=start",
+                        "start=%lu", start, "idx=%lu", from, NULL);
+                goto out;
+            }
+
+            end2 = (end <= max_ts) ? end : max_ts;
+
+            /* Check if end falls out of same HTIME file. The end
+             * falling to a different htime file or changelog
+             * disable-enable is detected only after 20 seconds.
+             * This is required because, applications generally
+             * asks historical changelogs till current time and
+             * it is possible changelog is not rolled over yet.
+             * So, buffer time of default rollover time plus 5
+             * seconds is subtracted.  If the application requests
+             * the end time with in half a minute of changelog
+             * disable, it's not detected as changelog disable and
+             * it's application's responsibility to retry after
+             * 20 seconds before confirming it as partial history.
+             */
+            if ((end - 20) > max_ts) {
+                partial_history = _gf_true;
+            }
+
+            /**
+             * search @end2 in htime file returning it's index (@to)
+             */
+            to = gf_history_b_search(fd, end2, 0, total_changelog - 1, len);
+
+            if (gf_history_check(fd, to, end2, len) != 0) {
                 ret = -1;
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_LIB_MSG_GET_TIME_ERROR, "for=end",
+                        "start=%lu", end2, "idx=%lu", to, NULL);
                 goto out;
-        }
+            }
 
-        ret = pthread_attr_init (&attr);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_LIB_MSG_PTHREAD_ERROR,
-                        "Pthread init failed");
-                return -1;
-        }
+            ret = gf_history_get_timestamp(fd, from, len, &ts1);
+            if (ret == -1)
+                goto out;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl) {
-                ret = -1;
+            ret = gf_history_get_timestamp(fd, to, len, &ts2);
+            if (ret == -1)
                 goto out;
-        }
 
-        hist_jnl = (gf_changelog_journal_t *) jnl->hist_jnl;
-        if (!hist_jnl) {
+            gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_LIB_MSG_FINAL_INFO,
+                    "from=%lu", ts1, "to=%lu", ts2, "changes=%lu",
+                    (to - from + 1), NULL);
+
+            hist_data = GF_CALLOC(1, sizeof(gf_changelog_history_data_t),
+                                  gf_changelog_mt_history_data_t);
+
+            hist_data->htime_fd = fd;
+            hist_data->from = from;
+            hist_data->to = to;
+            hist_data->len = len;
+            hist_data->n_parallel = n_parallel;
+            hist_data->this = this;
+
+            ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+            if (ret != 0) {
+                gf_msg(this->name, GF_LOG_ERROR, ret,
+                       CHANGELOG_LIB_MSG_PTHREAD_ERROR,
+                       "unable to sets the detach"
+                       " state attribute");
                 ret = -1;
                 goto out;
-        }
+            }
 
-        /* basic sanity check */
-        if (start > end || n_parallel <= 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_HIST_FAILED, "Sanity check failed",
-                         "start=%lu", start,
-                         "end=%lu", end,
-                         "thread_count=%d", n_parallel,
-                         NULL);
+            /* spawn a thread for background parsing & publishing */
+            ret = gf_thread_create(&consume_th, &attr, gf_history_consume,
+                                   hist_data, "cloghcon");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ret,
+                       CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED,
+                       "creation of consume parent-thread"
+                       " failed.");
                 ret = -1;
                 goto out;
-        }
+            }
 
-        /* cap parallelism count */
-        if (n_parallel > MAX_PARALLELS)
-                n_parallel = MAX_PARALLELS;
+            goto out;
 
-        CHANGELOG_FILL_HTIME_DIR (changelog_dir, htime_dir);
-
-        dirp = sys_opendir (htime_dir);
-        if (dirp == NULL) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_LIB_MSG_HTIME_ERROR,
-                         "open dir on htime failed",
-                         "path=%s", htime_dir,
-                         NULL);
-                ret = -1;
-                goto out;
+        } else { /* end of range check */
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_LIB_MSG_HIST_FAILED, "start=%lu", start,
+                    "end=%lu", end, "chlog_min=%lu", min_ts, "chlog_max=%lu",
+                    max_ts, NULL);
         }
-
-        for (;;) {
-
-                errno = 0;
-
-                entry = sys_readdir (dirp, scratch);
-
-                if (!entry || errno != 0)
-                        break;
-
-                ret = gf_changelog_extract_min_max (entry->d_name, htime_dir,
-                                                    &fd, &total_changelog,
-                                                    &min_ts, &max_ts);
-                if (ret) {
-                        if (-2 == ret)
-                                continue;
-                        goto out;
-                }
-
-                if (start >= min_ts && start < max_ts) {
-                        /**
-                         * TODO: handle short reads later...
-                         */
-                        n_read = sys_read (fd, buffer, PATH_MAX);
-                        if (n_read < 0) {
-                                ret = -1;
-                                gf_msg (this->name, GF_LOG_ERROR, errno,
-                                        CHANGELOG_LIB_MSG_READ_ERROR,
-                                        "unable to read htime file");
-                                goto out;
-                        }
-
-                        len = strlen (buffer);
-
-                        /**
-                         * search @start in the htime file returning it's index
-                         * (@from)
-                         */
-                        from = gf_history_b_search (fd, start, 0,
-                                                   total_changelog - 1, len);
-
-                        /* ensuring correctness of gf_b_search */
-                        if (gf_history_check (fd, from, start, len) != 0) {
-                                ret = -1;
-                                gf_smsg (this->name, GF_LOG_ERROR, 0,
-                                         CHANGELOG_LIB_MSG_GET_TIME_ERROR,
-                                         "wrong result for start",
-                                         "start=%lu", start,
-                                         "idx=%lu", from,
-                                         NULL);
-                                goto out;
-                        }
-
-                        end2 = (end <= max_ts) ? end : max_ts;
-
-                        /**
-                         * search @end2 in htime file returning it's index (@to)
-                         */
-                        to = gf_history_b_search (fd, end2,
-                                                  0, total_changelog - 1, len);
-
-                        if (gf_history_check (fd, to, end2, len) != 0) {
-                                ret = -1;
-                                gf_smsg (this->name, GF_LOG_ERROR, 0,
-                                         CHANGELOG_LIB_MSG_GET_TIME_ERROR,
-                                         "wrong result for end",
-                                         "start=%lu", end2,
-                                         "idx=%lu", to,
-                                         NULL);
-                                goto out;
-                        }
-
-                        ret = gf_history_get_timestamp (fd, from, len, &ts1);
-                        if (ret == -1)
-                                goto out;
-
-                        ret = gf_history_get_timestamp (fd, to, len, &ts2);
-                        if (ret == -1)
-                                goto out;
-
-                        gf_smsg (this->name, GF_LOG_INFO, 0,
-                                 CHANGELOG_LIB_MSG_TOTAL_LOG_INFO,
-                                 "FINAL",
-                                 "from=%lu", ts1,
-                                 "to=%lu", ts2,
-                                 "changes=%lu", (to - from + 1),
-                                 NULL);
-
-                        hist_data =  GF_CALLOC (1,
-                                     sizeof (gf_changelog_history_data_t),
-                                     gf_changelog_mt_history_data_t);
-
-                        hist_data->htime_fd   = fd;
-                        hist_data->from       = from;
-                        hist_data->to         = to;
-                        hist_data->len        = len;
-                        hist_data->n_parallel = n_parallel;
-                        hist_data->this       = this;
-
-                        ret = pthread_attr_setdetachstate
-                                (&attr, PTHREAD_CREATE_DETACHED);
-                        if (ret != 0) {
-                                gf_msg (this->name, GF_LOG_ERROR, ret,
-                                        CHANGELOG_LIB_MSG_PTHREAD_ERROR,
-                                        "unable to sets the detach"
-                                        " state attribute");
-                                ret = -1;
-                                goto out;
-                        }
-
-                        /* spawn a thread for background parsing & publishing */
-                        ret = gf_thread_create (&consume_th, &attr,
-                                                gf_history_consume, hist_data,
-                                                "cloghcon");
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, ret,
-                                        CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED
-                                        , "creation of consume parent-thread"
-                                        " failed.");
-                                ret = -1;
-                                goto out;
-                        }
-
-                        goto out;
-
-                } else {/* end of range check */
-                        gf_smsg (this->name, GF_LOG_ERROR, errno,
-                                 CHANGELOG_LIB_MSG_HIST_FAILED,
-                                 "Requested changelog "
-                                 "range is not available.",
-                                 "start=%lu", start,
-                                 "chlog_min=%lu", min_ts,
-                                 "chlog_max=%lu", max_ts,
-                                 NULL);
-                        ret = -2;
-                        goto out;
-                }
-        } /* end of readdir() */
+    } /* end of readdir() */
 
 out:
-        if (dirp != NULL)
-                (void) sys_closedir (dirp);
+    if (dirp != NULL)
+        (void)sys_closedir(dirp);
 
-        if (ret < 0) {
-                if (fd != -1)
-                        (void) sys_close (fd);
-                GF_FREE (hist_data);
-                (void) pthread_attr_destroy (&attr);
+    if (ret < 0) {
+        if (fd != -1)
+            (void)sys_close(fd);
+        GF_FREE(hist_data);
+        (void)pthread_attr_destroy(&attr);
 
-                return ret;
-        }
+        return ret;
+    }
 
-        hist_jnl->hist_done = 1;
-        *actual_end = ts2;
+    hist_jnl->hist_done = 1;
+    *actual_end = ts2;
 
-        return ret;
+    if (partial_history) {
+        ret = 1;
+    }
+
+    return ret;
 }
diff --git a/xlators/features/changelog/src/changelog-barrier.c b/xlators/features/changelog/src/changelog-barrier.c
index ac1eb0e4397..0fb89ddb127 100644
--- a/xlators/features/changelog/src/changelog-barrier.c
+++ b/xlators/features/changelog/src/changelog-barrier.c
@@ -10,125 +10,122 @@
 
 #include "changelog-helpers.h"
 #include "changelog-messages.h"
-#include "call-stub.h"
+#include <glusterfs/call-stub.h>
 
 /* Enqueue a stub*/
 void
-__chlog_barrier_enqueue (xlator_t *this, call_stub_t *stub)
+__chlog_barrier_enqueue(xlator_t *this, call_stub_t *stub)
 {
-        changelog_priv_t *priv    = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        list_add_tail (&stub->list, &priv->queue);
-        priv->queue_size++;
+    list_add_tail(&stub->list, &priv->queue);
+    priv->queue_size++;
 
-        return;
+    return;
 }
 
 /* Dequeue a stub */
 call_stub_t *
-__chlog_barrier_dequeue (xlator_t *this, struct list_head *queue)
+__chlog_barrier_dequeue(xlator_t *this, struct list_head *queue)
 {
-        call_stub_t      *stub            = NULL;
-        changelog_priv_t *priv            = NULL;
+    call_stub_t *stub = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (list_empty (queue))
-                goto out;
+    if (list_empty(queue))
+        goto out;
 
-        stub = list_entry (queue->next, call_stub_t, list);
-        list_del_init (&stub->list);
+    stub = list_entry(queue->next, call_stub_t, list);
+    list_del_init(&stub->list);
 
 out:
-        return stub;
+    return stub;
 }
 
 /* Dequeue all the stubs and call corresponding resume functions */
 void
-chlog_barrier_dequeue_all (xlator_t *this, struct list_head *queue)
+chlog_barrier_dequeue_all(xlator_t *this, struct list_head *queue)
 {
-        call_stub_t            *stub    = NULL;
+    call_stub_t *stub = NULL;
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                CHANGELOG_MSG_BARRIER_INFO,
-                "Dequeuing all the changelog barriered fops");
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS,
+            NULL);
 
-        while ((stub = __chlog_barrier_dequeue (this, queue)))
-                call_resume (stub);
+    while ((stub = __chlog_barrier_dequeue(this, queue)))
+        call_resume(stub);
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                CHANGELOG_MSG_BARRIER_INFO,
-                "Dequeuing changelog barriered fops is finished");
-        return;
+    gf_smsg(this->name, GF_LOG_INFO, 0,
+            CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS_FINISHED, NULL);
+    return;
 }
 
 /* Function called on changelog barrier timeout */
 void
-chlog_barrier_timeout (void *data)
+chlog_barrier_timeout(void *data)
 {
-        xlator_t               *this    = NULL;
-        changelog_priv_t       *priv    = NULL;
-        struct list_head        queue   = {0,};
+    xlator_t *this = NULL;
+    changelog_priv_t *priv = NULL;
+    struct list_head queue = {
+        0,
+    };
 
-        this = data;
-        THIS = this;
-        priv = this->private;
+    this = data;
+    THIS = this;
+    priv = this->private;
 
-        INIT_LIST_HEAD (&queue);
+    INIT_LIST_HEAD(&queue);
 
-        gf_msg (this->name, GF_LOG_ERROR, 0,
-                CHANGELOG_MSG_BARRIER_ERROR,
-                "Disabling changelog barrier because of the timeout.");
+    gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_BARRIER_TIMEOUT, NULL);
 
-        LOCK (&priv->lock);
-        {
-                __chlog_barrier_disable (this, &queue);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        __chlog_barrier_disable(this, &queue);
+    }
+    UNLOCK(&priv->lock);
 
-        chlog_barrier_dequeue_all (this, &queue);
+    chlog_barrier_dequeue_all(this, &queue);
 
-        return;
+    return;
 }
 
 /* Disable changelog barrier enable flag */
 void
-__chlog_barrier_disable (xlator_t *this, struct list_head *queue)
+__chlog_barrier_disable(xlator_t *this, struct list_head *queue)
 {
-        changelog_priv_t  *priv   = this->private;
-        GF_ASSERT (priv);
+    changelog_priv_t *priv = this->private;
+    GF_ASSERT(priv);
 
-        if (priv->timer) {
-                gf_timer_call_cancel (this->ctx, priv->timer);
-                priv->timer = NULL;
-        }
+    if (priv->timer) {
+        gf_timer_call_cancel(this->ctx, priv->timer);
+        priv->timer = NULL;
+    }
 
-        list_splice_init (&priv->queue, queue);
-        priv->queue_size = 0;
-        priv->barrier_enabled = _gf_false;
+    list_splice_init(&priv->queue, queue);
+    priv->queue_size = 0;
+    priv->barrier_enabled = _gf_false;
 }
 
 /* Enable chagelog barrier enable with timer */
 int
-__chlog_barrier_enable (xlator_t *this, changelog_priv_t *priv)
+__chlog_barrier_enable(xlator_t *this, changelog_priv_t *priv)
 {
-        int             ret     = -1;
-
-        priv->timer = gf_timer_call_after (this->ctx, priv->timeout,
-                                           chlog_barrier_timeout, (void *)this);
-        if (!priv->timer) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        CHANGELOG_MSG_BARRIER_ERROR,
-                        "Couldn't add changelog barrier timeout event.");
-                goto out;
-        }
-
-        priv->barrier_enabled = _gf_true;
-        ret = 0;
+    int ret = -1;
+
+    priv->timer = gf_timer_call_after(this->ctx, priv->timeout,
+                                      chlog_barrier_timeout, (void *)this);
+    if (!priv->timer) {
+        gf_smsg(this->name, GF_LOG_CRITICAL, 0,
+                CHANGELOG_MSG_TIMEOUT_ADD_FAILED, NULL);
+        goto out;
+    }
+
+    priv->barrier_enabled = _gf_true;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/changelog/src/changelog-encoders.c b/xlators/features/changelog/src/changelog-encoders.c
index 95030236636..63754516c2e 100644
--- a/xlators/features/changelog/src/changelog-encoders.c
+++ b/xlators/features/changelog/src/changelog-encoders.c
@@ -11,117 +11,117 @@
 #include "changelog-encoders.h"
 
 size_t
-entry_fn (void *data, char *buffer, gf_boolean_t encode)
+entry_fn(void *data, char *buffer, gf_boolean_t encode)
 {
-        char    *tmpbuf = NULL;
-        size_t  bufsz  = 0;
-        struct changelog_entry_fields *ce = NULL;
-
-        ce = (struct changelog_entry_fields *) data;
-
-        if (encode) {
-                tmpbuf = uuid_utoa (ce->cef_uuid);
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, tmpbuf, strlen (tmpbuf));
-        } else {
-                CHANGELOG_FILL_BUFFER (buffer, bufsz,
-                                       ce->cef_uuid, sizeof (uuid_t));
-        }
-
-        CHANGELOG_FILL_BUFFER (buffer, bufsz, "/", 1);
-        CHANGELOG_FILL_BUFFER (buffer, bufsz,
-                               ce->cef_bname, strlen (ce->cef_bname));
-        return bufsz;
+    char *tmpbuf = NULL;
+    size_t bufsz = 0;
+    struct changelog_entry_fields *ce = NULL;
+
+    ce = (struct changelog_entry_fields *)data;
+
+    if (encode) {
+        tmpbuf = uuid_utoa(ce->cef_uuid);
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, tmpbuf, strlen(tmpbuf));
+    } else {
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_uuid, sizeof(uuid_t));
+    }
+
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, "/", 1);
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_bname, strlen(ce->cef_bname));
+    return bufsz;
 }
 
 size_t
-del_entry_fn (void *data, char *buffer, gf_boolean_t encode)
+del_entry_fn(void *data, char *buffer, gf_boolean_t encode)
 {
-        char    *tmpbuf = NULL;
-        size_t  bufsz  = 0;
-        struct changelog_entry_fields *ce = NULL;
-
-        ce = (struct changelog_entry_fields *) data;
-
-        if (encode) {
-                tmpbuf = uuid_utoa (ce->cef_uuid);
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, tmpbuf, strlen (tmpbuf));
-        } else {
-                CHANGELOG_FILL_BUFFER (buffer, bufsz,
-                                       ce->cef_uuid, sizeof (uuid_t));
-        }
-
-        CHANGELOG_FILL_BUFFER (buffer, bufsz, "/", 1);
-        CHANGELOG_FILL_BUFFER (buffer, bufsz,
-                               ce->cef_bname, strlen (ce->cef_bname));
-        CHANGELOG_FILL_BUFFER (buffer, bufsz, "\0", 1);
-
-        if (ce->cef_path[0] == '\0') {
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, "\0", 1);
-        } else {
-                CHANGELOG_FILL_BUFFER (buffer, bufsz,
-                                       ce->cef_path, strlen (ce->cef_path));
-        }
-
-        return bufsz;
+    char *tmpbuf = NULL;
+    size_t bufsz = 0;
+    struct changelog_entry_fields *ce = NULL;
+
+    ce = (struct changelog_entry_fields *)data;
+
+    if (encode) {
+        tmpbuf = uuid_utoa(ce->cef_uuid);
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, tmpbuf, strlen(tmpbuf));
+    } else {
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_uuid, sizeof(uuid_t));
+    }
+
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, "/", 1);
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_bname, strlen(ce->cef_bname));
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, "\0", 1);
+
+    if (ce->cef_path[0] == '\0') {
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, "\0", 1);
+    } else {
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_path,
+                              strlen(ce->cef_path));
+    }
+
+    return bufsz;
 }
 
 size_t
-fop_fn (void *data, char *buffer, gf_boolean_t encode)
+fop_fn(void *data, char *buffer, gf_boolean_t encode)
 {
-        char buf[10]          = {0,};
-        size_t         bufsz = 0;
-        glusterfs_fop_t fop   = 0;
+    char buf[10] = {
+        0,
+    };
+    size_t bufsz = 0;
+    glusterfs_fop_t fop = 0;
 
-        fop = *(glusterfs_fop_t *) data;
+    fop = *(glusterfs_fop_t *)data;
 
-        if (encode) {
-                (void) snprintf (buf, sizeof (buf), "%d", fop);
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
-        } else
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, &fop, sizeof (fop));
+    if (encode) {
+        (void)snprintf(buf, sizeof(buf), "%d", fop);
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, buf, strlen(buf));
+    } else
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, &fop, sizeof(fop));
 
-        return bufsz;
+    return bufsz;
 }
 
 size_t
-number_fn (void *data, char *buffer, gf_boolean_t encode)
+number_fn(void *data, char *buffer, gf_boolean_t encode)
 {
-        size_t       bufsz = 0;
-        unsigned int nr    = 0;
-        char buf[20]       = {0,};
+    size_t bufsz = 0;
+    unsigned int nr = 0;
+    char buf[20] = {
+        0,
+    };
 
-        nr = *(unsigned int *) data;
+    nr = *(unsigned int *)data;
 
-        if (encode) {
-                (void) snprintf (buf, sizeof (buf), "%u", nr);
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
-        } else
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, &nr, sizeof (unsigned int));
+    if (encode) {
+        (void)snprintf(buf, sizeof(buf), "%u", nr);
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, buf, strlen(buf));
+    } else
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, &nr, sizeof(unsigned int));
 
-        return bufsz;
+    return bufsz;
 }
 
 void
-entry_free_fn (void *data)
+entry_free_fn(void *data)
 {
-        changelog_opt_t *co = data;
+    changelog_opt_t *co = data;
 
-        if (!co)
-                return;
+    if (!co)
+        return;
 
-        GF_FREE (co->co_entry.cef_bname);
+    GF_FREE(co->co_entry.cef_bname);
 }
 
 void
-del_entry_free_fn (void *data)
+del_entry_free_fn(void *data)
 {
-        changelog_opt_t *co = data;
+    changelog_opt_t *co = data;
 
-        if (!co)
-                return;
+    if (!co)
+        return;
 
-        GF_FREE (co->co_entry.cef_bname);
-        GF_FREE (co->co_entry.cef_path);
+    GF_FREE(co->co_entry.cef_bname);
+    GF_FREE(co->co_entry.cef_path);
 }
 
 /**
@@ -129,108 +129,104 @@ del_entry_free_fn (void *data)
  */
 
 static void
-changelog_encode_write_xtra (changelog_log_data_t *cld,
-                             char *buffer, size_t *off, gf_boolean_t encode)
+changelog_encode_write_xtra(changelog_log_data_t *cld, char *buffer,
+                            size_t *off, gf_boolean_t encode)
 {
-        int              i      = 0;
-        size_t           offset = 0;
-        void            *data   = NULL;
-        changelog_opt_t *co     = NULL;
-
-        offset = *off;
-
-        co = (changelog_opt_t *) cld->cld_ptr;
-
-        for (; i < cld->cld_xtra_records; i++, co++) {
-                CHANGELOG_FILL_BUFFER (buffer, offset, "\0", 1);
-
-                switch (co->co_type) {
-                case CHANGELOG_OPT_REC_FOP:
-                        data = &co->co_fop;
-                        break;
-                case CHANGELOG_OPT_REC_ENTRY:
-                        data = &co->co_entry;
-                        break;
-                case CHANGELOG_OPT_REC_UINT32:
-                        data = &co->co_uint32;
-                        break;
-                }
-
-                if (co->co_convert)
-                        offset += co->co_convert (data,
-                                                  buffer + offset, encode);
-                else /* no coversion: write it out as it is */
-                        CHANGELOG_FILL_BUFFER (buffer, offset,
-                                               data, co->co_len);
+    int i = 0;
+    size_t offset = 0;
+    void *data = NULL;
+    changelog_opt_t *co = NULL;
+
+    offset = *off;
+
+    co = (changelog_opt_t *)cld->cld_ptr;
+
+    for (; i < cld->cld_xtra_records; i++, co++) {
+        CHANGELOG_FILL_BUFFER(buffer, offset, "\0", 1);
+
+        switch (co->co_type) {
+            case CHANGELOG_OPT_REC_FOP:
+                data = &co->co_fop;
+                break;
+            case CHANGELOG_OPT_REC_ENTRY:
+                data = &co->co_entry;
+                break;
+            case CHANGELOG_OPT_REC_UINT32:
+                data = &co->co_uint32;
+                break;
         }
 
-        *off = offset;
+        if (co->co_convert)
+            offset += co->co_convert(data, buffer + offset, encode);
+        else /* no coversion: write it out as it is */
+            CHANGELOG_FILL_BUFFER(buffer, offset, data, co->co_len);
+    }
+
+    *off = offset;
 }
 
 int
-changelog_encode_ascii (xlator_t *this, changelog_log_data_t *cld)
+changelog_encode_ascii(xlator_t *this, changelog_log_data_t *cld)
 {
-        size_t            off      = 0;
-        size_t            gfid_len = 0;
-        char             *gfid_str = NULL;
-        char             *buffer   = NULL;
-        changelog_priv_t *priv     = NULL;
+    size_t off = 0;
+    size_t gfid_len = 0;
+    char *gfid_str = NULL;
+    char *buffer = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        gfid_str = uuid_utoa (cld->cld_gfid);
-        gfid_len = strlen (gfid_str);
+    gfid_str = uuid_utoa(cld->cld_gfid);
+    gfid_len = strlen(gfid_str);
 
-        /* extra bytes for decorations */
-        buffer = alloca (gfid_len + cld->cld_ptr_len + 10);
-        CHANGELOG_STORE_ASCII (priv, buffer,
-                               off, gfid_str, gfid_len, cld);
+    /* extra bytes for decorations */
+    buffer = alloca(gfid_len + cld->cld_ptr_len + 10);
+    CHANGELOG_STORE_ASCII(priv, buffer, off, gfid_str, gfid_len, cld);
 
-        if (cld->cld_xtra_records)
-                changelog_encode_write_xtra (cld, buffer, &off, _gf_true);
+    if (cld->cld_xtra_records)
+        changelog_encode_write_xtra(cld, buffer, &off, _gf_true);
 
-        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
+    CHANGELOG_FILL_BUFFER(buffer, off, "\0", 1);
 
-        return changelog_write_change (priv, buffer, off);
+    return changelog_write_change(priv, buffer, off);
 }
 
 int
-changelog_encode_binary (xlator_t *this, changelog_log_data_t *cld)
+changelog_encode_binary(xlator_t *this, changelog_log_data_t *cld)
 {
-        size_t            off    = 0;
-        char             *buffer = NULL;
-        changelog_priv_t *priv   = NULL;
+    size_t off = 0;
+    char *buffer = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        /* extra bytes for decorations */
-        buffer = alloca (sizeof (uuid_t) + cld->cld_ptr_len + 10);
-        CHANGELOG_STORE_BINARY (priv, buffer, off, cld->cld_gfid, cld);
+    /* extra bytes for decorations */
+    buffer = alloca(sizeof(uuid_t) + cld->cld_ptr_len + 10);
+    CHANGELOG_STORE_BINARY(priv, buffer, off, cld->cld_gfid, cld);
 
-        if (cld->cld_xtra_records)
-                changelog_encode_write_xtra (cld, buffer, &off, _gf_false);
+    if (cld->cld_xtra_records)
+        changelog_encode_write_xtra(cld, buffer, &off, _gf_false);
 
-        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
+    CHANGELOG_FILL_BUFFER(buffer, off, "\0", 1);
 
-        return changelog_write_change (priv, buffer, off);
+    return changelog_write_change(priv, buffer, off);
 }
 
-static struct changelog_encoder
-cb_encoder[] = {
-        [CHANGELOG_ENCODE_BINARY] =
+static struct changelog_encoder cb_encoder[] = {
+    [CHANGELOG_ENCODE_BINARY] =
         {
-                .encoder = CHANGELOG_ENCODE_BINARY,
-                .encode = changelog_encode_binary,
+            .encoder = CHANGELOG_ENCODE_BINARY,
+            .encode = changelog_encode_binary,
         },
-        [CHANGELOG_ENCODE_ASCII] =
+    [CHANGELOG_ENCODE_ASCII] =
         {
-                .encoder = CHANGELOG_ENCODE_ASCII,
-                .encode = changelog_encode_ascii,
+            .encoder = CHANGELOG_ENCODE_ASCII,
+            .encode = changelog_encode_ascii,
         },
 };
 
 void
 changelog_encode_change(changelog_priv_t *priv)
 {
-        priv->ce = &cb_encoder[priv->encode_mode];
+    priv->ce = &cb_encoder[priv->encode_mode];
 }
diff --git a/xlators/features/changelog/src/changelog-encoders.h b/xlators/features/changelog/src/changelog-encoders.h
index d6a50cc9ef7..26252696d56 100644
--- a/xlators/features/changelog/src/changelog-encoders.h
+++ b/xlators/features/changelog/src/changelog-encoders.h
@@ -11,41 +11,39 @@
 #ifndef _CHANGELOG_ENCODERS_H
 #define _CHANGELOG_ENCODERS_H
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "changelog-helpers.h"
 
-#define CHANGELOG_STORE_ASCII(priv, buf, off, gfid, gfid_len, cld) do { \
-                CHANGELOG_FILL_BUFFER (buffer, off,                     \
-                                       priv->maps[cld->cld_type], 1);   \
-                CHANGELOG_FILL_BUFFER (buffer,                          \
-                                       off, gfid, gfid_len);            \
-        } while (0)
+#define CHANGELOG_STORE_ASCII(priv, buf, off, gfid, gfid_len, cld)             \
+    do {                                                                       \
+        CHANGELOG_FILL_BUFFER(buffer, off, priv->maps[cld->cld_type], 1);      \
+        CHANGELOG_FILL_BUFFER(buffer, off, gfid, gfid_len);                    \
+    } while (0)
 
-#define CHANGELOG_STORE_BINARY(priv, buf, off, gfid, cld) do {          \
-                CHANGELOG_FILL_BUFFER (buffer, off,                     \
-                                       priv->maps[cld->cld_type], 1);   \
-                CHANGELOG_FILL_BUFFER (buffer,                          \
-                                       off, gfid, sizeof (uuid_t));     \
-        } while (0)
+#define CHANGELOG_STORE_BINARY(priv, buf, off, gfid, cld)                      \
+    do {                                                                       \
+        CHANGELOG_FILL_BUFFER(buffer, off, priv->maps[cld->cld_type], 1);      \
+        CHANGELOG_FILL_BUFFER(buffer, off, gfid, sizeof(uuid_t));              \
+    } while (0)
 
 size_t
-entry_fn (void *data, char *buffer, gf_boolean_t encode);
+entry_fn(void *data, char *buffer, gf_boolean_t encode);
 size_t
-del_entry_fn (void *data, char *buffer, gf_boolean_t encode);
+del_entry_fn(void *data, char *buffer, gf_boolean_t encode);
 size_t
-fop_fn (void *data, char *buffer, gf_boolean_t encode);
+fop_fn(void *data, char *buffer, gf_boolean_t encode);
 size_t
-number_fn (void *data, char *buffer, gf_boolean_t encode);
+number_fn(void *data, char *buffer, gf_boolean_t encode);
 void
-entry_free_fn (void *data);
+entry_free_fn(void *data);
 void
-del_entry_free_fn (void *data);
+del_entry_free_fn(void *data);
 int
-changelog_encode_binary (xlator_t *, changelog_log_data_t *);
+changelog_encode_binary(xlator_t *, changelog_log_data_t *);
 int
-changelog_encode_ascii (xlator_t *, changelog_log_data_t *);
+changelog_encode_ascii(xlator_t *, changelog_log_data_t *);
 void
 changelog_encode_change(changelog_priv_t *);
 
diff --git a/xlators/features/changelog/src/changelog-ev-handle.c b/xlators/features/changelog/src/changelog-ev-handle.c
index b5681944567..aa94459de5a 100644
--- a/xlators/features/changelog/src/changelog-ev-handle.c
+++ b/xlators/features/changelog/src/changelog-ev-handle.c
@@ -14,19 +14,19 @@
 
 struct rpc_clnt_program changelog_ev_program;
 
-#define NR_IOVEC  (MAX_IOVEC - 3)
+#define NR_IOVEC (MAX_IOVEC - 3)
 struct ev_rpc_vec {
-        int count;
-        struct iovec vector[NR_IOVEC];
+    int count;
+    struct iovec vector[NR_IOVEC];
 
-        /* sequence number */
-        unsigned long seq;
+    /* sequence number */
+    unsigned long seq;
 };
 
 struct ev_rpc {
-        rbuf_list_t     *rlist;
-        struct rpc_clnt *rpc;
-        struct ev_rpc_vec vec;
+    rbuf_list_t *rlist;
+    struct rpc_clnt *rpc;
+    struct ev_rpc_vec vec;
 };
 
 /**
@@ -35,217 +35,229 @@ struct ev_rpc {
  * intelligence can be built into the server.
  */
 int
-changelog_event_dispatch_cbk (struct rpc_req *req,
-                              struct iovec *iov, int count, void *myframe)
+changelog_event_dispatch_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                             void *myframe)
 {
-        return 0;
+    return 0;
 }
 
 /* dispatcher RPC */
 int
-changelog_dispatch_vec (call_frame_t *frame, xlator_t *this,
-                        struct rpc_clnt *rpc, struct ev_rpc_vec *vec)
+changelog_dispatch_vec(call_frame_t *frame, xlator_t *this,
+                       struct rpc_clnt *rpc, struct ev_rpc_vec *vec)
 {
-         struct timeval      tv  = {0,};
-         changelog_event_req req = {0,};
-
-         (void) gettimeofday (&tv, NULL);
-
-         /**
-          * Event dispatch RPC header contains a sequence number for each
-          * dispatch. This allows the receiver to order the request before
-          * processing.
-          */
-         req.seq     = vec->seq;
-         req.tv_sec  = tv.tv_sec;
-         req.tv_usec = tv.tv_usec;
-
-         return changelog_rpc_sumbit_req (rpc, (void *)&req,
-                                          frame, &changelog_ev_program,
-                                          CHANGELOG_REV_PROC_EVENT,
-                                          vec->vector, vec->count, NULL,
-                                          this, changelog_event_dispatch_cbk,
-                                          (xdrproc_t) xdr_changelog_event_req);
- }
-
- int
- changelog_event_dispatch_rpc (call_frame_t *frame, xlator_t *this, void *data)
- {
-         int                idx      = 0;
-         int                count    = 0;
-         int                ret      = 0;
-         unsigned long      sequence = 0;
-         rbuf_iovec_t      *rvec     = NULL;
-         struct ev_rpc     *erpc     = NULL;
-         struct rlist_iter  riter    = {{0,},};
-
-         /* dispatch NR_IOVEC IO vectors at a time. */
-
-         erpc = data;
-         sequence = erpc->rlist->seq[0];
-
-         rlist_iter_init (&riter, erpc->rlist);
-
-         rvec_for_each_entry (rvec, &riter) {
-                 idx = count % NR_IOVEC;
-                 if (++count == NR_IOVEC) {
-                         erpc->vec.vector[idx] = rvec->iov;
-                         erpc->vec.seq = sequence++;
-                         erpc->vec.count = NR_IOVEC;
-
-                         ret = changelog_dispatch_vec (frame, this,
-                                                       erpc->rpc, &erpc->vec);
-                         if (ret)
-                                 break;
-                         count = 0;
-                         continue;
-                 }
-
-                 erpc->vec.vector[idx] = rvec->iov;
-         }
-
-         if (ret)
-                 goto error_return;
-
-         idx = count % NR_IOVEC;
-         if (idx) {
-                 erpc->vec.seq = sequence;
-                 erpc->vec.count = idx;
-
-                 ret = changelog_dispatch_vec (frame, this,
-                                               erpc->rpc, &erpc->vec);
-         }
-
- error_return:
-         return ret;
+    struct timeval tv = {
+        0,
+    };
+    changelog_event_req req = {
+        0,
+    };
+
+    (void)gettimeofday(&tv, NULL);
+
+    /**
+     * Event dispatch RPC header contains a sequence number for each
+     * dispatch. This allows the receiver to order the request before
+     * processing.
+     */
+    req.seq = vec->seq;
+    req.tv_sec = tv.tv_sec;
+    req.tv_usec = tv.tv_usec;
+
+    return changelog_rpc_sumbit_req(
+        rpc, (void *)&req, frame, &changelog_ev_program,
+        CHANGELOG_REV_PROC_EVENT, vec->vector, vec->count, NULL, this,
+        changelog_event_dispatch_cbk, (xdrproc_t)xdr_changelog_event_req);
 }
 
 int
-changelog_rpc_notify (struct rpc_clnt *rpc,
-                      void *mydata, rpc_clnt_event_t event, void *data)
+changelog_event_dispatch_rpc(call_frame_t *frame, xlator_t *this, void *data)
 {
-        xlator_t                *this      = NULL;
-        changelog_rpc_clnt_t    *crpc      = NULL;
-        changelog_clnt_t        *c_clnt    = NULL;
-        changelog_priv_t        *priv      = NULL;
-        changelog_ev_selector_t *selection = NULL;
+    int idx = 0;
+    int count = 0;
+    int ret = 0;
+    unsigned long sequence = 0;
+    rbuf_iovec_t *rvec = NULL;
+    struct ev_rpc *erpc = NULL;
+    struct rlist_iter riter = {
+        {
+            0,
+        },
+    };
 
-        crpc = mydata;
-        this = crpc->this;
-        c_clnt = crpc->c_clnt;
+    /* dispatch NR_IOVEC IO vectors at a time. */
 
-        priv = this->private;
+    erpc = data;
+    sequence = erpc->rlist->seq[0];
 
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-                rpc_clnt_set_connected (&rpc->conn);
-                selection = &priv->ev_selection;
+    rlist_iter_init(&riter, erpc->rlist);
 
-                LOCK (&c_clnt->wait_lock);
-                {
-                        LOCK (&c_clnt->active_lock);
-                        {
-                                changelog_select_event (this, selection,
-                                                        crpc->filter);
-                                list_move_tail (&crpc->list, &c_clnt->active);
-                        }
-                        UNLOCK (&c_clnt->active_lock);
-                }
-                UNLOCK (&c_clnt->wait_lock);
+    rvec_for_each_entry(rvec, &riter)
+    {
+        idx = count % NR_IOVEC;
+        if (++count == NR_IOVEC) {
+            erpc->vec.vector[idx] = rvec->iov;
+            erpc->vec.seq = sequence++;
+            erpc->vec.count = NR_IOVEC;
 
+            ret = changelog_dispatch_vec(frame, this, erpc->rpc, &erpc->vec);
+            if (ret)
                 break;
-        case RPC_CLNT_DISCONNECT:
-                rpc_clnt_disable (crpc->rpc);
+            count = 0;
+            continue;
+        }
+
+        erpc->vec.vector[idx] = rvec->iov;
+    }
 
-                /* rpc_clnt_disable doesn't unref the rpc. It just marks
-                 * the rpc as disabled and cancels reconnection timer.
-                 * Hence unref the rpc object to free it.
-                 */
-                rpc_clnt_unref (crpc->rpc);
+    if (ret)
+        goto error_return;
 
-                if (priv)
-                        selection = &priv->ev_selection;
+    idx = count % NR_IOVEC;
+    if (idx) {
+        erpc->vec.seq = sequence;
+        erpc->vec.count = idx;
+
+        ret = changelog_dispatch_vec(frame, this, erpc->rpc, &erpc->vec);
+    }
+
+error_return:
+    return ret;
+}
 
-                LOCK (&crpc->lock);
+int
+changelog_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                     void *data)
+{
+    xlator_t *this = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_ev_selector_t *selection = NULL;
+    uint64_t clntcnt = 0;
+    uint64_t xprtcnt = 0;
+
+    crpc = mydata;
+    this = crpc->this;
+    c_clnt = crpc->c_clnt;
+
+    priv = this->private;
+
+    switch (event) {
+        case RPC_CLNT_CONNECT:
+            selection = &priv->ev_selection;
+            GF_ATOMIC_INC(priv->clntcnt);
+
+            LOCK(&c_clnt->wait_lock);
+            {
+                LOCK(&c_clnt->active_lock);
                 {
-                        if (selection)
-                                changelog_deselect_event (this, selection,
-                                                          crpc->filter);
-                        changelog_set_disconnect_flag (crpc, _gf_true);
+                    changelog_select_event(this, selection, crpc->filter);
+                    list_move_tail(&crpc->list, &c_clnt->active);
                 }
-                UNLOCK (&crpc->lock);
+                UNLOCK(&c_clnt->active_lock);
+            }
+            UNLOCK(&c_clnt->wait_lock);
 
-                break;
+            break;
+        case RPC_CLNT_DISCONNECT:
+            rpc_clnt_disable(crpc->rpc);
+
+            /* rpc_clnt_disable doesn't unref the rpc. It just marks
+             * the rpc as disabled and cancels reconnection timer.
+             * Hence unref the rpc object to free it.
+             */
+            rpc_clnt_unref(crpc->rpc);
+
+            if (priv)
+                selection = &priv->ev_selection;
+
+            LOCK(&crpc->lock);
+            {
+                if (selection)
+                    changelog_deselect_event(this, selection, crpc->filter);
+                changelog_set_disconnect_flag(crpc, _gf_true);
+            }
+            UNLOCK(&crpc->lock);
+            LOCK(&c_clnt->active_lock);
+            {
+                list_del_init(&crpc->list);
+            }
+            UNLOCK(&c_clnt->active_lock);
+
+            break;
         case RPC_CLNT_MSG:
         case RPC_CLNT_DESTROY:
-                /* Free up mydata */
-                changelog_rpc_clnt_unref (crpc);
-                break;
+            /* Free up mydata */
+            changelog_rpc_clnt_unref(crpc);
+            clntcnt = GF_ATOMIC_DEC(priv->clntcnt);
+            xprtcnt = GF_ATOMIC_GET(priv->xprtcnt);
+            if (this->cleanup_starting) {
+                if (!clntcnt && !xprtcnt)
+                    changelog_process_cleanup_event(this);
+            }
+            break;
         case RPC_CLNT_PING:
-                break;
-        }
+            break;
+    }
 
-        return 0;
+    return 0;
 }
 
 void *
-changelog_ev_connector (void *data)
+changelog_ev_connector(void *data)
 {
-        xlator_t             *this   = NULL;
-        changelog_clnt_t     *c_clnt = NULL;
-        changelog_rpc_clnt_t *crpc   = NULL;
+    xlator_t *this = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
 
-        c_clnt = data;
-        this = c_clnt->this;
+    c_clnt = data;
+    this = c_clnt->this;
 
-        while (1) {
-                pthread_mutex_lock (&c_clnt->pending_lock);
-                {
-                        while (list_empty (&c_clnt->pending))
-                                pthread_cond_wait (&c_clnt->pending_cond,
-                                                   &c_clnt->pending_lock);
-                        crpc = list_first_entry (&c_clnt->pending,
-                                                 changelog_rpc_clnt_t, list);
-                        crpc->rpc =
-                                changelog_rpc_client_init (this, crpc,
-                                                           crpc->sock,
-                                                           changelog_rpc_notify);
-                        if (!crpc->rpc) {
-                                gf_smsg (this->name, GF_LOG_ERROR, 0,
-                                         CHANGELOG_MSG_RPC_CONNECT_ERROR,
-                                         "failed to connect back",
-                                         "path=%s", crpc->sock,
-                                         NULL);
-                                crpc->cleanup (crpc);
-                                goto mutex_unlock;
-                        }
-
-                        LOCK (&c_clnt->wait_lock);
-                        {
-                                list_move_tail (&crpc->list, &c_clnt->waitq);
-                        }
-                        UNLOCK (&c_clnt->wait_lock);
-                }
-        mutex_unlock:
-                pthread_mutex_unlock (&c_clnt->pending_lock);
+    while (1) {
+        pthread_mutex_lock(&c_clnt->pending_lock);
+        {
+            while (list_empty(&c_clnt->pending))
+                pthread_cond_wait(&c_clnt->pending_cond, &c_clnt->pending_lock);
+            crpc = list_first_entry(&c_clnt->pending, changelog_rpc_clnt_t,
+                                    list);
+            crpc->rpc = changelog_rpc_client_init(this, crpc, crpc->sock,
+                                                  changelog_rpc_notify);
+            if (!crpc->rpc) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_MSG_RPC_CONNECT_ERROR, "path=%s", crpc->sock,
+                        NULL);
+                crpc->cleanup(crpc);
+                goto mutex_unlock;
+            }
+
+            LOCK(&c_clnt->wait_lock);
+            {
+                list_move_tail(&crpc->list, &c_clnt->waitq);
+            }
+            UNLOCK(&c_clnt->wait_lock);
         }
+    mutex_unlock:
+        pthread_mutex_unlock(&c_clnt->pending_lock);
+    }
 
-        return NULL;
+    return NULL;
 }
 
 void
-changelog_ev_cleanup_connections (xlator_t *this, changelog_clnt_t *c_clnt)
+changelog_ev_cleanup_connections(xlator_t *this, changelog_clnt_t *c_clnt)
 {
-        changelog_rpc_clnt_t *crpc = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
 
-        /* cleanup active connections */
-        LOCK (&c_clnt->active_lock);
+    /* cleanup active connections */
+    LOCK(&c_clnt->active_lock);
+    {
+        list_for_each_entry(crpc, &c_clnt->active, list)
         {
-                list_for_each_entry (crpc, &c_clnt->active, list) {
-                        rpc_clnt_disable (crpc->rpc);
-                }
+            rpc_clnt_disable(crpc->rpc);
         }
-        UNLOCK (&c_clnt->active_lock);
+    }
+    UNLOCK(&c_clnt->active_lock);
 }
 
 /**
@@ -256,147 +268,145 @@ changelog_ev_cleanup_connections (xlator_t *this, changelog_clnt_t *c_clnt)
  */
 
 static changelog_rpc_clnt_t *
-get_client (changelog_clnt_t *c_clnt, struct list_head **next)
+get_client(changelog_clnt_t *c_clnt, struct list_head **next)
 {
-        changelog_rpc_clnt_t *crpc = NULL;
-
-        LOCK (&c_clnt->active_lock);
-        {
-                if (*next == &c_clnt->active)
-                        goto unblock;
-                crpc = list_entry (*next, changelog_rpc_clnt_t, list);
-                /* ref rpc as DISCONNECT might unref the rpc asynchronously */
-                changelog_rpc_clnt_ref (crpc);
-                rpc_clnt_ref (crpc->rpc);
-                *next = (*next)->next;
-        }
- unblock:
-        UNLOCK (&c_clnt->active_lock);
-
-        return crpc;
+    changelog_rpc_clnt_t *crpc = NULL;
+
+    LOCK(&c_clnt->active_lock);
+    {
+        if (*next == &c_clnt->active)
+            goto unblock;
+        crpc = list_entry(*next, changelog_rpc_clnt_t, list);
+        /* ref rpc as DISCONNECT might unref the rpc asynchronously */
+        changelog_rpc_clnt_ref(crpc);
+        rpc_clnt_ref(crpc->rpc);
+        *next = (*next)->next;
+    }
+unblock:
+    UNLOCK(&c_clnt->active_lock);
+
+    return crpc;
 }
 
 static void
-put_client (changelog_clnt_t *c_clnt, changelog_rpc_clnt_t *crpc)
+put_client(changelog_clnt_t *c_clnt, changelog_rpc_clnt_t *crpc)
 {
-        LOCK (&c_clnt->active_lock);
-        {
-                rpc_clnt_unref (crpc->rpc);
-                changelog_rpc_clnt_unref (crpc);
-        }
-        UNLOCK (&c_clnt->active_lock);
+    LOCK(&c_clnt->active_lock);
+    {
+        rpc_clnt_unref(crpc->rpc);
+        changelog_rpc_clnt_unref(crpc);
+    }
+    UNLOCK(&c_clnt->active_lock);
 }
 
 void
-_dispatcher (rbuf_list_t *rlist, void *arg)
+_dispatcher(rbuf_list_t *rlist, void *arg)
 {
-        xlator_t             *this   = NULL;
-        changelog_clnt_t     *c_clnt = NULL;
-        changelog_rpc_clnt_t *crpc   = NULL;
-        struct ev_rpc         erpc   = {0,};
-        struct list_head     *next   = NULL;
-
-        c_clnt = arg;
-        this = c_clnt->this;
-
-        erpc.rlist = rlist;
-        next = c_clnt->active.next;
-
-        while (1) {
-                crpc = get_client (c_clnt, &next);
-                if (!crpc)
-                        break;
-                erpc.rpc = crpc->rpc;
-                (void) changelog_invoke_rpc (this, crpc->rpc,
-                                             &changelog_ev_program,
-                                             CHANGELOG_REV_PROC_EVENT, &erpc);
-                put_client (c_clnt, crpc);
-        }
+    xlator_t *this = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
+    struct ev_rpc erpc = {
+        0,
+    };
+    struct list_head *next = NULL;
+
+    c_clnt = arg;
+    this = c_clnt->this;
+
+    erpc.rlist = rlist;
+    next = c_clnt->active.next;
+
+    while (1) {
+        crpc = get_client(c_clnt, &next);
+        if (!crpc)
+            break;
+        erpc.rpc = crpc->rpc;
+        (void)changelog_invoke_rpc(this, crpc->rpc, &changelog_ev_program,
+                                   CHANGELOG_REV_PROC_EVENT, &erpc);
+        put_client(c_clnt, crpc);
+    }
 }
 
 /** this is called under rotbuff's lock */
 void
-sequencer (rbuf_list_t *rlist, void *mydata)
+sequencer(rbuf_list_t *rlist, void *mydata)
 {
-        unsigned long     range  = 0;
-        changelog_clnt_t *c_clnt = 0;
+    unsigned long range = 0;
+    changelog_clnt_t *c_clnt = 0;
 
-        c_clnt = mydata;
+    c_clnt = mydata;
 
-        range = (RLIST_ENTRY_COUNT (rlist)) / NR_IOVEC;
-        if ((RLIST_ENTRY_COUNT (rlist)) % NR_IOVEC)
-                range++;
-        RLIST_STORE_SEQ (rlist, c_clnt->sequence, range);
+    range = (RLIST_ENTRY_COUNT(rlist)) / NR_IOVEC;
+    if ((RLIST_ENTRY_COUNT(rlist)) % NR_IOVEC)
+        range++;
+    RLIST_STORE_SEQ(rlist, c_clnt->sequence, range);
 
-        c_clnt->sequence += range;
+    c_clnt->sequence += range;
 }
 
 void *
-changelog_ev_dispatch (void *data)
+changelog_ev_dispatch(void *data)
 {
-        int                   ret    = 0;
-        void                 *opaque = NULL;
-        xlator_t             *this   = NULL;
-        changelog_clnt_t     *c_clnt = NULL;
-        struct timeval        tv     = {0,};
-
-        c_clnt = data;
-        this = c_clnt->this;
-
-        while (1) {
-                /* TODO: change this to be pthread cond based.. later */
-
-                tv.tv_sec = 1;
-                tv.tv_usec = 0;
-                select (0, NULL, NULL, NULL, &tv);
-
-                ret = rbuf_get_buffer (c_clnt->rbuf,
-                                       &opaque, sequencer, c_clnt);
-                if (ret != RBUF_CONSUMABLE) {
-                        if (ret != RBUF_EMPTY)
-                                gf_smsg (this->name, GF_LOG_WARNING, 0,
-                                         CHANGELOG_MSG_BUFFER_STARVATION_ERROR,
-                                         "Failed to get buffer for RPC dispatch",
-                                         "rbuf_retval=%d", ret,
-                                         NULL);
-                        continue;
-                }
-
-                ret = rbuf_wait_for_completion (c_clnt->rbuf,
-                                                opaque, _dispatcher, c_clnt);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                CHANGELOG_MSG_PUT_BUFFER_FAILED,
-                                "failed to put buffer after consumption");
-
+    int ret = 0;
+    void *opaque = NULL;
+    xlator_t *this = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    struct timeval tv = {
+        0,
+    };
+
+    c_clnt = data;
+    this = c_clnt->this;
+
+    while (1) {
+        /* TODO: change this to be pthread cond based.. later */
+
+        tv.tv_sec = 1;
+        tv.tv_usec = 0;
+        select(0, NULL, NULL, NULL, &tv);
+
+        ret = rbuf_get_buffer(c_clnt->rbuf, &opaque, sequencer, c_clnt);
+        if (ret != RBUF_CONSUMABLE) {
+            if (ret != RBUF_EMPTY)
+                gf_smsg(this->name, GF_LOG_WARNING, 0,
+                        CHANGELOG_MSG_BUFFER_STARVATION_ERROR,
+                        "Failed to get buffer for RPC dispatch",
+                        "rbuf_retval=%d", ret, NULL);
+            continue;
         }
 
-        return NULL;
+        ret = rbuf_wait_for_completion(c_clnt->rbuf, opaque, _dispatcher,
+                                       c_clnt);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    CHANGELOG_MSG_PUT_BUFFER_FAILED, NULL);
+    }
+
+    return NULL;
 }
 
 void
-changelog_ev_queue_connection (changelog_clnt_t *c_clnt,
-                               changelog_rpc_clnt_t *crpc)
+changelog_ev_queue_connection(changelog_clnt_t *c_clnt,
+                              changelog_rpc_clnt_t *crpc)
 {
-        pthread_mutex_lock (&c_clnt->pending_lock);
-        {
-                list_add_tail (&crpc->list, &c_clnt->pending);
-                pthread_cond_signal (&c_clnt->pending_cond);
-        }
-        pthread_mutex_unlock (&c_clnt->pending_lock);
+    pthread_mutex_lock(&c_clnt->pending_lock);
+    {
+        list_add_tail(&crpc->list, &c_clnt->pending);
+        pthread_cond_signal(&c_clnt->pending_cond);
+    }
+    pthread_mutex_unlock(&c_clnt->pending_lock);
 }
 
 struct rpc_clnt_procedure changelog_ev_procs[CHANGELOG_REV_PROC_MAX] = {
-        [CHANGELOG_REV_PROC_NULL]  = {"NULL", NULL},
-        [CHANGELOG_REV_PROC_EVENT] = {
-                "EVENT DISPATCH", changelog_event_dispatch_rpc
-        },
+    [CHANGELOG_REV_PROC_NULL] = {"NULL", NULL},
+    [CHANGELOG_REV_PROC_EVENT] = {"EVENT DISPATCH",
+                                  changelog_event_dispatch_rpc},
 };
 
 struct rpc_clnt_program changelog_ev_program = {
-        .progname  = "CHANGELOG EVENT DISPATCHER",
-        .prognum   = CHANGELOG_REV_RPC_PROCNUM,
-        .progver   = CHANGELOG_REV_RPC_PROCVER,
-        .numproc   = CHANGELOG_REV_PROC_MAX,
-        .proctable = changelog_ev_procs,
+    .progname = "CHANGELOG EVENT DISPATCHER",
+    .prognum = CHANGELOG_REV_RPC_PROCNUM,
+    .progver = CHANGELOG_REV_RPC_PROCVER,
+    .numproc = CHANGELOG_REV_PROC_MAX,
+    .proctable = changelog_ev_procs,
 };
diff --git a/xlators/features/changelog/src/changelog-ev-handle.h b/xlators/features/changelog/src/changelog-ev-handle.h
index e89af8793a8..cc1af58a276 100644
--- a/xlators/features/changelog/src/changelog-ev-handle.h
+++ b/xlators/features/changelog/src/changelog-ev-handle.h
@@ -11,68 +11,67 @@
 #ifndef __CHANGELOG_EV_HANDLE_H
 #define __CHANGELOG_EV_HANDLE_H
 
-#include "list.h"
-#include "xlator.h"
+#include <glusterfs/list.h>
+#include <glusterfs/xlator.h>
 #include "rpc-clnt.h"
 
-#include "rot-buffs.h"
+#include <glusterfs/rot-buffs.h>
 
 struct changelog_clnt;
 
 typedef struct changelog_rpc_clnt {
-        xlator_t *this;
+    xlator_t *this;
 
-        gf_lock_t lock;
+    gf_lock_t lock;
 
-        gf_atomic_t   ref;
-        gf_boolean_t  disconnected;
+    gf_atomic_t ref;
+    gf_boolean_t disconnected;
 
-        unsigned int filter;
-        char sock[UNIX_PATH_MAX];
+    unsigned int filter;
+    char sock[UNIX_PATH_MAX];
 
-        struct changelog_clnt *c_clnt;   /* back pointer to list holder */
+    struct changelog_clnt *c_clnt; /* back pointer to list holder */
 
-        struct rpc_clnt *rpc;            /* RPC client endpoint */
+    struct rpc_clnt *rpc; /* RPC client endpoint */
 
-        struct list_head list;           /* ->pending, ->waitq, ->active */
+    struct list_head list; /* ->pending, ->waitq, ->active */
 
-        void (*cleanup)
-        (struct changelog_rpc_clnt *);   /* cleanup handler */
+    void (*cleanup)(struct changelog_rpc_clnt *); /* cleanup handler */
 } changelog_rpc_clnt_t;
 
 static inline void
-changelog_rpc_clnt_ref (changelog_rpc_clnt_t *crpc)
+changelog_rpc_clnt_ref(changelog_rpc_clnt_t *crpc)
 {
-        GF_ATOMIC_INC (crpc->ref);
+    GF_ATOMIC_INC(crpc->ref);
 }
 
 static inline void
-changelog_set_disconnect_flag (changelog_rpc_clnt_t *crpc, gf_boolean_t flag)
+changelog_set_disconnect_flag(changelog_rpc_clnt_t *crpc, gf_boolean_t flag)
 {
-        crpc->disconnected = flag;
+    crpc->disconnected = flag;
 }
 
 static inline int
-changelog_rpc_clnt_is_disconnected (changelog_rpc_clnt_t *crpc)
+changelog_rpc_clnt_is_disconnected(changelog_rpc_clnt_t *crpc)
 {
-        return (crpc->disconnected == _gf_true);
+    return (crpc->disconnected == _gf_true);
 }
 
 static inline void
-changelog_rpc_clnt_unref (changelog_rpc_clnt_t *crpc)
+changelog_rpc_clnt_unref(changelog_rpc_clnt_t *crpc)
 {
-        gf_boolean_t gone = _gf_false;
-        uint64_t     ref  = 0;
+    gf_boolean_t gone = _gf_false;
+    uint64_t ref = 0;
 
-        ref = GF_ATOMIC_DEC (crpc->ref);
+    ref = GF_ATOMIC_DEC(crpc->ref);
 
-        if (!ref && changelog_rpc_clnt_is_disconnected (crpc)) {
-                list_del (&crpc->list);
-                gone = _gf_true;
-        }
+    if (!ref && changelog_rpc_clnt_is_disconnected(crpc)) {
+        list_del(&crpc->list);
+        gone = _gf_true;
+    }
 
-        if (gone)
-                crpc->cleanup (crpc);
+    if (gone)
+        crpc->cleanup(crpc);
 }
 
 /**
@@ -100,35 +99,38 @@ changelog_rpc_clnt_unref (changelog_rpc_clnt_t *crpc)
  */
 
 typedef struct changelog_clnt {
-        xlator_t *this;
+    xlator_t *this;
 
-        /* pending connections */
-        pthread_mutex_t pending_lock;
-        pthread_cond_t pending_cond;
-        struct list_head pending;
+    /* pending connections */
+    pthread_mutex_t pending_lock;
+    pthread_cond_t pending_cond;
+    struct list_head pending;
 
-        /* current active connections */
-        gf_lock_t active_lock;
-        struct list_head active;
+    /* current active connections */
+    gf_lock_t active_lock;
+    struct list_head active;
 
-        gf_lock_t wait_lock;
-        struct list_head waitq;
+    gf_lock_t wait_lock;
+    struct list_head waitq;
 
-        /* consumer part of rot-buffs */
-        rbuf_t *rbuf;
-        unsigned long sequence;
+    /* consumer part of rot-buffs */
+    rbuf_t *rbuf;
+    unsigned long sequence;
 } changelog_clnt_t;
 
-void *changelog_ev_connector (void *);
+void *
+changelog_ev_connector(void *);
 
-void *changelog_ev_dispatch (void *);
+void *
+changelog_ev_dispatch(void *);
 
 /* APIs */
 void
-changelog_ev_queue_connection (changelog_clnt_t *, changelog_rpc_clnt_t *);
+changelog_ev_queue_connection(changelog_clnt_t *, changelog_rpc_clnt_t *);
 
 void
-changelog_ev_cleanup_connections (xlator_t *, changelog_clnt_t *);
+changelog_ev_cleanup_connections(xlator_t *, changelog_clnt_t *);
 
+void
+changelog_process_cleanup_event(xlator_t *);
 #endif
-
diff --git a/xlators/features/changelog/src/changelog-helpers.c b/xlators/features/changelog/src/changelog-helpers.c
index 306287ecd11..e561997d858 100644
--- a/xlators/features/changelog/src/changelog-helpers.c
+++ b/xlators/features/changelog/src/changelog-helpers.c
@@ -8,11 +8,11 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-#include "logging.h"
-#include "iobuf.h"
-#include "syscall.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/iobuf.h>
+#include <glusterfs/syscall.h>
 
 #include "changelog-helpers.h"
 #include "changelog-encoders.h"
@@ -22,313 +22,281 @@
 #include "changelog-encoders.h"
 #include "changelog-rpc-common.h"
 #include <pthread.h>
+#include <time.h>
 
 static void
-changelog_cleanup_free_mutex (void *arg_mutex)
+changelog_cleanup_free_mutex(void *arg_mutex)
 {
-    pthread_mutex_t *p_mutex = (pthread_mutex_t*) arg_mutex;
+    pthread_mutex_t *p_mutex = (pthread_mutex_t *)arg_mutex;
 
     if (p_mutex)
-            pthread_mutex_unlock(p_mutex);
+        pthread_mutex_unlock(p_mutex);
 }
 
 int
-changelog_thread_cleanup (xlator_t *this, pthread_t thr_id)
+changelog_thread_cleanup(xlator_t *this, pthread_t thr_id)
 {
-        int   ret    = 0;
-        void *retval = NULL;
-
-        /* send a cancel request to the thread */
-        ret = pthread_cancel (thr_id);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_CANCEL_FAILED,
-                        "could not cancel thread");
-                goto out;
-        }
-
-        ret = pthread_join (thr_id, &retval);
-        if ((ret != 0) || (retval != PTHREAD_CANCELED)) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_PTHREAD_CANCEL_FAILED,
-                        "cancel request not adhered as expected");
-        }
+    int ret = 0;
+    void *retval = NULL;
+
+    /* send a cancel request to the thread */
+    ret = pthread_cancel(thr_id);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_CANCEL_FAILED, NULL);
+        goto out;
+    }
+
+    ret = pthread_join(thr_id, &retval);
+    if ((ret != 0) || (retval != PTHREAD_CANCELED)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_CANCEL_FAILED, NULL);
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 void *
-changelog_get_usable_buffer (changelog_local_t *local)
+changelog_get_usable_buffer(changelog_local_t *local)
 {
-        changelog_log_data_t *cld = NULL;
+    changelog_log_data_t *cld = NULL;
 
-        if (!local)
-                return NULL;
+    if (!local)
+        return NULL;
 
-        cld = &local->cld;
-        if (!cld->cld_iobuf)
-                return NULL;
+    cld = &local->cld;
+    if (!cld->cld_iobuf)
+        return NULL;
 
-        return cld->cld_iobuf->ptr;
+    return cld->cld_iobuf->ptr;
 }
 
 static int
-changelog_selector_index (unsigned int selector)
+changelog_selector_index(unsigned int selector)
 {
-        return (ffs (selector) - 1);
+    return (ffs(selector) - 1);
 }
 
 int
-changelog_ev_selected (xlator_t *this,
-                       changelog_ev_selector_t *selection,
-                       unsigned int selector)
+changelog_ev_selected(xlator_t *this, changelog_ev_selector_t *selection,
+                      unsigned int selector)
 {
-        int idx = 0;
-
-        idx = changelog_selector_index (selector);
-        gf_msg_debug (this->name, 0,
-                      "selector ref count for %d (idx: %d): %d",
-                      selector, idx, selection->ref[idx]);
-        /* this can be lockless */
-        return (idx < CHANGELOG_EV_SELECTION_RANGE
-                 && (selection->ref[idx] > 0));
+    int idx = 0;
+
+    idx = changelog_selector_index(selector);
+    gf_msg_debug(this->name, 0, "selector ref count for %d (idx: %d): %d",
+                 selector, idx, selection->ref[idx]);
+    /* this can be lockless */
+    return (idx < CHANGELOG_EV_SELECTION_RANGE && (selection->ref[idx] > 0));
 }
 
 void
-changelog_select_event (xlator_t *this,
-                        changelog_ev_selector_t *selection,
-                        unsigned int selector)
+changelog_select_event(xlator_t *this, changelog_ev_selector_t *selection,
+                       unsigned int selector)
 {
-        int idx = 0;
-
-        LOCK (&selection->reflock);
-        {
-                while (selector) {
-                        idx = changelog_selector_index (selector);
-                        if (idx < CHANGELOG_EV_SELECTION_RANGE) {
-                                selection->ref[idx]++;
-                                gf_msg_debug (this->name, 0,
-                                              "selecting event %d", idx);
-                        }
-                        selector &= ~(1 << idx);
-                }
-        }
-        UNLOCK (&selection->reflock);
+    int idx = 0;
+
+    LOCK(&selection->reflock);
+    {
+        while (selector) {
+            idx = changelog_selector_index(selector);
+            if (idx < CHANGELOG_EV_SELECTION_RANGE) {
+                selection->ref[idx]++;
+                gf_msg_debug(this->name, 0, "selecting event %d", idx);
+            }
+            selector &= ~(1 << idx);
+        }
+    }
+    UNLOCK(&selection->reflock);
 }
 
 void
-changelog_deselect_event (xlator_t *this,
-                          changelog_ev_selector_t *selection,
-                          unsigned int selector)
+changelog_deselect_event(xlator_t *this, changelog_ev_selector_t *selection,
+                         unsigned int selector)
 {
-        int idx = 0;
-
-        LOCK (&selection->reflock);
-        {
-                while (selector) {
-                        idx = changelog_selector_index (selector);
-                        if (idx < CHANGELOG_EV_SELECTION_RANGE) {
-                                selection->ref[idx]--;
-                                gf_msg_debug (this->name, 0,
-                                              "de-selecting event %d", idx);
-                        }
-                        selector &= ~(1 << idx);
-                }
-        }
-        UNLOCK (&selection->reflock);
+    int idx = 0;
+
+    LOCK(&selection->reflock);
+    {
+        while (selector) {
+            idx = changelog_selector_index(selector);
+            if (idx < CHANGELOG_EV_SELECTION_RANGE) {
+                selection->ref[idx]--;
+                gf_msg_debug(this->name, 0, "de-selecting event %d", idx);
+            }
+            selector &= ~(1 << idx);
+        }
+    }
+    UNLOCK(&selection->reflock);
 }
 
 int
-changelog_init_event_selection (xlator_t *this,
-                                changelog_ev_selector_t *selection)
+changelog_init_event_selection(xlator_t *this,
+                               changelog_ev_selector_t *selection)
 {
-        int ret = 0;
-        int j = CHANGELOG_EV_SELECTION_RANGE;
+    int ret = 0;
+    int j = CHANGELOG_EV_SELECTION_RANGE;
 
-        ret = LOCK_INIT (&selection->reflock);
-        if (ret != 0)
-                return -1;
-
-        LOCK (&selection->reflock);
-        {
-                while (j--) {
-                        selection->ref[j] = 0;
-                }
-        }
-        UNLOCK (&selection->reflock);
-
-        return 0;
-}
-
-int
-changelog_cleanup_event_selection (xlator_t *this,
-                                   changelog_ev_selector_t *selection)
-{
-        int j = CHANGELOG_EV_SELECTION_RANGE;
+    ret = LOCK_INIT(&selection->reflock);
+    if (ret != 0)
+        return -1;
 
-        LOCK (&selection->reflock);
-        {
-                while (j--) {
-                        if (selection->ref[j] > 0)
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        CHANGELOG_MSG_CLEANUP_ON_ACTIVE_REF,
-                                        "changelog event selection cleaning up "
-                                        " on active references");
-                }
+    LOCK(&selection->reflock);
+    {
+        while (j--) {
+            selection->ref[j] = 0;
         }
-        UNLOCK (&selection->reflock);
+    }
+    UNLOCK(&selection->reflock);
 
-        return LOCK_DESTROY (&selection->reflock);
+    return 0;
 }
 
 static void
-changelog_perform_dispatch (xlator_t *this,
-                            changelog_priv_t *priv, void *mem, size_t size)
+changelog_perform_dispatch(xlator_t *this, changelog_priv_t *priv, void *mem,
+                           size_t size)
 {
-        char *buf    = NULL;
-        void *opaque = NULL;
-
-        buf = rbuf_reserve_write_area (priv->rbuf, size, &opaque);
-        if (!buf) {
-                gf_msg_callingfn (this->name,
-                                  GF_LOG_WARNING, 0,
-                                  CHANGELOG_MSG_DISPATCH_EVENT_FAILED,
-                                  "failed to dispatch event");
-                return;
-        }
+    char *buf = NULL;
+    void *opaque = NULL;
+
+    buf = rbuf_reserve_write_area(priv->rbuf, size, &opaque);
+    if (!buf) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0,
+                         CHANGELOG_MSG_DISPATCH_EVENT_FAILED,
+                         "failed to dispatch event");
+        return;
+    }
 
-        memcpy (buf, mem, size);
-        rbuf_write_complete (opaque);
+    memcpy(buf, mem, size);
+    rbuf_write_complete(opaque);
 }
 
 void
-changelog_dispatch_event (xlator_t *this,
-                          changelog_priv_t *priv, changelog_event_t *ev)
+changelog_dispatch_event(xlator_t *this, changelog_priv_t *priv,
+                         changelog_event_t *ev)
 {
-        changelog_ev_selector_t *selection = NULL;
+    changelog_ev_selector_t *selection = NULL;
 
-        selection = &priv->ev_selection;
-        if (changelog_ev_selected (this, selection, ev->ev_type)) {
-                changelog_perform_dispatch (this, priv, ev, CHANGELOG_EV_SIZE);
-        }
+    selection = &priv->ev_selection;
+    if (changelog_ev_selected(this, selection, ev->ev_type)) {
+        changelog_perform_dispatch(this, priv, ev, CHANGELOG_EV_SIZE);
+    }
 }
 
 void
-changelog_set_usable_record_and_length (changelog_local_t *local,
-                                        size_t len, int xr)
+changelog_set_usable_record_and_length(changelog_local_t *local, size_t len,
+                                       int xr)
 {
-        changelog_log_data_t *cld = NULL;
+    changelog_log_data_t *cld = NULL;
 
-        cld = &local->cld;
+    cld = &local->cld;
 
-        cld->cld_ptr_len = len;
-        cld->cld_xtra_records = xr;
+    cld->cld_ptr_len = len;
+    cld->cld_xtra_records = xr;
 }
 
 void
-changelog_local_cleanup (xlator_t *xl, changelog_local_t *local)
+changelog_local_cleanup(xlator_t *xl, changelog_local_t *local)
 {
-        int                   i   = 0;
-        changelog_opt_t      *co  = NULL;
-        changelog_log_data_t *cld = NULL;
+    int i = 0;
+    changelog_opt_t *co = NULL;
+    changelog_log_data_t *cld = NULL;
 
-        if (!local)
-                return;
+    if (!local)
+        return;
 
-        cld = &local->cld;
+    cld = &local->cld;
 
-        /* cleanup dynamic allocation for extra records */
-        if (cld->cld_xtra_records) {
-                co = (changelog_opt_t *) cld->cld_ptr;
-                for (; i < cld->cld_xtra_records; i++, co++)
-                        if (co->co_free)
-                                co->co_free (co);
-        }
+    /* cleanup dynamic allocation for extra records */
+    if (cld->cld_xtra_records) {
+        co = (changelog_opt_t *)cld->cld_ptr;
+        for (; i < cld->cld_xtra_records; i++, co++)
+            if (co->co_free)
+                co->co_free(co);
+    }
 
-        CHANGELOG_IOBUF_UNREF (cld->cld_iobuf);
+    CHANGELOG_IOBUF_UNREF(cld->cld_iobuf);
 
-        if (local->inode)
-                inode_unref (local->inode);
+    if (local->inode)
+        inode_unref(local->inode);
 
-        mem_put (local);
+    mem_put(local);
 }
 
 int
-changelog_write (int fd, char *buffer, size_t len)
+changelog_write(int fd, char *buffer, size_t len)
 {
-        ssize_t size = 0;
-        size_t written = 0;
+    ssize_t size = 0;
+    size_t written = 0;
 
-        while (written < len) {
-                size = sys_write (fd,
-                              buffer + written, len - written);
-                if (size <= 0)
-                        break;
+    while (written < len) {
+        size = sys_write(fd, buffer + written, len - written);
+        if (size <= 0)
+            break;
 
-                written += size;
-        }
+        written += size;
+    }
 
-        return (written != len);
+    return (written != len);
 }
 
 int
-htime_update (xlator_t *this,
-              changelog_priv_t *priv, unsigned long ts,
-              char * buffer)
+htime_update(xlator_t *this, changelog_priv_t *priv, time_t ts, char *buffer)
 {
-        char changelog_path[PATH_MAX+1]   = {0,};
-        int len                           = -1;
-        char x_value[25]                  = {0,};
-        /* time stamp(10) + : (1) + rolltime (12 ) + buffer (2) */
-        int ret                           = 0;
-
-        if (priv->htime_fd ==-1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_HTIME_ERROR,
-                        "Htime fd not available for updation");
-                ret = -1;
-                goto out;
-        }
-        strncpy (changelog_path, buffer, PATH_MAX);
-        len = strlen (changelog_path);
-        changelog_path[len] = '\0'; /* redundant */
-
-        if (changelog_write (priv->htime_fd, (void*) changelog_path, len+1 ) < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_HTIME_ERROR,
-                        "Htime file content write failed");
-                ret =-1;
-                goto out;
-        }
-
-        snprintf (x_value, sizeof x_value, "%lu:%d",
-                 ts, priv->rollover_count);
-
-        if (sys_fsetxattr (priv->htime_fd, HTIME_KEY, x_value,
-                           strlen (x_value), XATTR_REPLACE)) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_HTIME_ERROR,
-                         "Htime xattr updation failed with XATTR_REPLACE",
-                         "changelog=%s", changelog_path,
-                         NULL);
-
-                if (sys_fsetxattr (priv->htime_fd, HTIME_KEY, x_value,
-                                   strlen (x_value), 0)) {
-                        gf_smsg (this->name, GF_LOG_ERROR, errno,
-                                 CHANGELOG_MSG_HTIME_ERROR,
-                                 "Htime xattr updation failed",
-                                 "changelog=%s", changelog_path,
-                                 NULL);
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        priv->rollover_count +=1;
+    char changelog_path[PATH_MAX + 1] = {
+        0,
+    };
+    int len = -1;
+    char x_value[25] = {
+        0,
+    };
+    /* time stamp(10) + : (1) + rolltime (12 ) + buffer (2) */
+    int ret = 0;
+
+    if (priv->htime_fd == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_HTIME_ERROR,
+                "reason=fd not available", NULL);
+        ret = -1;
+        goto out;
+    }
+    len = snprintf(changelog_path, PATH_MAX, "%s", buffer);
+    if (len >= PATH_MAX) {
+        ret = -1;
+        goto out;
+    }
+    if (changelog_write(priv->htime_fd, (void *)changelog_path, len + 1) < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_HTIME_ERROR,
+                "reason=write failed", NULL);
+        ret = -1;
+        goto out;
+    }
+
+    len = snprintf(x_value, sizeof(x_value), "%ld:%d", ts,
+                   priv->rollover_count);
+    if (len >= sizeof(x_value)) {
+        ret = -1;
+        goto out;
+    }
+
+    if (sys_fsetxattr(priv->htime_fd, HTIME_KEY, x_value, len, XATTR_REPLACE)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_HTIME_ERROR,
+                "reason=xattr updation failed", "XATTR_REPLACE=true",
+                "changelog=%s", changelog_path, NULL);
+
+        if (sys_fsetxattr(priv->htime_fd, HTIME_KEY, x_value, len, 0)) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_HTIME_ERROR,
+                    "reason=xattr updation failed", "changelog=%s",
+                    changelog_path, NULL);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    priv->rollover_count += 1;
 
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -340,43 +308,45 @@ out:
  * 0 : If NOT empty, proceed usual.
  */
 int
-cl_is_empty (xlator_t *this, int fd)
+cl_is_empty(xlator_t *this, int fd)
 {
-        int             ret             = -1;
-        size_t          elen            = 0;
-        int             encoding        = -1;
-        char            buffer[1024]    = {0,};
-        struct stat     stbuf           = {0,};
-        int             major_version   = -1;
-        int             minor_version   = -1;
-
-        ret = sys_fstat (fd, &stbuf);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_FSTAT_OP_FAILED,
-                        "Could not stat (CHANGELOG)");
-                goto out;
-        }
-
-        ret = sys_lseek (fd, 0, SEEK_SET);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_LSEEK_OP_FAILED,
-                        "Could not lseek (CHANGELOG)");
-                goto out;
-        }
-
-        CHANGELOG_GET_HEADER_INFO (fd, buffer, 1024, encoding,
-                                   major_version, minor_version, elen);
-
-        if (elen == stbuf.st_size) {
-                ret = 1;
-        } else {
-                ret = 0;
-        }
+    int ret = -1;
+    size_t elen = 0;
+    int encoding = -1;
+    char buffer[1024] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+    int major_version = -1;
+    int minor_version = -1;
+
+    ret = sys_fstat(fd, &stbuf);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FSTAT_OP_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = sys_lseek(fd, 0, SEEK_SET);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_LSEEK_OP_FAILED,
+                NULL);
+        goto out;
+    }
+
+    CHANGELOG_GET_HEADER_INFO(fd, buffer, sizeof(buffer), encoding,
+                              major_version, minor_version, elen);
+
+    if (elen == stbuf.st_size) {
+        ret = 1;
+    } else {
+        ret = 0;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -388,162 +358,172 @@ out:
  * -1 : Error
  */
 int
-update_path (xlator_t *this, char *cl_path)
+update_path(xlator_t *this, char *cl_path)
 {
-        char low_cl[]     = "changelog";
-        char up_cl[]      = "CHANGELOG";
-        char *found       = NULL;
-        int  ret          = -1;
-
-        found = strstr(cl_path, up_cl);
-
-        if (found == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_LSEEK_OP_FAILED,
-                        "Could not find CHANGELOG in changelog path");
-                goto out;
-        } else {
-                strncpy(found, low_cl, strlen(low_cl));
-        }
-
-        ret = 0;
+    const char low_cl[] = "changelog";
+    const char up_cl[] = "CHANGELOG";
+    char *found = NULL;
+    int ret = -1;
+
+    found = strstr(cl_path, up_cl);
+
+    if (found == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PATH_NOT_FOUND,
+                NULL);
+        goto out;
+    } else {
+        memcpy(found, low_cl, sizeof(low_cl) - 1);
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-changelog_rollover_changelog (xlator_t *this,
-                              changelog_priv_t *priv, unsigned long ts)
+changelog_rollover_changelog(xlator_t *this, changelog_priv_t *priv, time_t ts)
 {
-        int   ret             = -1;
-        int   notify          = 0;
-        int   cl_empty_flag   = 0;
-        char  ofile[PATH_MAX] = {0,};
-        char  nfile[PATH_MAX] = {0,};
-        changelog_event_t ev  = {0,};
-
-        if (priv->changelog_fd != -1) {
-                ret = sys_fsync (priv->changelog_fd);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_FSYNC_OP_FAILED,
-                                "fsync failed");
-                }
-                ret = cl_is_empty (this, priv->changelog_fd);
-                if (ret == 1) {
-                        cl_empty_flag = 1;
-                } else if (ret == -1) {
-                        /* Log error but proceed as usual */
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                CHANGELOG_MSG_DETECT_EMPTY_CHANGELOG_FAILED,
-                                "Error detecting empty changelog");
-                }
-                sys_close (priv->changelog_fd);
-                priv->changelog_fd = -1;
-        }
-
-        (void) snprintf (ofile, PATH_MAX,
-                         "%s/"CHANGELOG_FILE_NAME, priv->changelog_dir);
-        (void) snprintf (nfile, PATH_MAX,
-                         "%s/"CHANGELOG_FILE_NAME".%lu",
-                         priv->changelog_dir, ts);
-
-        if (cl_empty_flag == 1) {
-                ret = sys_unlink (ofile);
-                if (ret) {
-                        gf_smsg (this->name, GF_LOG_ERROR, errno,
-                                 CHANGELOG_MSG_UNLINK_OP_FAILED,
-                                 "error unlinking empty changelog",
-                                 "path=%s", ofile,
-                                 NULL);
-                        ret = 0;  /* Error in unlinking empty changelog should
-                                     not break further changelog operation, so
-                                     reset return value to 0*/
-                }
-        } else {
-                ret = sys_rename (ofile, nfile);
+    int ret = -1;
+    int notify = 0;
+    int cl_empty_flag = 0;
+    struct tm *gmt;
+    char yyyymmdd[40];
+    char ofile[PATH_MAX] = {
+        0,
+    };
+    char nfile[PATH_MAX] = {
+        0,
+    };
+    char nfile_dir[PATH_MAX] = {
+        0,
+    };
+    changelog_event_t ev = {
+        0,
+    };
+
+    if (priv->changelog_fd != -1) {
+        ret = sys_fsync(priv->changelog_fd);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_FSYNC_OP_FAILED, NULL);
+        }
+        ret = cl_is_empty(this, priv->changelog_fd);
+        if (ret == 1) {
+            cl_empty_flag = 1;
+        } else if (ret == -1) {
+            /* Log error but proceed as usual */
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    CHANGELOG_MSG_DETECT_EMPTY_CHANGELOG_FAILED, NULL);
+        }
+        sys_close(priv->changelog_fd);
+        priv->changelog_fd = -1;
+    }
+
+    /* Get GMT time. */
+    gmt = gmtime(&ts);
+
+    strftime(yyyymmdd, sizeof(yyyymmdd), "%Y/%m/%d", gmt);
+
+    (void)snprintf(ofile, PATH_MAX, "%s/" CHANGELOG_FILE_NAME,
+                   priv->changelog_dir);
+    (void)snprintf(nfile, PATH_MAX, "%s/%s/" CHANGELOG_FILE_NAME ".%ld",
+                   priv->changelog_dir, yyyymmdd, ts);
+    (void)snprintf(nfile_dir, PATH_MAX, "%s/%s", priv->changelog_dir, yyyymmdd);
+
+    if (cl_empty_flag == 1) {
+        ret = sys_unlink(ofile);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_UNLINK_OP_FAILED, "path=%s", ofile, NULL);
+            ret = 0; /* Error in unlinking empty changelog should
+                        not break further changelog operation, so
+                        reset return value to 0*/
+        }
+    } else {
+        ret = sys_rename(ofile, nfile);
+
+        /* Changelog file rename gets ENOENT when parent dir doesn't exist */
+        if (errno == ENOENT) {
+            ret = mkdir_p(nfile_dir, 0600, _gf_true);
+
+            if ((ret == -1) && (EEXIST != errno)) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        CHANGELOG_MSG_MKDIR_ERROR, "%s", nfile_dir, NULL);
+                goto out;
+            }
 
-                if (ret && (errno == ENOENT)) {
-                        ret = 0;
-                        goto out;
-                }
-                if (ret) {
-                        gf_smsg (this->name, GF_LOG_ERROR, errno,
-                                 CHANGELOG_MSG_RENAME_ERROR,
-                                 "error renaming",
-                                 "from=%s", ofile,
-                                 "to=%s", nfile,
-                                 NULL);
-                }
+            ret = sys_rename(ofile, nfile);
         }
 
-        if (!ret && (cl_empty_flag == 0)) {
-                        notify = 1;
+        if (ret && (errno == ENOENT)) {
+            ret = 0;
+            goto out;
         }
-
-        if (!ret) {
-                if (cl_empty_flag) {
-                        update_path (this, nfile);
-                }
-                ret = htime_update (this, priv, ts, nfile);
-                if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR,
-                                0, CHANGELOG_MSG_HTIME_ERROR,
-                                "could not update htime file");
-                        goto out;
-                }
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_RENAME_ERROR,
+                    "from=%s", ofile, "to=%s", nfile, NULL);
         }
+    }
+
+    if (!ret && (cl_empty_flag == 0)) {
+        notify = 1;
+    }
 
-        if (notify) {
-                ev.ev_type = CHANGELOG_OP_TYPE_JOURNAL;
-                memcpy (ev.u.journal.path, nfile, strlen (nfile) + 1);
-                changelog_dispatch_event (this, priv, &ev);
+    if (!ret) {
+        if (cl_empty_flag) {
+            update_path(this, nfile);
         }
- out:
-        /* If this is explicit rollover initiated by snapshot,
-         * wakeup reconfigure thread waiting for changelog to
-         * rollover. This should happen even in failure cases as
-         * well otherwise snapshot will timeout and fail. Hence
-         * moved under out.
-         */
-        if (priv->explicit_rollover) {
-                priv->explicit_rollover = _gf_false;
-
-                pthread_mutex_lock (&priv->bn.bnotify_mutex);
-                {
-                        if (ret) {
-                                priv->bn.bnotify_error = _gf_true;
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_MSG_EXPLICIT_ROLLOVER_FAILED,
-                                        "Fail snapshot because of "
-                                        "previous errors");
-                        } else {
-                                gf_smsg (this->name, GF_LOG_INFO, 0,
-                                         CHANGELOG_MSG_BNOTIFY_INFO, "Explicit "
-                                         "rollover changelog signaling "
-                                         "bnotify",
-                                         "changelog=%s", nfile,
-                                         NULL);
-                        }
-                        priv->bn.bnotify = _gf_false;
-                        pthread_cond_signal (&priv->bn.bnotify_cond);
-                }
-                pthread_mutex_unlock (&priv->bn.bnotify_mutex);
+        ret = htime_update(this, priv, ts, nfile);
+        if (ret == -1) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_HTIME_ERROR,
+                    NULL);
+            goto out;
         }
-        return ret;
+    }
+
+    if (notify) {
+        ev.ev_type = CHANGELOG_OP_TYPE_JOURNAL;
+        memcpy(ev.u.journal.path, nfile, strlen(nfile) + 1);
+        changelog_dispatch_event(this, priv, &ev);
+    }
+out:
+    /* If this is explicit rollover initiated by snapshot,
+     * wakeup reconfigure thread waiting for changelog to
+     * rollover. This should happen even in failure cases as
+     * well otherwise snapshot will timeout and fail. Hence
+     * moved under out.
+     */
+    if (priv->explicit_rollover) {
+        priv->explicit_rollover = _gf_false;
+
+        pthread_mutex_lock(&priv->bn.bnotify_mutex);
+        {
+            if (ret) {
+                priv->bn.bnotify_error = _gf_true;
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_MSG_EXPLICIT_ROLLOVER_FAILED, NULL);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_BNOTIFY_INFO,
+                        "changelog=%s", nfile, NULL);
+            }
+            priv->bn.bnotify = _gf_false;
+            pthread_cond_signal(&priv->bn.bnotify_cond);
+        }
+        pthread_mutex_unlock(&priv->bn.bnotify_mutex);
+    }
+    return ret;
 }
 
 int
-filter_cur_par_dirs (const struct dirent *entry)
+filter_cur_par_dirs(const struct dirent *entry)
 {
-        if (entry == NULL)
-                return 0;
+    if (entry == NULL)
+        return 0;
 
-        if ((strcmp(entry->d_name, ".") == 0) ||
-            (strcmp(entry->d_name, "..") == 0))
-                return 0;
-        else
-                return 1;
+    if ((strcmp(entry->d_name, ".") == 0) || (strcmp(entry->d_name, "..") == 0))
+        return 0;
+    else
+        return 1;
 }
 
 /*
@@ -556,295 +536,284 @@ filter_cur_par_dirs (const struct dirent *entry)
  */
 
 int
-find_current_htime (int ht_dir_fd, const char *ht_dir_path, char *ht_file_bname)
+find_current_htime(int ht_dir_fd, const char *ht_dir_path, char *ht_file_bname)
 {
-        struct dirent       **namelist = NULL;
-        int                   ret      = 0;
-        int                   cnt      = 0;
-        int                   i        = 0;
-        xlator_t             *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (ht_dir_path);
-
-        cnt = scandir (ht_dir_path, &namelist, filter_cur_par_dirs, alphasort);
-        if (cnt < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_SCAN_DIR_FAILED,
-                        "scandir failed");
-        } else if (cnt > 0) {
-                strncpy (ht_file_bname, namelist[cnt - 1]->d_name, NAME_MAX);
-                ht_file_bname[NAME_MAX - 1] = 0;
-
-                if (sys_fsetxattr (ht_dir_fd, HTIME_CURRENT, ht_file_bname,
-                    strlen (ht_file_bname), 0)) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_FSETXATTR_FAILED,
-                                "fsetxattr failed: HTIME_CURRENT");
-                        ret = -1;
-                        goto out;
-                }
-
-                if (sys_fsync (ht_dir_fd) < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_FSYNC_OP_FAILED,
-                                "fsync failed");
-                        ret = -1;
-                        goto out;
-                }
-        }
+    struct dirent **namelist = NULL;
+    int ret = 0;
+    int cnt = 0;
+    int i = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(ht_dir_path);
+
+    cnt = scandir(ht_dir_path, &namelist, filter_cur_par_dirs, alphasort);
+    if (cnt < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_SCAN_DIR_FAILED,
+                NULL);
+    } else if (cnt > 0) {
+        if (snprintf(ht_file_bname, NAME_MAX, "%s",
+                     namelist[cnt - 1]->d_name) >= NAME_MAX) {
+            ret = -1;
+            goto out;
+        }
+        if (sys_fsetxattr(ht_dir_fd, HTIME_CURRENT, ht_file_bname,
+                          strlen(ht_file_bname), 0)) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_FSETXATTR_FAILED, "HTIME_CURRENT", NULL);
+            ret = -1;
+            goto out;
+        }
+
+        if (sys_fsync(ht_dir_fd) < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_FSYNC_OP_FAILED, NULL);
+            ret = -1;
+            goto out;
+        }
+    }
 
- out:
-        for (i = 0; i < cnt; i++)
-                free (namelist[i]);
-        free (namelist);
+out:
+    for (i = 0; i < cnt; i++)
+        free(namelist[i]);
+    free(namelist);
 
-        if (ret)
-                cnt = ret;
+    if (ret)
+        cnt = ret;
 
-        return cnt;
+    return cnt;
 }
 
 /* Returns 0 on successful open of htime file
  * returns -1 on failure or error
  */
 int
-htime_open (xlator_t *this,
-              changelog_priv_t *priv, unsigned long ts)
+htime_open(xlator_t *this, changelog_priv_t *priv, time_t ts)
 {
-        int ht_file_fd                  = -1;
-        int ht_dir_fd                   = -1;
-        int ret                         = 0;
-        int cnt                         = 0;
-        char ht_dir_path[PATH_MAX]      = {0,};
-        char ht_file_path[PATH_MAX]     = {0,};
-        char ht_file_bname[NAME_MAX]    = {0,};
-        char x_value[NAME_MAX]          = {0,};
-        int flags                       = 0;
-        unsigned long min_ts            = 0;
-        unsigned long max_ts            = 0;
-        unsigned long total             = 0;
-        unsigned long total1            = 0;
-        ssize_t size                    = 0;
-        struct stat stat_buf            = {0,};
-        unsigned long record_len        = 0;
-
-        CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, ht_dir_path);
-
-        /* Open htime directory to get HTIME_CURRENT */
-        ht_dir_fd = open (ht_dir_path, O_RDONLY);
-        if (ht_dir_fd == -1) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_OPEN_FAILED, "open failed",
-                         "path=%s", ht_dir_path,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-
-        size = sys_fgetxattr (ht_dir_fd, HTIME_CURRENT, ht_file_bname,
-                             sizeof (ht_file_bname));
-        if (size < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_FGETXATTR_FAILED, "Error extracting"
-                        " HTIME_CURRENT.");
-
-                /* If upgrade scenario, find the latest HTIME.TSTAMP file
-                 * and use the same. If error, create a new HTIME.TSTAMP
-                 * file.
-                 */
-                cnt = find_current_htime (ht_dir_fd, ht_dir_path,
-                                           ht_file_bname);
-                if (cnt <= 0) {
-                        gf_msg (this->name, GF_LOG_INFO, errno,
-                                CHANGELOG_MSG_HTIME_INFO,
-                                "HTIME_CURRENT not found. Changelog enabled"
-                                " before init");
-                        return htime_create (this, priv, ts);
-                }
-
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_HTIME_ERROR, "Error extracting"
-                        " HTIME_CURRENT.");
-        }
-
-        gf_smsg (this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_HTIME_INFO,
-                 "HTIME_CURRENT",
-                 "path=%s", ht_file_bname, NULL);
-        (void) snprintf (ht_file_path, PATH_MAX, "%s/%s",
-                         ht_dir_path, ht_file_bname);
-
-        /* Open in append mode as existing htime file is used */
-        flags |= (O_RDWR | O_SYNC | O_APPEND);
-        ht_file_fd = open (ht_file_path, flags,
-                        S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (ht_file_fd < 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_OPEN_FAILED,
-                         "unable to open htime file",
-                         "path=%s", ht_file_path,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-
-        /* save this htime_fd in priv->htime_fd */
-        priv->htime_fd = ht_file_fd;
-
-        ret = sys_fstat (ht_file_fd, &stat_buf);
-        if (ret < 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_HTIME_ERROR,
-                         "unable to stat htime file",
-                         "path=%s", ht_file_path,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-
-        /* Initialize rollover-number in priv to current number */
-        size = sys_fgetxattr (ht_file_fd, HTIME_KEY, x_value, sizeof (x_value));
-        if (size < 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_FGETXATTR_FAILED, "error extracting max"
-                         " timstamp from htime file",
-                         "path=%s", ht_file_path,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-
-        sscanf (x_value, "%lu:%lu", &max_ts, &total);
-
-        /* 22 = 1(/) + 20(CHANGELOG.TIMESTAMP) + 1(\x00) */
-        record_len = strlen(priv->changelog_dir) + 22;
-        total1 = stat_buf.st_size/record_len;
-        if (total != total1) {
-                gf_smsg (this->name, GF_LOG_INFO, 0,
-                         CHANGELOG_MSG_TOTAL_LOG_INFO,
-                         "Mismatch of changelog count. "
-                         "INIT CASE",
-                         "xattr_total=%lu", total,
-                         "size_total=%lu", total1,
-                         NULL);
-        }
-
-        gf_smsg (this->name, GF_LOG_INFO, 0,
-                 CHANGELOG_MSG_TOTAL_LOG_INFO,
-                 "INIT CASE",
-                 "min=%lu", min_ts,
-                 "max=%lu", max_ts,
-                 "total_changelogs=%lu", total,
-                 NULL);
-
-        if (total < total1)
-                priv->rollover_count = total1 + 1;
-        else
-                priv->rollover_count = total + 1;
+    int ht_file_fd = -1;
+    int ht_dir_fd = -1;
+    int ret = 0;
+    int cnt = 0;
+    char ht_dir_path[PATH_MAX] = {
+        0,
+    };
+    char ht_file_path[PATH_MAX] = {
+        0,
+    };
+    char ht_file_bname[NAME_MAX] = {
+        0,
+    };
+    char x_value[NAME_MAX] = {
+        0,
+    };
+    int flags = 0;
+    unsigned long min_ts = 0;
+    unsigned long max_ts = 0;
+    unsigned long total = 0;
+    unsigned long total1 = 0;
+    ssize_t size = 0;
+    struct stat stat_buf = {
+        0,
+    };
+    unsigned long record_len = 0;
+    int32_t len = 0;
+
+    CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, ht_dir_path);
+
+    /* Open htime directory to get HTIME_CURRENT */
+    ht_dir_fd = open(ht_dir_path, O_RDONLY);
+    if (ht_dir_fd == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", ht_dir_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    size = sys_fgetxattr(ht_dir_fd, HTIME_CURRENT, ht_file_bname,
+                         sizeof(ht_file_bname));
+    if (size < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FGETXATTR_FAILED,
+                "name=HTIME_CURRENT", NULL);
+
+        /* If upgrade scenario, find the latest HTIME.TSTAMP file
+         * and use the same. If error, create a new HTIME.TSTAMP
+         * file.
+         */
+        cnt = find_current_htime(ht_dir_fd, ht_dir_path, ht_file_bname);
+        if (cnt <= 0) {
+            gf_smsg(this->name, GF_LOG_INFO, errno,
+                    CHANGELOG_MSG_NO_HTIME_CURRENT, NULL);
+            sys_close(ht_dir_fd);
+            return htime_create(this, priv, ts);
+        }
+
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_HTIME_CURRENT_ERROR, NULL);
+    }
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_HTIME_CURRENT, "path=%s",
+            ht_file_bname, NULL);
+    len = snprintf(ht_file_path, PATH_MAX, "%s/%s", ht_dir_path, ht_file_bname);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Open in append mode as existing htime file is used */
+    flags |= (O_RDWR | O_SYNC | O_APPEND);
+    ht_file_fd = open(ht_file_path, flags,
+                      S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (ht_file_fd < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", ht_file_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    /* save this htime_fd in priv->htime_fd */
+    priv->htime_fd = ht_file_fd;
+
+    ret = sys_fstat(ht_file_fd, &stat_buf);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_HTIME_STAT_ERROR,
+                "path=%s", ht_file_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    /* Initialize rollover-number in priv to current number */
+    size = sys_fgetxattr(ht_file_fd, HTIME_KEY, x_value, sizeof(x_value));
+    if (size < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FGETXATTR_FAILED,
+                "name=%s", HTIME_KEY, "path=%s", ht_file_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    sscanf(x_value, "%lu:%lu", &max_ts, &total);
+
+    /* 22 = 1(/) + 20(CHANGELOG.TIMESTAMP) + 1(\x00) */
+    record_len = strlen(priv->changelog_dir) + 22;
+    total1 = stat_buf.st_size / record_len;
+    if (total != total1) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_TOTAL_LOG_INFO,
+                "xattr_total=%lu", total, "size_total=%lu", total1, NULL);
+    }
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_TOTAL_LOG_INFO, "min=%lu",
+            min_ts, "max=%lu", max_ts, "total_changelogs=%lu", total, NULL);
+
+    if (total < total1)
+        priv->rollover_count = total1 + 1;
+    else
+        priv->rollover_count = total + 1;
 
 out:
-        if (ht_dir_fd != -1)
-                sys_close (ht_dir_fd);
-        return ret;
+    if (ht_dir_fd != -1)
+        sys_close(ht_dir_fd);
+    return ret;
 }
 
 /* Returns 0 on successful creation of htime file
  * returns -1 on failure or error
  */
 int
-htime_create (xlator_t *this,
-              changelog_priv_t *priv, unsigned long ts)
+htime_create(xlator_t *this, changelog_priv_t *priv, time_t ts)
 {
-        int ht_file_fd                      = -1;
-        int ht_dir_fd                       = -1;
-        int ret                             = 0;
-        char ht_dir_path[PATH_MAX]          = {0,};
-        char ht_file_path[PATH_MAX]         = {0,};
-        char ht_file_bname[NAME_MAX + 1]    = {0,};
-        int flags                           = 0;
-
-        gf_smsg (this->name, GF_LOG_INFO, 0,
-                 CHANGELOG_MSG_HTIME_INFO, "Changelog enable: Creating new "
-                 "HTIME file",
-                 "name=%lu", ts,
-                 NULL);
-
-        CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, ht_dir_path);
-
-        /* get the htime file name in ht_file_path */
-        (void) snprintf (ht_file_path,PATH_MAX,"%s/%s.%lu",ht_dir_path,
-                        HTIME_FILE_NAME, ts);
-
-        flags |= (O_CREAT | O_RDWR | O_SYNC);
-        ht_file_fd = open (ht_file_path, flags,
-                        S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (ht_file_fd < 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_OPEN_FAILED,
-                         "unable to create htime file",
-                         "path=%s", ht_file_path,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-
-        if (sys_fsetxattr (ht_file_fd, HTIME_KEY, HTIME_INITIAL_VALUE,
-                       sizeof (HTIME_INITIAL_VALUE)-1,  0)) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_FSETXATTR_FAILED,
-                        "Htime xattr initialization failed");
-                ret = -1;
-                goto out;
-        }
-
-        ret = sys_fsync (ht_file_fd);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_FSYNC_OP_FAILED,
-                        "fsync failed");
-                goto out;
-        }
-
-        /* Set xattr HTIME_CURRENT on htime directory to htime filename */
-        ht_dir_fd = open (ht_dir_path, O_RDONLY);
-        if (ht_dir_fd == -1) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_OPEN_FAILED, "open failed",
-                         "path=%s", ht_dir_path,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-
-        (void) snprintf (ht_file_bname, sizeof (ht_file_bname), "%s.%lu",
-                         HTIME_FILE_NAME, ts);
-        if (sys_fsetxattr (ht_dir_fd, HTIME_CURRENT, ht_file_bname,
-            strlen (ht_file_bname), 0)) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_FSETXATTR_FAILED, "fsetxattr failed:"
-                        " HTIME_CURRENT");
-                ret = -1;
-                goto out;
-        }
-
-        ret = sys_fsync (ht_dir_fd);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_FSYNC_OP_FAILED,
-                        "fsync failed");
-                goto out;
-        }
-
-        /* save this htime_fd in priv->htime_fd */
-        priv->htime_fd = ht_file_fd;
-        /* initialize rollover-number in priv to 1 */
-        priv->rollover_count = 1;
+    int ht_file_fd = -1;
+    int ht_dir_fd = -1;
+    int ret = 0;
+    char ht_dir_path[PATH_MAX] = {
+        0,
+    };
+    char ht_file_path[PATH_MAX] = {
+        0,
+    };
+    char ht_file_bname[NAME_MAX + 1] = {
+        0,
+    };
+    int flags = 0;
+    int32_t len = 0;
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_NEW_HTIME_FILE,
+            "name=%ld", ts, NULL);
+
+    CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, ht_dir_path);
+
+    /* get the htime file name in ht_file_path */
+    len = snprintf(ht_file_path, PATH_MAX, "%s/%s.%ld", ht_dir_path,
+                   HTIME_FILE_NAME, ts);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    flags |= (O_CREAT | O_RDWR | O_SYNC);
+    ht_file_fd = open(ht_file_path, flags,
+                      S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (ht_file_fd < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", ht_file_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (sys_fsetxattr(ht_file_fd, HTIME_KEY, HTIME_INITIAL_VALUE,
+                      sizeof(HTIME_INITIAL_VALUE) - 1, 0)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_XATTR_INIT_FAILED, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = sys_fsync(ht_file_fd);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FSYNC_OP_FAILED,
+                NULL);
+        goto out;
+    }
+
+    /* save this htime_fd in priv->htime_fd */
+    priv->htime_fd = ht_file_fd;
+
+    ht_file_fd = -1;
+
+    /* Set xattr HTIME_CURRENT on htime directory to htime filename */
+    ht_dir_fd = open(ht_dir_path, O_RDONLY);
+    if (ht_dir_fd == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", ht_dir_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    (void)snprintf(ht_file_bname, sizeof(ht_file_bname), "%s.%ld",
+                   HTIME_FILE_NAME, ts);
+    if (sys_fsetxattr(ht_dir_fd, HTIME_CURRENT, ht_file_bname,
+                      strlen(ht_file_bname), 0)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FSETXATTR_FAILED,
+                " HTIME_CURRENT", NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = sys_fsync(ht_dir_fd);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FSYNC_OP_FAILED,
+                NULL);
+        goto out;
+    }
+
+    /* initialize rollover-number in priv to 1 */
+    priv->rollover_count = 1;
 
 out:
-        if (ht_dir_fd != -1)
-                sys_close (ht_dir_fd);
-        return ret;
+    if (ht_dir_fd != -1)
+        sys_close(ht_dir_fd);
+    if (ht_file_fd != -1)
+        sys_close(ht_file_fd);
+    return ret;
 }
 
 /* Description:
@@ -856,49 +825,53 @@ out:
  *      -1  : On failure.
  */
 int
-changelog_snap_open (xlator_t *this,
-                         changelog_priv_t *priv)
+changelog_snap_open(xlator_t *this, changelog_priv_t *priv)
 {
-        int fd                        = -1;
-        int ret                       = 0;
-        int flags                     = 0;
-        char buffer[1024]             = {0,};
-        char c_snap_path[PATH_MAX]    = {0,};
-        char csnap_dir_path[PATH_MAX] = {0,};
-
-        CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir_path);
-
-        (void) snprintf (c_snap_path, PATH_MAX,
-                        "%s/"CSNAP_FILE_NAME,
-                        csnap_dir_path);
-
-        flags |= (O_CREAT | O_RDWR | O_TRUNC);
-
-        fd = open (c_snap_path, flags,
-                        S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (fd < 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_OPEN_FAILED, "unable to open file",
-                         "path=%s", c_snap_path,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-        priv->c_snap_fd = fd;
-
-        (void) snprintf (buffer, 1024, CHANGELOG_HEADER,
-                        CHANGELOG_VERSION_MAJOR,
-                        CHANGELOG_VERSION_MINOR,
-                        priv->ce->encoder);
-        ret = changelog_snap_write_change (priv, buffer, strlen (buffer));
-        if (ret < 0) {
-                sys_close (priv->c_snap_fd);
-                priv->c_snap_fd = -1;
-                goto out;
-        }
+    int fd = -1;
+    int ret = 0;
+    int flags = 0;
+    char buffer[1024] = {
+        0,
+    };
+    char c_snap_path[PATH_MAX] = {
+        0,
+    };
+    char csnap_dir_path[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir_path);
+
+    len = snprintf(c_snap_path, PATH_MAX, "%s/" CSNAP_FILE_NAME,
+                   csnap_dir_path);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    flags |= (O_CREAT | O_RDWR | O_TRUNC);
+
+    fd = open(c_snap_path, flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (fd < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", c_snap_path, NULL);
+        ret = -1;
+        goto out;
+    }
+    priv->c_snap_fd = fd;
+
+    (void)snprintf(buffer, 1024, CHANGELOG_HEADER, CHANGELOG_VERSION_MAJOR,
+                   CHANGELOG_VERSION_MINOR, priv->ce->encoder);
+    ret = changelog_snap_write_change(priv, buffer, strlen(buffer));
+    if (ret < 0) {
+        sys_close(priv->c_snap_fd);
+        priv->c_snap_fd = -1;
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -909,17 +882,15 @@ out:
  *      -1 : On Failure.
  */
 int
-changelog_snap_logging_start (xlator_t *this,
-                                  changelog_priv_t *priv)
+changelog_snap_logging_start(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
+    int ret = 0;
 
-        ret = changelog_snap_open (this, priv);
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                        CHANGELOG_MSG_SNAP_INFO,
-                        "Now starting to log in call path");
+    ret = changelog_snap_open(this, priv);
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_SNAP_INFO, "starting",
+            NULL);
 
-        return ret;
+    return ret;
 }
 
 /*
@@ -930,120 +901,104 @@ changelog_snap_logging_start (xlator_t *this,
  *      -1 : On Failure.
  */
 int
-changelog_snap_logging_stop (xlator_t *this,
-                changelog_priv_t *priv)
+changelog_snap_logging_stop(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret         = 0;
+    int ret = 0;
 
-        sys_close (priv->c_snap_fd);
-        priv->c_snap_fd = -1;
+    sys_close(priv->c_snap_fd);
+    priv->c_snap_fd = -1;
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                CHANGELOG_MSG_SNAP_INFO,
-                "Stopped to log in call path");
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_SNAP_INFO, "Stopped",
+            NULL);
 
-        return ret;
+    return ret;
 }
 
 int
-changelog_open_journal (xlator_t *this,
-                        changelog_priv_t *priv)
+changelog_open_journal(xlator_t *this, changelog_priv_t *priv)
 {
-        int fd                        = 0;
-        int ret                       = -1;
-        int flags                     = 0;
-        char buffer[1024]             = {0,};
-        char changelog_path[PATH_MAX] = {0,};
-
-        (void) snprintf (changelog_path, PATH_MAX,
-                         "%s/"CHANGELOG_FILE_NAME,
-                         priv->changelog_dir);
-
-        flags |= (O_CREAT | O_RDWR);
-        if (priv->fsync_interval == 0)
-                flags |= O_SYNC;
-
-        fd = open (changelog_path, flags,
-                   S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (fd < 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_OPEN_FAILED,
-                         "unable to open/create changelog file."
-                         " change-logging will be"
-                         " inactive",
-                         "path=%s", changelog_path,
-                         NULL);
-                goto out;
-        }
-
-        priv->changelog_fd = fd;
+    int fd = 0;
+    int ret = -1;
+    int flags = 0;
+    char buffer[1024] = {
+        0,
+    };
+    char changelog_path[PATH_MAX] = {
+        0,
+    };
+
+    (void)snprintf(changelog_path, PATH_MAX, "%s/" CHANGELOG_FILE_NAME,
+                   priv->changelog_dir);
+
+    flags |= (O_CREAT | O_RDWR);
+    if (priv->fsync_interval == 0)
+        flags |= O_SYNC;
+
+    fd = open(changelog_path, flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (fd < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", changelog_path, NULL);
+        goto out;
+    }
+
+    priv->changelog_fd = fd;
+
+    (void)snprintf(buffer, 1024, CHANGELOG_HEADER, CHANGELOG_VERSION_MAJOR,
+                   CHANGELOG_VERSION_MINOR, priv->ce->encoder);
+    ret = changelog_write_change(priv, buffer, strlen(buffer));
+    if (ret) {
+        sys_close(priv->changelog_fd);
+        priv->changelog_fd = -1;
+        goto out;
+    }
+
+    ret = 0;
 
-        (void) snprintf (buffer, 1024, CHANGELOG_HEADER,
-                         CHANGELOG_VERSION_MAJOR,
-                         CHANGELOG_VERSION_MINOR,
-                         priv->ce->encoder);
-        ret = changelog_write_change (priv, buffer, strlen (buffer));
-        if (ret) {
-                sys_close (priv->changelog_fd);
-                priv->changelog_fd = -1;
-                goto out;
-        }
-
-        ret = 0;
-
- out:
-        return ret;
+out:
+    return ret;
 }
 
 int
-changelog_start_next_change (xlator_t *this,
-                             changelog_priv_t *priv,
-                             unsigned long ts, gf_boolean_t finale)
+changelog_start_next_change(xlator_t *this, changelog_priv_t *priv, time_t ts,
+                            gf_boolean_t finale)
 {
-        int ret = -1;
+    int ret = -1;
 
-        ret = changelog_rollover_changelog (this, priv, ts);
+    ret = changelog_rollover_changelog(this, priv, ts);
 
-        if (!ret && !finale)
-                ret = changelog_open_journal (this, priv);
+    if (!ret && !finale)
+        ret = changelog_open_journal(this, priv);
 
-        return ret;
+    return ret;
 }
 
 /**
  * return the length of entry
  */
 size_t
-changelog_entry_length ()
+changelog_entry_length()
 {
-        return sizeof (changelog_log_data_t);
+    return sizeof(changelog_log_data_t);
 }
 
-int
-changelog_fill_rollover_data (changelog_log_data_t *cld, gf_boolean_t is_last)
+void
+changelog_fill_rollover_data(changelog_log_data_t *cld, gf_boolean_t is_last)
 {
-        struct timeval tv = {0,};
-
-        cld->cld_type = CHANGELOG_TYPE_ROLLOVER;
-
-        if (gettimeofday (&tv, NULL))
-                return -1;
-
-        cld->cld_roll_time = (unsigned long) tv.tv_sec;
-        cld->cld_finale = is_last;
-        return 0;
+    cld->cld_type = CHANGELOG_TYPE_ROLLOVER;
+    cld->cld_roll_time = gf_time();
+    cld->cld_finale = is_last;
 }
 
 int
-changelog_snap_write_change (changelog_priv_t *priv, char *buffer, size_t len)
+changelog_snap_write_change(changelog_priv_t *priv, char *buffer, size_t len)
 {
-        return changelog_write (priv->c_snap_fd, buffer, len);
+    return changelog_write(priv->c_snap_fd, buffer, len);
 }
 
 int
-changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len)
+changelog_write_change(changelog_priv_t *priv, char *buffer, size_t len)
 {
-        return changelog_write (priv->changelog_fd, buffer, len);
+    return changelog_write(priv->changelog_fd, buffer, len);
 }
 
 /*
@@ -1056,255 +1011,230 @@ changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len)
  *     -1 : On Failure.
  */
 int
-changelog_snap_handle_ascii_change (xlator_t *this,
-                                    changelog_log_data_t *cld)
+changelog_snap_handle_ascii_change(xlator_t *this, changelog_log_data_t *cld)
 {
-        size_t            off      = 0;
-        size_t            gfid_len = 0;
-        char             *gfid_str = NULL;
-        char             *buffer   = NULL;
-        changelog_priv_t *priv     = NULL;
-        int               ret      = 0;
-
-        if (this == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        priv = this->private;
-
-        if (priv == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        gfid_str = uuid_utoa (cld->cld_gfid);
-        gfid_len = strlen (gfid_str);
-
-        /*  extra bytes for decorations */
-        buffer = alloca (gfid_len + cld->cld_ptr_len + 10);
-        CHANGELOG_STORE_ASCII (priv, buffer,
-                        off, gfid_str, gfid_len, cld);
-
-        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
-
-        ret = changelog_snap_write_change (priv, buffer, off);
-
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_WRITE_FAILED,
-                        "error writing csnap to disk");
-        }
-        gf_msg (this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_SNAP_INFO,
-                        "Successfully wrote to csnap");
-        ret = 0;
+    size_t off = 0;
+    size_t gfid_len = 0;
+    char *gfid_str = NULL;
+    char *buffer = NULL;
+    changelog_priv_t *priv = NULL;
+    int ret = 0;
+
+    if (this == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    priv = this->private;
+
+    if (priv == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    gfid_str = uuid_utoa(cld->cld_gfid);
+    gfid_len = strlen(gfid_str);
+
+    /*  extra bytes for decorations */
+    buffer = alloca(gfid_len + cld->cld_ptr_len + 10);
+    CHANGELOG_STORE_ASCII(priv, buffer, off, gfid_str, gfid_len, cld);
+
+    CHANGELOG_FILL_BUFFER(buffer, off, "\0", 1);
+
+    ret = changelog_snap_write_change(priv, buffer, off);
+
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_WRITE_FAILED,
+                "csnap", NULL);
+    }
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_WROTE_TO_CSNAP, NULL);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-changelog_handle_change (xlator_t *this,
-                         changelog_priv_t *priv, changelog_log_data_t *cld)
+changelog_handle_change(xlator_t *this, changelog_priv_t *priv,
+                        changelog_log_data_t *cld)
 {
-        int ret = 0;
-
-        if (CHANGELOG_TYPE_IS_ROLLOVER (cld->cld_type)) {
-                changelog_encode_change (priv);
-                ret = changelog_start_next_change (this, priv,
-                                                   cld->cld_roll_time,
-                                                   cld->cld_finale);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_MSG_GET_TIME_OP_FAILED,
-                                "Problem rolling over changelog(s)");
-                goto out;
-        }
+    int ret = 0;
 
-        /**
-         * case when there is reconfigure done (disabling changelog) and there
-         * are still fops that have updates in prgress.
-         */
-        if (priv->changelog_fd == -1)
-                return 0;
-
-        if (CHANGELOG_TYPE_IS_FSYNC (cld->cld_type)) {
-                ret = sys_fsync (priv->changelog_fd);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_FSYNC_OP_FAILED,
-                                "fsync failed");
-                }
-                goto out;
-        }
+    if (CHANGELOG_TYPE_IS_ROLLOVER(cld->cld_type)) {
+        changelog_encode_change(priv);
+        ret = changelog_start_next_change(this, priv, cld->cld_roll_time,
+                                          cld->cld_finale);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    CHANGELOG_MSG_GET_TIME_OP_FAILED, NULL);
+        goto out;
+    }
+
+    /**
+     * case when there is reconfigure done (disabling changelog) and there
+     * are still fops that have updates in prgress.
+     */
+    if (priv->changelog_fd == -1)
+        return 0;
 
-        ret = priv->ce->encode (this, cld);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_WRITE_FAILED,
-                        "error writing changelog to disk");
+    if (CHANGELOG_TYPE_IS_FSYNC(cld->cld_type)) {
+        ret = sys_fsync(priv->changelog_fd);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_FSYNC_OP_FAILED, NULL);
         }
+        goto out;
+    }
 
- out:
-        return ret;
+    ret = priv->ce->encode(this, cld);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_WRITE_FAILED,
+                "changelog", NULL);
+    }
+
+out:
+    return ret;
 }
 
 changelog_local_t *
-changelog_local_init (xlator_t *this, inode_t *inode,
-                      uuid_t gfid, int xtra_records,
-                      gf_boolean_t update_flag)
+changelog_local_init(xlator_t *this, inode_t *inode, uuid_t gfid,
+                     int xtra_records, gf_boolean_t update_flag)
 {
-        changelog_local_t *local = NULL;
-        struct iobuf      *iobuf = NULL;
+    changelog_local_t *local = NULL;
+    struct iobuf *iobuf = NULL;
+
+    /**
+     * We relax the presence of inode if @update_flag is true.
+     * The caller (implementation of the fop) needs to be careful to
+     * not blindly use local->inode.
+     */
+    if (!update_flag && !inode) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0,
+                         CHANGELOG_MSG_INODE_NOT_FOUND,
+                         "inode needed for version checking !!!");
 
-        /**
-         * We relax the presence of inode if @update_flag is true.
-         * The caller (implmentation of the fop) needs to be careful to
-         * not blindly use local->inode.
-         */
-        if (!update_flag && !inode) {
-                gf_msg_callingfn (this->name, GF_LOG_WARNING, 0,
-                                  CHANGELOG_MSG_INODE_NOT_FOUND,
-                                  "inode needed for version checking !!!");
-                goto out;
-        }
+        goto out;
+    }
 
-        if (xtra_records) {
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool,
-                                    xtra_records * CHANGELOG_OPT_RECORD_LEN);
-                if (!iobuf)
-                        goto out;
-        }
+    if (xtra_records) {
+        iobuf = iobuf_get2(this->ctx->iobuf_pool,
+                           xtra_records * CHANGELOG_OPT_RECORD_LEN);
+        if (!iobuf)
+            goto out;
+    }
 
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                CHANGELOG_IOBUF_UNREF (iobuf);
-                goto out;
-        }
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        CHANGELOG_IOBUF_UNREF(iobuf);
+        goto out;
+    }
 
-        local->update_no_check = update_flag;
+    local->update_no_check = update_flag;
 
-        gf_uuid_copy (local->cld.cld_gfid, gfid);
+    gf_uuid_copy(local->cld.cld_gfid, gfid);
 
-        local->cld.cld_iobuf = iobuf;
-        local->cld.cld_xtra_records = 0; /* set by the caller */
+    local->cld.cld_iobuf = iobuf;
+    local->cld.cld_xtra_records = 0; /* set by the caller */
 
-        if (inode)
-                local->inode = inode_ref (inode);
+    if (inode)
+        local->inode = inode_ref(inode);
 
- out:
-        return local;
+out:
+    return local;
 }
 
 int
-changelog_forget (xlator_t *this, inode_t *inode)
+changelog_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t ctx_addr = 0;
-        changelog_inode_ctx_t *ctx = NULL;
+    uint64_t ctx_addr = 0;
+    changelog_inode_ctx_t *ctx = NULL;
 
-        inode_ctx_del (inode, this, &ctx_addr);
-        if (!ctx_addr)
-                return 0;
+    inode_ctx_del(inode, this, &ctx_addr);
+    if (!ctx_addr)
+        return 0;
 
-        ctx = (changelog_inode_ctx_t *) (long) ctx_addr;
-        GF_FREE (ctx);
+    ctx = (changelog_inode_ctx_t *)(long)ctx_addr;
+    GF_FREE(ctx);
 
-        return 0;
+    return 0;
 }
 
 int
-changelog_inject_single_event (xlator_t *this,
-                               changelog_priv_t *priv,
-                               changelog_log_data_t *cld)
+changelog_inject_single_event(xlator_t *this, changelog_priv_t *priv,
+                              changelog_log_data_t *cld)
 {
-        return priv->cd.dispatchfn (this, priv, priv->cd.cd_data, cld, NULL);
+    return priv->cd.dispatchfn(this, priv, priv->cd.cd_data, cld, NULL);
 }
 
 /* Wait till all the black fops are drained */
 void
-changelog_drain_black_fops (xlator_t *this, changelog_priv_t *priv)
+changelog_drain_black_fops(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
-
-        /* clean up framework of pthread_mutex is required here as
-         * 'reconfigure' terminates the changelog_rollover thread
-         * on graph change.
-         */
-        pthread_cleanup_push (changelog_cleanup_free_mutex,
-                                        &priv->dm.drain_black_mutex);
-        ret = pthread_mutex_lock (&priv->dm.drain_black_mutex);
-        if (ret)
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_PTHREAD_ERROR, "pthread error",
-                         "error=%d", ret,
-                         NULL);
-        while (priv->dm.black_fop_cnt > 0) {
-                gf_msg_debug (this->name, 0,
-                              "Condtional wait on black fops: %ld",
-                              priv->dm.black_fop_cnt);
-                priv->dm.drain_wait_black = _gf_true;
-                ret = pthread_cond_wait (&priv->dm.drain_black_cond,
-                                         &priv->dm.drain_black_mutex);
-                if (ret)
-                        gf_smsg (this->name, GF_LOG_ERROR, errno,
-                                 CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED,
-                                 "pthread cond wait failed",
-                                 "error=%d", ret,
-                                 NULL);
-        }
-        priv->dm.drain_wait_black = _gf_false;
-        ret = pthread_mutex_unlock (&priv->dm.drain_black_mutex);
+    int ret = 0;
+
+    /* clean up framework of pthread_mutex is required here as
+     * 'reconfigure' terminates the changelog_rollover thread
+     * on graph change.
+     */
+    pthread_cleanup_push(changelog_cleanup_free_mutex,
+                         &priv->dm.drain_black_mutex);
+    ret = pthread_mutex_lock(&priv->dm.drain_black_mutex);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PTHREAD_ERROR,
+                "error=%d", ret, NULL);
+    while (priv->dm.black_fop_cnt > 0) {
+        gf_msg_debug(this->name, 0, "Conditional wait on black fops: %ld",
+                     priv->dm.black_fop_cnt);
+        priv->dm.drain_wait_black = _gf_true;
+        ret = pthread_cond_wait(&priv->dm.drain_black_cond,
+                                &priv->dm.drain_black_mutex);
         if (ret)
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_PTHREAD_ERROR, "pthread error",
-                         "error=%d", ret,
-                         NULL);
-        pthread_cleanup_pop (0);
-        gf_msg_debug (this->name, 0,
-                      "Woke up: Conditional wait on black fops");
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED, "error=%d", ret,
+                    NULL);
+    }
+    priv->dm.drain_wait_black = _gf_false;
+    ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PTHREAD_ERROR,
+                "error=%d", ret, NULL);
+    pthread_cleanup_pop(0);
+    gf_msg_debug(this->name, 0, "Woke up: Conditional wait on black fops");
 }
 
 /* Wait till all the white  fops are drained */
 void
-changelog_drain_white_fops (xlator_t *this, changelog_priv_t *priv)
+changelog_drain_white_fops(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
-
-        /* clean up framework of pthread_mutex is required here as
-         * 'reconfigure' terminates the changelog_rollover thread
-         * on graph change.
-         */
-        pthread_cleanup_push (changelog_cleanup_free_mutex,
-                                        &priv->dm.drain_white_mutex);
-        ret = pthread_mutex_lock (&priv->dm.drain_white_mutex);
+    int ret = 0;
+
+    /* clean up framework of pthread_mutex is required here as
+     * 'reconfigure' terminates the changelog_rollover thread
+     * on graph change.
+     */
+    pthread_cleanup_push(changelog_cleanup_free_mutex,
+                         &priv->dm.drain_white_mutex);
+    ret = pthread_mutex_lock(&priv->dm.drain_white_mutex);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PTHREAD_ERROR,
+                "error=%d", ret, NULL);
+    while (priv->dm.white_fop_cnt > 0) {
+        gf_msg_debug(this->name, 0, "Conditional wait on white fops : %ld",
+                     priv->dm.white_fop_cnt);
+        priv->dm.drain_wait_white = _gf_true;
+        ret = pthread_cond_wait(&priv->dm.drain_white_cond,
+                                &priv->dm.drain_white_mutex);
         if (ret)
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_PTHREAD_ERROR, "pthread error",
-                         "error=%d", ret,
-                         NULL);
-        while (priv->dm.white_fop_cnt > 0) {
-                gf_msg_debug (this->name, 0,
-                              "Condtional wait on white fops : %ld",
-                        priv->dm.white_fop_cnt);
-                priv->dm.drain_wait_white = _gf_true;
-                ret = pthread_cond_wait (&priv->dm.drain_white_cond,
-                                         &priv->dm.drain_white_mutex);
-                if (ret)
-                        gf_smsg (this->name, GF_LOG_ERROR, errno,
-                                 CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED,
-                                 "pthread cond wait failed",
-                                 "error=%d", ret,
-                                 NULL);
-        }
-        priv->dm.drain_wait_white = _gf_false;
-        ret = pthread_mutex_unlock (&priv->dm.drain_white_mutex);
-        if (ret)
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_PTHREAD_ERROR, "pthread error",
-                         "error=%d", ret,
-                         NULL);
-        pthread_cleanup_pop (0);
-        gf_msg_debug (this->name, 0,
-                      "Woke up: Conditional wait on white fops");
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED, "error=%d", ret,
+                    NULL);
+    }
+    priv->dm.drain_wait_white = _gf_false;
+    ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PTHREAD_ERROR,
+                "error=%d", ret, NULL);
+    pthread_cleanup_pop(0);
+    gf_msg_debug(this->name, 0, "Woke up: Conditional wait on white fops");
 }
 
 /**
@@ -1312,193 +1242,194 @@ changelog_drain_white_fops (xlator_t *this, changelog_priv_t *priv)
  * a certain time etc..). move them into separate routine.
  */
 void *
-changelog_rollover (void *data)
+changelog_rollover(void *data)
 {
-        int                     ret             = 0;
-        xlator_t               *this            = NULL;
-        struct timespec         tv              = {0,};
-        changelog_log_data_t    cld             = {0,};
-        changelog_time_slice_t *slice           = NULL;
-        changelog_priv_t       *priv            = data;
-
-        this = priv->cr.this;
-        slice = &priv->slice;
-
-        while (1) {
-                (void) pthread_testcancel();
-
-                tv.tv_sec  = time (NULL) + priv->rollover_time;
-                tv.tv_nsec = 0;
-                ret = 0; /* Reset ret to zero */
-
-               /* The race between actual rollover and explicit rollover is
-                * handled. If actual rollover is being done and the
-                * explicit rollover event comes, the event is not missed.
-                * Since explicit rollover sets 'cr.notify' to true, this
-                * thread doesn't wait on 'pthread_cond_timedwait'.
-                */
-                pthread_cleanup_push (changelog_cleanup_free_mutex,
-                                      &priv->cr.lock);
-                pthread_mutex_lock (&priv->cr.lock);
-                {
-                        while (ret == 0 && !priv->cr.notify)
-                                ret = pthread_cond_timedwait (&priv->cr.cond,
-                                                              &priv->cr.lock,
-                                                              &tv);
-                        if (ret == 0)
-                                priv->cr.notify = _gf_false;
-                }
-                pthread_mutex_unlock (&priv->cr.lock);
-                pthread_cleanup_pop (0);
-
-                if (ret == 0) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                CHANGELOG_MSG_BARRIER_INFO,
-                                "Explicit wakeup on barrier notify");
-                        priv->explicit_rollover = _gf_true;
-                } else if (ret && ret != ETIMEDOUT) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CHANGELOG_MSG_SELECT_FAILED,
-                                "pthread_cond_timedwait failed");
-                        continue;
-                } else if (ret && ret == ETIMEDOUT) {
-                        gf_msg_debug (this->name, 0, "Wokeup on timeout");
-                }
-
-               /* Reading curent_color without lock is fine here
-                * as it is only modified here and is next to reading.
-                */
-                if (priv->current_color == FOP_COLOR_BLACK) {
-                        LOCK(&priv->lock);
-                                priv->current_color = FOP_COLOR_WHITE;
-                        UNLOCK(&priv->lock);
-                        gf_msg_debug (this->name, 0, "Black fops"
-                                      " to be drained:%ld",
-                                      priv->dm.black_fop_cnt);
-                        changelog_drain_black_fops (this, priv);
-                } else {
-                        LOCK(&priv->lock);
-                                priv->current_color = FOP_COLOR_BLACK;
-                        UNLOCK(&priv->lock);
-                        gf_msg_debug (this->name, 0, "White fops"
-                                      " to be drained:%ld",
-                                      priv->dm.white_fop_cnt);
-                        changelog_drain_white_fops (this, priv);
-                }
-
-                /* Adding delay of 1 second only during explicit rollover:
-                 *
-                 * Changelog rollover can happen either due to actual
-                 * or the explict rollover during snapshot. Actual
-                 * rollover is controlled by tuneable called 'rollover-time'.
-                 * The minimum granularity for rollover-time is 1 second.
-                 * Explicit rollover is asynchronous in nature and happens
-                 * during snapshot.
-                 *
-                 * Basically, rollover renames the current CHANGELOG file
-                 * to CHANGELOG.TIMESTAMP. Let's assume, at time 't1',
-                 * actual and explicit rollover raced against  each
-                 * other and actual rollover won the race renaming the
-                 * CHANGELOG file to CHANGELOG.t1 and opens a new
-                 * CHANGELOG file. There is high chance that, an immediate
-                 * explicit rollover at time 't1' can happen with in the same
-                 * second to rename CHANGELOG file to CHANGELOG.t1 resulting in
-                 * purging the earlier CHANGELOG.t1 file created by actual
-                 * rollover. So adding a delay of 1 second guarantees unique
-                 * CHANGELOG.TIMESTAMP during  explicit rollover.
-                 */
-                if (priv->explicit_rollover == _gf_true)
-                        sleep (1);
-
-                ret = changelog_fill_rollover_data (&cld, _gf_false);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_MSG_GET_TIME_OP_FAILED,
-                                "failed to fill rollover data");
-                        continue;
-                }
+    int ret = 0;
+    xlator_t *this = NULL;
+    struct timespec tv = {
+        0,
+    };
+    changelog_log_data_t cld = {
+        0,
+    };
+    changelog_time_slice_t *slice = NULL;
+    changelog_priv_t *priv = data;
+
+    this = priv->cr.this;
+    slice = &priv->slice;
+
+    while (1) {
+        (void)pthread_testcancel();
+
+        tv.tv_sec = gf_time() + priv->rollover_time;
+        tv.tv_nsec = 0;
+        ret = 0; /* Reset ret to zero */
+
+        /* The race between actual rollover and explicit rollover is
+         * handled. If actual rollover is being done and the
+         * explicit rollover event comes, the event is not missed.
+         * Since explicit rollover sets 'cr.notify' to true, this
+         * thread doesn't wait on 'pthread_cond_timedwait'.
+         */
+        pthread_cleanup_push(changelog_cleanup_free_mutex, &priv->cr.lock);
+        pthread_mutex_lock(&priv->cr.lock);
+        {
+            while (ret == 0 && !priv->cr.notify)
+                ret = pthread_cond_timedwait(&priv->cr.cond, &priv->cr.lock,
+                                             &tv);
+            if (ret == 0)
+                priv->cr.notify = _gf_false;
+        }
+        pthread_mutex_unlock(&priv->cr.lock);
+        pthread_cleanup_pop(0);
+
+        if (ret == 0) {
+            gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_BARRIER_INFO,
+                    NULL);
+            priv->explicit_rollover = _gf_true;
+        } else if (ret && ret != ETIMEDOUT) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_SELECT_FAILED, NULL);
+            continue;
+        } else if (ret && ret == ETIMEDOUT) {
+            gf_msg_debug(this->name, 0, "Wokeup on timeout");
+        }
+
+        /* Reading curent_color without lock is fine here
+         * as it is only modified here and is next to reading.
+         */
+        if (priv->current_color == FOP_COLOR_BLACK) {
+            LOCK(&priv->lock);
+            priv->current_color = FOP_COLOR_WHITE;
+            UNLOCK(&priv->lock);
+            gf_msg_debug(this->name, 0,
+                         "Black fops"
+                         " to be drained:%ld",
+                         priv->dm.black_fop_cnt);
+            changelog_drain_black_fops(this, priv);
+        } else {
+            LOCK(&priv->lock);
+            priv->current_color = FOP_COLOR_BLACK;
+            UNLOCK(&priv->lock);
+            gf_msg_debug(this->name, 0,
+                         "White fops"
+                         " to be drained:%ld",
+                         priv->dm.white_fop_cnt);
+            changelog_drain_white_fops(this, priv);
+        }
+
+        /* Adding delay of 1 second only during explicit rollover:
+         *
+         * Changelog rollover can happen either due to actual
+         * or the explicit rollover during snapshot. Actual
+         * rollover is controlled by tuneable called 'rollover-time'.
+         * The minimum granularity for rollover-time is 1 second.
+         * Explicit rollover is asynchronous in nature and happens
+         * during snapshot.
+         *
+         * Basically, rollover renames the current CHANGELOG file
+         * to CHANGELOG.TIMESTAMP. Let's assume, at time 't1',
+         * actual and explicit rollover raced against  each
+         * other and actual rollover won the race renaming the
+         * CHANGELOG file to CHANGELOG.t1 and opens a new
+         * CHANGELOG file. There is high chance that, an immediate
+         * explicit rollover at time 't1' can happen with in the same
+         * second to rename CHANGELOG file to CHANGELOG.t1 resulting in
+         * purging the earlier CHANGELOG.t1 file created by actual
+         * rollover. So adding a delay of 1 second guarantees unique
+         * CHANGELOG.TIMESTAMP during  explicit rollover.
+         */
+        if (priv->explicit_rollover == _gf_true)
+            sleep(1);
 
-                _mask_cancellation ();
+        changelog_fill_rollover_data(&cld, _gf_false);
 
-                LOCK (&priv->lock);
-                {
-                        ret = changelog_inject_single_event (this, priv, &cld);
-                        if (!ret)
-                                SLICE_VERSION_UPDATE (slice);
-                }
-                UNLOCK (&priv->lock);
+        _mask_cancellation();
 
-                _unmask_cancellation ();
+        LOCK(&priv->lock);
+        {
+            ret = changelog_inject_single_event(this, priv, &cld);
+            if (!ret)
+                SLICE_VERSION_UPDATE(slice);
         }
+        UNLOCK(&priv->lock);
 
-        return NULL;
+        _unmask_cancellation();
+    }
+
+    return NULL;
 }
 
 void *
-changelog_fsync_thread (void *data)
+changelog_fsync_thread(void *data)
 {
-        int                   ret  = 0;
-        xlator_t             *this = NULL;
-        struct timeval        tv   = {0,};
-        changelog_log_data_t  cld  = {0,};
-        changelog_priv_t     *priv = data;
-
-        this = priv->cf.this;
-        cld.cld_type = CHANGELOG_TYPE_FSYNC;
-
-        while (1) {
-                (void) pthread_testcancel();
-
-                tv.tv_sec  = priv->fsync_interval;
-                tv.tv_usec = 0;
+    int ret = 0;
+    xlator_t *this = NULL;
+    struct timeval tv = {
+        0,
+    };
+    changelog_log_data_t cld = {
+        0,
+    };
+    changelog_priv_t *priv = data;
+
+    this = priv->cf.this;
+    cld.cld_type = CHANGELOG_TYPE_FSYNC;
+
+    while (1) {
+        (void)pthread_testcancel();
+
+        tv.tv_sec = priv->fsync_interval;
+        tv.tv_usec = 0;
+
+        ret = select(0, NULL, NULL, NULL, &tv);
+        if (ret)
+            continue;
 
-                ret = select (0, NULL, NULL, NULL, &tv);
-                if (ret)
-                        continue;
+        _mask_cancellation();
 
-                _mask_cancellation ();
+        ret = changelog_inject_single_event(this, priv, &cld);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    CHANGELOG_MSG_INJECT_FSYNC_FAILED, NULL);
 
-                ret = changelog_inject_single_event (this, priv, &cld);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_MSG_INJECT_FSYNC_FAILED,
-                                "failed to inject fsync event");
+        _unmask_cancellation();
+    }
 
-                _unmask_cancellation ();
-        }
-
-        return NULL;
+    return NULL;
 }
 
 /* macros for inode/changelog version checks */
 
-#define INODE_VERSION_UPDATE(priv, inode, iver, slice, type) do {       \
-                LOCK (&inode->lock);                                    \
-                {                                                       \
-                        LOCK (&priv->lock);                             \
-                        {                                               \
-                                *iver = slice->changelog_version[type]; \
-                        }                                               \
-                        UNLOCK (&priv->lock);                           \
-                }                                                       \
-                UNLOCK (&inode->lock);                                  \
-        } while (0)
-
-#define INODE_VERSION_EQUALS_SLICE(priv, ver, slice, type, upd) do {    \
-                LOCK (&priv->lock);                                     \
-                {                                                       \
-                        upd = (ver == slice->changelog_version[type])   \
-                                ? _gf_false : _gf_true;                 \
-                }                                                       \
-                UNLOCK (&priv->lock);                                   \
-        } while (0)
+#define INODE_VERSION_UPDATE(priv, inode, iver, slice, type)                   \
+    do {                                                                       \
+        LOCK(&inode->lock);                                                    \
+        {                                                                      \
+            LOCK(&priv->lock);                                                 \
+            {                                                                  \
+                *iver = slice->changelog_version[type];                        \
+            }                                                                  \
+            UNLOCK(&priv->lock);                                               \
+        }                                                                      \
+        UNLOCK(&inode->lock);                                                  \
+    } while (0)
+
+#define INODE_VERSION_EQUALS_SLICE(priv, ver, slice, type, upd)                \
+    do {                                                                       \
+        LOCK(&priv->lock);                                                     \
+        {                                                                      \
+            upd = (ver == slice->changelog_version[type]) ? _gf_false          \
+                                                          : _gf_true;          \
+        }                                                                      \
+        UNLOCK(&priv->lock);                                                   \
+    } while (0)
 
 static int
-__changelog_inode_ctx_set (xlator_t *this,
-                           inode_t *inode, changelog_inode_ctx_t *ctx)
+__changelog_inode_ctx_set(xlator_t *this, inode_t *inode,
+                          changelog_inode_ctx_t *ctx)
 {
-        uint64_t ctx_addr = (uint64_t) ctx;
-        return __inode_ctx_set (inode, this, &ctx_addr);
+    uint64_t ctx_addr = (uint64_t)(uintptr_t)ctx;
+    return __inode_ctx_set(inode, this, &ctx_addr);
 }
 
 /**
@@ -1506,56 +1437,53 @@ __changelog_inode_ctx_set (xlator_t *this,
  * for a particular type.
  */
 changelog_inode_ctx_t *
-__changelog_inode_ctx_get (xlator_t *this,
-                           inode_t *inode, unsigned long **iver,
-                           unsigned long *version, changelog_log_type type)
+__changelog_inode_ctx_get(xlator_t *this, inode_t *inode, unsigned long **iver,
+                          unsigned long *version, changelog_log_type type)
 {
-        int                    ret      = 0;
-        uint64_t               ctx_addr = 0;
-        changelog_inode_ctx_t *ctx      = NULL;
-
-        ret = __inode_ctx_get (inode, this, &ctx_addr);
-        if (ret < 0)
-                ctx_addr = 0;
-        if (ctx_addr != 0) {
-                ctx = (changelog_inode_ctx_t *) (long)ctx_addr;
-                goto out;
-        }
-
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_changelog_mt_inode_ctx_t);
-        if (!ctx)
-                goto out;
+    int ret = 0;
+    uint64_t ctx_addr = 0;
+    changelog_inode_ctx_t *ctx = NULL;
+
+    ret = __inode_ctx_get(inode, this, &ctx_addr);
+    if (ret < 0)
+        ctx_addr = 0;
+    if (ctx_addr != 0) {
+        ctx = (changelog_inode_ctx_t *)(long)ctx_addr;
+        goto out;
+    }
+
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_changelog_mt_inode_ctx_t);
+    if (!ctx)
+        goto out;
+
+    ret = __changelog_inode_ctx_set(this, inode, ctx);
+    if (ret) {
+        GF_FREE(ctx);
+        ctx = NULL;
+    }
 
-        ret = __changelog_inode_ctx_set (this, inode, ctx);
-        if (ret) {
-                GF_FREE (ctx);
-                ctx = NULL;
-        }
-
- out:
-        if (ctx && iver && version) {
-                *iver = CHANGELOG_INODE_VERSION_TYPE (ctx, type);
-                *version = **iver;
-        }
+out:
+    if (ctx && iver && version) {
+        *iver = CHANGELOG_INODE_VERSION_TYPE(ctx, type);
+        *version = **iver;
+    }
 
-        return ctx;
+    return ctx;
 }
 
 static changelog_inode_ctx_t *
-changelog_inode_ctx_get (xlator_t *this,
-                         inode_t *inode, unsigned long **iver,
-                         unsigned long *version, changelog_log_type type)
+changelog_inode_ctx_get(xlator_t *this, inode_t *inode, unsigned long **iver,
+                        unsigned long *version, changelog_log_type type)
 {
-        changelog_inode_ctx_t *ctx = NULL;
+    changelog_inode_ctx_t *ctx = NULL;
 
-        LOCK (&inode->lock);
-        {
-                ctx = __changelog_inode_ctx_get (this,
-                                                 inode, iver, version, type);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ctx = __changelog_inode_ctx_get(this, inode, iver, version, type);
+    }
+    UNLOCK(&inode->lock);
 
-        return ctx;
+    return ctx;
 }
 
 /**
@@ -1659,59 +1587,57 @@ changelog_inode_ctx_get (xlator_t *this,
  * signifies an update was recorded in the current time slice).
  */
 void
-changelog_update (xlator_t *this, changelog_priv_t *priv,
-                  changelog_local_t *local, changelog_log_type type)
+changelog_update(xlator_t *this, changelog_priv_t *priv,
+                 changelog_local_t *local, changelog_log_type type)
 {
-        int                     ret        = 0;
-        unsigned long          *iver       = NULL;
-        unsigned long           version    = 0;
-        inode_t                *inode      = NULL;
-        changelog_time_slice_t *slice      = NULL;
-        changelog_inode_ctx_t  *ctx        = NULL;
-        changelog_log_data_t   *cld_0      = NULL;
-        changelog_log_data_t   *cld_1      = NULL;
-        changelog_local_t      *next_local = NULL;
-        gf_boolean_t            need_upd   = _gf_true;
-
-        slice = &priv->slice;
-
-        /**
-         * for fops that do not require inode version checking
-         */
-        if (local->update_no_check)
-                goto update;
+    int ret = 0;
+    unsigned long *iver = NULL;
+    unsigned long version = 0;
+    inode_t *inode = NULL;
+    changelog_time_slice_t *slice = NULL;
+    changelog_inode_ctx_t *ctx = NULL;
+    changelog_log_data_t *cld_0 = NULL;
+    changelog_log_data_t *cld_1 = NULL;
+    changelog_local_t *next_local = NULL;
+    gf_boolean_t need_upd = _gf_true;
 
-        inode = local->inode;
+    slice = &priv->slice;
 
-        ctx = changelog_inode_ctx_get (this,
-                                       inode, &iver, &version, type);
-        if (!ctx)
-                goto update;
+    /**
+     * for fops that do not require inode version checking
+     */
+    if (local->update_no_check)
+        goto update;
 
-        INODE_VERSION_EQUALS_SLICE (priv, version, slice, type, need_upd);
+    inode = local->inode;
 
- update:
-        if (need_upd) {
-                cld_0 = &local->cld;
-                cld_0->cld_type = type;
+    ctx = changelog_inode_ctx_get(this, inode, &iver, &version, type);
+    if (!ctx)
+        goto update;
 
-                if ( (next_local = local->prev_entry) != NULL ) {
-                        cld_1 = &next_local->cld;
-                        cld_1->cld_type = type;
-                }
+    INODE_VERSION_EQUALS_SLICE(priv, version, slice, type, need_upd);
 
-                ret = priv->cd.dispatchfn (this, priv,
-                                           priv->cd.cd_data, cld_0, cld_1);
+update:
+    if (need_upd) {
+        cld_0 = &local->cld;
+        cld_0->cld_type = type;
 
-                /**
-                 * update after the dispatcher has successfully done
-                 * it's job.
-                 */
-                if (!local->update_no_check && iver && !ret)
-                        INODE_VERSION_UPDATE (priv, inode, iver, slice, type);
+        if ((next_local = local->prev_entry) != NULL) {
+            cld_1 = &next_local->cld;
+            cld_1->cld_type = type;
         }
 
-        return;
+        ret = priv->cd.dispatchfn(this, priv, priv->cd.cd_data, cld_0, cld_1);
+
+        /**
+         * update after the dispatcher has successfully done
+         * it's job.
+         */
+        if (!local->update_no_check && iver && !ret)
+            INODE_VERSION_UPDATE(priv, inode, iver, slice, type);
+    }
+
+    return;
 }
 
 /* Begin: Geo-rep snapshot dependency changes */
@@ -1727,228 +1653,221 @@ changelog_update (xlator_t *this, changelog_priv_t *priv,
  */
 
 void
-changelog_color_fop_and_inc_cnt (xlator_t *this, changelog_priv_t *priv,
-                                                 changelog_local_t *local)
+changelog_color_fop_and_inc_cnt(xlator_t *this, changelog_priv_t *priv,
+                                changelog_local_t *local)
 {
-        if (!priv || !local)
-                return;
+    if (!priv || !local)
+        return;
 
-        LOCK (&priv->lock);
-        {
-                local->color = priv->current_color;
-                changelog_inc_fop_cnt (this, priv, local);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        local->color = priv->current_color;
+        changelog_inc_fop_cnt(this, priv, local);
+    }
+    UNLOCK(&priv->lock);
 }
 
 /* Increments the respective fop counter based on the fop color */
 void
-changelog_inc_fop_cnt (xlator_t *this, changelog_priv_t *priv,
-                                       changelog_local_t *local)
+changelog_inc_fop_cnt(xlator_t *this, changelog_priv_t *priv,
+                      changelog_local_t *local)
 {
-        int ret = 0;
-
-        if (local) {
-                if (local->color == FOP_COLOR_BLACK) {
-                        ret = pthread_mutex_lock (&priv->dm.drain_black_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                        {
-                                priv->dm.black_fop_cnt++;
-                        }
-                        ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                } else {
-                        ret = pthread_mutex_lock (&priv->dm.drain_white_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                        {
-                                priv->dm.white_fop_cnt++;
-                        }
-                        ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                }
-        }
- out:
-        return;
+    int ret = 0;
+
+    if (local) {
+        if (local->color == FOP_COLOR_BLACK) {
+            ret = pthread_mutex_lock(&priv->dm.drain_black_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+            {
+                priv->dm.black_fop_cnt++;
+            }
+            ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+        } else {
+            ret = pthread_mutex_lock(&priv->dm.drain_white_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+            {
+                priv->dm.white_fop_cnt++;
+            }
+            ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+        }
+    }
+out:
+    return;
 }
 
 /* Decrements the respective fop counter based on the fop color */
 void
-changelog_dec_fop_cnt (xlator_t *this, changelog_priv_t *priv,
-                                       changelog_local_t *local)
+changelog_dec_fop_cnt(xlator_t *this, changelog_priv_t *priv,
+                      changelog_local_t *local)
 {
-        int ret = 0;
-
-        if (local) {
-                if (local->color == FOP_COLOR_BLACK) {
-                        ret = pthread_mutex_lock (&priv->dm.drain_black_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                        {
-                                priv->dm.black_fop_cnt--;
-                                if (priv->dm.black_fop_cnt == 0 &&
-                                    priv->dm.drain_wait_black == _gf_true) {
-                                        ret = pthread_cond_signal (
-                                                    &priv->dm.drain_black_cond);
-                                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret,
-                                                                           out);
-                                        gf_msg_debug (this->name, 0,
-                                                      "Signalled "
-                                                      "draining of black");
-                                }
-                        }
-                        ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                } else {
-                        ret = pthread_mutex_lock (&priv->dm.drain_white_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                        {
-                                priv->dm.white_fop_cnt--;
-                                if (priv->dm.white_fop_cnt == 0 &&
-                                    priv->dm.drain_wait_white == _gf_true) {
-                                        ret = pthread_cond_signal (
-                                                    &priv->dm.drain_white_cond);
-                                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret,
-                                                                           out);
-                                        gf_msg_debug (this->name, 0,
-                                                      "Signalled "
-                                                      "draining of white");
-                                }
-                        }
-                        ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
+    int ret = 0;
+
+    if (local) {
+        if (local->color == FOP_COLOR_BLACK) {
+            ret = pthread_mutex_lock(&priv->dm.drain_black_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+            {
+                priv->dm.black_fop_cnt--;
+                if (priv->dm.black_fop_cnt == 0 &&
+                    priv->dm.drain_wait_black == _gf_true) {
+                    ret = pthread_cond_signal(&priv->dm.drain_black_cond);
+                    CHANGELOG_PTHREAD_ERROR_HANDLE_2(
+                        ret, out, priv->dm.drain_black_mutex);
+                    gf_msg_debug(this->name, 0,
+                                 "Signalled "
+                                 "draining of black");
                 }
+            }
+            ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+        } else {
+            ret = pthread_mutex_lock(&priv->dm.drain_white_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+            {
+                priv->dm.white_fop_cnt--;
+                if (priv->dm.white_fop_cnt == 0 &&
+                    priv->dm.drain_wait_white == _gf_true) {
+                    ret = pthread_cond_signal(&priv->dm.drain_white_cond);
+                    CHANGELOG_PTHREAD_ERROR_HANDLE_2(
+                        ret, out, priv->dm.drain_white_mutex);
+                    gf_msg_debug(this->name, 0,
+                                 "Signalled "
+                                 "draining of white");
+                }
+            }
+            ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
         }
- out:
-        return;
+    }
+out:
+    return;
 }
 
 /* Write to a pipe setup between changelog main thread and changelog
  * rollover thread to initiate explicit rollover of changelog journal.
  */
 int
-changelog_barrier_notify (changelog_priv_t *priv, char *buf)
+changelog_barrier_notify(changelog_priv_t *priv, char *buf)
 {
-        int ret = 0;
-
-        pthread_mutex_lock (&priv->cr.lock);
-        {
-                ret = pthread_cond_signal (&priv->cr.cond);
-                priv->cr.notify = _gf_true;
-        }
-        pthread_mutex_unlock (&priv->cr.lock);
-        return ret;
+    int ret = 0;
+
+    pthread_mutex_lock(&priv->cr.lock);
+    {
+        ret = pthread_cond_signal(&priv->cr.cond);
+        priv->cr.notify = _gf_true;
+    }
+    pthread_mutex_unlock(&priv->cr.lock);
+    return ret;
 }
 
 /* Clean up flags set on barrier notification */
 void
-changelog_barrier_cleanup (xlator_t *this, changelog_priv_t *priv,
-                                                struct list_head *queue)
+changelog_barrier_cleanup(xlator_t *this, changelog_priv_t *priv,
+                          struct list_head *queue)
 {
-        int ret = 0;
-
-        LOCK (&priv->bflags.lock);
-                priv->bflags.barrier_ext = _gf_false;
-        UNLOCK (&priv->bflags.lock);
-
-        ret = pthread_mutex_lock (&priv->bn.bnotify_mutex);
-        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-        {
-                priv->bn.bnotify = _gf_false;
-        }
-        ret = pthread_mutex_unlock (&priv->bn.bnotify_mutex);
-        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-
-        /* Disable changelog barrier and dequeue fops */
-        LOCK (&priv->lock);
-        {
-                if (priv->barrier_enabled == _gf_true)
-                        __chlog_barrier_disable (this, queue);
-                else
-                        ret = -1;
-        }
-        UNLOCK (&priv->lock);
-        if (ret == 0)
-                chlog_barrier_dequeue_all(this, queue);
+    int ret = 0;
+
+    LOCK(&priv->bflags.lock);
+    priv->bflags.barrier_ext = _gf_false;
+    UNLOCK(&priv->bflags.lock);
+
+    ret = pthread_mutex_lock(&priv->bn.bnotify_mutex);
+    CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+    {
+        priv->bn.bnotify = _gf_false;
+    }
+    ret = pthread_mutex_unlock(&priv->bn.bnotify_mutex);
+    CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+
+    /* Disable changelog barrier and dequeue fops */
+    LOCK(&priv->lock);
+    {
+        if (priv->barrier_enabled == _gf_true)
+            __chlog_barrier_disable(this, queue);
+        else
+            ret = -1;
+    }
+    UNLOCK(&priv->lock);
+    if (ret == 0)
+        chlog_barrier_dequeue_all(this, queue);
 
- out:
-        return;
+out:
+    return;
 }
 /* End: Geo-Rep snapshot dependency changes */
 
 int32_t
-changelog_fill_entry_buf (call_frame_t *frame, xlator_t *this,
-                          loc_t *loc, changelog_local_t **local)
+changelog_fill_entry_buf(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                         changelog_local_t **local)
 {
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
-        char             *dup_path = NULL;
-        char             *bname    = NULL;
-        inode_t          *parent   = NULL;
-
-        GF_ASSERT (this);
-
-        parent = inode_parent (loc->inode, 0, 0);
-        if (!parent) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_INODE_NOT_FOUND,
-                         "Parent inode not found",
-                         "gfid=%s", uuid_utoa (loc->inode->gfid),
-                         NULL);
-                goto err;
-        }
-
-        CHANGELOG_INIT_NOCHECK (this, *local, loc->inode, loc->inode->gfid, 5);
-        if (!(*local)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_LOCAL_INIT_FAILED, "changelog local"
-                        " initiatilization failed");
-                goto err;
-        }
-
-        co = changelog_get_usable_buffer (*local);
-        if (!co) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_NO_MEMORY,
-                        "Failed to get buffer");
-                goto err;
-        }
-
-        if (loc->inode->ia_type == IA_IFDIR) {
-                CHANGLOG_FILL_FOP_NUMBER (co, GF_FOP_MKDIR, fop_fn, xtra_len);
-                co++;
-                CHANGELOG_FILL_UINT32 (co, S_IFDIR|0755, number_fn, xtra_len);
-                co++;
-        } else {
-                CHANGLOG_FILL_FOP_NUMBER (co, GF_FOP_CREATE, fop_fn, xtra_len);
-                co++;
-                CHANGELOG_FILL_UINT32 (co, S_IFREG|0644, number_fn, xtra_len);
-                co++;
-        }
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
+    char *dup_path = NULL;
+    char *bname = NULL;
+    inode_t *parent = NULL;
+
+    GF_ASSERT(this);
+
+    parent = inode_parent(loc->inode, 0, 0);
+    if (!parent) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_INODE_NOT_FOUND,
+                "type=parent", "gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+        goto err;
+    }
+
+    CHANGELOG_INIT_NOCHECK(this, *local, loc->inode, loc->inode->gfid, 5);
+    if (!(*local)) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_LOCAL_INIT_FAILED,
+                NULL);
+        goto err;
+    }
+
+    co = changelog_get_usable_buffer(*local);
+    if (!co) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_GET_BUFFER_FAILED,
+                NULL);
+        goto err;
+    }
+
+    if (loc->inode->ia_type == IA_IFDIR) {
+        CHANGLOG_FILL_FOP_NUMBER(co, GF_FOP_MKDIR, fop_fn, xtra_len);
         co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
+        CHANGELOG_FILL_UINT32(co, S_IFDIR | 0755, number_fn, xtra_len);
+        co++;
+    } else {
+        CHANGLOG_FILL_FOP_NUMBER(co, GF_FOP_CREATE, fop_fn, xtra_len);
+        co++;
+        CHANGELOG_FILL_UINT32(co, S_IFREG | 0644, number_fn, xtra_len);
         co++;
+    }
 
-        dup_path = gf_strdup (loc->path);
-        bname = basename (dup_path);
+    CHANGELOG_FILL_UINT32(co, frame->root->uid, number_fn, xtra_len);
+    co++;
 
-        CHANGELOG_FILL_ENTRY (co, parent->gfid, bname, entry_fn, entry_free_fn,
-                              xtra_len, err);
-        changelog_set_usable_record_and_length (*local, xtra_len, 5);
+    CHANGELOG_FILL_UINT32(co, frame->root->gid, number_fn, xtra_len);
+    co++;
 
-        if (dup_path)
-                GF_FREE (dup_path);
-        if (parent)
-                inode_unref (parent);
-        return 0;
+    dup_path = gf_strdup(loc->path);
+    bname = basename(dup_path);
+
+    CHANGELOG_FILL_ENTRY(co, parent->gfid, bname, entry_fn, entry_free_fn,
+                         xtra_len, err);
+    changelog_set_usable_record_and_length(*local, xtra_len, 5);
+
+    if (dup_path)
+        GF_FREE(dup_path);
+    if (parent)
+        inode_unref(parent);
+    return 0;
 
 err:
-        if (dup_path)
-                GF_FREE (dup_path);
-        if (parent)
-                inode_unref (parent);
-        return -1;
+    if (dup_path)
+        GF_FREE(dup_path);
+    if (parent)
+        inode_unref(parent);
+    return -1;
 }
 
 /*
@@ -1961,80 +1880,98 @@ err:
  */
 
 int
-resolve_pargfid_to_path (xlator_t *this, const uuid_t pgfid,
-                         char **path, char *bname)
+resolve_pargfid_to_path(xlator_t *this, const uuid_t pgfid, char **path,
+                        char *bname)
 {
-        char             *linkname                  = NULL;
-        char             *dir_handle                = NULL;
-        char             *pgfidstr                  = NULL;
-        char             *saveptr                   = NULL;
-        ssize_t           len                       = 0;
-        int               ret                       = 0;
-        uuid_t            tmp_gfid                  = {0, };
-        uuid_t            pargfid                   = {0, };
-        changelog_priv_t *priv                      = NULL;
-        char              gpath[PATH_MAX]           = {0,};
-        char              result[PATH_MAX]          = {0,};
-        char             *dir_name                  = NULL;
-        char              pre_dir_name[PATH_MAX]    = {0,};
-
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        gf_uuid_copy (pargfid, pgfid);
-        if (!path || gf_uuid_is_null (pargfid)) {
-                ret = -1;
-                goto out;
-        }
+    char *linkname = NULL;
+    char *dir_handle = NULL;
+    char *pgfidstr = NULL;
+    char *saveptr = NULL;
+    ssize_t len = 0;
+    int ret = 0;
+    uuid_t tmp_gfid = {
+        0,
+    };
+    uuid_t pargfid = {
+        0,
+    };
+    changelog_priv_t *priv = NULL;
+    char gpath[PATH_MAX] = {
+        0,
+    };
+    char result[PATH_MAX] = {
+        0,
+    };
+    char *dir_name = NULL;
+    char pre_dir_name[PATH_MAX] = {
+        0,
+    };
+
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    gf_uuid_copy(pargfid, pgfid);
+    if (!path || gf_uuid_is_null(pargfid)) {
+        ret = -1;
+        goto out;
+    }
+
+    if (__is_root_gfid(pargfid)) {
+        if (bname)
+            *path = gf_strdup(bname);
+        else
+            *path = gf_strdup(".");
+        return ret;
+    }
 
-        if (__is_root_gfid (pargfid)) {
-                if (bname)
-                        *path = gf_strdup (bname);
-                else
-                        *path = gf_strdup (".");
-                return ret;
-        }
+    dir_handle = alloca(PATH_MAX);
+    linkname = alloca(PATH_MAX);
+    (void)snprintf(gpath, PATH_MAX, "%s/.glusterfs/", priv->changelog_brick);
 
-        dir_handle = alloca (PATH_MAX);
-        linkname   = alloca (PATH_MAX);
-        (void) snprintf (gpath, PATH_MAX, "%s/.glusterfs/",
-                         priv->changelog_brick);
-
-        while (!(__is_root_gfid (pargfid))) {
-                snprintf (dir_handle, PATH_MAX, "%s/%02x/%02x/%s", gpath,
-                          pargfid[0], pargfid[1], uuid_utoa (pargfid));
-
-                len = sys_readlink (dir_handle, linkname, PATH_MAX);
-                if (len < 0) {
-                        gf_smsg (this->name, GF_LOG_ERROR, errno,
-                                 CHANGELOG_MSG_READLINK_OP_FAILED,
-                                 "could not read the "
-                                 "link from the gfid handle",
-                                 "handle=%s", dir_handle,
-                                 NULL);
-                        ret = -1;
-                        goto out;
-                }
+    while (!(__is_root_gfid(pargfid))) {
+        len = snprintf(dir_handle, PATH_MAX, "%s/%02x/%02x/%s", gpath,
+                       pargfid[0], pargfid[1], uuid_utoa(pargfid));
+        if ((len < 0) || (len >= PATH_MAX)) {
+            ret = -1;
+            goto out;
+        }
 
-                linkname[len] = '\0';
+        len = sys_readlink(dir_handle, linkname, PATH_MAX);
+        if (len < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_READLINK_OP_FAILED,
+                    "could not read the "
+                    "link from the gfid handle",
+                    "handle=%s", dir_handle, NULL);
+            ret = -1;
+            goto out;
+        }
 
-                pgfidstr = strtok_r (linkname + strlen("../../00/00/"), "/",
-                                     &saveptr);
-                dir_name = strtok_r (NULL, "/", &saveptr);
+        linkname[len] = '\0';
 
-                snprintf (result, PATH_MAX, "%s/%s", dir_name, pre_dir_name);
-                strncpy (pre_dir_name, result, sizeof(pre_dir_name));
+        pgfidstr = strtok_r(linkname + strlen("../../00/00/"), "/", &saveptr);
+        dir_name = strtok_r(NULL, "/", &saveptr);
 
-                gf_uuid_parse (pgfidstr, tmp_gfid);
-                gf_uuid_copy (pargfid, tmp_gfid);
+        len = snprintf(result, PATH_MAX, "%s/%s", dir_name, pre_dir_name);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            ret = -1;
+            goto out;
+        }
+        if (snprintf(pre_dir_name, len + 1, "%s", result) >= len + 1) {
+            ret = -1;
+            goto out;
         }
 
-        if (bname)
-                strncat (result, bname, strlen(bname) + 1);
+        gf_uuid_parse(pgfidstr, tmp_gfid);
+        gf_uuid_copy(pargfid, tmp_gfid);
+    }
 
-        *path = gf_strdup (result);
+    if (bname)
+        strncat(result, bname, strlen(bname) + 1);
+
+    *path = gf_strdup(result);
 
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/changelog/src/changelog-helpers.h b/xlators/features/changelog/src/changelog-helpers.h
index 39a792891dd..38fa7590c32 100644
--- a/xlators/features/changelog/src/changelog-helpers.h
+++ b/xlators/features/changelog/src/changelog-helpers.h
@@ -11,14 +11,14 @@
 #ifndef _CHANGELOG_HELPERS_H
 #define _CHANGELOG_HELPERS_H
 
-#include "locking.h"
-#include "timer.h"
+#include <glusterfs/locking.h>
+#include <glusterfs/timer.h>
 #include "pthread.h"
-#include "iobuf.h"
-#include "rot-buffs.h"
+#include <glusterfs/iobuf.h>
+#include <glusterfs/rot-buffs.h>
 
 #include "changelog-misc.h"
-#include "call-stub.h"
+#include <glusterfs/call-stub.h>
 
 #include "rpcsvc.h"
 #include "changelog-ev-handle.h"
@@ -30,44 +30,44 @@
  * the changelog entry
  */
 typedef struct changelog_log_data {
-        /* rollover related */
-        unsigned long cld_roll_time;
+    /* rollover related */
+    time_t cld_roll_time;
 
-        /* reopen changelog? */
-        gf_boolean_t cld_finale;
+    /* reopen changelog? */
+    gf_boolean_t cld_finale;
 
-        changelog_log_type cld_type;
+    changelog_log_type cld_type;
 
-        /**
-         * sincd gfid is _always_ a necessity, it's not a part
-         * of the iobuf. by doing this we do not add any overhead
-         * for data and metadata related fops.
-         */
-        uuid_t        cld_gfid;
+    /**
+     * sincd gfid is _always_ a necessity, it's not a part
+     * of the iobuf. by doing this we do not add any overhead
+     * for data and metadata related fops.
+     */
+    uuid_t cld_gfid;
 
-        /**
-         * iobufs are used for optionals records: pargfid, path,
-         * write offsets etc.. It's the fop implementers job
-         * to allocate (iobuf_get() in the fop) and get unref'ed
-         * in the callback (CHANGELOG_STACK_UNWIND).
-         */
-        struct iobuf *cld_iobuf;
+    /**
+     * iobufs are used for optionals records: pargfid, path,
+     * write offsets etc.. It's the fop implementers job
+     * to allocate (iobuf_get() in the fop) and get unref'ed
+     * in the callback (CHANGELOG_STACK_UNWIND).
+     */
+    struct iobuf *cld_iobuf;
 
 #define cld_ptr cld_iobuf->ptr
 
-        /**
-         * after allocation you can point this to the length of
-         * usable data, but make sure it does not exceed the
-         * the size of the requested iobuf.
-         */
-        size_t        cld_iobuf_len;
+    /**
+     * after allocation you can point this to the length of
+     * usable data, but make sure it does not exceed the
+     * the size of the requested iobuf.
+     */
+    size_t cld_iobuf_len;
 
 #define cld_ptr_len cld_iobuf_len
 
-        /**
-         * number of optional records
-         */
-        int cld_xtra_records;
+    /**
+     * number of optional records
+     */
+    int cld_xtra_records;
 } changelog_log_data_t;
 
 /**
@@ -77,55 +77,48 @@ typedef struct changelog_log_data {
 typedef struct changelog_priv changelog_priv_t;
 
 typedef struct changelog_dispatcher {
-        void *cd_data;
-        int (*dispatchfn) (xlator_t *, changelog_priv_t *, void *,
-                           changelog_log_data_t *, changelog_log_data_t *);
+    void *cd_data;
+    int (*dispatchfn)(xlator_t *, changelog_priv_t *, void *,
+                      changelog_log_data_t *, changelog_log_data_t *);
 } changelog_dispatcher_t;
 
 struct changelog_bootstrap {
-        changelog_mode_t mode;
-        int (*ctor) (xlator_t *, changelog_dispatcher_t *);
-        int (*dtor) (xlator_t *, changelog_dispatcher_t *);
+    changelog_mode_t mode;
+    int (*ctor)(xlator_t *, changelog_dispatcher_t *);
+    int (*dtor)(xlator_t *, changelog_dispatcher_t *);
 };
 
 struct changelog_encoder {
-        changelog_encoder_t encoder;
-        int (*encode) (xlator_t *, changelog_log_data_t *);
+    changelog_encoder_t encoder;
+    int (*encode)(xlator_t *, changelog_log_data_t *);
 };
 
-
 /* xlator private */
 
 typedef struct changelog_time_slice {
-        /**
-         * just in case we need nanosecond granularity some day.
-         * field is unused as of now (maybe we'd need it later).
-         */
-        struct timeval tv_start;
-
-        /**
-         * version of changelog file, incremented each time changes
-         * rollover.
-         */
-        unsigned long changelog_version[CHANGELOG_MAX_TYPE];
+    /**
+     * version of changelog file, incremented each time changes
+     * rollover.
+     */
+    unsigned long changelog_version[CHANGELOG_MAX_TYPE];
 } changelog_time_slice_t;
 
 typedef struct changelog_rollover {
-        /* rollover thread */
-        pthread_t rollover_th;
+    /* rollover thread */
+    pthread_t rollover_th;
 
-        xlator_t *this;
+    xlator_t *this;
 
-        pthread_mutex_t lock;
-        pthread_cond_t cond;
-        gf_boolean_t notify;
+    pthread_mutex_t lock;
+    pthread_cond_t cond;
+    gf_boolean_t notify;
 } changelog_rollover_t;
 
 typedef struct changelog_fsync {
-        /* fsync() thread */
-        pthread_t fsync_th;
+    /* fsync() thread */
+    pthread_t fsync_th;
 
-        xlator_t *this;
+    xlator_t *this;
 } changelog_fsync_t;
 
 /* Draining during changelog rollover (for geo-rep snapshot dependency):
@@ -145,198 +138,219 @@ typedef struct changelog_fsync {
  */
 
 typedef enum chlog_fop_color {
-         FOP_COLOR_BLACK,
-         FOP_COLOR_WHITE
+    FOP_COLOR_BLACK,
+    FOP_COLOR_WHITE
 } chlog_fop_color_t;
 
 /* Barrier notify variable */
 typedef struct barrier_notify {
-         pthread_mutex_t        bnotify_mutex;
-         pthread_cond_t         bnotify_cond;
-         gf_boolean_t           bnotify;
-         gf_boolean_t           bnotify_error;
+    pthread_mutex_t bnotify_mutex;
+    pthread_cond_t bnotify_cond;
+    gf_boolean_t bnotify;
+    gf_boolean_t bnotify_error;
 } barrier_notify_t;
 
 /* Two separate mutex and conditional variable set is used
  * to drain white and black fops. */
 
 typedef struct drain_mgmt {
-         pthread_mutex_t        drain_black_mutex;
-         pthread_cond_t         drain_black_cond;
-         pthread_mutex_t        drain_white_mutex;
-         pthread_cond_t         drain_white_cond;
-         /* Represents black fops count in-transit */
-         unsigned long          black_fop_cnt;
-         /* Represents white fops count in-transit */
-         unsigned long          white_fop_cnt;
-         gf_boolean_t           drain_wait_black;
-         gf_boolean_t           drain_wait_white;
+    pthread_mutex_t drain_black_mutex;
+    pthread_cond_t drain_black_cond;
+    pthread_mutex_t drain_white_mutex;
+    pthread_cond_t drain_white_cond;
+    /* Represents black fops count in-transit */
+    unsigned long black_fop_cnt;
+    /* Represents white fops count in-transit */
+    unsigned long white_fop_cnt;
+    gf_boolean_t drain_wait_black;
+    gf_boolean_t drain_wait_white;
 } drain_mgmt_t;
 
 /* External barrier as a result of snap on/off indicating flag*/
 typedef struct barrier_flags {
-        gf_lock_t lock;
-        gf_boolean_t barrier_ext;
+    gf_lock_t lock;
+    gf_boolean_t barrier_ext;
 } barrier_flags_t;
 
 /* Event selection */
 typedef struct changelog_ev_selector {
-        gf_lock_t reflock;
+    gf_lock_t reflock;
 
-        /**
-         * Array of references for each selection bit.
-         */
-        unsigned int ref[CHANGELOG_EV_SELECTION_RANGE];
+    /**
+     * Array of references for each selection bit.
+     */
+    unsigned int ref[CHANGELOG_EV_SELECTION_RANGE];
 } changelog_ev_selector_t;
 
-
 /* changelog's private structure */
 struct changelog_priv {
-        gf_boolean_t active;
+    /* changelog journalling */
+    gf_boolean_t active;
+
+    /* changelog live notifications */
+    gf_boolean_t rpc_active;
+
+    /* to generate unique socket file per brick */
+    char *changelog_brick;
+
+    /* logging directory */
+    char *changelog_dir;
 
-        /* to generate unique socket file per brick */
-        char *changelog_brick;
+    /* htime directory */
+    char *htime_dir;
 
-        /* logging directory */
-        char *changelog_dir;
+    /* one file for all changelog types */
+    int changelog_fd;
 
-        /* htime directory */
-        char *htime_dir;
+    /* htime fd for current changelog session */
+    int htime_fd;
 
-        /* one file for all changelog types */
-        int changelog_fd;
+    /*  c_snap_fd is fd for call-path changelog */
+    int c_snap_fd;
 
-        /* htime fd for current changelog session */
-        int htime_fd;
+    /* rollover_count used by htime */
+    int rollover_count;
 
-        /*  c_snap_fd is fd for call-path changelog */
-        int c_snap_fd;
+    gf_lock_t lock;
 
-        /* rollover_count used by htime */
-        int  rollover_count;
+    /*  lock to synchronize CSNAP updation */
+    gf_lock_t c_snap_lock;
 
-        gf_lock_t lock;
+    /* written end of the pipe */
+    int wfd;
 
-        /*  lock to synchronize CSNAP updation */
-        gf_lock_t c_snap_lock;
+    /* rollover time */
+    int32_t rollover_time;
 
-        /* written end of the pipe */
-        int wfd;
+    /* fsync() interval */
+    int32_t fsync_interval;
 
-        /* rollover time */
-        int32_t rollover_time;
+    /* changelog type maps */
+    const char *maps[CHANGELOG_MAX_TYPE];
 
-        /* fsync() interval */
-        int32_t fsync_interval;
+    /* time slicer */
+    changelog_time_slice_t slice;
 
-        /* changelog type maps */
-        const char *maps[CHANGELOG_MAX_TYPE];
+    /* context of the updater */
+    changelog_dispatcher_t cd;
 
-        /* time slicer */
-        changelog_time_slice_t slice;
+    /* context of the rollover thread */
+    changelog_rollover_t cr;
 
-        /* context of the updater */
-        changelog_dispatcher_t cd;
+    /* context of fsync thread */
+    changelog_fsync_t cf;
 
-        /* context of the rollover thread */
-        changelog_rollover_t cr;
+    /* operation mode */
+    changelog_mode_t op_mode;
 
-        /* context of fsync thread */
-        changelog_fsync_t cf;
+    /* bootstrap routine for 'current' logger */
+    struct changelog_bootstrap *cb;
 
-        /* operation mode */
-        changelog_mode_t op_mode;
+    /* encoder mode */
+    changelog_encoder_t encode_mode;
 
-        /* bootstrap routine for 'current' logger */
-        struct changelog_bootstrap *cb;
+    /* encoder */
+    struct changelog_encoder *ce;
 
-        /* encoder mode */
-        changelog_encoder_t encode_mode;
+    /**
+     * snapshot dependency changes
+     */
 
-        /* encoder */
-        struct changelog_encoder *ce;
+    /* Draining of fops*/
+    drain_mgmt_t dm;
 
-        /**
-         * snapshot dependency changes
-         */
+    /* Represents the active color. Initially by default black */
+    chlog_fop_color_t current_color;
 
-        /* Draining of fops*/
-        drain_mgmt_t dm;
+    /* flag to determine explicit rollover is triggered */
+    gf_boolean_t explicit_rollover;
 
-        /* Represents the active color. Initially by default black */
-        chlog_fop_color_t current_color;
+    /* barrier notification variable protected by mutex */
+    barrier_notify_t bn;
 
-        /* flag to determine explicit rollover is triggered */
-        gf_boolean_t explicit_rollover;
+    /* barrier on/off indicating flags */
+    barrier_flags_t bflags;
 
-        /* barrier notification variable protected by mutex */
-        barrier_notify_t bn;
+    /* changelog barrier on/off indicating flag */
+    gf_boolean_t barrier_enabled;
+    struct list_head queue;
+    uint32_t queue_size;
+    gf_timer_t *timer;
+    struct timespec timeout;
 
-        /* barrier on/off indicating flags */
-        barrier_flags_t bflags;
+    /**
+     * buffers, RPC, event selection, notifications and other
+     * beasts.
+     */
 
-        /* changelog barrier on/off indicating flag */
-        gf_boolean_t      barrier_enabled;
-        struct list_head  queue;
-        uint32_t          queue_size;
-        gf_timer_t       *timer;
-        struct timespec   timeout;
+    /* epoll pthread */
+    pthread_t poller;
 
-        /**
-         * buffers, RPC, event selection, notifications and other
-         * beasts.
-         */
+    /* rotational buffer */
+    rbuf_t *rbuf;
 
-        /* epoll pthread */
-        pthread_t poller;
+    /* changelog RPC server */
+    rpcsvc_t *rpc;
 
-        /* rotational buffer */
-        rbuf_t *rbuf;
+    /* event selection */
+    changelog_ev_selector_t ev_selection;
 
-        /* changelog RPC server */
-        rpcsvc_t *rpc;
+    /* client handling (reverse connection) */
+    pthread_t connector;
 
-        /* event selection */
-        changelog_ev_selector_t ev_selection;
+    int nr_dispatchers;
+    pthread_t *ev_dispatcher;
 
-        /* client handling (reverse connection) */
-        pthread_t connector;
+    changelog_clnt_t connections;
 
-        int nr_dispatchers;
-        pthread_t *ev_dispatcher;
+    /* glusterfind dependency to capture paths on deleted entries*/
+    gf_boolean_t capture_del_path;
 
-        changelog_clnt_t connections;
+    /* Save total no. of listners */
+    gf_atomic_t listnercnt;
 
-        /* glusterfind dependency to capture paths on deleted entries*/
-        gf_boolean_t capture_del_path;
+    /* Save total no. of xprt are associated with listner */
+    gf_atomic_t xprtcnt;
+
+    /* Save xprt list */
+    struct list_head xprt_list;
+
+    /* Save total no. of client connection */
+    gf_atomic_t clntcnt;
+
+    /* Save cleanup brick in victim */
+    xlator_t *victim;
+
+    /* Status to save cleanup notify status */
+    gf_boolean_t notify_down;
 };
 
 struct changelog_local {
-        inode_t              *inode;
-        gf_boolean_t          update_no_check;
+    inode_t *inode;
+    gf_boolean_t update_no_check;
 
-        changelog_log_data_t  cld;
+    changelog_log_data_t cld;
 
-        /**
-         * ->prev_entry is used in cases when there needs to be
-         * additional changelog entry for the parent (eg. rename)
-         * It's analogous to ->next in single linked list world,
-         * but we call it as ->prev_entry... ha ha ha
-         */
-        struct changelog_local *prev_entry;
+    /**
+     * ->prev_entry is used in cases when there needs to be
+     * additional changelog entry for the parent (eg. rename)
+     * It's analogous to ->next in single linked list world,
+     * but we call it as ->prev_entry... ha ha ha
+     */
+    struct changelog_local *prev_entry;
 
-        /* snap dependency changes */
-        chlog_fop_color_t color;
+    /* snap dependency changes */
+    chlog_fop_color_t color;
 };
 
 typedef struct changelog_local changelog_local_t;
 
 /* inode version is stored in inode ctx */
 typedef struct changelog_inode_ctx {
-        unsigned long iversion[CHANGELOG_MAX_TYPE];
+    unsigned long iversion[CHANGELOG_MAX_TYPE];
 } changelog_inode_ctx_t;
 
-#define CHANGELOG_INODE_VERSION_TYPE(ctx, type)  &(ctx->iversion[type])
+#define CHANGELOG_INODE_VERSION_TYPE(ctx, type) &(ctx->iversion[type])
 
 /**
  * Optional Records:
@@ -344,269 +358,276 @@ typedef struct changelog_inode_ctx {
  *  @changelog_opt_t struct. The array is allocated via @iobufs.
  */
 typedef enum {
-        CHANGELOG_OPT_REC_FOP,
-        CHANGELOG_OPT_REC_ENTRY,
-        CHANGELOG_OPT_REC_UINT32,
+    CHANGELOG_OPT_REC_FOP,
+    CHANGELOG_OPT_REC_ENTRY,
+    CHANGELOG_OPT_REC_UINT32,
 } changelog_optional_rec_type_t;
 
 struct changelog_entry_fields {
-        uuid_t  cef_uuid;
-        char   *cef_bname;
-        char   *cef_path;
+    uuid_t cef_uuid;
+    char *cef_bname;
+    char *cef_path;
 };
 
 typedef struct {
-        /**
-         * @co_covert can be used to do post-processing of the record before
-         * it's persisted to the CHANGELOG. If this is NULL, then the record
-         * is persisted as per it's in memory format.
-         */
-        size_t (*co_convert) (void *data, char *buffer, gf_boolean_t encode);
-
-        /* release routines */
-        void (*co_free) (void *data);
-
-        /* type of the field */
-        changelog_optional_rec_type_t co_type;
-
-        /**
-         * sizeof of the 'valid' field in the union. This field is not used if
-         * @co_convert is specified.
-         */
-        size_t co_len;
-
-        union {
-                unsigned int                  co_uint32;
-                glusterfs_fop_t               co_fop;
-                struct changelog_entry_fields co_entry;
-        };
+    /**
+     * @co_covert can be used to do post-processing of the record before
+     * it's persisted to the CHANGELOG. If this is NULL, then the record
+     * is persisted as per it's in memory format.
+     */
+    size_t (*co_convert)(void *data, char *buffer, gf_boolean_t encode);
+
+    /* release routines */
+    void (*co_free)(void *data);
+
+    /* type of the field */
+    changelog_optional_rec_type_t co_type;
+
+    /**
+     * sizeof of the 'valid' field in the union. This field is not used if
+     * @co_convert is specified.
+     */
+    size_t co_len;
+
+    union {
+        unsigned int co_uint32;
+        glusterfs_fop_t co_fop;
+        struct changelog_entry_fields co_entry;
+    };
 } changelog_opt_t;
 
-#define CHANGELOG_OPT_RECORD_LEN  sizeof (changelog_opt_t)
+#define CHANGELOG_OPT_RECORD_LEN sizeof(changelog_opt_t)
 
 /**
  * helpers routines
  */
 
 int
-changelog_thread_cleanup (xlator_t *this, pthread_t thr_id);
+changelog_thread_cleanup(xlator_t *this, pthread_t thr_id);
 
 void *
-changelog_get_usable_buffer (changelog_local_t *local);
+changelog_get_usable_buffer(changelog_local_t *local);
 
 void
-changelog_set_usable_record_and_length (changelog_local_t *local,
-                                        size_t len, int xr);
+changelog_set_usable_record_and_length(changelog_local_t *local, size_t len,
+                                       int xr);
 void
-changelog_local_cleanup (xlator_t *xl, changelog_local_t *local);
+changelog_local_cleanup(xlator_t *xl, changelog_local_t *local);
 changelog_local_t *
-changelog_local_init (xlator_t *this, inode_t *inode, uuid_t gfid,
-                      int xtra_records, gf_boolean_t update_flag);
-int
-changelog_start_next_change (xlator_t *this,
-                             changelog_priv_t *priv,
-                             unsigned long ts, gf_boolean_t finale);
+changelog_local_init(xlator_t *this, inode_t *inode, uuid_t gfid,
+                     int xtra_records, gf_boolean_t update_flag);
 int
-changelog_open_journal (xlator_t *this, changelog_priv_t *priv);
+changelog_start_next_change(xlator_t *this, changelog_priv_t *priv, time_t ts,
+                            gf_boolean_t finale);
 int
-changelog_fill_rollover_data (changelog_log_data_t *cld, gf_boolean_t is_last);
+changelog_open_journal(xlator_t *this, changelog_priv_t *priv);
+void
+changelog_fill_rollover_data(changelog_log_data_t *cld, gf_boolean_t is_last);
 int
-changelog_inject_single_event (xlator_t *this,
-                               changelog_priv_t *priv,
-                               changelog_log_data_t *cld);
+changelog_inject_single_event(xlator_t *this, changelog_priv_t *priv,
+                              changelog_log_data_t *cld);
 size_t
-changelog_entry_length ();
+changelog_entry_length();
 int
-changelog_write (int fd, char *buffer, size_t len);
+changelog_write(int fd, char *buffer, size_t len);
 int
-changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len);
+changelog_write_change(changelog_priv_t *priv, char *buffer, size_t len);
 int
-changelog_handle_change (xlator_t *this,
-                         changelog_priv_t *priv, changelog_log_data_t *cld);
+changelog_handle_change(xlator_t *this, changelog_priv_t *priv,
+                        changelog_log_data_t *cld);
 void
-changelog_update (xlator_t *this, changelog_priv_t *priv,
-                  changelog_local_t *local, changelog_log_type type);
+changelog_update(xlator_t *this, changelog_priv_t *priv,
+                 changelog_local_t *local, changelog_log_type type);
 void *
-changelog_rollover (void *data);
+changelog_rollover(void *data);
 void *
-changelog_fsync_thread (void *data);
+changelog_fsync_thread(void *data);
 int
-changelog_forget (xlator_t *this, inode_t *inode);
+changelog_forget(xlator_t *this, inode_t *inode);
 int
-htime_update (xlator_t *this, changelog_priv_t *priv,
-              unsigned long ts, char * buffer);
+htime_update(xlator_t *this, changelog_priv_t *priv, time_t ts, char *buffer);
 int
-htime_open (xlator_t *this, changelog_priv_t *priv, unsigned long ts);
+htime_open(xlator_t *this, changelog_priv_t *priv, time_t ts);
 int
-htime_create (xlator_t *this, changelog_priv_t *priv, unsigned long ts);
+htime_create(xlator_t *this, changelog_priv_t *priv, time_t ts);
 
 /* Geo-Rep snapshot dependency changes */
 void
-changelog_color_fop_and_inc_cnt (xlator_t *this, changelog_priv_t *priv,
-                                                 changelog_local_t *local);
+changelog_color_fop_and_inc_cnt(xlator_t *this, changelog_priv_t *priv,
+                                changelog_local_t *local);
 void
-changelog_inc_fop_cnt (xlator_t *this, changelog_priv_t *priv,
-                                       changelog_local_t *local);
+changelog_inc_fop_cnt(xlator_t *this, changelog_priv_t *priv,
+                      changelog_local_t *local);
 void
-changelog_dec_fop_cnt (xlator_t *this, changelog_priv_t *priv,
-                                       changelog_local_t *local);
+changelog_dec_fop_cnt(xlator_t *this, changelog_priv_t *priv,
+                      changelog_local_t *local);
 int
-changelog_barrier_notify (changelog_priv_t *priv, char* buf);
+changelog_barrier_notify(changelog_priv_t *priv, char *buf);
 void
-changelog_barrier_cleanup (xlator_t *this, changelog_priv_t *priv,
-                                                struct list_head *queue);
+changelog_barrier_cleanup(xlator_t *this, changelog_priv_t *priv,
+                          struct list_head *queue);
 void
-changelog_drain_white_fops (xlator_t *this, changelog_priv_t *priv);
+changelog_drain_white_fops(xlator_t *this, changelog_priv_t *priv);
 void
-changelog_drain_black_fops (xlator_t *this, changelog_priv_t *priv);
+changelog_drain_black_fops(xlator_t *this, changelog_priv_t *priv);
 
 /* Crash consistency of changelog wrt snapshot */
 int
-changelog_snap_logging_stop ( xlator_t *this, changelog_priv_t *priv);
+changelog_snap_logging_stop(xlator_t *this, changelog_priv_t *priv);
 int
-changelog_snap_logging_start ( xlator_t *this, changelog_priv_t *priv);
+changelog_snap_logging_start(xlator_t *this, changelog_priv_t *priv);
 int
-changelog_snap_open ( xlator_t *this, changelog_priv_t *priv);
+changelog_snap_open(xlator_t *this, changelog_priv_t *priv);
 int
-changelog_snap_handle_ascii_change (xlator_t *this,
-                changelog_log_data_t *cld);
+changelog_snap_handle_ascii_change(xlator_t *this, changelog_log_data_t *cld);
 int
-changelog_snap_write_change (changelog_priv_t *priv, char *buffer, size_t len);
+changelog_snap_write_change(changelog_priv_t *priv, char *buffer, size_t len);
 
 /* Changelog barrier routines */
-void __chlog_barrier_enqueue (xlator_t *this, call_stub_t *stub);
-void __chlog_barrier_disable (xlator_t *this, struct list_head *queue);
-void chlog_barrier_dequeue_all (xlator_t *this, struct list_head *queue);
-call_stub_t *__chlog_barrier_dequeue (xlator_t *this, struct list_head *queue);
-int __chlog_barrier_enable (xlator_t *this, changelog_priv_t *priv);
+void
+__chlog_barrier_enqueue(xlator_t *this, call_stub_t *stub);
+void
+__chlog_barrier_disable(xlator_t *this, struct list_head *queue);
+void
+chlog_barrier_dequeue_all(xlator_t *this, struct list_head *queue);
+call_stub_t *
+__chlog_barrier_dequeue(xlator_t *this, struct list_head *queue);
+int
+__chlog_barrier_enable(xlator_t *this, changelog_priv_t *priv);
 
 int32_t
-changelog_fill_entry_buf (call_frame_t *frame, xlator_t *this,
-                          loc_t *loc, changelog_local_t **local);
+changelog_fill_entry_buf(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                         changelog_local_t **local);
 
 /* event selection routines */
-void changelog_select_event (xlator_t *,
-                                    changelog_ev_selector_t *, unsigned int);
-void changelog_deselect_event (xlator_t *,
-                                      changelog_ev_selector_t *, unsigned int);
-int changelog_init_event_selection (xlator_t *,
-                                           changelog_ev_selector_t *);
-int changelog_cleanup_event_selection (xlator_t *,
-                                              changelog_ev_selector_t *);
-int changelog_ev_selected (xlator_t *,
-                                  changelog_ev_selector_t *, unsigned int);
 void
-changelog_dispatch_event (xlator_t *, changelog_priv_t *, changelog_event_t *);
+changelog_select_event(xlator_t *, changelog_ev_selector_t *, unsigned int);
+void
+changelog_deselect_event(xlator_t *, changelog_ev_selector_t *, unsigned int);
+int
+changelog_init_event_selection(xlator_t *, changelog_ev_selector_t *);
+int
+changelog_ev_selected(xlator_t *, changelog_ev_selector_t *, unsigned int);
+void
+changelog_dispatch_event(xlator_t *, changelog_priv_t *, changelog_event_t *);
 
 changelog_inode_ctx_t *
-__changelog_inode_ctx_get (xlator_t *, inode_t *, unsigned long **,
-                           unsigned long *, changelog_log_type);
+__changelog_inode_ctx_get(xlator_t *, inode_t *, unsigned long **,
+                          unsigned long *, changelog_log_type);
 int
-resolve_pargfid_to_path (xlator_t *this, const uuid_t gfid, char **path,
-                         char *bname);
+resolve_pargfid_to_path(xlator_t *this, const uuid_t gfid, char **path,
+                        char *bname);
 
 /* macros */
 
-#define CHANGELOG_STACK_UNWIND(fop, frame, params ...) do {             \
-                changelog_local_t *__local = NULL;                      \
-                xlator_t          *__xl    = NULL;                      \
-                if (frame) {                                            \
-                        __local      = frame->local;                    \
-                        __xl         = frame->this;                     \
-                        frame->local = NULL;                            \
-                }                                                       \
-                STACK_UNWIND_STRICT (fop, frame, params);               \
-                if (__local && __local->prev_entry)                     \
-                        changelog_local_cleanup (__xl,                  \
-                                                 __local->prev_entry);  \
-                changelog_local_cleanup (__xl, __local);                \
-        } while (0)
-
-#define CHANGELOG_IOBUF_REF(iobuf) do {         \
-                if (iobuf)                      \
-                        iobuf_ref (iobuf);      \
-        } while (0)
-
-#define CHANGELOG_IOBUF_UNREF(iobuf) do {       \
-                if (iobuf)                      \
-                        iobuf_unref (iobuf);    \
-        } while (0)
-
-#define CHANGELOG_FILL_BUFFER(buffer, off, val, len) do {       \
-                memcpy (buffer + off, val, len);                \
-                off += len;                                     \
-        } while (0)
-
-#define SLICE_VERSION_UPDATE(slice) do {                \
-                int i = 0;                              \
-                for (; i < CHANGELOG_MAX_TYPE; i++) {   \
-                        slice->changelog_version[i]++;  \
-                }                                       \
-        } while (0)
-
-#define CHANGELOG_FILL_UINT32(co, number, converter, xlen) do { \
-                co->co_convert = converter;                     \
-                co->co_free = NULL;                             \
-                co->co_type = CHANGELOG_OPT_REC_UINT32;         \
-                co->co_uint32 = number;                         \
-                xlen += sizeof (unsigned int);                 \
-        } while (0)
-
-#define CHANGLOG_FILL_FOP_NUMBER(co, fop, converter, xlen) do { \
-                co->co_convert = converter;                     \
-                co->co_free = NULL;                             \
-                co->co_type = CHANGELOG_OPT_REC_FOP;            \
-                co->co_fop = fop;                               \
-                xlen += sizeof (fop);                           \
-        } while (0)
-
-#define CHANGELOG_FILL_ENTRY(co, pargfid, bname,                        \
-                             converter, freefn, xlen, label)            \
-        do {                                                            \
-                co->co_convert = converter;                             \
-                co->co_free = freefn;                                   \
-                co->co_type = CHANGELOG_OPT_REC_ENTRY;                  \
-                gf_uuid_copy (co->co_entry.cef_uuid, pargfid);          \
-                co->co_entry.cef_bname = gf_strdup(bname);              \
-                if (!co->co_entry.cef_bname)                            \
-                        goto label;                                     \
-                xlen += (UUID_CANONICAL_FORM_LEN + strlen (bname));     \
-        } while (0)
-
-#define CHANGELOG_FILL_ENTRY_DIR_PATH(co, pargfid, bname, converter,        \
-                                      del_freefn, xlen, label, capture_del) \
-        do {                                                                \
-                co->co_convert = converter;                                 \
-                co->co_free = del_freefn;                                   \
-                co->co_type = CHANGELOG_OPT_REC_ENTRY;                      \
-                gf_uuid_copy (co->co_entry.cef_uuid, pargfid);              \
-                co->co_entry.cef_bname = gf_strdup(bname);                  \
-                if (!co->co_entry.cef_bname)                                \
-                        goto label;                                         \
-                xlen += (UUID_CANONICAL_FORM_LEN + strlen (bname));         \
-                if (!capture_del || resolve_pargfid_to_path (this, pargfid, \
-                    &(co->co_entry.cef_path), co->co_entry.cef_bname)) {    \
-                        co->co_entry.cef_path = gf_strdup ("\0");           \
-                        xlen += 1;                                          \
-                } else {                                                    \
-                        xlen += (strlen (co->co_entry.cef_path));           \
-                }                                                           \
-        } while (0)
-
-#define CHANGELOG_INIT(this, local, inode, gfid, xrec)                  \
-        local = changelog_local_init (this, inode, gfid, xrec, _gf_false)
-
-#define CHANGELOG_INIT_NOCHECK(this, local, inode, gfid, xrec)          \
-        local = changelog_local_init (this, inode, gfid, xrec, _gf_true)
-
-#define CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, label) do {      \
-                if (!priv->active)                                   \
-                        goto label;                                  \
-                /* ignore rebalance process's activity. */           \
-                if ((frame->root->pid == GF_CLIENT_PID_DEFRAG) ||    \
-                    (frame->root->pid == GF_CLIENT_PID_TIER_DEFRAG)) \
-                        goto label;                                  \
-        } while (0)
+#define CHANGELOG_STACK_UNWIND(fop, frame, params...)                          \
+    do {                                                                       \
+        changelog_local_t *__local = NULL;                                     \
+        xlator_t *__xl = NULL;                                                 \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            __xl = frame->this;                                                \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        if (__local && __local->prev_entry)                                    \
+            changelog_local_cleanup(__xl, __local->prev_entry);                \
+        changelog_local_cleanup(__xl, __local);                                \
+    } while (0)
+
+#define CHANGELOG_IOBUF_REF(iobuf)                                             \
+    do {                                                                       \
+        if (iobuf)                                                             \
+            iobuf_ref(iobuf);                                                  \
+    } while (0)
+
+#define CHANGELOG_IOBUF_UNREF(iobuf)                                           \
+    do {                                                                       \
+        if (iobuf)                                                             \
+            iobuf_unref(iobuf);                                                \
+    } while (0)
+
+#define CHANGELOG_FILL_BUFFER(buffer, off, val, len)                           \
+    do {                                                                       \
+        memcpy(buffer + off, val, len);                                        \
+        off += len;                                                            \
+    } while (0)
+
+#define SLICE_VERSION_UPDATE(slice)                                            \
+    do {                                                                       \
+        int i = 0;                                                             \
+        for (; i < CHANGELOG_MAX_TYPE; i++) {                                  \
+            slice->changelog_version[i]++;                                     \
+        }                                                                      \
+    } while (0)
+
+#define CHANGELOG_FILL_UINT32(co, number, converter, xlen)                     \
+    do {                                                                       \
+        co->co_convert = converter;                                            \
+        co->co_free = NULL;                                                    \
+        co->co_type = CHANGELOG_OPT_REC_UINT32;                                \
+        co->co_uint32 = number;                                                \
+        xlen += sizeof(unsigned int);                                          \
+    } while (0)
+
+#define CHANGLOG_FILL_FOP_NUMBER(co, fop, converter, xlen)                     \
+    do {                                                                       \
+        co->co_convert = converter;                                            \
+        co->co_free = NULL;                                                    \
+        co->co_type = CHANGELOG_OPT_REC_FOP;                                   \
+        co->co_fop = fop;                                                      \
+        xlen += sizeof(fop);                                                   \
+    } while (0)
+
+#define CHANGELOG_FILL_ENTRY(co, pargfid, bname, converter, freefn, xlen,      \
+                             label)                                            \
+    do {                                                                       \
+        co->co_convert = converter;                                            \
+        co->co_free = freefn;                                                  \
+        co->co_type = CHANGELOG_OPT_REC_ENTRY;                                 \
+        gf_uuid_copy(co->co_entry.cef_uuid, pargfid);                          \
+        co->co_entry.cef_bname = gf_strdup(bname);                             \
+        if (!co->co_entry.cef_bname)                                           \
+            goto label;                                                        \
+        xlen += (UUID_CANONICAL_FORM_LEN + strlen(bname));                     \
+    } while (0)
+
+#define CHANGELOG_FILL_ENTRY_DIR_PATH(co, pargfid, bname, converter,           \
+                                      del_freefn, xlen, label, capture_del)    \
+    do {                                                                       \
+        co->co_convert = converter;                                            \
+        co->co_free = del_freefn;                                              \
+        co->co_type = CHANGELOG_OPT_REC_ENTRY;                                 \
+        gf_uuid_copy(co->co_entry.cef_uuid, pargfid);                          \
+        co->co_entry.cef_bname = gf_strdup(bname);                             \
+        if (!co->co_entry.cef_bname)                                           \
+            goto label;                                                        \
+        xlen += (UUID_CANONICAL_FORM_LEN + strlen(bname));                     \
+        if (!capture_del ||                                                    \
+            resolve_pargfid_to_path(this, pargfid, &(co->co_entry.cef_path),   \
+                                    co->co_entry.cef_bname)) {                 \
+            co->co_entry.cef_path = gf_strdup("\0");                           \
+            xlen += 1;                                                         \
+        } else {                                                               \
+            xlen += (strlen(co->co_entry.cef_path));                           \
+        }                                                                      \
+    } while (0)
+
+#define CHANGELOG_INIT(this, local, inode, gfid, xrec)                         \
+    local = changelog_local_init(this, inode, gfid, xrec, _gf_false)
+
+#define CHANGELOG_INIT_NOCHECK(this, local, inode, gfid, xrec)                 \
+    local = changelog_local_init(this, inode, gfid, xrec, _gf_true)
+
+#define CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, label)                     \
+    do {                                                                       \
+        if (!priv->active)                                                     \
+            goto label;                                                        \
+        /* ignore rebalance process's activity. */                             \
+        if ((frame->root->pid == GF_CLIENT_PID_DEFRAG) ||                      \
+            (frame->root->pid == GF_CLIENT_PID_TIER_DEFRAG))                   \
+            goto label;                                                        \
+    } while (0)
 
 /* If it is a METADATA entry and fop num being GF_FOP_NULL, don't
  * log in the changelog as it is of no use. And also if it is
@@ -615,66 +636,81 @@ resolve_pargfid_to_path (xlator_t *this, const uuid_t gfid, char **path,
  * to same changelog will be missed. Hence check for boundary
  * condition.
  */
-#define CHANGELOG_OP_BOUNDARY_CHECK(frame, label) do {          \
-                if (frame->root->op <= GF_FOP_NULL ||           \
-                    frame->root->op >= GF_FOP_MAXVALUE)         \
-                        goto label;                             \
-        } while (0)
+#define CHANGELOG_OP_BOUNDARY_CHECK(frame, label)                              \
+    do {                                                                       \
+        if (frame->root->op <= GF_FOP_NULL ||                                  \
+            frame->root->op >= GF_FOP_MAXVALUE)                                \
+            goto label;                                                        \
+    } while (0)
 
 /**
  * ignore internal fops for all clients except AFR self-heal daemon
  */
-#define CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, dict, label) do {    \
-                if ((frame->root->pid != GF_CLIENT_PID_SELF_HEALD)      \
-                    && dict                                             \
-                    && dict_get (dict, GLUSTERFS_INTERNAL_FOP_KEY))     \
-                        goto label;                                     \
-        } while (0)
-
-#define CHANGELOG_COND_GOTO(priv, cond, label) do {                    \
-                if (!priv->active || cond)                             \
-                        goto label;                                    \
-        } while (0)
+#define CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, dict, label)                \
+    do {                                                                       \
+        if ((frame->root->pid != GF_CLIENT_PID_SELF_HEALD) && dict &&          \
+            dict_get(dict, GLUSTERFS_INTERNAL_FOP_KEY))                        \
+            goto label;                                                        \
+    } while (0)
+
+#define CHANGELOG_COND_GOTO(priv, cond, label)                                 \
+    do {                                                                       \
+        if (!priv->active || cond)                                             \
+            goto label;                                                        \
+    } while (0)
 
 /* Begin: Geo-Rep snapshot dependency changes */
 
-#define DICT_ERROR         -1
-#define BARRIER_OFF         0
-#define BARRIER_ON          1
-#define DICT_DEFAULT        2
-
-#define CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, label) do {                      \
-                if (!priv->active) {                                           \
-                        gf_msg (this->name, GF_LOG_WARNING, 0,                 \
-                                CHANGELOG_MSG_NOT_ACTIVE,                      \
-                                "Changelog is not active, return success");    \
-                        ret = 0;                                               \
-                        goto label;                                            \
-                }                                                              \
-        } while (0)
+#define DICT_ERROR -1
+#define BARRIER_OFF 0
+#define BARRIER_ON 1
+#define DICT_DEFAULT 2
+
+#define CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, label)                           \
+    do {                                                                       \
+        if (!priv->active) {                                                   \
+            gf_smsg(this->name, GF_LOG_WARNING, 0,                             \
+                    CHANGELOG_MSG_CHANGELOG_NOT_ACTIVE, NULL);                 \
+            ret = 0;                                                           \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
 
 /* Log pthread error and goto label */
-#define CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, label) do {                      \
-                if (ret) {                                                     \
-                        gf_smsg (this->name, GF_LOG_ERROR,                     \
-                                 0, CHANGELOG_MSG_PTHREAD_ERROR,               \
-                                 "pthread error", "error=%d", ret, NULL);      \
-                        ret = -1;                                              \
-                        goto label;                                            \
-                }                                                              \
-        } while (0);
+#define CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, label)                           \
+    do {                                                                       \
+        if (ret) {                                                             \
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_PTHREAD_ERROR,  \
+                    "error=%d", ret, NULL);                                    \
+            ret = -1;                                                          \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0);
 
 /* Log pthread error, set flag and goto label */
-#define CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, label, flag) do {                \
-                if (ret) {                                                     \
-                        gf_smsg (this->name, GF_LOG_ERROR, 0,                  \
-                                 CHANGELOG_MSG_PTHREAD_ERROR,                  \
-                                 "pthread error", "error=%d", ret, NULL);      \
-                        ret = -1;                                              \
-                        flag = _gf_true;                                       \
-                        goto label;                                            \
-                }                                                              \
-        } while (0)
+#define CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, label, flag)                     \
+    do {                                                                       \
+        if (ret) {                                                             \
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_PTHREAD_ERROR,  \
+                    "error=%d", ret, NULL);                                    \
+            ret = -1;                                                          \
+            flag = _gf_true;                                                   \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+/* Log pthread error, unlock mutex and goto label */
+#define CHANGELOG_PTHREAD_ERROR_HANDLE_2(ret, label, mutex)                    \
+    do {                                                                       \
+        if (ret) {                                                             \
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_PTHREAD_ERROR,  \
+                    "error=%d", ret, NULL);                                    \
+            ret = -1;                                                          \
+            pthread_mutex_unlock(&mutex);                                      \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
 /* End: Geo-Rep snapshot dependency changes */
 
 #endif /* _CHANGELOG_HELPERS_H */
diff --git a/xlators/features/changelog/src/changelog-mem-types.h b/xlators/features/changelog/src/changelog-mem-types.h
index 33fea31b979..a2d8a9cbe93 100644
--- a/xlators/features/changelog/src/changelog-mem-types.h
+++ b/xlators/features/changelog/src/changelog-mem-types.h
@@ -11,24 +11,24 @@
 #ifndef _CHANGELOG_MEM_TYPES_H
 #define _CHANGELOG_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_changelog_mem_types {
-        gf_changelog_mt_priv_t                     = gf_common_mt_end + 1,
-        gf_changelog_mt_str_t                      = gf_common_mt_end + 2,
-        gf_changelog_mt_batch_t                    = gf_common_mt_end + 3,
-        gf_changelog_mt_rt_t                       = gf_common_mt_end + 4,
-        gf_changelog_mt_inode_ctx_t                = gf_common_mt_end + 5,
-        gf_changelog_mt_rpc_clnt_t                 = gf_common_mt_end + 6,
-        gf_changelog_mt_libgfchangelog_t           = gf_common_mt_end + 7,
-        gf_changelog_mt_libgfchangelog_entry_t     = gf_common_mt_end + 8,
-        gf_changelog_mt_libgfchangelog_rl_t        = gf_common_mt_end + 9,
-        gf_changelog_mt_changelog_buffer_t         = gf_common_mt_end + 10,
-        gf_changelog_mt_history_data_t             = gf_common_mt_end + 11,
-        gf_changelog_mt_libgfchangelog_call_pool_t = gf_common_mt_end + 12,
-        gf_changelog_mt_libgfchangelog_event_t     = gf_common_mt_end + 13,
-        gf_changelog_mt_ev_dispatcher_t            = gf_common_mt_end + 14,
-        gf_changelog_mt_end
+    gf_changelog_mt_priv_t = gf_common_mt_end + 1,
+    gf_changelog_mt_str_t = gf_common_mt_end + 2,
+    gf_changelog_mt_batch_t = gf_common_mt_end + 3,
+    gf_changelog_mt_rt_t = gf_common_mt_end + 4,
+    gf_changelog_mt_inode_ctx_t = gf_common_mt_end + 5,
+    gf_changelog_mt_rpc_clnt_t = gf_common_mt_end + 6,
+    gf_changelog_mt_libgfchangelog_t = gf_common_mt_end + 7,
+    gf_changelog_mt_libgfchangelog_entry_t = gf_common_mt_end + 8,
+    gf_changelog_mt_libgfchangelog_rl_t = gf_common_mt_end + 9,
+    gf_changelog_mt_changelog_buffer_t = gf_common_mt_end + 10,
+    gf_changelog_mt_history_data_t = gf_common_mt_end + 11,
+    gf_changelog_mt_libgfchangelog_call_pool_t = gf_common_mt_end + 12,
+    gf_changelog_mt_libgfchangelog_event_t = gf_common_mt_end + 13,
+    gf_changelog_mt_ev_dispatcher_t = gf_common_mt_end + 14,
+    gf_changelog_mt_end
 };
 
 #endif
diff --git a/xlators/features/changelog/src/changelog-messages.h b/xlators/features/changelog/src/changelog-messages.h
index bbef2f2bde5..cb0e16c85d8 100644
--- a/xlators/features/changelog/src/changelog-messages.h
+++ b/xlators/features/changelog/src/changelog-messages.h
@@ -11,7 +11,7 @@
 #ifndef _CHANGELOG_MESSAGES_H_
 #define _CHANGELOG_MESSAGES_H_
 
-#include "glfs-message-id.h"
+#include <glusterfs/glfs-message-id.h>
 
 /* To add new message IDs, append new identifiers at the end of the list.
  *
@@ -23,63 +23,150 @@
  * glfs-message-id.h.
  */
 
-GLFS_MSGID(CHANGELOG,
-        CHANGELOG_MSG_OPEN_FAILED,
-        CHANGELOG_MSG_NO_MEMORY,
-        CHANGELOG_MSG_VOL_MISCONFIGURED,
-        CHANGELOG_MSG_RENAME_ERROR,
-        CHANGELOG_MSG_READ_ERROR,
-        CHANGELOG_MSG_HTIME_ERROR,
-        CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
-        CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED,
-        CHANGELOG_MSG_CHILD_MISCONFIGURED,
-        CHANGELOG_MSG_DIR_OPTIONS_NOT_SET,
-        CHANGELOG_MSG_CLOSE_ERROR,
-        CHANGELOG_MSG_PIPE_CREATION_ERROR,
-        CHANGELOG_MSG_DICT_GET_FAILED,
-        CHANGELOG_MSG_BARRIER_INFO,
-        CHANGELOG_MSG_BARRIER_ERROR,
-        CHANGELOG_MSG_GET_TIME_OP_FAILED,
-        CHANGELOG_MSG_WRITE_FAILED,
-        CHANGELOG_MSG_PTHREAD_ERROR,
-        CHANGELOG_MSG_INODE_NOT_FOUND,
-        CHANGELOG_MSG_FSYNC_OP_FAILED,
-        CHANGELOG_MSG_TOTAL_LOG_INFO,
-        CHANGELOG_MSG_SNAP_INFO,
-        CHANGELOG_MSG_SELECT_FAILED,
-        CHANGELOG_MSG_FCNTL_FAILED,
-        CHANGELOG_MSG_BNOTIFY_INFO,
-        CHANGELOG_MSG_ENTRY_BUF_INFO,
-        CHANGELOG_MSG_NOT_ACTIVE,
-        CHANGELOG_MSG_LOCAL_INIT_FAILED,
-        CHANGELOG_MSG_NOTIFY_REGISTER_FAILED,
-        CHANGELOG_MSG_PROGRAM_NAME_REG_FAILED,
-        CHANGELOG_MSG_HANDLE_PROBE_ERROR,
-        CHANGELOG_MSG_SET_FD_CONTEXT,
-        CHANGELOG_MSG_FREEUP_FAILED,
-        CHANGELOG_MSG_HTIME_INFO,
-        CHANGELOG_MSG_RPC_SUBMIT_REPLY_FAILED,
-        CHANGELOG_MSG_RPC_BUILD_ERROR,
-        CHANGELOG_MSG_RPC_CONNECT_ERROR,
-        CHANGELOG_MSG_RPC_START_ERROR,
-        CHANGELOG_MSG_BUFFER_STARVATION_ERROR,
-        CHANGELOG_MSG_SCAN_DIR_FAILED,
-        CHANGELOG_MSG_FSETXATTR_FAILED,
-        CHANGELOG_MSG_FGETXATTR_FAILED,
-        CHANGELOG_MSG_CLEANUP_ON_ACTIVE_REF,
-        CHANGELOG_MSG_DISPATCH_EVENT_FAILED,
-        CHANGELOG_MSG_PUT_BUFFER_FAILED,
-        CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED,
-        CHANGELOG_MSG_PTHREAD_CANCEL_FAILED,
-        CHANGELOG_MSG_INJECT_FSYNC_FAILED,
-        CHANGELOG_MSG_CREATE_FRAME_FAILED,
-        CHANGELOG_MSG_FSTAT_OP_FAILED,
-        CHANGELOG_MSG_LSEEK_OP_FAILED,
-        CHANGELOG_MSG_STRSTR_OP_FAILED,
-        CHANGELOG_MSG_UNLINK_OP_FAILED,
-        CHANGELOG_MSG_DETECT_EMPTY_CHANGELOG_FAILED,
-        CHANGELOG_MSG_READLINK_OP_FAILED,
-        CHANGELOG_MSG_EXPLICIT_ROLLOVER_FAILED
-);
+GLFS_MSGID(
+    CHANGELOG, CHANGELOG_MSG_OPEN_FAILED, CHANGELOG_MSG_BARRIER_FOP_FAILED,
+    CHANGELOG_MSG_VOL_MISCONFIGURED, CHANGELOG_MSG_RENAME_ERROR,
+    CHANGELOG_MSG_READ_ERROR, CHANGELOG_MSG_HTIME_ERROR,
+    CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
+    CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED, CHANGELOG_MSG_CHILD_MISCONFIGURED,
+    CHANGELOG_MSG_DIR_OPTIONS_NOT_SET, CHANGELOG_MSG_CLOSE_ERROR,
+    CHANGELOG_MSG_PIPE_CREATION_ERROR, CHANGELOG_MSG_DICT_GET_FAILED,
+    CHANGELOG_MSG_BARRIER_INFO, CHANGELOG_MSG_BARRIER_ERROR,
+    CHANGELOG_MSG_GET_TIME_OP_FAILED, CHANGELOG_MSG_WRITE_FAILED,
+    CHANGELOG_MSG_PTHREAD_ERROR, CHANGELOG_MSG_INODE_NOT_FOUND,
+    CHANGELOG_MSG_FSYNC_OP_FAILED, CHANGELOG_MSG_TOTAL_LOG_INFO,
+    CHANGELOG_MSG_SNAP_INFO, CHANGELOG_MSG_SELECT_FAILED,
+    CHANGELOG_MSG_FCNTL_FAILED, CHANGELOG_MSG_BNOTIFY_INFO,
+    CHANGELOG_MSG_ENTRY_BUF_INFO, CHANGELOG_MSG_CHANGELOG_NOT_ACTIVE,
+    CHANGELOG_MSG_LOCAL_INIT_FAILED, CHANGELOG_MSG_NOTIFY_REGISTER_FAILED,
+    CHANGELOG_MSG_PROGRAM_NAME_REG_FAILED, CHANGELOG_MSG_HANDLE_PROBE_ERROR,
+    CHANGELOG_MSG_SET_FD_CONTEXT, CHANGELOG_MSG_FREEUP_FAILED,
+    CHANGELOG_MSG_RECONFIGURE, CHANGELOG_MSG_RPC_SUBMIT_REPLY_FAILED,
+    CHANGELOG_MSG_RPC_BUILD_ERROR, CHANGELOG_MSG_RPC_CONNECT_ERROR,
+    CHANGELOG_MSG_RPC_START_ERROR, CHANGELOG_MSG_BUFFER_STARVATION_ERROR,
+    CHANGELOG_MSG_SCAN_DIR_FAILED, CHANGELOG_MSG_FSETXATTR_FAILED,
+    CHANGELOG_MSG_FGETXATTR_FAILED, CHANGELOG_MSG_CLEANUP_ON_ACTIVE_REF,
+    CHANGELOG_MSG_DISPATCH_EVENT_FAILED, CHANGELOG_MSG_PUT_BUFFER_FAILED,
+    CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED, CHANGELOG_MSG_PTHREAD_CANCEL_FAILED,
+    CHANGELOG_MSG_INJECT_FSYNC_FAILED, CHANGELOG_MSG_CREATE_FRAME_FAILED,
+    CHANGELOG_MSG_FSTAT_OP_FAILED, CHANGELOG_MSG_LSEEK_OP_FAILED,
+    CHANGELOG_MSG_STRSTR_OP_FAILED, CHANGELOG_MSG_UNLINK_OP_FAILED,
+    CHANGELOG_MSG_DETECT_EMPTY_CHANGELOG_FAILED,
+    CHANGELOG_MSG_READLINK_OP_FAILED, CHANGELOG_MSG_EXPLICIT_ROLLOVER_FAILED,
+    CHANGELOG_MSG_RPCSVC_NOTIFY_FAILED, CHANGELOG_MSG_MEMORY_INIT_FAILED,
+    CHANGELOG_MSG_NO_MEMORY, CHANGELOG_MSG_HTIME_STAT_ERROR,
+    CHANGELOG_MSG_HTIME_CURRENT_ERROR, CHANGELOG_MSG_BNOTIFY_COND_INFO,
+    CHANGELOG_MSG_NO_HTIME_CURRENT, CHANGELOG_MSG_HTIME_CURRENT,
+    CHANGELOG_MSG_NEW_HTIME_FILE, CHANGELOG_MSG_MKDIR_ERROR,
+    CHANGELOG_MSG_PATH_NOT_FOUND, CHANGELOG_MSG_XATTR_INIT_FAILED,
+    CHANGELOG_MSG_WROTE_TO_CSNAP, CHANGELOG_MSG_UNUSED_0,
+    CHANGELOG_MSG_GET_BUFFER_FAILED, CHANGELOG_MSG_BARRIER_STATE_NOTIFY,
+    CHANGELOG_MSG_BARRIER_DISABLED, CHANGELOG_MSG_BARRIER_ALREADY_DISABLED,
+    CHANGELOG_MSG_BARRIER_ON_ERROR, CHANGELOG_MSG_BARRIER_ENABLE,
+    CHANGELOG_MSG_BARRIER_KEY_NOT_FOUND, CHANGELOG_MSG_ERROR_IN_DICT_GET,
+    CHANGELOG_MSG_UNUSED_1, CHANGELOG_MSG_UNUSED_2,
+    CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS,
+    CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS_FINISHED,
+    CHANGELOG_MSG_BARRIER_TIMEOUT, CHANGELOG_MSG_TIMEOUT_ADD_FAILED,
+    CHANGELOG_MSG_CLEANUP_ALREADY_SET);
 
+#define CHANGELOG_MSG_BARRIER_FOP_FAILED_STR                                   \
+    "failed to barrier FOPs, disabling changelog barrier"
+#define CHANGELOG_MSG_MEMORY_INIT_FAILED_STR "memory accounting init failed"
+#define CHANGELOG_MSG_NO_MEMORY_STR "failed to create local memory pool"
+#define CHANGELOG_MSG_ENTRY_BUF_INFO_STR                                       \
+    "Entry cannot be captured for gfid, Capturing DATA entry."
+#define CHANGELOG_MSG_PTHREAD_ERROR_STR "pthread error"
+#define CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED_STR "pthread_mutex_init failed"
+#define CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED_STR "pthread_cond_init failed"
+#define CHANGELOG_MSG_HTIME_ERROR_STR "failed to update HTIME file"
+#define CHANGELOG_MSG_HTIME_STAT_ERROR_STR "unable to stat htime file"
+#define CHANGELOG_MSG_HTIME_CURRENT_ERROR_STR "Error extracting HTIME_CURRENT."
+#define CHANGELOG_MSG_UNLINK_OP_FAILED_STR "error unlinking empty changelog"
+#define CHANGELOG_MSG_RENAME_ERROR_STR "error renaming"
+#define CHANGELOG_MSG_MKDIR_ERROR_STR "unable to create directory"
+#define CHANGELOG_MSG_BNOTIFY_INFO_STR                                         \
+    "Explicit rollover changelog signaling bnotify"
+#define CHANGELOG_MSG_BNOTIFY_COND_INFO_STR "Woke up: bnotify conditional wait"
+#define CHANGELOG_MSG_RECONFIGURE_STR "Reconfigure: Changelog Enable"
+#define CHANGELOG_MSG_NO_HTIME_CURRENT_STR                                     \
+    "HTIME_CURRENT not found. Changelog enabled before init"
+#define CHANGELOG_MSG_HTIME_CURRENT_STR "HTIME_CURRENT"
+#define CHANGELOG_MSG_NEW_HTIME_FILE_STR                                       \
+    "Changelog enable: Creating new HTIME file"
+#define CHANGELOG_MSG_FGETXATTR_FAILED_STR "fgetxattr failed"
+#define CHANGELOG_MSG_TOTAL_LOG_INFO_STR "changelog info"
+#define CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED_STR "pthread cond wait failed"
+#define CHANGELOG_MSG_INODE_NOT_FOUND_STR "inode not found"
+#define CHANGELOG_MSG_READLINK_OP_FAILED_STR                                   \
+    "could not read the link from the gfid handle"
+#define CHANGELOG_MSG_OPEN_FAILED_STR "unable to open file"
+#define CHANGELOG_MSG_RPC_CONNECT_ERROR_STR "failed to connect back"
+#define CHANGELOG_MSG_BUFFER_STARVATION_ERROR_STR                              \
+    "Failed to get buffer for RPC dispatch"
+#define CHANGELOG_MSG_PTHREAD_CANCEL_FAILED_STR "could not cancel thread"
+#define CHANGELOG_MSG_FSTAT_OP_FAILED_STR "Could not stat (CHANGELOG)"
+#define CHANGELOG_MSG_LSEEK_OP_FAILED_STR "Could not lseek (changelog)"
+#define CHANGELOG_MSG_PATH_NOT_FOUND_STR                                       \
+    "Could not find CHANGELOG in changelog path"
+#define CHANGELOG_MSG_FSYNC_OP_FAILED_STR "fsync failed"
+#define CHANGELOG_MSG_DETECT_EMPTY_CHANGELOG_FAILED_STR                        \
+    "Error detecting empty changelog"
+#define CHANGELOG_MSG_EXPLICIT_ROLLOVER_FAILED_STR                             \
+    "Fail snapshot because of previous errors"
+#define CHANGELOG_MSG_SCAN_DIR_FAILED_STR "scandir failed"
+#define CHANGELOG_MSG_FSETXATTR_FAILED_STR "fsetxattr failed"
+#define CHANGELOG_MSG_XATTR_INIT_FAILED_STR "Htime xattr initialization failed"
+#define CHANGELOG_MSG_SNAP_INFO_STR "log in call path"
+#define CHANGELOG_MSG_WRITE_FAILED_STR "error writing to disk"
+#define CHANGELOG_MSG_WROTE_TO_CSNAP_STR "Successfully wrote to csnap"
+#define CHANGELOG_MSG_GET_TIME_OP_FAILED_STR "Problem rolling over changelog(s)"
+#define CHANGELOG_MSG_BARRIER_INFO_STR "Explicit wakeup on barrier notify"
+#define CHANGELOG_MSG_SELECT_FAILED_STR "pthread_cond_timedwait failed"
+#define CHANGELOG_MSG_INJECT_FSYNC_FAILED_STR "failed to inject fsync event"
+#define CHANGELOG_MSG_LOCAL_INIT_FAILED_STR                                    \
+    "changelog local initialization failed"
+#define CHANGELOG_MSG_GET_BUFFER_FAILED_STR "Failed to get buffer"
+#define CHANGELOG_MSG_SET_FD_CONTEXT_STR                                       \
+    "could not set fd context(for release cbk)"
+#define CHANGELOG_MSG_DICT_GET_FAILED_STR "Barrier failed"
+#define CHANGELOG_MSG_BARRIER_STATE_NOTIFY_STR "Barrier notification"
+#define CHANGELOG_MSG_BARRIER_ERROR_STR                                        \
+    "Received another barrier off notification while already off"
+#define CHANGELOG_MSG_BARRIER_DISABLED_STR "disabled changelog barrier"
+#define CHANGELOG_MSG_BARRIER_ALREADY_DISABLED_STR                             \
+    "Changelog barrier already disabled"
+#define CHANGELOG_MSG_BARRIER_ON_ERROR_STR                                     \
+    "Received another barrier on notification when last one is not served yet"
+#define CHANGELOG_MSG_BARRIER_ENABLE_STR "Enabled changelog barrier"
+#define CHANGELOG_MSG_BARRIER_KEY_NOT_FOUND_STR "barrier key not found"
+#define CHANGELOG_MSG_ERROR_IN_DICT_GET_STR                                    \
+    "Something went wrong in dict_get_str_boolean"
+#define CHANGELOG_MSG_DIR_OPTIONS_NOT_SET_STR "changelog-dir option is not set"
+#define CHANGELOG_MSG_FREEUP_FAILED_STR "could not cleanup bootstrapper"
+#define CHANGELOG_MSG_CHILD_MISCONFIGURED_STR                                  \
+    "translator needs a single subvolume"
+#define CHANGELOG_MSG_VOL_MISCONFIGURED_STR                                    \
+    "dangling volume. please check volfile"
+#define CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS_STR                               \
+    "Dequeuing all the changelog barriered fops"
+#define CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS_FINISHED_STR                      \
+    "Dequeuing changelog barriered fops is finished"
+#define CHANGELOG_MSG_BARRIER_TIMEOUT_STR                                      \
+    "Disabling changelog barrier because of the timeout"
+#define CHANGELOG_MSG_TIMEOUT_ADD_FAILED_STR                                   \
+    "Couldn't add changelog barrier timeout event"
+#define CHANGELOG_MSG_RPC_BUILD_ERROR_STR "failed to build rpc options"
+#define CHANGELOG_MSG_NOTIFY_REGISTER_FAILED_STR "failed to register notify"
+#define CHANGELOG_MSG_RPC_START_ERROR_STR "failed to start rpc"
+#define CHANGELOG_MSG_CREATE_FRAME_FAILED_STR "failed to create frame"
+#define CHANGELOG_MSG_RPC_SUBMIT_REPLY_FAILED_STR "failed to serialize reply"
+#define CHANGELOG_MSG_PROGRAM_NAME_REG_FAILED_STR "cannot register program"
+#define CHANGELOG_MSG_CHANGELOG_NOT_ACTIVE_STR                                 \
+    "Changelog is not active, return success"
+#define CHANGELOG_MSG_PUT_BUFFER_FAILED_STR                                    \
+    "failed to put buffer after consumption"
+#define CHANGELOG_MSG_CLEANUP_ALREADY_SET_STR                                  \
+    "cleanup_starting flag is already set for xl"
+#define CHANGELOG_MSG_HANDLE_PROBE_ERROR_STR "xdr decoding error"
 #endif /* !_CHANGELOG_MESSAGES_H_ */
diff --git a/xlators/features/changelog/src/changelog-misc.h b/xlators/features/changelog/src/changelog-misc.h
index 93af201879e..e2addc09414 100644
--- a/xlators/features/changelog/src/changelog-misc.h
+++ b/xlators/features/changelog/src/changelog-misc.h
@@ -11,10 +11,10 @@
 #ifndef _CHANGELOG_MISC_H
 #define _CHANGELOG_MISC_H
 
-#include "glusterfs.h"
-#include "common-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/common-utils.h>
 
-#define CHANGELOG_MAX_TYPE  4
+#define CHANGELOG_MAX_TYPE 4
 #define CHANGELOG_FILE_NAME "CHANGELOG"
 #define HTIME_FILE_NAME "HTIME"
 #define CSNAP_FILE_NAME "CHANGELOG.SNAP"
@@ -22,111 +22,110 @@
 #define HTIME_CURRENT "trusted.glusterfs.current_htime"
 #define HTIME_INITIAL_VALUE "0:0"
 
-#define CHANGELOG_VERSION_MAJOR  1
-#define CHANGELOG_VERSION_MINOR  2
+#define CHANGELOG_VERSION_MAJOR 1
+#define CHANGELOG_VERSION_MINOR 2
 
-#define CHANGELOG_UNIX_SOCK  DEFAULT_VAR_RUN_DIRECTORY"/changelog-%s.sock"
-#define CHANGELOG_TMP_UNIX_SOCK  DEFAULT_VAR_RUN_DIRECTORY"/.%s%lu.sock"
+#define CHANGELOG_UNIX_SOCK DEFAULT_VAR_RUN_DIRECTORY "/changelog-%s.sock"
+#define CHANGELOG_TMP_UNIX_SOCK DEFAULT_VAR_RUN_DIRECTORY "/.%s%lu.sock"
 
 /**
  * header starts with the version and the format of the changelog.
  * 'version' not much of a use now.
  */
-#define CHANGELOG_HEADER                                                \
-        "GlusterFS Changelog | version: v%d.%d | encoding : %d\n"
-
-#define CHANGELOG_MAKE_SOCKET_PATH(brick_path, sockpath, len) do {      \
-                char xxh64[GF_XXH64_DIGEST_LENGTH*2+1] = {0,};          \
-                gf_xxh64_wrapper ((unsigned char *)brick_path,          \
-                                  strlen(brick_path),                   \
-                                  GF_XXHSUM64_DEFAULT_SEED, xxh64);     \
-                (void) snprintf (sockpath, len,                         \
-                                 CHANGELOG_UNIX_SOCK, xxh64);           \
-        } while (0)
-
-#define CHANGELOG_MAKE_TMP_SOCKET_PATH(brick_path, sockpath, len) do {  \
-                unsigned long pid = 0;                                  \
-                char xxh64[GF_XXH64_DIGEST_LENGTH*2+1] = {0,};          \
-                pid = (unsigned long) getpid ();                        \
-                gf_xxh64_wrapper ((unsigned char *)brick_path,          \
-                                  strlen(brick_path),                   \
-                                  GF_XXHSUM64_DEFAULT_SEED, xxh64);     \
-                (void) snprintf (sockpath,                              \
-                                 len, CHANGELOG_TMP_UNIX_SOCK,          \
-                                 xxh64, pid);                           \
-        } while (0)
-
+#define CHANGELOG_HEADER                                                       \
+    "GlusterFS Changelog | version: v%d.%d | encoding : %d\n"
+
+#define CHANGELOG_MAKE_SOCKET_PATH(brick_path, sockpath, len)                  \
+    do {                                                                       \
+        char xxh64[GF_XXH64_DIGEST_LENGTH * 2 + 1] = {                         \
+            0,                                                                 \
+        };                                                                     \
+        gf_xxh64_wrapper((unsigned char *)brick_path, strlen(brick_path),      \
+                         GF_XXHSUM64_DEFAULT_SEED, xxh64);                     \
+        (void)snprintf(sockpath, len, CHANGELOG_UNIX_SOCK, xxh64);             \
+    } while (0)
+
+#define CHANGELOG_MAKE_TMP_SOCKET_PATH(brick_path, sockpath, len)              \
+    do {                                                                       \
+        unsigned long pid = 0;                                                 \
+        char xxh64[GF_XXH64_DIGEST_LENGTH * 2 + 1] = {                         \
+            0,                                                                 \
+        };                                                                     \
+        pid = (unsigned long)getpid();                                         \
+        gf_xxh64_wrapper((unsigned char *)brick_path, strlen(brick_path),      \
+                         GF_XXHSUM64_DEFAULT_SEED, xxh64);                     \
+        (void)snprintf(sockpath, len, CHANGELOG_TMP_UNIX_SOCK, xxh64, pid);    \
+    } while (0)
 
 /**
  * ... used by libgfchangelog.
  */
-#define CHANGELOG_GET_HEADER_INFO(fd, buffer, len, enc, maj, min, elen) do { \
-                FILE *fp;                                               \
-                int fd_dup;                                             \
-                                                                        \
-                enc = -1;                                               \
-                maj = -1;                                               \
-                min = -1;                                               \
-                fd_dup = dup (fd);                                      \
-                                                                        \
-                if (fd_dup != -1) {                                     \
-                        fp = fdopen (fd_dup, "r");                      \
-                        if (fp) {                                       \
-                                if (fgets (buffer, len, fp)) {          \
-                                        elen = strlen (buffer);         \
-                                        sscanf (buffer,                 \
-                                                CHANGELOG_HEADER,       \
-                                                &maj, &min, &enc);      \
-                                }                                       \
-                                fclose (fp);                            \
-                        } else {                                        \
-                                sys_close (fd_dup);                     \
-                        }                                               \
-                }                                                       \
-        } while (0)
-
-#define CHANGELOG_FILL_HTIME_DIR(changelog_dir, path) do {              \
-                strncpy (path, changelog_dir, sizeof (path) - 1);       \
-                strcat (path, "/htime");                                \
-        } while(0)
-
-#define CHANGELOG_FILL_CSNAP_DIR(changelog_dir, path) do {              \
-                strncpy (path, changelog_dir, sizeof (path) - 1);       \
-                strcat (path, "/csnap");                                \
-        } while(0)
+#define CHANGELOG_GET_HEADER_INFO(fd, buffer, len, enc, maj, min, elen)        \
+    do {                                                                       \
+        FILE *fp;                                                              \
+        int fd_dup;                                                            \
+                                                                               \
+        enc = -1;                                                              \
+        maj = -1;                                                              \
+        min = -1;                                                              \
+        fd_dup = dup(fd);                                                      \
+                                                                               \
+        if (fd_dup != -1) {                                                    \
+            fp = fdopen(fd_dup, "r");                                          \
+            if (fp) {                                                          \
+                if (fgets(buffer, len, fp)) {                                  \
+                    elen = strlen(buffer);                                     \
+                    sscanf(buffer, CHANGELOG_HEADER, &maj, &min, &enc);        \
+                }                                                              \
+                fclose(fp);                                                    \
+            } else {                                                           \
+                sys_close(fd_dup);                                             \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
+
+#define CHANGELOG_FILL_HTIME_DIR(changelog_dir, path)                          \
+    do {                                                                       \
+        snprintf(path, sizeof(path), "%s/htime", changelog_dir);               \
+    } while (0)
+
+#define CHANGELOG_FILL_CSNAP_DIR(changelog_dir, path)                          \
+    do {                                                                       \
+        snprintf(path, sizeof(path), "%s/csnap", changelog_dir);               \
+    } while (0)
 /**
  * everything after 'CHANGELOG_TYPE_METADATA_XATTR' are internal types
  * (ie. none of the fops trigger this type of event), hence
  * CHANGELOG_MAX_TYPE = 4
  */
 typedef enum {
-        CHANGELOG_TYPE_DATA = 0,
-        CHANGELOG_TYPE_METADATA,
-        CHANGELOG_TYPE_ENTRY,
-        CHANGELOG_TYPE_METADATA_XATTR,
-        CHANGELOG_TYPE_ROLLOVER,
-        CHANGELOG_TYPE_FSYNC,
+    CHANGELOG_TYPE_DATA = 0,
+    CHANGELOG_TYPE_METADATA,
+    CHANGELOG_TYPE_ENTRY,
+    CHANGELOG_TYPE_METADATA_XATTR,
+    CHANGELOG_TYPE_ROLLOVER,
+    CHANGELOG_TYPE_FSYNC,
 } changelog_log_type;
 
 /* operation modes - RT for now */
 typedef enum {
-        CHANGELOG_MODE_RT = 0,
+    CHANGELOG_MODE_RT = 0,
 } changelog_mode_t;
 
 /* encoder types */
 
 typedef enum {
-        CHANGELOG_ENCODE_MIN = 0,
-        CHANGELOG_ENCODE_BINARY,
-        CHANGELOG_ENCODE_ASCII,
-        CHANGELOG_ENCODE_MAX,
+    CHANGELOG_ENCODE_MIN = 0,
+    CHANGELOG_ENCODE_BINARY,
+    CHANGELOG_ENCODE_ASCII,
+    CHANGELOG_ENCODE_MAX,
 } changelog_encoder_t;
 
-#define CHANGELOG_VALID_ENCODING(enc)                                   \
-        (enc > CHANGELOG_ENCODE_MIN && enc < CHANGELOG_ENCODE_MAX)
+#define CHANGELOG_VALID_ENCODING(enc)                                          \
+    (enc > CHANGELOG_ENCODE_MIN && enc < CHANGELOG_ENCODE_MAX)
 
-#define CHANGELOG_TYPE_IS_ENTRY(type)  (type == CHANGELOG_TYPE_ENTRY)
-#define CHANGELOG_TYPE_IS_ROLLOVER(type)  (type == CHANGELOG_TYPE_ROLLOVER)
-#define CHANGELOG_TYPE_IS_FSYNC(type)  (type == CHANGELOG_TYPE_FSYNC)
+#define CHANGELOG_TYPE_IS_ENTRY(type) (type == CHANGELOG_TYPE_ENTRY)
+#define CHANGELOG_TYPE_IS_ROLLOVER(type) (type == CHANGELOG_TYPE_ROLLOVER)
+#define CHANGELOG_TYPE_IS_FSYNC(type) (type == CHANGELOG_TYPE_FSYNC)
 
 #endif /* _CHANGELOG_MISC_H */
diff --git a/xlators/features/changelog/src/changelog-rpc-common.c b/xlators/features/changelog/src/changelog-rpc-common.c
index 1745f218b08..125246a17e1 100644
--- a/xlators/features/changelog/src/changelog-rpc-common.c
+++ b/xlators/features/changelog/src/changelog-rpc-common.c
@@ -11,7 +11,7 @@
 #include "changelog-rpc-common.h"
 #include "changelog-messages.h"
 
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 /**
 *****************************************************
                   Client Interface
@@ -24,65 +24,63 @@
  */
 
 void *
-changelog_rpc_poller (void *arg)
+changelog_rpc_poller(void *arg)
 {
-        xlator_t *this = arg;
+    xlator_t *this = arg;
 
-        (void) event_dispatch (this->ctx->event_pool);
-        return NULL;
+    (void)gf_event_dispatch(this->ctx->event_pool);
+    return NULL;
 }
 
 struct rpc_clnt *
-changelog_rpc_client_init (xlator_t *this, void *cbkdata,
-                           char *sockfile, rpc_clnt_notify_t fn)
+changelog_rpc_client_init(xlator_t *this, void *cbkdata, char *sockfile,
+                          rpc_clnt_notify_t fn)
 {
-        int              ret         = 0;
-        struct rpc_clnt *rpc         = NULL;
-        dict_t          *options     = NULL;
-
-        if (!cbkdata)
-                cbkdata = this;
-
-        options = dict_new ();
-        if (!options)
-                goto error_return;
-
-        ret = rpc_transport_unix_options_build (&options, sockfile, 0);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_RPC_BUILD_ERROR,
-                        "failed to build rpc options");
-                goto dealloc_dict;
-        }
-
-        rpc = rpc_clnt_new (options, this, this->name, 16);
-        if (!rpc)
-                goto dealloc_dict;
-
-        ret = rpc_clnt_register_notify (rpc, fn, cbkdata);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_NOTIFY_REGISTER_FAILED,
-                        "failed to register notify");
-                goto dealloc_rpc_clnt;
-        }
-
-        ret = rpc_clnt_start (rpc);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_RPC_START_ERROR,
-                        "failed to start rpc");
-                goto dealloc_rpc_clnt;
-        }
-
-        return rpc;
-
- dealloc_rpc_clnt:
-        rpc_clnt_unref (rpc);
- dealloc_dict:
-        dict_unref (options);
- error_return:
-        return NULL;
+    int ret = 0;
+    struct rpc_clnt *rpc = NULL;
+    dict_t *options = NULL;
+
+    if (!cbkdata)
+        cbkdata = this;
+
+    options = dict_new();
+    if (!options)
+        goto error_return;
+
+    ret = rpc_transport_unix_options_build(options, sockfile, 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_RPC_BUILD_ERROR,
+                NULL);
+        goto dealloc_dict;
+    }
+
+    rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!rpc)
+        goto dealloc_dict;
+
+    ret = rpc_clnt_register_notify(rpc, fn, cbkdata);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                CHANGELOG_MSG_NOTIFY_REGISTER_FAILED, NULL);
+        goto dealloc_rpc_clnt;
+    }
+
+    ret = rpc_clnt_start(rpc);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_RPC_START_ERROR,
+                NULL);
+        goto dealloc_rpc_clnt;
+    }
+
+    dict_unref(options);
+    return rpc;
+
+dealloc_rpc_clnt:
+    rpc_clnt_unref(rpc);
+dealloc_dict:
+    dict_unref(options);
+error_return:
+    return NULL;
 }
 
 /**
@@ -90,96 +88,96 @@ changelog_rpc_client_init (xlator_t *this, void *cbkdata,
  * RPC server.
  */
 int
-changelog_rpc_sumbit_req (struct rpc_clnt *rpc, void *req,
-                          call_frame_t *frame, rpc_clnt_prog_t *prog,
-                          int procnum, struct iovec *payload, int payloadcnt,
-                          struct iobref *iobref, xlator_t *this,
-                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+changelog_rpc_sumbit_req(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+                         rpc_clnt_prog_t *prog, int procnum,
+                         struct iovec *payload, int payloadcnt,
+                         struct iobref *iobref, xlator_t *this,
+                         fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
-        int           ret        = 0;
-        int           count      = 0;
-        struct iovec  iov        = {0, };
-        struct iobuf *iobuf      = NULL;
-        char          new_iobref = 0;
-        ssize_t       xdr_size   = 0;
-
-        GF_ASSERT (this);
+    int ret = 0;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    char new_iobref = 0;
+    ssize_t xdr_size = 0;
 
-        if (req) {
-                xdr_size = xdr_sizeof (xdrproc, req);
+    GF_ASSERT(this);
 
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                };
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
 
-                if (!iobref) {
-                        iobref = iobref_new ();
-                        if (!iobref) {
-                                goto out;
-                        }
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
 
-                        new_iobref = 1;
-                }
+        if (!iobref) {
+            iobref = iobref_new();
+            if (!iobref) {
+                goto out;
+            }
 
-                iobref_add (iobref, iobuf);
+            new_iobref = 1;
+        }
 
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_size (iobuf);
+        iobref_add(iobref, iobuf);
 
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        goto out;
-                }
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_size(iobuf);
 
-                iov.iov_len = ret;
-                count = 1;
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            goto out;
         }
 
-        ret = rpc_clnt_submit (rpc, prog, procnum, cbkfn, &iov, count,
-                               payload, payloadcnt, iobref, frame, NULL,
-                               0, NULL, 0, NULL);
+        iov.iov_len = ret;
+        count = 1;
+    }
+
+    ret = rpc_clnt_submit(rpc, prog, procnum, cbkfn, &iov, count, payload,
+                          payloadcnt, iobref, frame, NULL, 0, NULL, 0, NULL);
 
- out:
-        if (new_iobref)
-                iobref_unref (iobref);
-        if (iobuf)
-                iobuf_unref (iobuf);
-        return ret;
+out:
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
 }
 
 /**
  * Entry point to perform a remote procedure call
  */
 int
-changelog_invoke_rpc (xlator_t *this, struct rpc_clnt *rpc,
-                      rpc_clnt_prog_t *prog, int procidx, void *arg)
+changelog_invoke_rpc(xlator_t *this, struct rpc_clnt *rpc,
+                     rpc_clnt_prog_t *prog, int procidx, void *arg)
 {
-        int                   ret   = 0;
-        call_frame_t         *frame = NULL;
-        rpc_clnt_procedure_t *proc  = NULL;
-
-        if (!this || !prog)
-                goto error_return;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_CREATE_FRAME_FAILED,
-                        "failed to create frame");
-                goto error_return;
-        }
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+
+    if (!this || !prog)
+        goto error_return;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_CREATE_FRAME_FAILED,
+                NULL);
+        goto error_return;
+    }
 
-        proc = &prog->proctable[procidx];
-        if (proc->fn)
-                ret = proc->fn (frame, this, arg);
+    proc = &prog->proctable[procidx];
+    if (proc->fn)
+        ret = proc->fn(frame, this, arg);
 
-        STACK_DESTROY (frame->root);
-        return ret;
+    STACK_DESTROY(frame->root);
+    return ret;
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 /**
@@ -189,165 +187,173 @@ changelog_invoke_rpc (xlator_t *this, struct rpc_clnt *rpc,
 */
 
 struct iobuf *
-__changelog_rpc_serialize_reply (rpcsvc_request_t *req, void *arg,
-                                 struct iovec *outmsg, xdrproc_t xdrproc)
+__changelog_rpc_serialize_reply(rpcsvc_request_t *req, void *arg,
+                                struct iovec *outmsg, xdrproc_t xdrproc)
 {
-        struct iobuf *iob      = NULL;
-        ssize_t       retlen   = 0;
-        ssize_t       rsp_size = 0;
+    struct iobuf *iob = NULL;
+    ssize_t retlen = 0;
+    ssize_t rsp_size = 0;
 
-        rsp_size = xdr_sizeof (xdrproc, arg);
-        iob = iobuf_get2 (req->svc->ctx->iobuf_pool, rsp_size);
-        if (!iob)
-                goto error_return;
+    rsp_size = xdr_sizeof(xdrproc, arg);
+    iob = iobuf_get2(req->svc->ctx->iobuf_pool, rsp_size);
+    if (!iob)
+        goto error_return;
 
-        iobuf_to_iovec (iob, outmsg);
+    iobuf_to_iovec(iob, outmsg);
 
-        retlen = xdr_serialize_generic (*outmsg, arg, xdrproc);
-        if (retlen == -1)
-                goto unref_iob;
+    retlen = xdr_serialize_generic(*outmsg, arg, xdrproc);
+    if (retlen == -1)
+        goto unref_iob;
 
-        outmsg->iov_len = retlen;
-        return iob;
+    outmsg->iov_len = retlen;
+    return iob;
 
- unref_iob:
-        iobuf_unref (iob);
- error_return:
-        return NULL;
+unref_iob:
+    iobuf_unref(iob);
+error_return:
+    return NULL;
 }
 
 int
-changelog_rpc_sumbit_reply (rpcsvc_request_t *req,
-                            void *arg, struct iovec *payload, int payloadcount,
-                            struct iobref *iobref, xdrproc_t xdrproc)
+changelog_rpc_sumbit_reply(rpcsvc_request_t *req, void *arg,
+                           struct iovec *payload, int payloadcount,
+                           struct iobref *iobref, xdrproc_t xdrproc)
 {
-        int           ret        = -1;
-        struct iobuf *iob        = NULL;
-        struct iovec  iov        = {0,};
-        char          new_iobref = 0;
-
-        if (!req)
-                goto return_ret;
-
-        if (!iobref) {
-                iobref = iobref_new ();
-                if (!iobref)
-                        goto return_ret;
-                new_iobref = 1;
-        }
-
-        iob = __changelog_rpc_serialize_reply (req, arg, &iov, xdrproc);
-        if (!iob)
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_RPC_SUBMIT_REPLY_FAILED,
-                        "failed to serialize reply");
-        else
-                iobref_add (iobref, iob);
-
-        ret = rpcsvc_submit_generic (req, &iov,
-                                     1, payload, payloadcount, iobref);
-
-        if (new_iobref)
-                iobref_unref (iobref);
-        if (iob)
-                iobuf_unref (iob);
- return_ret:
-        return ret;
+    int ret = -1;
+    struct iobuf *iob = NULL;
+    struct iovec iov = {
+        0,
+    };
+    char new_iobref = 0;
+
+    if (!req)
+        goto return_ret;
+
+    if (!iobref) {
+        iobref = iobref_new();
+        if (!iobref)
+            goto return_ret;
+        new_iobref = 1;
+    }
+
+    iob = __changelog_rpc_serialize_reply(req, arg, &iov, xdrproc);
+    if (!iob)
+        gf_smsg("", GF_LOG_ERROR, 0, CHANGELOG_MSG_RPC_SUBMIT_REPLY_FAILED,
+                NULL);
+    else
+        iobref_add(iobref, iob);
+
+    ret = rpcsvc_submit_generic(req, &iov, 1, payload, payloadcount, iobref);
+
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iob)
+        iobuf_unref(iob);
+return_ret:
+    return ret;
 }
 
 void
-changelog_rpc_server_destroy (xlator_t *this, rpcsvc_t *rpc, char *sockfile,
-                              rpcsvc_notify_t fn, struct rpcsvc_program **progs)
+changelog_rpc_server_destroy(xlator_t *this, rpcsvc_t *rpc, char *sockfile,
+                             rpcsvc_notify_t fn, struct rpcsvc_program **progs)
 {
-        rpcsvc_listener_t      *listener = NULL;
-        rpcsvc_listener_t      *next     = NULL;
-        struct rpcsvc_program  *prog     = NULL;
-
-        while (*progs) {
-                prog = *progs;
-                (void) rpcsvc_program_unregister (rpc, prog);
-                progs++;
+    rpcsvc_listener_t *listener = NULL;
+    rpcsvc_listener_t *next = NULL;
+    struct rpcsvc_program *prog = NULL;
+    rpc_transport_t *trans = NULL;
+
+    if (!rpc)
+        return;
+
+    while (*progs) {
+        prog = *progs;
+        (void)rpcsvc_program_unregister(rpc, prog);
+        progs++;
+    }
+
+    list_for_each_entry_safe(listener, next, &rpc->listeners, list)
+    {
+        if (listener->trans) {
+            trans = listener->trans;
+            rpc_transport_disconnect(trans, _gf_false);
         }
+    }
 
-        list_for_each_entry_safe (listener, next, &rpc->listeners, list) {
-                rpcsvc_listener_destroy (listener);
-        }
+    (void)rpcsvc_unregister_notify(rpc, fn, this);
 
-        (void) rpcsvc_unregister_notify (rpc, fn, this);
-        sys_unlink (sockfile);
+    /* TODO Avoid freeing rpc object in case of brick multiplex
+       after freeing rpc object svc->rpclock corrupted and it takes
+       more time to detach a brick
+    */
+    if (!this->cleanup_starting) {
         if (rpc->rxpool) {
-                mem_pool_destroy (rpc->rxpool);
-                rpc->rxpool = NULL;
+            mem_pool_destroy(rpc->rxpool);
+            rpc->rxpool = NULL;
         }
-
-        GF_FREE (rpc);
+        GF_FREE(rpc);
+    }
 }
 
 rpcsvc_t *
-changelog_rpc_server_init (xlator_t *this, char *sockfile, void *cbkdata,
-                           rpcsvc_notify_t fn, struct rpcsvc_program **progs)
+changelog_rpc_server_init(xlator_t *this, char *sockfile, void *cbkdata,
+                          rpcsvc_notify_t fn, struct rpcsvc_program **progs)
 {
-        int                    ret     = 0;
-        rpcsvc_t              *rpc     = NULL;
-        dict_t                *options = NULL;
-        struct rpcsvc_program *prog    = NULL;
-
-        if (!cbkdata)
-                cbkdata = this;
-
-        options = dict_new ();
-        if (!options)
-                goto error_return;
-
-        ret = rpcsvc_transport_unix_options_build (&options, sockfile);
-        if (ret)
-                goto dealloc_dict;
-
-        rpc = rpcsvc_init (this, this->ctx, options, 8);
-        if (rpc == NULL) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_RPC_START_ERROR,
-                        "failed to init rpc");
-                goto dealloc_dict;
-        }
+    int ret = 0;
+    rpcsvc_t *rpc = NULL;
+    dict_t *options = NULL;
+    struct rpcsvc_program *prog = NULL;
 
-        ret = rpcsvc_register_notify (rpc, fn, cbkdata);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_NOTIFY_REGISTER_FAILED,
-                        "failed to register notify function");
-                goto dealloc_rpc;
-        }
+    if (!cbkdata)
+        cbkdata = this;
 
-        ret = rpcsvc_create_listeners (rpc, options, this->name);
-        if (ret != 1) {
-                gf_msg_debug (this->name,
-                              0, "failed to create listeners");
-                goto dealloc_rpc;
-        }
+    options = dict_new();
+    if (!options)
+        return NULL;
 
-        while (*progs) {
-                prog = *progs;
-                ret = rpcsvc_program_register (rpc, prog, _gf_false);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_MSG_PROGRAM_NAME_REG_FAILED,
-                                "cannot register program "
-                                "(name: %s, prognum: %d, pogver: %d)",
-                                prog->progname, prog->prognum, prog->progver);
-                        goto dealloc_rpc;
-                }
-
-                progs++;
+    ret = rpcsvc_transport_unix_options_build(options, sockfile);
+    if (ret)
+        goto dealloc_dict;
+
+    rpc = rpcsvc_init(this, this->ctx, options, 8);
+    if (rpc == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_RPC_START_ERROR,
+                NULL);
+        goto dealloc_dict;
+    }
+
+    ret = rpcsvc_register_notify(rpc, fn, cbkdata);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                CHANGELOG_MSG_NOTIFY_REGISTER_FAILED, NULL);
+        goto dealloc_rpc;
+    }
+
+    ret = rpcsvc_create_listeners(rpc, options, this->name);
+    if (ret != 1) {
+        gf_msg_debug(this->name, 0, "failed to create listeners");
+        goto dealloc_rpc;
+    }
+
+    while (*progs) {
+        prog = *progs;
+        ret = rpcsvc_program_register(rpc, prog, _gf_false);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    CHANGELOG_MSG_PROGRAM_NAME_REG_FAILED, "name%s",
+                    prog->progname, "prognum=%d", prog->prognum, "pogver=%d",
+                    prog->progver, NULL);
+            goto dealloc_rpc;
         }
 
-        dict_unref (options);
-        return rpc;
+        progs++;
+    }
 
- dealloc_rpc:
-        GF_FREE (rpc);
- dealloc_dict:
-        dict_unref (options);
- error_return:
-        return NULL;
+    dict_unref(options);
+    return rpc;
+
+dealloc_rpc:
+    GF_FREE(rpc);
+dealloc_dict:
+    dict_unref(options);
+    return NULL;
 }
diff --git a/xlators/features/changelog/src/changelog-rpc-common.h b/xlators/features/changelog/src/changelog-rpc-common.h
index 95c850c9400..4d9aa2c694b 100644
--- a/xlators/features/changelog/src/changelog-rpc-common.h
+++ b/xlators/features/changelog/src/changelog-rpc-common.h
@@ -13,8 +13,8 @@
 
 #include "rpcsvc.h"
 #include "rpc-clnt.h"
-#include "event.h"
-#include "call-stub.h"
+#include <glusterfs/gf-event.h>
+#include <glusterfs/call-stub.h>
 
 #include "changelog-xdr.h"
 #include "xdr-generic.h"
@@ -24,61 +24,62 @@
 /**
  * Let's keep this non-configurable for now.
  */
-#define NR_ROTT_BUFFS  4
+#define NR_ROTT_BUFFS 4
 #define NR_DISPATCHERS (NR_ROTT_BUFFS - 1)
 
 enum changelog_rpc_procnum {
-        CHANGELOG_RPC_PROC_NULL    = 0,
-        CHANGELOG_RPC_PROBE_FILTER = 1,
-        CHANGELOG_RPC_PROC_MAX     = 2,
+    CHANGELOG_RPC_PROC_NULL = 0,
+    CHANGELOG_RPC_PROBE_FILTER = 1,
+    CHANGELOG_RPC_PROC_MAX = 2,
 };
 
-#define CHANGELOG_RPC_PROGNUM   1885957735
-#define CHANGELOG_RPC_PROGVER   1
+#define CHANGELOG_RPC_PROGNUM 1885957735
+#define CHANGELOG_RPC_PROGVER 1
 
 /**
  * reverse connection: data xfer path
  */
 enum changelog_reverse_rpc_procnum {
-        CHANGELOG_REV_PROC_NULL  = 0,
-        CHANGELOG_REV_PROC_EVENT = 1,
-        CHANGELOG_REV_PROC_MAX   = 2,
+    CHANGELOG_REV_PROC_NULL = 0,
+    CHANGELOG_REV_PROC_EVENT = 1,
+    CHANGELOG_REV_PROC_MAX = 2,
 };
 
-#define CHANGELOG_REV_RPC_PROCNUM   1886350951
-#define CHANGELOG_REV_RPC_PROCVER   1
+#define CHANGELOG_REV_RPC_PROCNUM 1886350951
+#define CHANGELOG_REV_RPC_PROCVER 1
 
 typedef struct changelog_rpc {
-        rpcsvc_t        *svc;
-        struct rpc_clnt *rpc;
-        char             sock[UNIX_PATH_MAX];  /* tied to server */
+    rpcsvc_t *svc;
+    struct rpc_clnt *rpc;
+    char sock[UNIX_PATH_MAX]; /* tied to server */
 } changelog_rpc_t;
 
 /* event poller */
-void *changelog_rpc_poller (void *);
+void *
+changelog_rpc_poller(void *);
 
 /* CLIENT API */
 struct rpc_clnt *
-changelog_rpc_client_init (xlator_t *, void *, char *, rpc_clnt_notify_t);
+changelog_rpc_client_init(xlator_t *, void *, char *, rpc_clnt_notify_t);
 
 int
-changelog_rpc_sumbit_req (struct rpc_clnt *, void *, call_frame_t *,
-                          rpc_clnt_prog_t *, int , struct iovec *, int,
-                          struct iobref *, xlator_t *, fop_cbk_fn_t, xdrproc_t);
+changelog_rpc_sumbit_req(struct rpc_clnt *, void *, call_frame_t *,
+                         rpc_clnt_prog_t *, int, struct iovec *, int,
+                         struct iobref *, xlator_t *, fop_cbk_fn_t, xdrproc_t);
 
 int
-changelog_invoke_rpc (xlator_t *, struct rpc_clnt *,
-                      rpc_clnt_prog_t *, int , void *);
+changelog_invoke_rpc(xlator_t *, struct rpc_clnt *, rpc_clnt_prog_t *, int,
+                     void *);
 
 /* SERVER API */
 int
-changelog_rpc_sumbit_reply (rpcsvc_request_t *, void *,
-                            struct iovec *, int, struct iobref *, xdrproc_t);
+changelog_rpc_sumbit_reply(rpcsvc_request_t *, void *, struct iovec *, int,
+                           struct iobref *, xdrproc_t);
 rpcsvc_t *
-changelog_rpc_server_init (xlator_t *, char *, void*,
-                           rpcsvc_notify_t, struct rpcsvc_program **);
+changelog_rpc_server_init(xlator_t *, char *, void *, rpcsvc_notify_t,
+                          struct rpcsvc_program **);
 void
-changelog_rpc_server_destroy (xlator_t *, rpcsvc_t *, char *,
-                              rpcsvc_notify_t, struct rpcsvc_program **);
+changelog_rpc_server_destroy(xlator_t *, rpcsvc_t *, char *, rpcsvc_notify_t,
+                             struct rpcsvc_program **);
 
 #endif
diff --git a/xlators/features/changelog/src/changelog-rpc.c b/xlators/features/changelog/src/changelog-rpc.c
index 852c0694f9a..440b88091a6 100644
--- a/xlators/features/changelog/src/changelog-rpc.c
+++ b/xlators/features/changelog/src/changelog-rpc.c
@@ -8,231 +8,346 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 #include "changelog-rpc.h"
 #include "changelog-mem-types.h"
 #include "changelog-ev-handle.h"
 
-struct rpcsvc_program *changelog_programs[];
+static struct rpcsvc_program *changelog_programs[];
 
 static void
-changelog_cleanup_dispatchers (xlator_t *this,
-                               changelog_priv_t *priv, int count)
+changelog_cleanup_dispatchers(xlator_t *this, changelog_priv_t *priv, int count)
 {
-        for (count--; count >= 0; count--) {
-                (void) changelog_thread_cleanup
-                        (this, priv->ev_dispatcher[count]);
-                priv->ev_dispatcher[count] = 0;
-        }
+    for (count--; count >= 0; count--) {
+        (void)changelog_thread_cleanup(this, priv->ev_dispatcher[count]);
+        priv->ev_dispatcher[count] = 0;
+    }
 }
 
 int
-changelog_cleanup_rpc_threads (xlator_t *this, changelog_priv_t *priv)
+changelog_cleanup_rpc_threads(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
-        changelog_clnt_t *conn = NULL;
-
-        conn = &priv->connections;
-        if (!conn)
-                return 0;
-
-        /** terminate RPC thread(s) */
-        ret = changelog_thread_cleanup (this, priv->connector);
-        if (ret != 0)
-                goto error_return;
-        priv->connector = 0;
-
-        /** terminate dispatcher thread(s) */
-        changelog_cleanup_dispatchers (this, priv, priv->nr_dispatchers);
-
-        /* TODO: what about pending and waiting connections? */
-        changelog_ev_cleanup_connections (this, conn);
-
-        /* destroy locks */
-        ret = pthread_mutex_destroy (&conn->pending_lock);
-        if (ret != 0)
-                goto error_return;
-        ret = pthread_cond_destroy (&conn->pending_cond);
-        if (ret != 0)
-                goto error_return;
-        ret = LOCK_DESTROY (&conn->active_lock);
-        if (ret != 0)
-                goto error_return;
-        ret = LOCK_DESTROY (&conn->wait_lock);
-        if (ret != 0)
-                goto error_return;
+    int ret = 0;
+    changelog_clnt_t *conn = NULL;
+
+    conn = &priv->connections;
+    if (!conn)
         return 0;
 
- error_return:
-        return -1;
+    /** terminate RPC thread(s) */
+    ret = changelog_thread_cleanup(this, priv->connector);
+    if (ret != 0)
+        goto error_return;
+    priv->connector = 0;
+
+    /** terminate dispatcher thread(s) */
+    changelog_cleanup_dispatchers(this, priv, priv->nr_dispatchers);
+
+    /* destroy locks */
+    ret = pthread_mutex_destroy(&conn->pending_lock);
+    if (ret != 0)
+        goto error_return;
+    ret = pthread_cond_destroy(&conn->pending_cond);
+    if (ret != 0)
+        goto error_return;
+    ret = LOCK_DESTROY(&conn->active_lock);
+    if (ret != 0)
+        goto error_return;
+    ret = LOCK_DESTROY(&conn->wait_lock);
+    if (ret != 0)
+        goto error_return;
+    return 0;
+
+error_return:
+    return -1;
 }
 
 static int
-changelog_init_rpc_threads (xlator_t *this, changelog_priv_t *priv,
-                            rbuf_t *rbuf, int nr_dispatchers)
+changelog_init_rpc_threads(xlator_t *this, changelog_priv_t *priv, rbuf_t *rbuf,
+                           int nr_dispatchers)
+{
+    int j = 0;
+    int ret = 0;
+    changelog_clnt_t *conn = NULL;
+
+    conn = &priv->connections;
+
+    conn->this = this;
+    conn->rbuf = rbuf;
+    conn->sequence = 1; /* start with sequence number one */
+
+    INIT_LIST_HEAD(&conn->pending);
+    INIT_LIST_HEAD(&conn->active);
+    INIT_LIST_HEAD(&conn->waitq);
+
+    ret = pthread_mutex_init(&conn->pending_lock, NULL);
+    if (ret)
+        goto error_return;
+    ret = pthread_cond_init(&conn->pending_cond, NULL);
+    if (ret)
+        goto cleanup_pending_lock;
+
+    ret = LOCK_INIT(&conn->active_lock);
+    if (ret)
+        goto cleanup_pending_cond;
+    ret = LOCK_INIT(&conn->wait_lock);
+    if (ret)
+        goto cleanup_active_lock;
+
+    /* spawn reverse connection thread */
+    ret = gf_thread_create(&priv->connector, NULL, changelog_ev_connector, conn,
+                           "clogecon");
+    if (ret != 0)
+        goto cleanup_wait_lock;
+
+    /* spawn dispatcher thread(s) */
+    priv->ev_dispatcher = GF_CALLOC(nr_dispatchers, sizeof(pthread_t),
+                                    gf_changelog_mt_ev_dispatcher_t);
+    if (!priv->ev_dispatcher)
+        goto cleanup_connector;
+
+    /* spawn dispatcher threads */
+    for (; j < nr_dispatchers; j++) {
+        ret = gf_thread_create(&priv->ev_dispatcher[j], NULL,
+                               changelog_ev_dispatch, conn, "clogd%03hx",
+                               j & 0x3ff);
+        if (ret != 0) {
+            changelog_cleanup_dispatchers(this, priv, j);
+            break;
+        }
+    }
+
+    if (ret != 0)
+        goto cleanup_connector;
+
+    priv->nr_dispatchers = nr_dispatchers;
+    return 0;
+
+cleanup_connector:
+    (void)pthread_cancel(priv->connector);
+cleanup_wait_lock:
+    LOCK_DESTROY(&conn->wait_lock);
+cleanup_active_lock:
+    LOCK_DESTROY(&conn->active_lock);
+cleanup_pending_cond:
+    (void)pthread_cond_destroy(&conn->pending_cond);
+cleanup_pending_lock:
+    (void)pthread_mutex_destroy(&conn->pending_lock);
+error_return:
+    return -1;
+}
+
+int
+changelog_rpcsvc_notify(rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+                        void *data)
 {
-        int               j    = 0;
-        int               ret  = 0;
-        changelog_clnt_t *conn = NULL;
-        char              thread_name[GF_THREAD_NAMEMAX] = {0,};
-
-
-        conn = &priv->connections;
-
-        conn->this = this;
-        conn->rbuf = rbuf;
-        conn->sequence = 1; /* start with sequence number one */
-
-        INIT_LIST_HEAD (&conn->pending);
-        INIT_LIST_HEAD (&conn->active);
-        INIT_LIST_HEAD (&conn->waitq);
-
-        ret = pthread_mutex_init (&conn->pending_lock, NULL);
-        if (ret)
-                goto error_return;
-        ret = pthread_cond_init (&conn->pending_cond, NULL);
-        if (ret)
-                goto cleanup_pending_lock;
-
-        ret = LOCK_INIT (&conn->active_lock);
-        if (ret)
-                goto cleanup_pending_cond;
-        ret = LOCK_INIT (&conn->wait_lock);
-        if (ret)
-                goto cleanup_active_lock;
-
-        /* spawn reverse connection thread */
-        ret = gf_thread_create (&priv->connector,
-                                NULL, changelog_ev_connector, conn, "clogecon");
-        if (ret != 0)
-                goto cleanup_wait_lock;
-
-        /* spawn dispatcher thread(s) */
-        priv->ev_dispatcher = GF_CALLOC (nr_dispatchers, sizeof(pthread_t),
-                                         gf_changelog_mt_ev_dispatcher_t);
-        if (!priv->ev_dispatcher)
-                goto cleanup_connector;
-
-        /* spawn dispatcher threads */
-        for (; j < nr_dispatchers; j++) {
-                snprintf (thread_name, sizeof(thread_name),
-                          "%s%d", "clogd", j);
-                ret = gf_thread_create (&priv->ev_dispatcher[j],
-                                        NULL, changelog_ev_dispatch, conn,
-                                        thread_name);
-                if (ret != 0) {
-                        changelog_cleanup_dispatchers (this, priv, j);
-                        break;
+    xlator_t *this = NULL;
+    rpc_transport_t *trans = NULL;
+    rpc_transport_t *xprt = NULL;
+    rpc_transport_t *xp_next = NULL;
+    changelog_priv_t *priv = NULL;
+    uint64_t listnercnt = 0;
+    uint64_t xprtcnt = 0;
+    uint64_t clntcnt = 0;
+    rpcsvc_listener_t *listener = NULL;
+    rpcsvc_listener_t *next = NULL;
+    gf_boolean_t listner_found = _gf_false;
+    socket_private_t *sockpriv = NULL;
+
+    if (!xl || !data || !rpc) {
+        gf_msg_callingfn("changelog", GF_LOG_WARNING, 0,
+                         CHANGELOG_MSG_RPCSVC_NOTIFY_FAILED,
+                         "Calling rpc_notify without initializing");
+        goto out;
+    }
+
+    this = xl;
+    trans = data;
+    priv = this->private;
+
+    if (!priv) {
+        gf_msg_callingfn("changelog", GF_LOG_WARNING, 0,
+                         CHANGELOG_MSG_RPCSVC_NOTIFY_FAILED,
+                         "Calling rpc_notify without priv initializing");
+        goto out;
+    }
+
+    if (event == RPCSVC_EVENT_ACCEPT) {
+        GF_ATOMIC_INC(priv->xprtcnt);
+        LOCK(&priv->lock);
+        {
+            list_add_tail(&trans->list, &priv->xprt_list);
+        }
+        UNLOCK(&priv->lock);
+        goto out;
+    }
+
+    if (event == RPCSVC_EVENT_DISCONNECT) {
+        list_for_each_entry_safe(listener, next, &rpc->listeners, list)
+        {
+            if (listener && listener->trans) {
+                if (listener->trans == trans) {
+                    listnercnt = GF_ATOMIC_DEC(priv->listnercnt);
+                    listner_found = _gf_true;
+                    rpcsvc_listener_destroy(listener);
                 }
+            }
         }
 
-        if (ret != 0)
-                goto cleanup_connector;
+        if (listnercnt > 0) {
+            goto out;
+        }
+        if (listner_found) {
+            LOCK(&priv->lock);
+            list_for_each_entry_safe(xprt, xp_next, &priv->xprt_list, list)
+            {
+                sockpriv = (socket_private_t *)(xprt->private);
+                gf_log("changelog", GF_LOG_INFO,
+                       "Send disconnect"
+                       " on socket %d",
+                       sockpriv->sock);
+                rpc_transport_disconnect(xprt, _gf_false);
+            }
+            UNLOCK(&priv->lock);
+            goto out;
+        }
+        LOCK(&priv->lock);
+        {
+            list_del_init(&trans->list);
+        }
+        UNLOCK(&priv->lock);
 
-        priv->nr_dispatchers = nr_dispatchers;
-        return 0;
+        xprtcnt = GF_ATOMIC_DEC(priv->xprtcnt);
+        clntcnt = GF_ATOMIC_GET(priv->clntcnt);
+        if (!xprtcnt && !clntcnt) {
+            changelog_process_cleanup_event(this);
+        }
+    }
 
- cleanup_connector:
-        (void) pthread_cancel (priv->connector);
- cleanup_wait_lock:
-        LOCK_DESTROY (&conn->wait_lock);
- cleanup_active_lock:
-        LOCK_DESTROY (&conn->active_lock);
- cleanup_pending_cond:
-        (void) pthread_cond_destroy (&conn->pending_cond);
- cleanup_pending_lock:
-        (void) pthread_mutex_destroy (&conn->pending_lock);
- error_return:
-        return -1;
+out:
+    return 0;
 }
 
-int
-changelog_rpcsvc_notify (rpcsvc_t *rpc,
-                         void *xl, rpcsvc_event_t event, void *data)
+void
+changelog_process_cleanup_event(xlator_t *this)
 {
-        return 0;
+    gf_boolean_t cleanup_notify = _gf_false;
+    changelog_priv_t *priv = NULL;
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
+
+    if (!this)
+        return;
+    priv = this->private;
+    if (!priv)
+        return;
+
+    LOCK(&priv->lock);
+    {
+        cleanup_notify = priv->notify_down;
+        priv->notify_down = _gf_true;
+    }
+    UNLOCK(&priv->lock);
+
+    if (priv->victim && !cleanup_notify) {
+        default_notify(this, GF_EVENT_PARENT_DOWN, priv->victim);
+
+        if (priv->rpc) {
+            /* sockfile path could have been saved to avoid this */
+            CHANGELOG_MAKE_SOCKET_PATH(priv->changelog_brick, sockfile,
+                                       UNIX_PATH_MAX);
+            sys_unlink(sockfile);
+            (void)rpcsvc_unregister_notify(priv->rpc, changelog_rpcsvc_notify,
+                                           this);
+            if (priv->rpc->rxpool) {
+                mem_pool_destroy(priv->rpc->rxpool);
+                priv->rpc->rxpool = NULL;
+            }
+            GF_FREE(priv->rpc);
+            priv->rpc = NULL;
+        }
+    }
 }
 
 void
-changelog_destroy_rpc_listner (xlator_t *this, changelog_priv_t *priv)
+changelog_destroy_rpc_listner(xlator_t *this, changelog_priv_t *priv)
 {
-        char sockfile[UNIX_PATH_MAX] = {0,};
-
-        /* sockfile path could have been saved to avoid this */
-        CHANGELOG_MAKE_SOCKET_PATH (priv->changelog_brick,
-                                    sockfile, UNIX_PATH_MAX);
-        changelog_rpc_server_destroy (this,
-                                      priv->rpc, sockfile,
-                                      changelog_rpcsvc_notify,
-                                      changelog_programs);
-        (void) changelog_cleanup_rpc_threads (this, priv);
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
+
+    /* sockfile path could have been saved to avoid this */
+    CHANGELOG_MAKE_SOCKET_PATH(priv->changelog_brick, sockfile, UNIX_PATH_MAX);
+    changelog_rpc_server_destroy(this, priv->rpc, sockfile,
+                                 changelog_rpcsvc_notify, changelog_programs);
 }
 
 rpcsvc_t *
-changelog_init_rpc_listener (xlator_t *this, changelog_priv_t *priv,
+changelog_init_rpc_listener(xlator_t *this, changelog_priv_t *priv,
                             rbuf_t *rbuf, int nr_dispatchers)
 {
-        int ret = 0;
-        char sockfile[UNIX_PATH_MAX] = {0,};
-        rpcsvc_t *svcp;
-
-        ret = changelog_init_rpc_threads (this, priv, rbuf, nr_dispatchers);
-        if (ret)
-                return NULL;
-
-        CHANGELOG_MAKE_SOCKET_PATH (priv->changelog_brick,
-                                    sockfile, UNIX_PATH_MAX);
-        (void) sys_unlink (sockfile);
-        svcp = changelog_rpc_server_init (this, sockfile, NULL,
-                                          changelog_rpcsvc_notify,
-                                          changelog_programs);
-        return svcp;
+    int ret = 0;
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
+    rpcsvc_t *svcp;
+
+    ret = changelog_init_rpc_threads(this, priv, rbuf, nr_dispatchers);
+    if (ret)
+        return NULL;
+
+    CHANGELOG_MAKE_SOCKET_PATH(priv->changelog_brick, sockfile, UNIX_PATH_MAX);
+    (void)sys_unlink(sockfile);
+    svcp = changelog_rpc_server_init(
+        this, sockfile, NULL, changelog_rpcsvc_notify, changelog_programs);
+    return svcp;
 }
 
 void
-changelog_rpc_clnt_cleanup (changelog_rpc_clnt_t *crpc)
+changelog_rpc_clnt_cleanup(changelog_rpc_clnt_t *crpc)
 {
-        if (!crpc)
-                return;
-        crpc->c_clnt = NULL;
-        LOCK_DESTROY (&crpc->lock);
-        GF_FREE (crpc);
+    if (!crpc)
+        return;
+    crpc->c_clnt = NULL;
+    LOCK_DESTROY(&crpc->lock);
+    GF_FREE(crpc);
 }
 
 static changelog_rpc_clnt_t *
-changelog_rpc_clnt_init (xlator_t *this,
-                         changelog_probe_req *rpc_req, changelog_clnt_t *c_clnt)
+changelog_rpc_clnt_init(xlator_t *this, changelog_probe_req *rpc_req,
+                        changelog_clnt_t *c_clnt)
 {
-        int ret = 0;
-        changelog_rpc_clnt_t *crpc = NULL;
-
-        crpc = GF_CALLOC (1, sizeof (*crpc), gf_changelog_mt_rpc_clnt_t);
-        if (!crpc)
-                goto error_return;
-        INIT_LIST_HEAD (&crpc->list);
-
-        /* Take a ref, the last unref will be on RPC_CLNT_DESTROY
-         * which comes as a result of last rpc_clnt_unref.
-         */
-        GF_ATOMIC_INIT (crpc->ref, 1);
-        changelog_set_disconnect_flag (crpc, _gf_false);
-
-        crpc->filter = rpc_req->filter;
-        (void) memcpy (crpc->sock, rpc_req->sock, strlen (rpc_req->sock));
-
-        crpc->this = this;
-        crpc->c_clnt = c_clnt;
-        crpc->cleanup = changelog_rpc_clnt_cleanup;
-
-        ret = LOCK_INIT (&crpc->lock);
-        if (ret != 0)
-                goto dealloc_crpc;
-        return crpc;
-
- dealloc_crpc:
-        GF_FREE (crpc);
- error_return:
-        return NULL;
+    int ret = 0;
+    changelog_rpc_clnt_t *crpc = NULL;
+
+    crpc = GF_CALLOC(1, sizeof(*crpc), gf_changelog_mt_rpc_clnt_t);
+    if (!crpc)
+        goto error_return;
+    INIT_LIST_HEAD(&crpc->list);
+
+    /* Take a ref, the last unref will be on RPC_CLNT_DESTROY
+     * which comes as a result of last rpc_clnt_unref.
+     */
+    GF_ATOMIC_INIT(crpc->ref, 1);
+    changelog_set_disconnect_flag(crpc, _gf_false);
+
+    crpc->filter = rpc_req->filter;
+    (void)memcpy(crpc->sock, rpc_req->sock, strlen(rpc_req->sock));
+
+    crpc->this = this;
+    crpc->c_clnt = c_clnt;
+    crpc->cleanup = changelog_rpc_clnt_cleanup;
+
+    ret = LOCK_INIT(&crpc->lock);
+    if (ret != 0)
+        goto dealloc_crpc;
+    return crpc;
+
+dealloc_crpc:
+    GF_FREE(crpc);
+error_return:
+    return NULL;
 }
 
 /**
@@ -246,81 +361,80 @@ changelog_rpc_clnt_init (xlator_t *this,
  */
 
 int
-changelog_handle_probe (rpcsvc_request_t *req)
+changelog_handle_probe(rpcsvc_request_t *req)
 {
-        int                   ret     = 0;
-        xlator_t             *this    = NULL;
-        rpcsvc_t             *svc     = NULL;
-        changelog_priv_t     *priv    = NULL;
-        changelog_clnt_t     *c_clnt  = NULL;
-        changelog_rpc_clnt_t *crpc    = NULL;
-
-        changelog_probe_req   rpc_req = {0,};
-        changelog_probe_rsp   rpc_rsp = {0,};
-
-
-        this = req->trans->xl;
-        if (this->cleanup_starting) {
-                gf_msg (this->name, GF_LOG_DEBUG, 0,
-                        CHANGELOG_MSG_HANDLE_PROBE_ERROR,
-                        "cleanup_starting flag is already set for xl");
-                return 0;
-        }
-
-        ret = xdr_to_generic (req->msg[0],
-                              &rpc_req, (xdrproc_t)xdr_changelog_probe_req);
-        if (ret < 0) {
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_HANDLE_PROBE_ERROR,
-                        "xdr decoding error");
-                req->rpc_err = GARBAGE_ARGS;
-                goto handle_xdr_error;
-        }
-
-        /* ->xl hidden in rpcsvc */
-        svc    = rpcsvc_request_service (req);
-        this   = svc->xl;
-        priv   = this->private;
-        c_clnt = &priv->connections;
-
-        crpc = changelog_rpc_clnt_init (this, &rpc_req, c_clnt);
-        if (!crpc)
-                goto handle_xdr_error;
-
-        changelog_ev_queue_connection (c_clnt, crpc);
-        rpc_rsp.op_ret = 0;
-
-        goto submit_rpc;
-
- handle_xdr_error:
-        rpc_rsp.op_ret = -1;
- submit_rpc:
-        (void) changelog_rpc_sumbit_reply (req, &rpc_rsp, NULL, 0, NULL,
-                                           (xdrproc_t)xdr_changelog_probe_rsp);
+    int ret = 0;
+    xlator_t *this = NULL;
+    rpcsvc_t *svc = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
+
+    changelog_probe_req rpc_req = {
+        0,
+    };
+    changelog_probe_rsp rpc_rsp = {
+        0,
+    };
+
+    this = req->trans->xl;
+    if (this->cleanup_starting) {
+        gf_smsg(this->name, GF_LOG_DEBUG, 0, CHANGELOG_MSG_CLEANUP_ALREADY_SET,
+                NULL);
         return 0;
+    }
+
+    ret = xdr_to_generic(req->msg[0], &rpc_req,
+                         (xdrproc_t)xdr_changelog_probe_req);
+    if (ret < 0) {
+        gf_smsg("", GF_LOG_ERROR, 0, CHANGELOG_MSG_HANDLE_PROBE_ERROR, NULL);
+        req->rpc_err = GARBAGE_ARGS;
+        goto handle_xdr_error;
+    }
+
+    /* ->xl hidden in rpcsvc */
+    svc = rpcsvc_request_service(req);
+    this = svc->xl;
+    priv = this->private;
+    c_clnt = &priv->connections;
+
+    crpc = changelog_rpc_clnt_init(this, &rpc_req, c_clnt);
+    if (!crpc)
+        goto handle_xdr_error;
+
+    changelog_ev_queue_connection(c_clnt, crpc);
+    rpc_rsp.op_ret = 0;
+
+    goto submit_rpc;
+
+handle_xdr_error:
+    rpc_rsp.op_ret = -1;
+submit_rpc:
+    (void)changelog_rpc_sumbit_reply(req, &rpc_rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_changelog_probe_rsp);
+    return 0;
 }
 
 /**
  * RPC declarations
  */
 
-rpcsvc_actor_t changelog_svc_actors[CHANGELOG_RPC_PROC_MAX] = {
-        [CHANGELOG_RPC_PROBE_FILTER] = {
-                "CHANGELOG PROBE FILTER", CHANGELOG_RPC_PROBE_FILTER,
-                changelog_handle_probe, NULL, 0, DRC_NA
-        },
+static rpcsvc_actor_t changelog_svc_actors[CHANGELOG_RPC_PROC_MAX] = {
+    [CHANGELOG_RPC_PROBE_FILTER] = {"CHANGELOG PROBE FILTER",
+                                    changelog_handle_probe, NULL,
+                                    CHANGELOG_RPC_PROBE_FILTER, DRC_NA, 0},
 };
 
-struct rpcsvc_program changelog_svc_prog = {
-        .progname  = CHANGELOG_RPC_PROGNAME,
-        .prognum   = CHANGELOG_RPC_PROGNUM,
-        .progver   = CHANGELOG_RPC_PROGVER,
-        .numactors = CHANGELOG_RPC_PROC_MAX,
-        .actors    = changelog_svc_actors,
-        .synctask  = _gf_true,
+static struct rpcsvc_program changelog_svc_prog = {
+    .progname = CHANGELOG_RPC_PROGNAME,
+    .prognum = CHANGELOG_RPC_PROGNUM,
+    .progver = CHANGELOG_RPC_PROGVER,
+    .numactors = CHANGELOG_RPC_PROC_MAX,
+    .actors = changelog_svc_actors,
+    .synctask = _gf_true,
 };
 
-struct rpcsvc_program *changelog_programs[] = {
-        &changelog_svc_prog,
-        NULL,
+static struct rpcsvc_program *changelog_programs[] = {
+    &changelog_svc_prog,
+    NULL,
 };
diff --git a/xlators/features/changelog/src/changelog-rpc.h b/xlators/features/changelog/src/changelog-rpc.h
index 9f1e2d223a3..b1707565249 100644
--- a/xlators/features/changelog/src/changelog-rpc.h
+++ b/xlators/features/changelog/src/changelog-rpc.h
@@ -11,21 +11,21 @@
 #ifndef __CHANGELOG_RPC_H
 #define __CHANGELOG_RPC_H
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "changelog-helpers.h"
 
 /* one time */
 #include "socket.h"
 #include "changelog-rpc-common.h"
 
-#define CHANGELOG_RPC_PROGNAME  "GlusterFS Changelog"
+#define CHANGELOG_RPC_PROGNAME "GlusterFS Changelog"
 
 rpcsvc_t *
-changelog_init_rpc_listener (xlator_t *, changelog_priv_t *, rbuf_t *, int);
+changelog_init_rpc_listener(xlator_t *, changelog_priv_t *, rbuf_t *, int);
 
 void
-changelog_destroy_rpc_listner (xlator_t *, changelog_priv_t *);
+changelog_destroy_rpc_listner(xlator_t *, changelog_priv_t *);
 
 int
-changelog_cleanup_rpc_threads (xlator_t *this, changelog_priv_t *priv);
+changelog_cleanup_rpc_threads(xlator_t *this, changelog_priv_t *priv);
 #endif
diff --git a/xlators/features/changelog/src/changelog-rt.c b/xlators/features/changelog/src/changelog-rt.c
index c262820c64c..841545ae359 100644
--- a/xlators/features/changelog/src/changelog-rt.c
+++ b/xlators/features/changelog/src/changelog-rt.c
@@ -8,60 +8,59 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-#include "logging.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/logging.h>
 
 #include "changelog-rt.h"
 #include "changelog-mem-types.h"
 
 int
-changelog_rt_init (xlator_t *this, changelog_dispatcher_t *cd)
+changelog_rt_init(xlator_t *this, changelog_dispatcher_t *cd)
 {
-        changelog_rt_t *crt = NULL;
+    changelog_rt_t *crt = NULL;
 
-        crt = GF_CALLOC (1, sizeof (*crt),
-                         gf_changelog_mt_rt_t);
-        if (!crt)
-                return -1;
+    crt = GF_CALLOC(1, sizeof(*crt), gf_changelog_mt_rt_t);
+    if (!crt)
+        return -1;
 
-        LOCK_INIT (&crt->lock);
+    LOCK_INIT(&crt->lock);
 
-        cd->cd_data = crt;
-        cd->dispatchfn = &changelog_rt_enqueue;
+    cd->cd_data = crt;
+    cd->dispatchfn = &changelog_rt_enqueue;
 
-        return 0;
+    return 0;
 }
 
 int
-changelog_rt_fini (xlator_t *this, changelog_dispatcher_t *cd)
+changelog_rt_fini(xlator_t *this, changelog_dispatcher_t *cd)
 {
-        changelog_rt_t *crt = NULL;
+    changelog_rt_t *crt = NULL;
 
-        crt = cd->cd_data;
+    crt = cd->cd_data;
 
-        LOCK_DESTROY (&crt->lock);
-        GF_FREE (crt);
+    LOCK_DESTROY(&crt->lock);
+    GF_FREE(crt);
 
-        return 0;
+    return 0;
 }
 
 int
-changelog_rt_enqueue (xlator_t *this, changelog_priv_t *priv, void *cbatch,
-                      changelog_log_data_t *cld_0, changelog_log_data_t *cld_1)
+changelog_rt_enqueue(xlator_t *this, changelog_priv_t *priv, void *cbatch,
+                     changelog_log_data_t *cld_0, changelog_log_data_t *cld_1)
 {
-        int             ret = 0;
-        changelog_rt_t *crt = NULL;
+    int ret = 0;
+    changelog_rt_t *crt = NULL;
 
-        crt = (changelog_rt_t *) cbatch;
+    crt = (changelog_rt_t *)cbatch;
 
-        LOCK (&crt->lock);
-        {
-                ret = changelog_handle_change (this, priv, cld_0);
-                if (!ret && cld_1)
-                        ret = changelog_handle_change (this, priv, cld_1);
-        }
-        UNLOCK (&crt->lock);
+    LOCK(&crt->lock);
+    {
+        ret = changelog_handle_change(this, priv, cld_0);
+        if (!ret && cld_1)
+            ret = changelog_handle_change(this, priv, cld_1);
+    }
+    UNLOCK(&crt->lock);
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/changelog/src/changelog-rt.h b/xlators/features/changelog/src/changelog-rt.h
index 1fc2bbc5bb9..28b9827d85b 100644
--- a/xlators/features/changelog/src/changelog-rt.h
+++ b/xlators/features/changelog/src/changelog-rt.h
@@ -11,23 +11,23 @@
 #ifndef _CHANGELOG_RT_H
 #define _CHANGELOG_RT_H
 
-#include "locking.h"
-#include "timer.h"
+#include <glusterfs/locking.h>
+#include <glusterfs/timer.h>
 #include "pthread.h"
 
 #include "changelog-helpers.h"
 
 /* unused as of now - may be you would need it later */
 typedef struct changelog_rt {
-        gf_lock_t lock;
+    gf_lock_t lock;
 } changelog_rt_t;
 
 int
-changelog_rt_init (xlator_t *this, changelog_dispatcher_t *cd);
+changelog_rt_init(xlator_t *this, changelog_dispatcher_t *cd);
 int
-changelog_rt_fini (xlator_t *this, changelog_dispatcher_t *cd);
+changelog_rt_fini(xlator_t *this, changelog_dispatcher_t *cd);
 int
-changelog_rt_enqueue (xlator_t *this, changelog_priv_t *priv, void *cbatch,
-                      changelog_log_data_t *cld_0, changelog_log_data_t *cld_1);
+changelog_rt_enqueue(xlator_t *this, changelog_priv_t *priv, void *cbatch,
+                     changelog_log_data_t *cld_0, changelog_log_data_t *cld_1);
 
 #endif /* _CHANGELOG_RT_H */
diff --git a/xlators/features/changelog/src/changelog.c b/xlators/features/changelog/src/changelog.c
index 12997181da4..6a6e5af859e 100644
--- a/xlators/features/changelog/src/changelog.c
+++ b/xlators/features/changelog/src/changelog.c
@@ -8,11 +8,11 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-#include "syscall.h"
-#include "logging.h"
-#include "iobuf.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/iobuf.h>
 
 #include "changelog-rt.h"
 
@@ -26,15 +26,20 @@
 #include "changelog-rpc.h"
 #include "errno.h"
 
-static struct changelog_bootstrap
-cb_bootstrap[] = {
-        {
-                .mode = CHANGELOG_MODE_RT,
-                .ctor = changelog_rt_init,
-                .dtor = changelog_rt_fini,
-        },
+static struct changelog_bootstrap cb_bootstrap[] = {
+    {
+        .mode = CHANGELOG_MODE_RT,
+        .ctor = changelog_rt_init,
+        .dtor = changelog_rt_fini,
+    },
 };
 
+static int
+changelog_init_rpc(xlator_t *this, changelog_priv_t *priv);
+
+static int
+changelog_init(xlator_t *this, changelog_priv_t *priv);
+
 /* Entry operations - TYPE III */
 
 /**
@@ -46,1111 +51,1052 @@ cb_bootstrap[] = {
 /* rmdir */
 
 int32_t
-changelog_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+changelog_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
-                                preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(rmdir, frame, op_ret, op_errno, preparent,
+                           postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_rmdir_resume (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, int xflags, dict_t *xdata)
+changelog_rmdir_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       int xflags, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Dequeue rmdir");
-                      changelog_color_fop_and_inc_cnt (this, priv,
-                      frame->local);
-        STACK_WIND (frame, changelog_rmdir_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rmdir,
-                    loc, xflags, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeue rmdir");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, xflags, xdata);
+    return 0;
 }
 
 int32_t
-changelog_rmdir (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int xflags, dict_t *xdata)
-{
-        size_t                  xtra_len                = 0;
-        changelog_priv_t       *priv                    = NULL;
-        changelog_opt_t        *co                      = NULL;
-        call_stub_t            *stub                    = NULL;
-        struct list_head        queue                   = {0, };
-        gf_boolean_t            barrier_enabled         = _gf_false;
-
-        INIT_LIST_HEAD (&queue);
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local,
-                                NULL, loc->inode->gfid, 2);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-
-        co++;
-        if (priv->capture_del_path) {
-                CHANGELOG_FILL_ENTRY_DIR_PATH (co, loc->pargfid, loc->name,
-                                               del_entry_fn, del_entry_free_fn,
-                                              xtra_len, wind, _gf_true);
+changelog_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+                dict_t *xdata)
+{
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    INIT_LIST_HEAD(&queue);
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, loc->inode->gfid, 2);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+
+    co++;
+    if (priv->capture_del_path) {
+        CHANGELOG_FILL_ENTRY_DIR_PATH(co, loc->pargfid, loc->name, del_entry_fn,
+                                      del_entry_free_fn, xtra_len, wind,
+                                      _gf_true);
+    } else {
+        CHANGELOG_FILL_ENTRY_DIR_PATH(co, loc->pargfid, loc->name, del_entry_fn,
+                                      del_entry_free_fn, xtra_len, wind,
+                                      _gf_false);
+    }
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 2);
+
+    /* changelog barrier */
+    /* Color assignment and increment of fop_cnt for rmdir/unlink/rename
+     * should be made with in priv lock if changelog barrier is not enabled.
+     * Because if counter is not incremented yet, draining wakes up and
+     * publishes the changelog but later these fops might hit the disk and
+     * present in snapped volume but where as the intention is these fops
+     * should not be present in snapped volume.
+     */
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_rmdir_stub(frame, changelog_rmdir_resume, loc, xflags,
+                                  xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
         } else {
-                CHANGELOG_FILL_ENTRY_DIR_PATH (co, loc->pargfid, loc->name,
-                                               del_entry_fn, del_entry_free_fn,
-                                              xtra_len, wind, _gf_false);
-        }
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
-
-/* changelog barrier */
-        /* Color assignment and increment of fop_cnt for rmdir/unlink/rename
-         * should be made with in priv lock if changelog barrier is not enabled.
-         * Because if counter is not incremented yet, draining wakes up and
-         * publishes the changelog but later these fops might hit the disk and
-         * present in snapped volume but where as the intention is these fops
-         * should not be present in snapped volume.
-         */
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_rmdir_stub (frame, changelog_rmdir_resume,
-                                                loc, xflags, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueue rmdir");
-                goto out;
-        }
-        if (barrier_enabled && !stub) {
-                gf_smsg (this->name, GF_LOG_ERROR, ENOMEM,
-                         CHANGELOG_MSG_NO_MEMORY,
-                         "Failed to barrier FOPs, disabling changelog barrier",
-                         "fop=rmdir",
-                         NULL);
-                chlog_barrier_dequeue_all (this, &queue);
-        }
-
-/* changelog barrier */
-
- wind:
-        STACK_WIND (frame, changelog_rmdir_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rmdir,
-                    loc, xflags, xdata);
- out:
-        return 0;
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueue rmdir");
+        goto out;
+    }
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=rmdir", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+    /* changelog barrier */
+
+wind:
+    STACK_WIND(frame, changelog_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, xflags, xdata);
+out:
+    return 0;
 }
 
 /* unlink */
 
 int32_t
-changelog_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                      struct iatt *postparent, dict_t *xdata)
+changelog_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (unlink, frame, op_ret, op_errno,
-                                preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(unlink, frame, op_ret, op_errno, preparent,
+                           postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_unlink_resume (call_frame_t *frame, xlator_t *this,
-                         loc_t *loc, int xflags, dict_t *xdata)
+changelog_unlink_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        int xflags, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Dequeue unlink");
-                      changelog_color_fop_and_inc_cnt
-                      (this, priv, frame->local);
-        STACK_WIND (frame, changelog_unlink_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
-                    loc, xflags, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeue unlink");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflags, xdata);
+    return 0;
 }
 
 int32_t
-changelog_unlink (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, int xflags, dict_t *xdata)
-{
-        size_t                       xtra_len        = 0;
-        changelog_priv_t             *priv           = NULL;
-        changelog_opt_t              *co             = NULL;
-        call_stub_t                  *stub           = NULL;
-        struct list_head             queue           = {0, };
-        gf_boolean_t                 barrier_enabled = _gf_false;
-        dht_changelog_rename_info_t  *info           = NULL;
-        int                          ret             = 0;
-        char                        *old_name        = NULL;
-        char                        *new_name        = NULL;
-        char                         *nname             = NULL;
-
-        INIT_LIST_HEAD (&queue);
-        priv = this->private;
-
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        ret = dict_get_bin (xdata, DHT_CHANGELOG_RENAME_OP_KEY, (void **)&info);
-        if (!ret) {     /* special case: unlink considered as rename */
-                /* 3 == fop + oldloc + newloc */
-                old_name = alloca (info->oldname_len);
-                new_name = alloca (info->newname_len);
-                CHANGELOG_INIT_NOCHECK (this, frame->local,
-                                        NULL, loc->inode->gfid, 3);
-
-                co = changelog_get_usable_buffer (frame->local);
-                if (!co)
-                        goto wind;
-
-                CHANGLOG_FILL_FOP_NUMBER (co, GF_FOP_RENAME, fop_fn, xtra_len);
-
-                co++;
-                strncpy (old_name, info->buffer, info->oldname_len);
-                CHANGELOG_FILL_ENTRY (co, info->old_pargfid, old_name,
-                                      entry_fn, entry_free_fn, xtra_len, wind);
-
-                co++;
-                /* new name resides just after old name */
-                nname = info->buffer + info->oldname_len;
-                strncpy (new_name, nname, info->newname_len);
-                CHANGELOG_FILL_ENTRY (co, info->new_pargfid, new_name,
-                                      entry_fn, entry_free_fn, xtra_len, wind);
-
-                changelog_set_usable_record_and_length (frame->local,
-                                                        xtra_len, 3);
-        } else {        /* default unlink */
-                CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, wind);
-                CHANGELOG_INIT_NOCHECK (this, frame->local, NULL,
-                                                        loc->inode->gfid, 2);
-
-                co = changelog_get_usable_buffer (frame->local);
-                if (!co)
-                        goto wind;
-
-                CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op,
-                                                fop_fn, xtra_len);
-
-                co++;
-                if (priv->capture_del_path) {
-                        CHANGELOG_FILL_ENTRY_DIR_PATH (co, loc->pargfid,
-                                     loc->name, del_entry_fn, del_entry_free_fn,
-                                     xtra_len, wind, _gf_true);
-                } else {
-                        CHANGELOG_FILL_ENTRY_DIR_PATH (co, loc->pargfid,
-                                     loc->name, del_entry_fn, del_entry_free_fn,
-                                     xtra_len, wind, _gf_false);
-                }
+changelog_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+                 dict_t *xdata)
+{
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+    dht_changelog_rename_info_t *info = NULL;
+    int ret = 0;
+    char *old_name = NULL;
+    char *new_name = NULL;
+    char *nname = NULL;
+
+    INIT_LIST_HEAD(&queue);
+    priv = this->private;
+
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_bin(xdata, DHT_CHANGELOG_RENAME_OP_KEY, (void **)&info);
+    if (!ret) { /* special case: unlink considered as rename */
+        /* 3 == fop + oldloc + newloc */
+        old_name = alloca(info->oldname_len);
+        new_name = alloca(info->newname_len);
+        CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, loc->inode->gfid, 3);
 
-                changelog_set_usable_record_and_length (frame->local,
-                                                        xtra_len, 2);
-        }
+        co = changelog_get_usable_buffer(frame->local);
+        if (!co)
+            goto wind;
 
-/* changelog barrier */
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_unlink_stub (frame, changelog_unlink_resume,
-                                                loc, xflags, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
+        CHANGLOG_FILL_FOP_NUMBER(co, GF_FOP_RENAME, fop_fn, xtra_len);
 
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueue unlink");
-                goto out;
-        }
-        if (barrier_enabled && !stub) {
-                gf_smsg (this->name, GF_LOG_ERROR, ENOMEM,
-                         CHANGELOG_MSG_NO_MEMORY,
-                         "Failed to barrier FOPs, disabling changelog barrier",
-                         "fop=unlink",
-                         NULL);
-                chlog_barrier_dequeue_all (this, &queue);
-        }
+        co++;
+        strncpy(old_name, info->buffer, info->oldname_len);
+        CHANGELOG_FILL_ENTRY(co, info->old_pargfid, old_name, entry_fn,
+                             entry_free_fn, xtra_len, wind);
 
-/* changelog barrier */
+        co++;
+        /* new name resides just after old name */
+        nname = info->buffer + info->oldname_len;
+        strncpy(new_name, nname, info->newname_len);
+        CHANGELOG_FILL_ENTRY(co, info->new_pargfid, new_name, entry_fn,
+                             entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length(frame->local, xtra_len, 3);
+    } else { /* default unlink */
+        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
+        CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, loc->inode->gfid, 2);
+
+        co = changelog_get_usable_buffer(frame->local);
+        if (!co)
+            goto wind;
 
- wind:
-        STACK_WIND (frame, changelog_unlink_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
-                    loc, xflags, xdata);
- out:
-        return 0;
+        CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        if (priv->capture_del_path) {
+            CHANGELOG_FILL_ENTRY_DIR_PATH(co, loc->pargfid, loc->name,
+                                          del_entry_fn, del_entry_free_fn,
+                                          xtra_len, wind, _gf_true);
+        } else {
+            CHANGELOG_FILL_ENTRY_DIR_PATH(co, loc->pargfid, loc->name,
+                                          del_entry_fn, del_entry_free_fn,
+                                          xtra_len, wind, _gf_false);
+        }
+
+        changelog_set_usable_record_and_length(frame->local, xtra_len, 2);
+    }
+
+    /* changelog barrier */
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_unlink_stub(frame, changelog_unlink_resume, loc, xflags,
+                                   xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueue unlink");
+        goto out;
+    }
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=unlink", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+    /* changelog barrier */
+
+wind:
+    STACK_WIND(frame, changelog_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflags, xdata);
+out:
+    return 0;
 }
 
 /* rename */
 
 int32_t
-changelog_rename_cbk (call_frame_t *frame,
-                      void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *buf, struct iatt *preoldparent,
-                      struct iatt *postoldparent, struct iatt *prenewparent,
-                      struct iatt *postnewparent, dict_t *xdata)
-{
-        changelog_priv_t                  *priv  = NULL;
-        changelog_local_t                 *local = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (rename, frame, op_ret, op_errno,
-                                buf, preoldparent, postoldparent,
-                                prenewparent, postnewparent, xdata);
-        return 0;
+changelog_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata)
+{
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(rename, frame, op_ret, op_errno, buf, preoldparent,
+                           postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_rename_resume (call_frame_t *frame, xlator_t *this,
-                         loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+changelog_rename_resume(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                        loc_t *newloc, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Dequeue rename");
-                      changelog_color_fop_and_inc_cnt
-                      (this, priv, frame->local);
-        STACK_WIND (frame, changelog_rename_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rename,
-                    oldloc, newloc, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeue rename");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 }
 
 int32_t
-changelog_rename (call_frame_t *frame, xlator_t *this,
-                  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        size_t               xtra_len        = 0;
-        changelog_priv_t     *priv           = NULL;
-        changelog_opt_t      *co             = NULL;
-        call_stub_t          *stub           = NULL;
-        struct list_head     queue           = {0, };
-        gf_boolean_t         barrier_enabled = _gf_false;
-        dht_changelog_rename_info_t  *info   = NULL;
-        int                  ret             = 0;
-
-        INIT_LIST_HEAD (&queue);
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        ret = dict_get_bin (xdata, DHT_CHANGELOG_RENAME_OP_KEY, (void **)&info);
-        if (ret && oldloc->inode->ia_type != IA_IFDIR)  {
-                /* xdata "NOT" set for a non-directory,
-                 * Special rename => avoid logging */
-               goto wind;
-        }
-
-        /* 3 == fop + oldloc + newloc */
-        CHANGELOG_INIT_NOCHECK (this, frame->local,
-                                NULL, oldloc->inode->gfid, 3);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-
-        co++;
-        CHANGELOG_FILL_ENTRY (co, oldloc->pargfid, oldloc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        co++;
-        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 3);
-/* changelog barrier */
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_rename_stub (frame, changelog_rename_resume,
-                                                oldloc, newloc, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueue rename");
-                goto out;
-        }
-        if (barrier_enabled && !stub) {
-                gf_smsg (this->name, GF_LOG_ERROR, ENOMEM,
-                         CHANGELOG_MSG_NO_MEMORY,
-                         "Failed to barrier FOPs, disabling changelog barrier",
-                         "fop=rename",
-                         NULL);
-                chlog_barrier_dequeue_all (this, &queue);
-        }
-/* changelog barrier */
-
- wind:
-        STACK_WIND (frame, changelog_rename_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rename,
-                    oldloc, newloc, xdata);
- out:
-        return 0;
+changelog_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                 loc_t *newloc, dict_t *xdata)
+{
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+    dht_changelog_rename_info_t *info = NULL;
+    int ret = 0;
+
+    INIT_LIST_HEAD(&queue);
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_bin(xdata, DHT_CHANGELOG_RENAME_OP_KEY, (void **)&info);
+    if (ret && oldloc->inode->ia_type != IA_IFDIR) {
+        /* xdata "NOT" set for a non-directory,
+         * Special rename => avoid logging */
+        goto wind;
+    }
+
+    /* 3 == fop + oldloc + newloc */
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, oldloc->inode->gfid, 3);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+
+    co++;
+    CHANGELOG_FILL_ENTRY(co, oldloc->pargfid, oldloc->name, entry_fn,
+                         entry_free_fn, xtra_len, wind);
+
+    co++;
+    CHANGELOG_FILL_ENTRY(co, newloc->pargfid, newloc->name, entry_fn,
+                         entry_free_fn, xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 3);
+    /* changelog barrier */
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_rename_stub(frame, changelog_rename_resume, oldloc,
+                                   newloc, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueue rename");
+        goto out;
+    }
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=rename", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+    /* changelog barrier */
+
+wind:
+    STACK_WIND(frame, changelog_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+out:
+    return 0;
 }
 
 /* link */
 
 int32_t
-changelog_link_cbk (call_frame_t *frame,
-                    void *cookie, xlator_t *this, int32_t op_ret,
-                    int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+changelog_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (link, frame, op_ret, op_errno,
-                                inode, buf, preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(link, frame, op_ret, op_errno, inode, buf, preparent,
+                           postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_link_resume (call_frame_t *frame, xlator_t *this,
-                        loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+changelog_link_resume(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                      loc_t *newloc, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Dequeuing link");
-                      changelog_color_fop_and_inc_cnt
-                      (this, priv, frame->local);
-        STACK_WIND (frame, changelog_link_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->link,
-                    oldloc, newloc, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeuing link");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 int32_t
-changelog_link (call_frame_t *frame,
-                xlator_t *this, loc_t *oldloc,
-                loc_t *newloc, dict_t *xdata)
-{
-        size_t            xtra_len         = 0;
-        changelog_priv_t *priv             = NULL;
-        changelog_opt_t  *co               = NULL;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, wind);
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, oldloc->gfid, 2);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-
-        co++;
-        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
-
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_link_stub (frame, changelog_link_resume,
-                                               oldloc, newloc, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueued link");
-                goto out;
-        }
-
-        if (barrier_enabled && !stub) {
-                gf_smsg (this->name, GF_LOG_ERROR, 0,
-                         CHANGELOG_MSG_NO_MEMORY,
-                         "Failed to barrier FOPs, disabling changelog barrier",
-                         "fop=link",
-                         NULL);
-                chlog_barrier_dequeue_all (this, &queue);
-        }
- wind:
-        STACK_WIND (frame, changelog_link_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->link,
-                    oldloc, newloc, xdata);
+changelog_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
+{
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+    CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, oldloc->gfid, 2);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+
+    co++;
+    CHANGELOG_FILL_ENTRY(co, newloc->pargfid, newloc->name, entry_fn,
+                         entry_free_fn, xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 2);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_link_stub(frame, changelog_link_resume, oldloc, newloc,
+                                 xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued link");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_BARRIER_FOP_FAILED,
+                "fop=link", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+wind:
+    STACK_WIND(frame, changelog_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 /* mkdir */
 
 int32_t
-changelog_mkdir_cbk (call_frame_t *frame,
-                     void *cookie, xlator_t *this, int32_t op_ret,
-                     int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+changelog_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
-                                inode, buf, preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, buf,
+                           preparent, postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_mkdir_resume (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, mode_t mode,
-                        mode_t umask, dict_t *xdata)
+changelog_mkdir_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       mode_t mode, mode_t umask, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Dequeuing mkdir");
-                      changelog_color_fop_and_inc_cnt
-                      (this, priv, frame->local);
-        STACK_WIND (frame, changelog_mkdir_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mkdir,
-                    loc, mode, umask, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeuing mkdir");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-changelog_mkdir (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
-{
-        int               ret              = -1;
-        uuid_t            gfid             = {0,};
-        size_t            xtra_len         = 0;
-        changelog_priv_t *priv             = NULL;
-        changelog_opt_t  *co               = NULL;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        ret = dict_get_gfuuid (xdata, "gfid-req", &gfid);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                              "failed to get gfid from dict");
-                goto wind;
-        }
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 5);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, S_IFDIR | mode, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 5);
-
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_mkdir_stub (frame, changelog_mkdir_resume,
-                                               loc, mode, umask, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueued mkdir");
-                goto out;
-        }
-
-        if (barrier_enabled && !stub) {
-                gf_smsg (this->name, GF_LOG_ERROR, ENOMEM,
-                         CHANGELOG_MSG_NO_MEMORY,
-                         "Failed to barrier FOPs, disabling changelog barrier",
-                         "fop=mkdir",
-                         NULL);
-                chlog_barrier_dequeue_all (this, &queue);
-        }
-
- wind:
-        STACK_WIND (frame, changelog_mkdir_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mkdir,
-                    loc, mode, umask, xdata);
+changelog_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                mode_t umask, dict_t *xdata)
+{
+    int ret = -1;
+    uuid_t gfid = {
+        0,
+    };
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_gfuuid(xdata, "gfid-req", &gfid);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to get gfid from dict");
+        goto wind;
+    }
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, gfid, 5);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, S_IFDIR | mode, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->uid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->gid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_ENTRY(co, loc->pargfid, loc->name, entry_fn, entry_free_fn,
+                         xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 5);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_mkdir_stub(frame, changelog_mkdir_resume, loc, mode,
+                                  umask, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued mkdir");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=mkdir", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+wind:
+    STACK_WIND(frame, changelog_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 /* symlink */
 
 int32_t
-changelog_symlink_cbk (call_frame_t *frame,
-                       void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                       struct iatt *postparent, dict_t *xdata)
+changelog_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
+                      struct iatt *buf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (symlink, frame, op_ret, op_errno,
-                                inode, buf, preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(symlink, frame, op_ret, op_errno, inode, buf,
+                           preparent, postparent, xdata);
+    return 0;
 }
 
-
 int32_t
-changelog_symlink_resume (call_frame_t *frame, xlator_t *this,
-                          const char *linkname, loc_t *loc,
-                          mode_t umask, dict_t *xdata)
+changelog_symlink_resume(call_frame_t *frame, xlator_t *this,
+                         const char *linkname, loc_t *loc, mode_t umask,
+                         dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Dequeuing symlink");
-                      changelog_color_fop_and_inc_cnt
-                      (this, priv, frame->local);
-        STACK_WIND (frame, changelog_symlink_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->symlink,
-                    linkname, loc, umask, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeuing symlink");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-changelog_symlink (call_frame_t *frame, xlator_t *this,
-                   const char *linkname, loc_t *loc,
-                   mode_t umask, dict_t *xdata)
-{
-        int               ret              = -1;
-        size_t            xtra_len         = 0;
-        uuid_t            gfid             = {0,};
-        changelog_priv_t *priv             = NULL;
-        changelog_opt_t  *co               = NULL;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        ret = dict_get_gfuuid (xdata, "gfid-req", &gfid);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                              "failed to get gfid from dict");
-                goto wind;
-        }
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 2);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
-
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_symlink_stub (frame,
-                                                 changelog_symlink_resume,
-                                                 linkname, loc, umask, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueued symlink");
-                goto out;
-        }
-
-        if (barrier_enabled && !stub) {
-                gf_smsg (this->name, GF_LOG_ERROR, ENOMEM,
-                         CHANGELOG_MSG_NO_MEMORY,
-                         "Failed to barrier FOPs, disabling changelog barrier",
-                         "fop=symlink",
-                         NULL);
-                chlog_barrier_dequeue_all (this, &queue);
-        }
-
- wind:
-        STACK_WIND (frame, changelog_symlink_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->symlink,
-                    linkname, loc, umask, xdata);
+changelog_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+                  loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    int ret = -1;
+    size_t xtra_len = 0;
+    uuid_t gfid = {
+        0,
+    };
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_gfuuid(xdata, "gfid-req", &gfid);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to get gfid from dict");
+        goto wind;
+    }
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, gfid, 2);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_ENTRY(co, loc->pargfid, loc->name, entry_fn, entry_free_fn,
+                         xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 2);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_symlink_stub(frame, changelog_symlink_resume, linkname,
+                                    loc, umask, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued symlink");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=symlink", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+wind:
+    STACK_WIND(frame, changelog_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 /* mknod */
 
 int32_t
-changelog_mknod_cbk (call_frame_t *frame,
-                     void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+changelog_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (mknod, frame, op_ret, op_errno,
-                                inode, buf, preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf,
+                           preparent, postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_mknod_resume (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, mode_t mode, dev_t rdev,
-                        mode_t umask, dict_t *xdata)
+changelog_mknod_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_msg_debug (this->name, 0, "Dequeuing mknod");
-                      changelog_color_fop_and_inc_cnt
-                      (this, priv, frame->local);
-        STACK_WIND (frame, changelog_mknod_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
-                    loc, mode, rdev, umask, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeuing mknod");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-changelog_mknod (call_frame_t *frame,
-                 xlator_t *this, loc_t *loc,
-                 mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
-{
-        int               ret              = -1;
-        uuid_t            gfid             = {0,};
-        size_t            xtra_len         = 0;
-        changelog_priv_t *priv             = NULL;
-        changelog_opt_t  *co               = NULL;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-
-        /* Check whether changelog active */
-        if (!(priv->active))
-                goto wind;
-
-        /* Check whether rebalance activity */
-        if (frame->root->pid == GF_CLIENT_PID_DEFRAG)
-                goto wind;
-
-        /* If tier-dht linkto is SET, ignore about verifiying :
-         * 1. Whether internal fop AND
-         * 2. Whether tier rebalance process activity (this will help in
-         * recording mknod if tier rebalance process calls this mknod) */
-        if (!(dict_get (xdata, "trusted.tier.tier-dht.linkto"))) {
-                CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, wind);
-                if (frame->root->pid == GF_CLIENT_PID_TIER_DEFRAG)
-                        goto wind;
-        }
-
-        ret = dict_get_gfuuid (xdata, "gfid-req", &gfid);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                              "failed to get gfid from dict");
-                goto wind;
-        }
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 5);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, mode, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 5);
-
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_mknod_stub (frame, changelog_mknod_resume,
-                                               loc, mode, dev, umask, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueued mknod");
-                goto out;
-        }
-
-        if (barrier_enabled && !stub) {
-                gf_smsg (this->name, GF_LOG_ERROR, ENOMEM,
-                         CHANGELOG_MSG_NO_MEMORY,
-                         "Failed to barrier FOPs, disabling changelog barrier",
-                         "fop=mknod",
-                         NULL);
-                chlog_barrier_dequeue_all (this, &queue);
-        }
-
- wind:
-        STACK_WIND (frame, changelog_mknod_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
-                    loc, mode, dev, umask, xdata);
+changelog_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                dev_t dev, mode_t umask, dict_t *xdata)
+{
+    int ret = -1;
+    uuid_t gfid = {
+        0,
+    };
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+
+    /* Check whether changelog active */
+    if (!(priv->active))
+        goto wind;
+
+    /* Check whether rebalance activity */
+    if (frame->root->pid == GF_CLIENT_PID_DEFRAG)
+        goto wind;
+
+    /* If tier-dht linkto is SET, ignore about verifiying :
+     * 1. Whether internal fop AND
+     * 2. Whether tier rebalance process activity (this will help in
+     * recording mknod if tier rebalance process calls this mknod) */
+    if (!(dict_get(xdata, "trusted.tier.tier-dht.linkto"))) {
+        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
+        if (frame->root->pid == GF_CLIENT_PID_TIER_DEFRAG)
+            goto wind;
+    }
+
+    ret = dict_get_gfuuid(xdata, "gfid-req", &gfid);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to get gfid from dict");
+        goto wind;
+    }
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, gfid, 5);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, mode, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->uid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->gid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_ENTRY(co, loc->pargfid, loc->name, entry_fn, entry_free_fn,
+                         xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 5);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_mknod_stub(frame, changelog_mknod_resume, loc, mode, dev,
+                                  umask, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued mknod");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=mknod", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+wind:
+    STACK_WIND(frame, changelog_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, dev, umask, xdata);
 out:
-        return 0;
+    return 0;
 }
 
-/* creat */
+/* create */
 
 int32_t
-changelog_create_cbk (call_frame_t *frame,
-                      void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      fd_t *fd, inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent,
-                      struct iatt *postparent, dict_t *xdata)
+changelog_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        int32_t ret = 0;
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
-        changelog_event_t  ev    = {0,};
-
-        priv  = this->private;
-        local = frame->local;
-
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
-
-        /* fill the event structure.. similar to open() */
-        ev.ev_type = CHANGELOG_OP_TYPE_CREATE;
-        gf_uuid_copy (ev.u.create.gfid, buf->ia_gfid);
-        ev.u.create.flags = fd->flags;
-        changelog_dispatch_event (this, priv, &ev);
-
-        if (changelog_ev_selected
-                   (this, &priv->ev_selection, CHANGELOG_OP_TYPE_RELEASE)) {
-                ret = fd_ctx_set (fd, this, (uint64_t)(long) 0x1);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                CHANGELOG_MSG_SET_FD_CONTEXT,
-                                "could not set fd context (for release cbk)");
-        }
-
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    int32_t ret = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
+    changelog_event_t ev = {
+        0,
+    };
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (create, frame,
-                                op_ret, op_errno, fd, inode,
-                                buf, preparent, postparent, xdata);
-        return 0;
-}
+    priv = this->private;
+    local = frame->local;
 
-int32_t
-changelog_create_resume (call_frame_t *frame, xlator_t *this,
-                         loc_t *loc, int32_t flags, mode_t mode,
-                         mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        changelog_priv_t *priv     = NULL;
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    /* fill the event structure.. similar to open() */
+    ev.ev_type = CHANGELOG_OP_TYPE_CREATE;
+    gf_uuid_copy(ev.u.create.gfid, buf->ia_gfid);
+    ev.u.create.flags = fd->flags;
+    changelog_dispatch_event(this, priv, &ev);
 
-        priv = this->private;
+    if (changelog_ev_selected(this, &priv->ev_selection,
+                              CHANGELOG_OP_TYPE_RELEASE)) {
+        ret = fd_ctx_set(fd, this, (uint64_t)(long)0x1);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, CHANGELOG_MSG_SET_FD_CONTEXT,
+                    NULL);
+    }
 
-        gf_msg_debug (this->name, 0, "Dequeuing create");
-                      changelog_color_fop_and_inc_cnt
-                      (this, priv, frame->local);
-        STACK_WIND (frame, changelog_create_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
-out:
-        return -1;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, buf,
+                           preparent, postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_create (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, int32_t flags, mode_t mode,
-                  mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        int               ret              = -1;
-        uuid_t            gfid             = {0,};
-        changelog_opt_t  *co               = NULL;
-        changelog_priv_t *priv             = NULL;
-        size_t            xtra_len         = 0;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        ret = dict_get_gfuuid (xdata, "gfid-req", &gfid);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                              "failed to get gfid from dict");
-                goto wind;
-        }
-
-        /* init with two extra records */
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 5);
-        if (!frame->local)
-                goto wind;
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, mode, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
+changelog_create_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                        dict_t *xdata)
+{
+    changelog_priv_t *priv = NULL;
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 5);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_create_stub (frame, changelog_create_resume,
-                                               loc, flags, mode, umask, fd,
-                                               xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
+    priv = this->private;
 
-        if (barrier_enabled && stub) {
-                gf_msg_debug (this->name, 0, "Enqueued create");
-                goto out;
-        }
+    gf_msg_debug(this->name, 0, "Dequeuing create");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 
-        if (barrier_enabled && !stub) {
-                gf_smsg (this->name, GF_LOG_ERROR, ENOMEM,
-                         CHANGELOG_MSG_NO_MEMORY,
-                         "Failed to barrier FOPs, disabling changelog barrier",
-                         "fop=create",
-                         NULL);
-                chlog_barrier_dequeue_all (this, &queue);
-        }
+out:
+    return -1;
+}
 
- wind:
-        STACK_WIND (frame, changelog_create_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
+int32_t
+changelog_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                 mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    int ret = -1;
+    uuid_t gfid = {
+        0,
+    };
+    changelog_opt_t *co = NULL;
+    changelog_priv_t *priv = NULL;
+    size_t xtra_len = 0;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_gfuuid(xdata, "gfid-req", &gfid);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to get gfid from dict");
+        goto wind;
+    }
+
+    /* init with two extra records */
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, gfid, 5);
+    if (!frame->local)
+        goto wind;
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, mode, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->uid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->gid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_ENTRY(co, loc->pargfid, loc->name, entry_fn, entry_free_fn,
+                         xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 5);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_create_stub(frame, changelog_create_resume, loc, flags,
+                                   mode, umask, fd, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued create");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=create", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+wind:
+    STACK_WIND(frame, changelog_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
 out:
-        return 0;
+    return 0;
 }
 
 /* }}} */
 
-
 /* Metadata modification fops - TYPE II */
 
 /* {{{ */
@@ -1158,268 +1104,253 @@ out:
 /* {f}setattr */
 
 int32_t
-changelog_fsetattr_cbk (call_frame_t *frame,
-                        void *cookie, xlator_t *this, int32_t op_ret,
-                        int32_t op_errno, struct iatt *preop_stbuf,
-                        struct iatt *postop_stbuf, dict_t *xdata)
+changelog_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                       dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
-
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
-                                preop_stbuf, postop_stbuf, xdata);
-
-        return 0;
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA);
 
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(fsetattr, frame, op_ret, op_errno, preop_stbuf,
+                           postop_stbuf, xdata);
 
+    return 0;
 }
 
 int32_t
-changelog_fsetattr (call_frame_t *frame,
-                    xlator_t *this, fd_t *fd,
-                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
+changelog_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 1);
-        if (!frame->local)
-                goto wind;
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 1);
+    if (!frame->local)
+        goto wind;
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_fsetattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetattr,
-                    fd, stbuf, valid, xdata);
-        return 0;
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
 }
 
 int32_t
-changelog_setattr_cbk (call_frame_t *frame,
-                       void *cookie, xlator_t *this, int32_t op_ret,
-                       int32_t op_errno, struct iatt *preop_stbuf,
-                       struct iatt *postop_stbuf, dict_t *xdata)
+changelog_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                      dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (setattr, frame, op_ret, op_errno,
-                                preop_stbuf, postop_stbuf, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(setattr, frame, op_ret, op_errno, preop_stbuf,
+                           postop_stbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_setattr (call_frame_t *frame,
-                   xlator_t *this, loc_t *loc,
-                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
+changelog_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
-        uuid_t            shard_root_gfid = {0,};
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
+    uuid_t shard_root_gfid = {
+        0,
+    };
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, wind);
+    CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
 
-        /* Do not record META on .shard */
-        gf_uuid_parse (SHARD_ROOT_GFID, shard_root_gfid);
-        if (gf_uuid_compare (loc->gfid, shard_root_gfid) == 0) {
-                goto wind;
-        }
+    /* Do not record META on .shard */
+    gf_uuid_parse(SHARD_ROOT_GFID, shard_root_gfid);
+    if (gf_uuid_compare(loc->gfid, shard_root_gfid) == 0) {
+        goto wind;
+    }
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        loc->inode, loc->inode->gfid, 1);
-        if (!frame->local)
-                goto wind;
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 1);
+    if (!frame->local)
+        goto wind;
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_setattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->setattr,
-                    loc, stbuf, valid, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
 }
 
 /* {f}removexattr */
 
 int32_t
-changelog_fremovexattr_cbk (call_frame_t *frame,
-                            void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+changelog_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_fremovexattr (call_frame_t *frame, xlator_t *this,
-                        fd_t *fd, const char *name, dict_t *xdata)
+changelog_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       const char *name, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 1);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_fremovexattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
 int32_t
-changelog_removexattr_cbk (call_frame_t *frame,
-                           void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+changelog_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_removexattr (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, const char *name, dict_t *xdata)
+changelog_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      const char *name, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        loc->inode, loc->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 1);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_removexattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 }
 
 /* {f}setxattr */
 
 int32_t
-changelog_setxattr_cbk (call_frame_t *frame,
-                        void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+changelog_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 /* changelog_handle_virtual_xattr:
@@ -1432,276 +1363,255 @@ changelog_setxattr_cbk (call_frame_t *frame,
  *           any other value: ENOTSUP is returned.
  */
 static void
-changelog_handle_virtual_xattr (call_frame_t *frame, xlator_t *this,
-                                loc_t *loc, dict_t *dict)
+changelog_handle_virtual_xattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                               dict_t *dict)
 {
-        changelog_priv_t     *priv         = NULL;
-        changelog_local_t    *local        = NULL;
-        int32_t               value        = 0;
-        int                   ret          = 0;
-        int                   dict_ret     = 0;
-        gf_boolean_t          valid        = _gf_false;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
+    int32_t value = 0;
+    int ret = 0;
+    int dict_ret = 0;
+    gf_boolean_t valid = _gf_false;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        dict_ret = dict_get_int32 (dict, GF_XATTR_TRIGGER_SYNC, &value);
+    dict_ret = dict_get_int32(dict, GF_XATTR_TRIGGER_SYNC, &value);
 
-        if ((dict_ret == 0 && value == 1) && ((loc->inode->ia_type == IA_IFDIR)
-            || (loc->inode->ia_type == IA_IFREG)))
-                valid = _gf_true;
+    if ((dict_ret == 0 && value == 1) && ((loc->inode->ia_type == IA_IFDIR) ||
+                                          (loc->inode->ia_type == IA_IFREG)))
+        valid = _gf_true;
 
-        if (valid) {
-                ret = changelog_fill_entry_buf (frame, this, loc, &local);
-                if (ret) {
-                        gf_smsg (this->name, GF_LOG_INFO, 0,
-                                 CHANGELOG_MSG_ENTRY_BUF_INFO,
-                                 "Entry cannot be"
-                                 " captured for gfid, Capturing DATA"
-                                 " entry.",
-                                 "gfid=%s", uuid_utoa (loc->inode->gfid),
-                                 NULL);
-                        goto unwind;
-                }
-                changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
-
- unwind:
-                /* Capture DATA only if it's a file. */
-                if (loc->inode->ia_type != IA_IFDIR)
-                        changelog_update (this, priv, frame->local,
-                                          CHANGELOG_TYPE_DATA);
-                /* Assign local to prev_entry, so unwind will take
-                 * care of cleanup. */
-                ((changelog_local_t *)(frame->local))->prev_entry = local;
-                CHANGELOG_STACK_UNWIND (setxattr, frame, 0, 0, NULL);
-                return;
-        } else {
-                CHANGELOG_STACK_UNWIND (setxattr, frame, -1, ENOTSUP, NULL);
-                return;
-        }
+    if (valid) {
+        ret = changelog_fill_entry_buf(frame, this, loc, &local);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_ENTRY_BUF_INFO,
+                    "gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+            goto unwind;
+        }
+        changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+    unwind:
+        /* Capture DATA only if it's a file. */
+        if (loc->inode->ia_type != IA_IFDIR)
+            changelog_update(this, priv, frame->local, CHANGELOG_TYPE_DATA);
+        /* Assign local to prev_entry, so unwind will take
+         * care of cleanup. */
+        ((changelog_local_t *)(frame->local))->prev_entry = local;
+        CHANGELOG_STACK_UNWIND(setxattr, frame, 0, 0, NULL);
+        return;
+    } else {
+        CHANGELOG_STACK_UNWIND(setxattr, frame, -1, ENOTSUP, NULL);
+        return;
+    }
 }
 
 int32_t
-changelog_setxattr (call_frame_t *frame,
-                    xlator_t *this, loc_t *loc,
-                    dict_t *dict, int32_t flags, dict_t *xdata)
+changelog_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *dict, int32_t flags, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        loc->inode, loc->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 1);
 
-        /* On setting this virtual xattr on a file, an explicit data
-         * sync is triggered from geo-rep as CREATE|DATA entry is
-         * recorded in changelog based on xattr value.
-         */
-        if (dict_get (dict, GF_XATTR_TRIGGER_SYNC)) {
-                changelog_handle_virtual_xattr (frame, this, loc, dict);
-                return 0;
-        }
+    /* On setting this virtual xattr on a file, an explicit data
+     * sync is triggered from geo-rep as CREATE|DATA entry is
+     * recorded in changelog based on xattr value.
+     */
+    if (dict_get(dict, GF_XATTR_TRIGGER_SYNC)) {
+        changelog_handle_virtual_xattr(frame, this, loc, dict);
+        return 0;
+    }
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_setxattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->setxattr,
-                    loc, dict, flags, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
 }
 
 int32_t
-changelog_fsetxattr_cbk (call_frame_t *frame,
-                         void *cookie, xlator_t *this, int32_t op_ret,
-                         int32_t op_errno, dict_t *xdata)
+changelog_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_fsetxattr (call_frame_t *frame,
-                     xlator_t *this, fd_t *fd, dict_t *dict,
-                     int32_t flags, dict_t *xdata)
+changelog_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                    int32_t flags, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+    CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 1);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_fsetxattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
 int32_t
-changelog_xattrop_cbk (call_frame_t *frame,
-                       void *cookie, xlator_t *this, int32_t op_ret,
-                       int32_t op_errno, dict_t *xattr, dict_t *xdata)
+changelog_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                      dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (xattrop, frame, op_ret, op_errno, xattr, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(xattrop, frame, op_ret, op_errno, xattr, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+changelog_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        changelog_priv_t *priv      = NULL;
-        changelog_opt_t  *co        = NULL;
-        size_t            xtra_len  = 0;
-        int               ret       = 0;
-        void             *size_attr = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
+    int ret = 0;
+    void *size_attr = NULL;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-        ret = dict_get_ptr (xattr, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
-        if (ret)
-                goto wind;
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+    ret = dict_get_ptr(xattr, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
+    if (ret)
+        goto wind;
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        loc->inode, loc->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 1);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_xattrop_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->xattrop,
-                    loc, optype, xattr, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, optype, xattr, xdata);
+    return 0;
 }
 
 int32_t
-changelog_fxattrop_cbk (call_frame_t *frame,
-                         void *cookie, xlator_t *this, int32_t op_ret,
-                         int32_t op_errno, dict_t *xattr, dict_t *xdata)
+changelog_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                       dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (fxattrop, frame,
-                                op_ret, op_errno, xattr, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, xattr, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_fxattrop (call_frame_t *frame,
-                     xlator_t *this, fd_t *fd, gf_xattrop_flags_t optype,
-                     dict_t *xattr, dict_t *xdata)
-{
-        changelog_priv_t *priv      = NULL;
-        changelog_opt_t  *co        = NULL;
-        size_t            xtra_len  = 0;
-        void             *size_attr = NULL;
-        int               ret       = 0;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-        ret = dict_get_ptr (xattr, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
-        if (ret)
-                goto wind;
+changelog_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
+    void *size_attr = NULL;
+    int ret = 0;
 
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+    ret = dict_get_ptr(xattr, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
+    if (ret)
+        goto wind;
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 1);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_fxattrop_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fxattrop,
-                    fd, optype, xattr, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, optype, xattr, xdata);
+    return 0;
 }
 /* }}} */
 
-
 /* Data modification fops - TYPE I */
 
 /* {{{ */
@@ -1709,164 +1619,151 @@ changelog_fxattrop (call_frame_t *frame,
 /* {f}truncate() */
 
 int32_t
-changelog_truncate_cbk (call_frame_t *frame,
-                        void *cookie, xlator_t *this, int32_t op_ret,
-                        int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+changelog_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_DATA);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (truncate, frame,
-                                op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                           xdata);
+    return 0;
 }
 
 int32_t
-changelog_truncate (call_frame_t *frame,
-                    xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
+changelog_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   off_t offset, dict_t *xdata)
 {
-        changelog_priv_t *priv = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        loc->inode, loc->inode->gfid, 0);
-        LOCK(&priv->c_snap_lock);
-        {
-                if (priv->c_snap_fd != -1 &&
-                    priv->barrier_enabled == _gf_true) {
-                        changelog_snap_handle_ascii_change (this,
-                              &( ((changelog_local_t *)(frame->local))->cld));
-                }
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 0);
+    LOCK(&priv->c_snap_lock);
+    {
+        if (priv->c_snap_fd != -1 && priv->barrier_enabled == _gf_true) {
+            changelog_snap_handle_ascii_change(
+                this, &(((changelog_local_t *)(frame->local))->cld));
         }
-        UNLOCK(&priv->c_snap_lock);
+    }
+    UNLOCK(&priv->c_snap_lock);
 
-
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_truncate_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
 int32_t
-changelog_ftruncate_cbk (call_frame_t *frame,
-                         void *cookie, xlator_t *this, int32_t op_ret,
-                         int32_t op_errno, struct iatt *prebuf,
-                         struct iatt *postbuf, dict_t *xdata)
+changelog_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_DATA);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (ftruncate, frame,
-                                op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                           xdata);
+    return 0;
 }
 
 int32_t
-changelog_ftruncate (call_frame_t *frame,
-                     xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
+changelog_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                    dict_t *xdata)
 {
-        changelog_priv_t *priv = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 0);
-        LOCK(&priv->c_snap_lock);
-        {
-                if (priv->c_snap_fd != -1 &&
-                    priv->barrier_enabled == _gf_true) {
-                        changelog_snap_handle_ascii_change (this,
-                              &( ((changelog_local_t *)(frame->local))->cld));
-                }
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 0);
+    LOCK(&priv->c_snap_lock);
+    {
+        if (priv->c_snap_fd != -1 && priv->barrier_enabled == _gf_true) {
+            changelog_snap_handle_ascii_change(
+                this, &(((changelog_local_t *)(frame->local))->cld));
         }
-        UNLOCK(&priv->c_snap_lock);
+    }
+    UNLOCK(&priv->c_snap_lock);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_ftruncate_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->ftruncate,
-                    fd, offset, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
 /* writev() */
 
 int32_t
-changelog_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf,
-                    dict_t *xdata)
+changelog_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret <= 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret <= 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_DATA);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (writev, frame,
-                                op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                           xdata);
+    return 0;
 }
 
 int32_t
-changelog_writev (call_frame_t *frame,
-                  xlator_t *this, fd_t *fd, struct iovec *vector,
-                  int32_t count, off_t offset, uint32_t flags,
-                  struct iobref *iobref, dict_t *xdata)
+changelog_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iovec *vector, int32_t count, off_t offset,
+                 uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        changelog_priv_t *priv = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 0);
-        LOCK(&priv->c_snap_lock);
-        {
-                if (priv->c_snap_fd != -1 &&
-                    priv->barrier_enabled == _gf_true) {
-                        changelog_snap_handle_ascii_change (this,
-                              &( ((changelog_local_t *)(frame->local))->cld));
-                }
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 0);
+    LOCK(&priv->c_snap_lock);
+    {
+        if (priv->c_snap_fd != -1 && priv->barrier_enabled == _gf_true) {
+            changelog_snap_handle_ascii_change(
+                this, &(((changelog_local_t *)(frame->local))->cld));
         }
-        UNLOCK(&priv->c_snap_lock);
+    }
+    UNLOCK(&priv->c_snap_lock);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_writev_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->writev, fd, vector,
-                    count, offset, flags, iobref, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 }
 
 /* }}} */
@@ -1875,81 +1772,79 @@ changelog_writev (call_frame_t *frame,
 
 /* {{{ */
 
-
-
 int
-changelog_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
-{
-        int                ret    = 0;
-        changelog_priv_t  *priv   = NULL;
-        changelog_event_t  ev     = {0,};
-        gf_boolean_t logopen = _gf_false;
-
-        priv = this->private;
-        if (frame->local) {
-                frame->local = NULL;
-                logopen = _gf_true;
-        }
-
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !logopen), unwind);
-
-        /* fill the event structure */
-        ev.ev_type = CHANGELOG_OP_TYPE_OPEN;
-        gf_uuid_copy (ev.u.open.gfid, fd->inode->gfid);
-        ev.u.open.flags = fd->flags;
-        changelog_dispatch_event (this, priv, &ev);
-
-        if (changelog_ev_selected
-                   (this, &priv->ev_selection, CHANGELOG_OP_TYPE_RELEASE)) {
-                ret = fd_ctx_set (fd, this, (uint64_t)(long) 0x1);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                CHANGELOG_MSG_SET_FD_CONTEXT,
-                                "could not set fd context (for release cbk)");
-        }
+changelog_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+{
+    int ret = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_event_t ev = {
+        0,
+    };
+    gf_boolean_t logopen = _gf_false;
+
+    priv = this->private;
+    if (frame->local) {
+        frame->local = NULL;
+        logopen = _gf_true;
+    }
+
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !logopen), unwind);
+
+    /* fill the event structure */
+    ev.ev_type = CHANGELOG_OP_TYPE_OPEN;
+    gf_uuid_copy(ev.u.open.gfid, fd->inode->gfid);
+    ev.u.open.flags = fd->flags;
+    changelog_dispatch_event(this, priv, &ev);
+
+    if (changelog_ev_selected(this, &priv->ev_selection,
+                              CHANGELOG_OP_TYPE_RELEASE)) {
+        ret = fd_ctx_set(fd, this, (uint64_t)(long)0x1);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, CHANGELOG_MSG_SET_FD_CONTEXT,
+                    NULL);
+    }
 
- unwind:
-        CHANGELOG_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
+unwind:
+    CHANGELOG_STACK_UNWIND(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
 int
-changelog_open (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int flags, fd_t *fd, dict_t *xdata)
+changelog_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+               fd_t *fd, dict_t *xdata)
 {
-        changelog_priv_t *priv = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        frame->local = (void *)0x1; /* do not dereference in ->cbk */
+    frame->local = (void *)0x1; /* do not dereference in ->cbk */
 
- wind:
-        STACK_WIND (frame, changelog_open_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->open, loc, flags, fd, xdata);
-        return 0;
+wind:
+    STACK_WIND(frame, changelog_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ */
 
-
 /* }}} */
 
 int32_t
-_changelog_generic_dispatcher (dict_t *dict,
-                               char *key, data_t *value, void *data)
+_changelog_generic_dispatcher(dict_t *dict, char *key, data_t *value,
+                              void *data)
 {
-        xlator_t *this = NULL;
-        changelog_priv_t *priv = NULL;
+    xlator_t *this = NULL;
+    changelog_priv_t *priv = NULL;
 
-        this = data;
-        priv = this->private;
+    this = data;
+    priv = this->private;
 
-        changelog_dispatch_event (this, priv, (changelog_event_t *)value->data);
-        return 0;
+    changelog_dispatch_event(this, priv, (changelog_event_t *)value->data);
+    return 0;
 }
 
 /**
@@ -1958,46 +1853,45 @@ _changelog_generic_dispatcher (dict_t *dict,
  * traverses the dictionary).
  */
 int32_t
-changelog_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+changelog_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        if (op != GF_IPC_TARGET_CHANGELOG)
-                goto wind;
+    if (op != GF_IPC_TARGET_CHANGELOG)
+        goto wind;
 
-        /* it's for us, do the job */
-        if (xdata)
-                (void) dict_foreach (xdata,
-                                     _changelog_generic_dispatcher, this);
+    /* it's for us, do the job */
+    if (xdata)
+        (void)dict_foreach(xdata, _changelog_generic_dispatcher, this);
 
-        STACK_UNWIND_STRICT (ipc, frame, 0, 0, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(ipc, frame, 0, 0, NULL);
+    return 0;
 
- wind:
-        STACK_WIND (frame, default_ipc_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ipc, op, xdata);
-        return 0;
+wind:
+    STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
 }
 
-
 /* {{{ */
 
 int32_t
-changelog_release (xlator_t *this, fd_t *fd)
+changelog_release(xlator_t *this, fd_t *fd)
 {
-        changelog_event_t ev = {0,};
-        changelog_priv_t *priv = NULL;
+    changelog_event_t ev = {
+        0,
+    };
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        ev.ev_type = CHANGELOG_OP_TYPE_RELEASE;
-        gf_uuid_copy (ev.u.release.gfid, fd->inode->gfid);
-        changelog_dispatch_event (this, priv, &ev);
+    ev.ev_type = CHANGELOG_OP_TYPE_RELEASE;
+    gf_uuid_copy(ev.u.release.gfid, fd->inode->gfid);
+    changelog_dispatch_event(this, priv, &ev);
 
-        (void) fd_ctx_del (fd, this, NULL);
+    (void)fd_ctx_del(fd, this, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /**
@@ -2012,1035 +1906,1084 @@ changelog_release (xlator_t *this, fd_t *fd)
  * needed if there are more operation modes in the future.
  */
 static void
-changelog_assign_opmode (changelog_priv_t *priv, char *mode)
+changelog_assign_opmode(changelog_priv_t *priv, char *mode)
 {
-        if ( strncmp (mode, "realtime", 8) == 0 ) {
-                priv->op_mode = CHANGELOG_MODE_RT;
-        }
+    if (strncmp(mode, "realtime", 8) == 0) {
+        priv->op_mode = CHANGELOG_MODE_RT;
+    }
 }
 
 static void
-changelog_assign_encoding (changelog_priv_t *priv, char *enc)
+changelog_assign_encoding(changelog_priv_t *priv, char *enc)
 {
-        if ( strncmp (enc, "binary", 6) == 0 ) {
-                priv->encode_mode = CHANGELOG_ENCODE_BINARY;
-        } else if ( strncmp (enc, "ascii", 5) == 0 ) {
-                priv->encode_mode = CHANGELOG_ENCODE_ASCII;
-        }
+    if (strncmp(enc, "binary", 6) == 0) {
+        priv->encode_mode = CHANGELOG_ENCODE_BINARY;
+    } else if (strncmp(enc, "ascii", 5) == 0) {
+        priv->encode_mode = CHANGELOG_ENCODE_ASCII;
+    }
 }
 
 static void
 changelog_assign_barrier_timeout(changelog_priv_t *priv, uint32_t timeout)
 {
-       LOCK (&priv->lock);
-       {
-               priv->timeout.tv_sec = timeout;
-       }
-       UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        priv->timeout.tv_sec = timeout;
+    }
+    UNLOCK(&priv->lock);
 }
 
 /* cleanup any helper threads that are running */
 static void
-changelog_cleanup_helper_threads (xlator_t *this, changelog_priv_t *priv)
+changelog_cleanup_helper_threads(xlator_t *this, changelog_priv_t *priv)
 {
-        if (priv->cr.rollover_th) {
-                (void) changelog_thread_cleanup (this, priv->cr.rollover_th);
-                priv->cr.rollover_th = 0;
-        }
+    if (priv->cr.rollover_th) {
+        (void)changelog_thread_cleanup(this, priv->cr.rollover_th);
+        priv->cr.rollover_th = 0;
+    }
 
-        if (priv->cf.fsync_th) {
-                (void) changelog_thread_cleanup (this, priv->cf.fsync_th);
-                priv->cf.fsync_th = 0;
-        }
+    if (priv->cf.fsync_th) {
+        (void)changelog_thread_cleanup(this, priv->cf.fsync_th);
+        priv->cf.fsync_th = 0;
+    }
 }
 
 /* spawn helper thread; cleaning up in case of errors */
 static int
-changelog_spawn_helper_threads (xlator_t *this, changelog_priv_t *priv)
-{
-        int ret = 0;
-
-        /* Geo-Rep snapshot dependency:
-         *
-         * To implement explicit rollover of changlog journal on barrier
-         * notification, a pipe is created to communicate between
-         * 'changelog_rollover' thread and changelog main thread. The select
-         * call used to wait till roll-over time in changelog_rollover thread
-         * is modified to wait on read end of the pipe. When barrier
-         * notification comes (i.e, in 'reconfigure'), select in
-         * changelog_rollover thread is woken up explicitly by writing into
-         * the write end of the pipe in 'reconfigure'.
-         */
-
-        priv->cr.notify = _gf_false;
-        priv->cr.this = this;
-        ret = gf_thread_create (&priv->cr.rollover_th,
-                                NULL, changelog_rollover, priv, "clogro");
-        if (ret)
-                goto out;
-
-        if (priv->fsync_interval) {
-                priv->cf.this = this;
-                ret = gf_thread_create (&priv->cf.fsync_th,
-                                        NULL, changelog_fsync_thread, priv,
-                                        "clogfsyn");
-        }
+changelog_spawn_helper_threads(xlator_t *this, changelog_priv_t *priv)
+{
+    int ret = 0;
+
+    /* Geo-Rep snapshot dependency:
+     *
+     * To implement explicit rollover of changlog journal on barrier
+     * notification, a pipe is created to communicate between
+     * 'changelog_rollover' thread and changelog main thread. The select
+     * call used to wait till roll-over time in changelog_rollover thread
+     * is modified to wait on read end of the pipe. When barrier
+     * notification comes (i.e, in 'reconfigure'), select in
+     * changelog_rollover thread is woken up explicitly by writing into
+     * the write end of the pipe in 'reconfigure'.
+     */
+
+    priv->cr.notify = _gf_false;
+    priv->cr.this = this;
+    ret = gf_thread_create(&priv->cr.rollover_th, NULL, changelog_rollover,
+                           priv, "clogro");
+    if (ret)
+        goto out;
+
+    if (priv->fsync_interval) {
+        priv->cf.this = this;
+        ret = gf_thread_create(&priv->cf.fsync_th, NULL, changelog_fsync_thread,
+                               priv, "clogfsyn");
+    }
+
+    if (ret)
+        changelog_cleanup_helper_threads(this, priv);
 
-        if (ret)
-                changelog_cleanup_helper_threads (this, priv);
-
- out:
-        return ret;
+out:
+    return ret;
 }
 
 int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        changelog_priv_t       *priv            = NULL;
-        dict_t                 *dict            = NULL;
-        char                    buf[1]          = {1};
-        int                     barrier         = DICT_DEFAULT;
-        gf_boolean_t            bclean_req      = _gf_false;
-        int                     ret             = 0;
-        int                     ret1            = 0;
-        struct list_head        queue           = {0, };
+notify(xlator_t *this, int event, void *data, ...)
+{
+    changelog_priv_t *priv = NULL;
+    dict_t *dict = NULL;
+    char buf[1] = {1};
+    int barrier = DICT_DEFAULT;
+    gf_boolean_t bclean_req = _gf_false;
+    int ret = 0;
+    int ret1 = 0;
+    struct list_head queue = {
+        0,
+    };
+    uint64_t xprtcnt = 0;
+    uint64_t clntcnt = 0;
+    changelog_clnt_t *conn = NULL;
+    gf_boolean_t cleanup_notify = _gf_false;
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
+    rpcsvc_listener_t *listener = NULL;
+    rpcsvc_listener_t *next = NULL;
+
+    INIT_LIST_HEAD(&queue);
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    if (event == GF_EVENT_PARENT_DOWN) {
+        priv->victim = data;
+        gf_log(this->name, GF_LOG_INFO,
+               "cleanup changelog rpc connection of brick %s",
+               priv->victim->name);
+
+        if (priv->rpc_active) {
+            this->cleanup_starting = 1;
+            changelog_destroy_rpc_listner(this, priv);
+            conn = &priv->connections;
+            if (conn)
+                changelog_ev_cleanup_connections(this, conn);
+            xprtcnt = GF_ATOMIC_GET(priv->xprtcnt);
+            clntcnt = GF_ATOMIC_GET(priv->clntcnt);
+            if (!xprtcnt && !clntcnt) {
+                LOCK(&priv->lock);
+                {
+                    cleanup_notify = priv->notify_down;
+                    priv->notify_down = _gf_true;
+                }
+                UNLOCK(&priv->lock);
+                if (priv->rpc) {
+                    list_for_each_entry_safe(listener, next,
+                                             &priv->rpc->listeners, list)
+                    {
+                        if (listener->trans) {
+                            rpc_transport_unref(listener->trans);
+                        }
+                    }
+                    rpcsvc_destroy(priv->rpc);
+                    priv->rpc = NULL;
+                }
+                CHANGELOG_MAKE_SOCKET_PATH(priv->changelog_brick, sockfile,
+                                           UNIX_PATH_MAX);
+                sys_unlink(sockfile);
+                if (!cleanup_notify)
+                    default_notify(this, GF_EVENT_PARENT_DOWN, data);
+            }
+        } else {
+            default_notify(this, GF_EVENT_PARENT_DOWN, data);
+        }
+        goto out;
+    }
 
-        INIT_LIST_HEAD (&queue);
+    if (event == GF_EVENT_TRANSLATOR_OP) {
+        dict = data;
 
-        priv = this->private;
-        if (!priv)
-                goto out;
+        barrier = dict_get_str_boolean(dict, "barrier", DICT_DEFAULT);
 
-        if (event == GF_EVENT_TRANSLATOR_OP) {
+        switch (barrier) {
+            case DICT_ERROR:
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_MSG_DICT_GET_FAILED, "dict_get_str_boolean",
+                        NULL);
+                ret = -1;
+                goto out;
 
-                dict = data;
+            case BARRIER_OFF:
+                gf_smsg(this->name, GF_LOG_INFO, 0,
+                        CHANGELOG_MSG_BARRIER_STATE_NOTIFY, "off", NULL);
 
-                barrier = dict_get_str_boolean (dict, "barrier", DICT_DEFAULT);
+                CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, out);
+                LOCK(&priv->c_snap_lock);
+                {
+                    changelog_snap_logging_stop(this, priv);
+                }
+                UNLOCK(&priv->c_snap_lock);
 
-                switch (barrier) {
-                case DICT_ERROR:
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_MSG_DICT_GET_FAILED,
-                                "Barrier dict_get_str_boolean failed");
+                LOCK(&priv->bflags.lock);
+                {
+                    if (priv->bflags.barrier_ext == _gf_false)
                         ret = -1;
-                        goto out;
+                }
+                UNLOCK(&priv->bflags.lock);
 
-                case BARRIER_OFF:
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                CHANGELOG_MSG_BARRIER_INFO,
-                                "Barrier off notification");
+                if (ret == -1) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            CHANGELOG_MSG_BARRIER_ERROR, NULL);
+                    goto out;
+                }
 
-                        CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, out);
-                        LOCK(&priv->c_snap_lock);
-                        {
-                                changelog_snap_logging_stop (this, priv);
-                        }
-                        UNLOCK(&priv->c_snap_lock);
+                /* Stop changelog barrier and dequeue all fops */
+                LOCK(&priv->lock);
+                {
+                    if (priv->barrier_enabled == _gf_true)
+                        __chlog_barrier_disable(this, &queue);
+                    else
+                        ret = -1;
+                }
+                UNLOCK(&priv->lock);
+                /* If ret = -1, then changelog barrier is already
+                 * disabled because of error or timeout.
+                 */
+                if (ret == 0) {
+                    chlog_barrier_dequeue_all(this, &queue);
+                    gf_smsg(this->name, GF_LOG_INFO, 0,
+                            CHANGELOG_MSG_BARRIER_DISABLED, NULL);
+                } else {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            CHANGELOG_MSG_BARRIER_ALREADY_DISABLED, NULL);
+                }
 
-                        LOCK (&priv->bflags.lock);
-                        {
-                                if (priv->bflags.barrier_ext == _gf_false)
-                                        ret = -1;
-                        }
-                        UNLOCK (&priv->bflags.lock);
-
-                        if (ret == -1 ) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_MSG_BARRIER_ERROR,
-                                        "Received another barrier off"
-                                        " notification while already off");
-                                goto out;
-                        }
+                LOCK(&priv->bflags.lock);
+                {
+                    priv->bflags.barrier_ext = _gf_false;
+                }
+                UNLOCK(&priv->bflags.lock);
 
-                        /* Stop changelog barrier and dequeue all fops */
-                        LOCK (&priv->lock);
-                        {
-                                if (priv->barrier_enabled == _gf_true)
-                                        __chlog_barrier_disable (this, &queue);
-                                else
-                                        ret = -1;
-                        }
-                        UNLOCK (&priv->lock);
-                        /* If ret = -1, then changelog barrier is already
-                         * disabled because of error or timeout.
-                         */
-                        if (ret == 0) {
-                                chlog_barrier_dequeue_all(this, &queue);
-                                gf_msg(this->name, GF_LOG_INFO, 0,
-                                       CHANGELOG_MSG_BARRIER_INFO,
-                                       "Disabled changelog barrier");
-                        } else {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_MSG_BARRIER_ERROR,
-                                        "Changelog barrier already disabled");
-                        }
+                goto out;
 
-                        LOCK (&priv->bflags.lock);
-                        {
-                                priv->bflags.barrier_ext = _gf_false;
-                        }
-                        UNLOCK (&priv->bflags.lock);
+            case BARRIER_ON:
+                gf_smsg(this->name, GF_LOG_INFO, 0,
+                        CHANGELOG_MSG_BARRIER_STATE_NOTIFY, "on", NULL);
 
-                        goto out;
+                CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, out);
+                LOCK(&priv->c_snap_lock);
+                {
+                    changelog_snap_logging_start(this, priv);
+                }
+                UNLOCK(&priv->c_snap_lock);
 
-                case BARRIER_ON:
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                CHANGELOG_MSG_BARRIER_INFO,
-                                "Barrier on notification");
+                LOCK(&priv->bflags.lock);
+                {
+                    if (priv->bflags.barrier_ext == _gf_true)
+                        ret = -1;
+                    else
+                        priv->bflags.barrier_ext = _gf_true;
+                }
+                UNLOCK(&priv->bflags.lock);
 
-                        CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, out);
-                        LOCK(&priv->c_snap_lock);
-                        {
-                                changelog_snap_logging_start (this, priv);
-                        }
-                        UNLOCK(&priv->c_snap_lock);
-
-                        LOCK (&priv->bflags.lock);
-                        {
-                                if (priv->bflags.barrier_ext == _gf_true)
-                                        ret = -1;
-                                else
-                                        priv->bflags.barrier_ext = _gf_true;
-                        }
-                        UNLOCK (&priv->bflags.lock);
-
-                        if (ret == -1 ) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_MSG_BARRIER_ERROR,
-                                        "Received another barrier on"
-                                        "notification when last one is"
-                                        "not served yet");
-                                goto out;
-                        }
+                if (ret == -1) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            CHANGELOG_MSG_BARRIER_ON_ERROR, NULL);
+                    goto out;
+                }
 
-                        ret = pthread_mutex_lock (&priv->bn.bnotify_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret, out,
-                                                                    bclean_req);
-                        {
-                                priv->bn.bnotify = _gf_true;
-                        }
-                        ret = pthread_mutex_unlock (&priv->bn.bnotify_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret, out,
-                                                                    bclean_req);
-
-                        /* Start changelog barrier */
-                        LOCK (&priv->lock);
-                        {
-                                ret = __chlog_barrier_enable (this, priv);
-                        }
-                        UNLOCK (&priv->lock);
-                        if (ret == -1) {
-                                changelog_barrier_cleanup (this, priv, &queue);
-                                goto out;
-                        }
+                ret = pthread_mutex_lock(&priv->bn.bnotify_mutex);
+                CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, out, bclean_req);
+                {
+                    priv->bn.bnotify = _gf_true;
+                }
+                ret = pthread_mutex_unlock(&priv->bn.bnotify_mutex);
+                CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, out, bclean_req);
 
-                        gf_msg(this->name, GF_LOG_INFO, 0,
-                               CHANGELOG_MSG_BARRIER_INFO,
-                               "Enabled changelog barrier");
-
-                        ret = changelog_barrier_notify(priv, buf);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CHANGELOG_MSG_WRITE_FAILED,
-                                        "Explicit roll over: write failed");
-                                changelog_barrier_cleanup (this, priv, &queue);
-                                ret = -1;
-                                goto out;
-                        }
+                /* Start changelog barrier */
+                LOCK(&priv->lock);
+                {
+                    ret = __chlog_barrier_enable(this, priv);
+                }
+                UNLOCK(&priv->lock);
+                if (ret == -1) {
+                    changelog_barrier_cleanup(this, priv, &queue);
+                    goto out;
+                }
 
-                        ret = pthread_mutex_lock (&priv->bn.bnotify_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret, out,
-                                                                    bclean_req);
-                        {
-                                /* The while condition check is required here to
-                                 * handle spurious wakeup of cond wait that can
-                                 * happen with pthreads. See man page */
-                                while (priv->bn.bnotify == _gf_true) {
-                                        ret = pthread_cond_wait (
-                                                       &priv->bn.bnotify_cond,
-                                                       &priv->bn.bnotify_mutex);
-                                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret,
-                                                                          out,
-                                                                    bclean_req);
-                                }
-                                if (priv->bn.bnotify_error == _gf_true) {
-                                        ret = -1;
-                                        priv->bn.bnotify_error = _gf_false;
-                                }
-                        }
-                        ret1 = pthread_mutex_unlock (&priv->bn.bnotify_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret1, out,
-                                                          bclean_req);
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                CHANGELOG_MSG_BNOTIFY_INFO,
-                                "Woke up: bnotify conditional wait");
-
-                        goto out;
-
-                case DICT_DEFAULT:
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CHANGELOG_MSG_DICT_GET_FAILED,
-                                "barrier key not found");
-                        ret = -1;
-                        goto out;
+                gf_smsg(this->name, GF_LOG_INFO, 0,
+                        CHANGELOG_MSG_BARRIER_ENABLE, NULL);
+
+                ret = changelog_barrier_notify(priv, buf);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            CHANGELOG_MSG_WRITE_FAILED, "Explicit roll over",
+                            NULL);
+                    changelog_barrier_cleanup(this, priv, &queue);
+                    ret = -1;
+                    goto out;
+                }
 
-                default:
-                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                                CHANGELOG_MSG_DICT_GET_FAILED,
-                                "Something went bad in dict_get_str_boolean");
+                ret = pthread_mutex_lock(&priv->bn.bnotify_mutex);
+                CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, out, bclean_req);
+                {
+                    /* The while condition check is required here to
+                     * handle spurious wakeup of cond wait that can
+                     * happen with pthreads. See man page */
+                    while (priv->bn.bnotify == _gf_true) {
+                        ret = pthread_cond_wait(&priv->bn.bnotify_cond,
+                                                &priv->bn.bnotify_mutex);
+                        CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, out, bclean_req);
+                    }
+                    if (priv->bn.bnotify_error == _gf_true) {
                         ret = -1;
-                        goto out;
+                        priv->bn.bnotify_error = _gf_false;
+                    }
                 }
-        } else {
-                ret = default_notify (this, event, data);
+                ret1 = pthread_mutex_unlock(&priv->bn.bnotify_mutex);
+                CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret1, out, bclean_req);
+                gf_smsg(this->name, GF_LOG_INFO, 0,
+                        CHANGELOG_MSG_BNOTIFY_COND_INFO, NULL);
+
+                goto out;
+
+            case DICT_DEFAULT:
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_MSG_BARRIER_KEY_NOT_FOUND, NULL);
+                ret = -1;
+                goto out;
+
+            default:
+                gf_smsg(this->name, GF_LOG_ERROR, EINVAL,
+                        CHANGELOG_MSG_ERROR_IN_DICT_GET, NULL);
+                ret = -1;
+                goto out;
         }
+    } else {
+        ret = default_notify(this, event, data);
+    }
 
- out:
-        if (bclean_req)
-                changelog_barrier_cleanup (this, priv, &queue);
+out:
+    if (bclean_req)
+        changelog_barrier_cleanup(this, priv, &queue);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_changelog_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                        CHANGELOG_MSG_NO_MEMORY, "Memory accounting"
-                        " init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_changelog_mt_end + 1);
 
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, ENOMEM,
+                CHANGELOG_MSG_MEMORY_INIT_FAILED, NULL);
         return ret;
+    }
+
+    return ret;
 }
 
 static int
-changelog_init (xlator_t *this, changelog_priv_t *priv)
+changelog_init(xlator_t *this, changelog_priv_t *priv)
 {
-        int                  i   = 0;
-        int                  ret = -1;
-        struct timeval       tv  = {0,};
-        changelog_log_data_t cld = {0,};
-
-        ret = gettimeofday (&tv, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CHANGELOG_MSG_GET_TIME_OP_FAILED,
-                        "gettimeofday() failure");
-                goto out;
-        }
-
-        priv->slice.tv_start = tv;
+    int i = 0;
+    int ret = 0;
+    changelog_log_data_t cld = {
+        0,
+    };
 
-        priv->maps[CHANGELOG_TYPE_DATA]           = "D ";
-        priv->maps[CHANGELOG_TYPE_METADATA]       = "M ";
-        priv->maps[CHANGELOG_TYPE_METADATA_XATTR] = "M ";
-        priv->maps[CHANGELOG_TYPE_ENTRY]          = "E ";
+    priv->maps[CHANGELOG_TYPE_DATA] = "D ";
+    priv->maps[CHANGELOG_TYPE_METADATA] = "M ";
+    priv->maps[CHANGELOG_TYPE_METADATA_XATTR] = "M ";
+    priv->maps[CHANGELOG_TYPE_ENTRY] = "E ";
 
-        for (; i < CHANGELOG_MAX_TYPE; i++) {
-                /* start with version 1 */
-                priv->slice.changelog_version[i] = 1;
-        }
+    for (; i < CHANGELOG_MAX_TYPE; i++) {
+        /* start with version 1 */
+        priv->slice.changelog_version[i] = 1;
+    }
 
-        if (!priv->active)
-                return ret;
+    if (!priv->active)
+        return ret;
 
-        /**
-         * start with a fresh changelog file every time. this is done
-         * in case there was an encoding change. so... things are kept
-         * simple here.
-         */
-        ret = changelog_fill_rollover_data (&cld, _gf_false);
-        if(ret)
-                goto out;
+    /**
+     * start with a fresh changelog file every time. this is done
+     * in case there was an encoding change. so... things are kept
+     * simple here.
+     */
+    changelog_fill_rollover_data(&cld, _gf_false);
 
-        ret = htime_open (this, priv, cld.cld_roll_time);
-        /* call htime open with cld's rollover_time */
-        if (ret)
-                goto out;
+    ret = htime_open(this, priv, cld.cld_roll_time);
+    /* call htime open with cld's rollover_time */
+    if (ret)
+        goto out;
 
-        LOCK (&priv->lock);
-        {
-                ret = changelog_inject_single_event (this, priv, &cld);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        ret = changelog_inject_single_event(this, priv, &cld);
+    }
+    UNLOCK(&priv->lock);
 
-        /* ... and finally spawn the helpers threads */
-        ret = changelog_spawn_helper_threads (this, priv);
+    /* ... and finally spawn the helpers threads */
+    ret = changelog_spawn_helper_threads(this, priv);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 /**
  * Init barrier related condition variables and locks
  */
 static int
-changelog_barrier_pthread_init (xlator_t *this, changelog_priv_t *priv)
-{
-        gf_boolean_t    bn_mutex_init         = _gf_false;
-        gf_boolean_t    bn_cond_init          = _gf_false;
-        gf_boolean_t    dm_mutex_black_init   = _gf_false;
-        gf_boolean_t    dm_cond_black_init    = _gf_false;
-        gf_boolean_t    dm_mutex_white_init   = _gf_false;
-        gf_boolean_t    dm_cond_white_init    = _gf_false;
-        gf_boolean_t    cr_mutex_init         = _gf_false;
-        gf_boolean_t    cr_cond_init          = _gf_false;
-        int             ret                   = 0;
-
-        if ((ret = pthread_mutex_init(&priv->bn.bnotify_mutex, NULL)) != 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
-                         "bnotify pthread_mutex_init failed",
-                         "ret=%d", ret,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-        bn_mutex_init = _gf_true;
-
-        if ((ret = pthread_cond_init(&priv->bn.bnotify_cond, NULL)) != 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED,
-                         "bnotify pthread_cond_init failed",
-                         "ret=%d", ret,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-        bn_cond_init = _gf_true;
-
-        if ((ret = pthread_mutex_init(&priv->dm.drain_black_mutex, NULL)) != 0)
-        {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
-                         "drain_black pthread_mutex_init failed",
-                         "ret=%d", ret,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-        dm_mutex_black_init = _gf_true;
-
-        if ((ret = pthread_cond_init(&priv->dm.drain_black_cond, NULL)) != 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED,
-                         "drain_black pthread_cond_init failed",
-                         "ret=%d", ret,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-        dm_cond_black_init = _gf_true;
-
-        if ((ret = pthread_mutex_init(&priv->dm.drain_white_mutex, NULL)) != 0)
-        {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
-                         "drain_white pthread_mutex_init failed",
-                         "ret=%d", ret,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-        dm_mutex_white_init = _gf_true;
-
-        if ((ret = pthread_cond_init(&priv->dm.drain_white_cond, NULL)) != 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED,
-                         "drain_white pthread_cond_init failed",
-                         "ret=%d", ret,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-        dm_cond_white_init = _gf_true;
-
-        if ((pthread_mutex_init(&priv->cr.lock, NULL)) != 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
-                         "changelog_rollover lock init failed",
-                         "ret=%d", ret,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-        cr_mutex_init = _gf_true;
-
-        if ((pthread_cond_init(&priv->cr.cond, NULL)) != 0) {
-                gf_smsg (this->name, GF_LOG_ERROR, errno,
-                         CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED,
-                         "changelog_rollover cond init failed",
-                         "ret=%d", ret,
-                         NULL);
-                ret = -1;
-                goto out;
-        }
-        cr_cond_init = _gf_true;
- out:
-        if (ret) {
-                if (bn_mutex_init)
-                        pthread_mutex_destroy(&priv->bn.bnotify_mutex);
-                if (bn_cond_init)
-                        pthread_cond_destroy (&priv->bn.bnotify_cond);
-                if (dm_mutex_black_init)
-                        pthread_mutex_destroy(&priv->dm.drain_black_mutex);
-                if (dm_cond_black_init)
-                        pthread_cond_destroy (&priv->dm.drain_black_cond);
-                if (dm_mutex_white_init)
-                        pthread_mutex_destroy(&priv->dm.drain_white_mutex);
-                if (dm_cond_white_init)
-                        pthread_cond_destroy (&priv->dm.drain_white_cond);
-                if (cr_mutex_init)
-                        pthread_mutex_destroy(&priv->cr.lock);
-                if (cr_cond_init)
-                        pthread_cond_destroy (&priv->cr.cond);
-        }
-        return ret;
+changelog_barrier_pthread_init(xlator_t *this, changelog_priv_t *priv)
+{
+    gf_boolean_t bn_mutex_init = _gf_false;
+    gf_boolean_t bn_cond_init = _gf_false;
+    gf_boolean_t dm_mutex_black_init = _gf_false;
+    gf_boolean_t dm_cond_black_init = _gf_false;
+    gf_boolean_t dm_mutex_white_init = _gf_false;
+    gf_boolean_t dm_cond_white_init = _gf_false;
+    gf_boolean_t cr_mutex_init = _gf_false;
+    gf_boolean_t cr_cond_init = _gf_false;
+    int ret = 0;
+
+    if ((ret = pthread_mutex_init(&priv->bn.bnotify_mutex, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED, "name=bnotify",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    bn_mutex_init = _gf_true;
+
+    if ((ret = pthread_cond_init(&priv->bn.bnotify_cond, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED, "name=bnotify",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    bn_cond_init = _gf_true;
+
+    if ((ret = pthread_mutex_init(&priv->dm.drain_black_mutex, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED, "name=drain_black",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    dm_mutex_black_init = _gf_true;
+
+    if ((ret = pthread_cond_init(&priv->dm.drain_black_cond, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED, "name=drain_black",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    dm_cond_black_init = _gf_true;
+
+    if ((ret = pthread_mutex_init(&priv->dm.drain_white_mutex, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED, "name=drain_white",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    dm_mutex_white_init = _gf_true;
+
+    if ((ret = pthread_cond_init(&priv->dm.drain_white_cond, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED, "name=drain_white",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    dm_cond_white_init = _gf_true;
+
+    if ((pthread_mutex_init(&priv->cr.lock, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
+                "name=changelog_rollover", "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    cr_mutex_init = _gf_true;
+
+    if ((pthread_cond_init(&priv->cr.cond, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED,
+                "changelog_rollover cond init failed", "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    cr_cond_init = _gf_true;
+out:
+    if (ret) {
+        if (bn_mutex_init)
+            pthread_mutex_destroy(&priv->bn.bnotify_mutex);
+        if (bn_cond_init)
+            pthread_cond_destroy(&priv->bn.bnotify_cond);
+        if (dm_mutex_black_init)
+            pthread_mutex_destroy(&priv->dm.drain_black_mutex);
+        if (dm_cond_black_init)
+            pthread_cond_destroy(&priv->dm.drain_black_cond);
+        if (dm_mutex_white_init)
+            pthread_mutex_destroy(&priv->dm.drain_white_mutex);
+        if (dm_cond_white_init)
+            pthread_cond_destroy(&priv->dm.drain_white_cond);
+        if (cr_mutex_init)
+            pthread_mutex_destroy(&priv->cr.lock);
+        if (cr_cond_init)
+            pthread_cond_destroy(&priv->cr.cond);
+    }
+    return ret;
 }
 
 /* Destroy barrier related condition variables and locks */
 static void
-changelog_barrier_pthread_destroy (changelog_priv_t *priv)
+changelog_barrier_pthread_destroy(changelog_priv_t *priv)
 {
-        pthread_mutex_destroy (&priv->bn.bnotify_mutex);
-        pthread_cond_destroy (&priv->bn.bnotify_cond);
-        pthread_mutex_destroy (&priv->dm.drain_black_mutex);
-        pthread_cond_destroy (&priv->dm.drain_black_cond);
-        pthread_mutex_destroy (&priv->dm.drain_white_mutex);
-        pthread_cond_destroy (&priv->dm.drain_white_cond);
-        pthread_mutex_destroy(&priv->cr.lock);
-        pthread_cond_destroy (&priv->cr.cond);
-        LOCK_DESTROY (&priv->bflags.lock);
+    pthread_mutex_destroy(&priv->bn.bnotify_mutex);
+    pthread_cond_destroy(&priv->bn.bnotify_cond);
+    pthread_mutex_destroy(&priv->dm.drain_black_mutex);
+    pthread_cond_destroy(&priv->dm.drain_black_cond);
+    pthread_mutex_destroy(&priv->dm.drain_white_mutex);
+    pthread_cond_destroy(&priv->dm.drain_white_cond);
+    pthread_mutex_destroy(&priv->cr.lock);
+    pthread_cond_destroy(&priv->cr.cond);
+    LOCK_DESTROY(&priv->bflags.lock);
 }
 
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        int                     ret            = 0;
-        char                   *tmp            = NULL;
-        changelog_priv_t       *priv           = NULL;
-        gf_boolean_t            active_earlier = _gf_true;
-        gf_boolean_t            active_now     = _gf_true;
-        changelog_time_slice_t *slice          = NULL;
-        changelog_log_data_t    cld            = {0,};
-        char    htime_dir[PATH_MAX]            = {0,};
-        char    csnap_dir[PATH_MAX]            = {0,};
-        struct timeval          tv             = {0,};
-        uint32_t                timeout        = 0;
-
-        priv = this->private;
-        if (!priv)
-                goto out;
-
-        ret = -1;
-        active_earlier = priv->active;
-
-        /* first stop the rollover and the fsync thread */
-        changelog_cleanup_helper_threads (this, priv);
-
-        GF_OPTION_RECONF ("changelog-dir", tmp, options, str, out);
-        if (!tmp) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_DIR_OPTIONS_NOT_SET,
-                        "\"changelog-dir\" option is not set");
-                goto out;
-        }
-
-        GF_FREE (priv->changelog_dir);
-        priv->changelog_dir = gf_strdup (tmp);
-        if (!priv->changelog_dir)
-                goto out;
+static void
+changelog_cleanup_rpc(xlator_t *this, changelog_priv_t *priv)
+{
+    /* terminate rpc server */
+    if (!this->cleanup_starting)
+        changelog_destroy_rpc_listner(this, priv);
 
-        ret = mkdir_p (priv->changelog_dir, 0600, _gf_true);
+    (void)changelog_cleanup_rpc_threads(this, priv);
+    /* cleanup rot buffs */
+    rbuf_dtor(priv->rbuf);
 
-        if (ret)
-                goto out;
-        CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, htime_dir);
-        ret = mkdir_p (htime_dir, 0600, _gf_true);
+    /* cleanup poller thread */
+    if (priv->poller)
+        (void)changelog_thread_cleanup(this, priv->poller);
+}
 
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    int ret = 0;
+    char *tmp = NULL;
+    changelog_priv_t *priv = NULL;
+    gf_boolean_t active_earlier = _gf_true;
+    gf_boolean_t active_now = _gf_true;
+    gf_boolean_t rpc_active_earlier = _gf_true;
+    gf_boolean_t rpc_active_now = _gf_true;
+    gf_boolean_t iniate_rpc = _gf_false;
+    changelog_time_slice_t *slice = NULL;
+    changelog_log_data_t cld = {
+        0,
+    };
+    char htime_dir[PATH_MAX] = {
+        0,
+    };
+    char csnap_dir[PATH_MAX] = {
+        0,
+    };
+    uint32_t timeout = 0;
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    ret = -1;
+    active_earlier = priv->active;
+    rpc_active_earlier = priv->rpc_active;
+
+    /* first stop the rollover and the fsync thread */
+    changelog_cleanup_helper_threads(this, priv);
+
+    GF_OPTION_RECONF("changelog-dir", tmp, options, str, out);
+    if (!tmp) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_DIR_OPTIONS_NOT_SET,
+                NULL);
+        goto out;
+    }
+
+    GF_FREE(priv->changelog_dir);
+    priv->changelog_dir = gf_strdup(tmp);
+    if (!priv->changelog_dir)
+        goto out;
+
+    ret = mkdir_p(priv->changelog_dir, 0600, _gf_true);
+
+    if (ret)
+        goto out;
+    CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, htime_dir);
+    ret = mkdir_p(htime_dir, 0600, _gf_true);
+
+    if (ret)
+        goto out;
+
+    CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir);
+    ret = mkdir_p(csnap_dir, 0600, _gf_true);
+
+    if (ret)
+        goto out;
+
+    GF_OPTION_RECONF("changelog", active_now, options, bool, out);
+    GF_OPTION_RECONF("changelog-notification", rpc_active_now, options, bool,
+                     out);
+
+    /* If journalling is enabled, enable rpc notifications */
+    if (active_now && !active_earlier) {
+        if (!rpc_active_earlier)
+            iniate_rpc = _gf_true;
+    }
+
+    if (rpc_active_now && !rpc_active_earlier) {
+        iniate_rpc = _gf_true;
+    }
+
+    /* TODO: Disable of changelog-notifications is not supported for now
+     * as there is no clean way of cleaning up of rpc resources
+     */
+
+    if (iniate_rpc) {
+        ret = changelog_init_rpc(this, priv);
         if (ret)
-                goto out;
+            goto out;
+        priv->rpc_active = _gf_true;
+    }
 
-        CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir);
-        ret = mkdir_p (csnap_dir, 0600, _gf_true);
+    /**
+     * changelog_handle_change() handles changes that could possibly
+     * have been submit changes before changelog deactivation.
+     */
+    if (!active_now)
+        priv->active = _gf_false;
 
-        if (ret)
-                goto out;
+    GF_OPTION_RECONF("op-mode", tmp, options, str, out);
+    changelog_assign_opmode(priv, tmp);
 
-        GF_OPTION_RECONF ("changelog", active_now, options, bool, out);
+    tmp = NULL;
 
-        /**
-         * changelog_handle_change() handles changes that could possibly
-         * have been submit changes before changelog deactivation.
-         */
-        if (!active_now)
-                priv->active = _gf_false;
+    GF_OPTION_RECONF("encoding", tmp, options, str, out);
+    changelog_assign_encoding(priv, tmp);
 
-        GF_OPTION_RECONF ("op-mode", tmp, options, str, out);
-        changelog_assign_opmode (priv, tmp);
+    GF_OPTION_RECONF("rollover-time", priv->rollover_time, options, int32, out);
+    GF_OPTION_RECONF("fsync-interval", priv->fsync_interval, options, int32,
+                     out);
+    GF_OPTION_RECONF("changelog-barrier-timeout", timeout, options, time, out);
+    changelog_assign_barrier_timeout(priv, timeout);
 
-        tmp = NULL;
+    GF_OPTION_RECONF("capture-del-path", priv->capture_del_path, options, bool,
+                     out);
 
-        GF_OPTION_RECONF ("encoding", tmp, options, str, out);
-        changelog_assign_encoding (priv, tmp);
+    if (active_now || active_earlier) {
+        changelog_fill_rollover_data(&cld, !active_now);
 
-        GF_OPTION_RECONF ("rollover-time",
-                          priv->rollover_time, options, int32, out);
-        GF_OPTION_RECONF ("fsync-interval",
-                          priv->fsync_interval, options, int32, out);
-        GF_OPTION_RECONF ("changelog-barrier-timeout",
-                          timeout, options, time, out);
-        changelog_assign_barrier_timeout (priv, timeout);
+        slice = &priv->slice;
 
-        GF_OPTION_RECONF ("capture-del-path", priv->capture_del_path, options,
-                          bool, out);
-
-        if (active_now || active_earlier) {
-                ret = changelog_fill_rollover_data (&cld, !active_now);
-                if (ret)
-                        goto out;
+        LOCK(&priv->lock);
+        {
+            ret = changelog_inject_single_event(this, priv, &cld);
+            if (!ret && active_now)
+                SLICE_VERSION_UPDATE(slice);
+        }
+        UNLOCK(&priv->lock);
 
-                slice = &priv->slice;
+        if (ret)
+            goto out;
 
-                LOCK (&priv->lock);
-                {
-                        ret = changelog_inject_single_event (this, priv, &cld);
-                        if (!ret && active_now)
-                                SLICE_VERSION_UPDATE (slice);
-                }
-                UNLOCK (&priv->lock);
-
-                if (ret)
-                        goto out;
-
-                if (active_now) {
-                        if (!active_earlier) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        CHANGELOG_MSG_HTIME_INFO,
-                                        "Reconfigure: Changelog Enable");
-                                if (gettimeofday(&tv, NULL) ) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                CHANGELOG_MSG_HTIME_ERROR,
-                                                 "unable to fetch htime");
-                                        ret = -1;
-                                        goto out;
-                                }
-                                htime_create (this, priv, tv.tv_sec);
-                        }
-                        ret = changelog_spawn_helper_threads (this, priv);
-                }
+        if (active_now) {
+            if (!active_earlier) {
+                gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_RECONFIGURE,
+                        NULL);
+                htime_create(this, priv, gf_time());
+            }
+            ret = changelog_spawn_helper_threads(this, priv);
         }
+    }
 
- out:
-        if (ret) {
-                /* TODO */
-        } else {
-                gf_msg_debug (this->name, 0,
-                              "changelog reconfigured");
-                if (active_now && priv)
-                        priv->active = _gf_true;
-        }
+out:
+    if (ret) {
+        /* TODO */
+    } else {
+        gf_msg_debug(this->name, 0, "changelog reconfigured");
+        if (active_now && priv)
+            priv->active = _gf_true;
+    }
 
-        return ret;
+    return ret;
 }
 
 static void
-changelog_freeup_options (xlator_t *this, changelog_priv_t *priv)
+changelog_freeup_options(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
+    int ret = 0;
 
-        ret = priv->cb->dtor (this, &priv->cd);
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_FREEUP_FAILED,
-                        "could not cleanup bootstrapper");
-        GF_FREE (priv->changelog_brick);
-        GF_FREE (priv->changelog_dir);
+    ret = priv->cb->dtor(this, &priv->cd);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_FREEUP_FAILED, NULL);
+    GF_FREE(priv->changelog_brick);
+    GF_FREE(priv->changelog_dir);
 }
 
 static int
-changelog_init_options (xlator_t *this, changelog_priv_t *priv)
+changelog_init_options(xlator_t *this, changelog_priv_t *priv)
 {
-        int       ret            = 0;
-        char     *tmp            = NULL;
-        uint32_t  timeout        = 0;
-        char htime_dir[PATH_MAX] = {0,};
-        char csnap_dir[PATH_MAX] = {0,};
+    int ret = 0;
+    char *tmp = NULL;
+    uint32_t timeout = 0;
+    char htime_dir[PATH_MAX] = {
+        0,
+    };
+    char csnap_dir[PATH_MAX] = {
+        0,
+    };
 
-        GF_OPTION_INIT ("changelog-brick", tmp, str, error_return);
-        priv->changelog_brick = gf_strdup (tmp);
-        if (!priv->changelog_brick)
-                goto error_return;
+    GF_OPTION_INIT("changelog-brick", tmp, str, error_return);
+    priv->changelog_brick = gf_strdup(tmp);
+    if (!priv->changelog_brick)
+        goto error_return;
 
-        tmp = NULL;
+    tmp = NULL;
 
-        GF_OPTION_INIT ("changelog-dir", tmp, str, dealloc_1);
-        priv->changelog_dir = gf_strdup (tmp);
-        if (!priv->changelog_dir)
-                goto dealloc_1;
+    GF_OPTION_INIT("changelog-dir", tmp, str, dealloc_1);
+    priv->changelog_dir = gf_strdup(tmp);
+    if (!priv->changelog_dir)
+        goto dealloc_1;
 
-        tmp = NULL;
+    tmp = NULL;
 
-        /**
-         * create the directory even if change-logging would be inactive
-         * so that consumers can _look_ into it (finding nothing...)
-         */
-        ret = mkdir_p (priv->changelog_dir, 0600, _gf_true);
+    /**
+     * create the directory even if change-logging would be inactive
+     * so that consumers can _look_ into it (finding nothing...)
+     */
+    ret = mkdir_p(priv->changelog_dir, 0600, _gf_true);
 
-        if (ret)
-                goto dealloc_2;
+    if (ret)
+        goto dealloc_2;
 
-        CHANGELOG_FILL_HTIME_DIR (priv->changelog_dir, htime_dir);
-        ret = mkdir_p (htime_dir, 0600, _gf_true);
-        if (ret)
-                goto dealloc_2;
+    CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, htime_dir);
+    ret = mkdir_p(htime_dir, 0600, _gf_true);
+    if (ret)
+        goto dealloc_2;
 
-        CHANGELOG_FILL_CSNAP_DIR (priv->changelog_dir, csnap_dir);
-        ret = mkdir_p (csnap_dir, 0600, _gf_true);
-        if (ret)
-                goto dealloc_2;
+    CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir);
+    ret = mkdir_p(csnap_dir, 0600, _gf_true);
+    if (ret)
+        goto dealloc_2;
 
-        GF_OPTION_INIT ("changelog", priv->active, bool, dealloc_2);
-        GF_OPTION_INIT ("capture-del-path", priv->capture_del_path,
-                        bool, dealloc_2);
+    GF_OPTION_INIT("changelog", priv->active, bool, dealloc_2);
+    GF_OPTION_INIT("changelog-notification", priv->rpc_active, bool, dealloc_2);
+    GF_OPTION_INIT("capture-del-path", priv->capture_del_path, bool, dealloc_2);
 
-        GF_OPTION_INIT ("op-mode", tmp, str, dealloc_2);
-        changelog_assign_opmode (priv, tmp);
+    GF_OPTION_INIT("op-mode", tmp, str, dealloc_2);
+    changelog_assign_opmode(priv, tmp);
 
-        tmp = NULL;
+    tmp = NULL;
 
-        GF_OPTION_INIT ("encoding", tmp, str, dealloc_2);
-        changelog_assign_encoding (priv, tmp);
-        changelog_encode_change (priv);
+    GF_OPTION_INIT("encoding", tmp, str, dealloc_2);
+    changelog_assign_encoding(priv, tmp);
+    changelog_encode_change(priv);
 
-        GF_OPTION_INIT ("rollover-time",
-                        priv->rollover_time, int32, dealloc_2);
+    GF_OPTION_INIT("rollover-time", priv->rollover_time, int32, dealloc_2);
 
-        GF_OPTION_INIT ("fsync-interval",
-                        priv->fsync_interval, int32, dealloc_2);
+    GF_OPTION_INIT("fsync-interval", priv->fsync_interval, int32, dealloc_2);
 
-        GF_OPTION_INIT ("changelog-barrier-timeout",
-                        timeout, time, dealloc_2);
-        changelog_assign_barrier_timeout (priv, timeout);
+    GF_OPTION_INIT("changelog-barrier-timeout", timeout, time, dealloc_2);
+    changelog_assign_barrier_timeout(priv, timeout);
 
-        GF_ASSERT (cb_bootstrap[priv->op_mode].mode == priv->op_mode);
-        priv->cb = &cb_bootstrap[priv->op_mode];
+    GF_ASSERT(cb_bootstrap[priv->op_mode].mode == priv->op_mode);
+    priv->cb = &cb_bootstrap[priv->op_mode];
 
-        /* ... now bootstrap the logger */
-        ret = priv->cb->ctor (this, &priv->cd);
-        if (ret)
-                goto dealloc_2;
+    /* ... now bootstrap the logger */
+    ret = priv->cb->ctor(this, &priv->cd);
+    if (ret)
+        goto dealloc_2;
 
-        priv->changelog_fd = -1;
+    priv->changelog_fd = -1;
 
-        return 0;
+    return 0;
 
- dealloc_2:
-        GF_FREE (priv->changelog_dir);
- dealloc_1:
-        GF_FREE (priv->changelog_brick);
- error_return:
-        return -1;
-}
-
-static void
-changelog_cleanup_rpc (xlator_t *this, changelog_priv_t *priv)
-{
-        /* terminate rpc server */
-        changelog_destroy_rpc_listner (this, priv);
-
-        /* cleanup rot buffs */
-        rbuf_dtor (priv->rbuf);
-
-        /* cleanup poller thread */
-        if (priv->poller)
-                (void) changelog_thread_cleanup (this, priv->poller);
+dealloc_2:
+    GF_FREE(priv->changelog_dir);
+dealloc_1:
+    GF_FREE(priv->changelog_brick);
+error_return:
+    return -1;
 }
 
 static int
-changelog_init_rpc (xlator_t *this, changelog_priv_t *priv)
+changelog_init_rpc(xlator_t *this, changelog_priv_t *priv)
 {
-        rpcsvc_t  *rpc      = NULL;
-        changelog_ev_selector_t *selection = NULL;
+    rpcsvc_t *rpc = NULL;
+    changelog_ev_selector_t *selection = NULL;
 
-        selection = &priv->ev_selection;
+    selection = &priv->ev_selection;
 
-        /* initialize event selection */
-        changelog_init_event_selection (this, selection);
+    /* initialize event selection */
+    changelog_init_event_selection(this, selection);
 
-        priv->rbuf = rbuf_init (NR_ROTT_BUFFS);
-        if (!priv->rbuf)
-                goto cleanup_thread;
+    priv->rbuf = rbuf_init(NR_ROTT_BUFFS);
+    if (!priv->rbuf)
+        goto cleanup_thread;
 
-        rpc = changelog_init_rpc_listener (this, priv,
-                                          priv->rbuf, NR_DISPATCHERS);
-        if (!rpc)
-                goto cleanup_rbuf;
-        priv->rpc = rpc;
+    rpc = changelog_init_rpc_listener(this, priv, priv->rbuf, NR_DISPATCHERS);
+    if (!rpc)
+        goto cleanup_rbuf;
+    priv->rpc = rpc;
 
-        return 0;
+    return 0;
 
- cleanup_rbuf:
-        rbuf_dtor (priv->rbuf);
- cleanup_thread:
-        if (priv->poller)
-                (void) changelog_thread_cleanup (this, priv->poller);
+cleanup_rbuf:
+    rbuf_dtor(priv->rbuf);
+cleanup_thread:
+    if (priv->poller)
+        (void)changelog_thread_cleanup(this, priv->poller);
 
-        return -1;
+    return -1;
 }
 
 int32_t
-init (xlator_t *this)
-{
-        int               ret  = -1;
-        changelog_priv_t *priv = NULL;
-
-        GF_VALIDATE_OR_GOTO ("changelog", this, error_return);
-
-        if (!this->children || this->children->next) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_CHILD_MISCONFIGURED,
-                        "translator needs a single subvolume");
-                goto error_return;
-        }
-
-        if (!this->parents) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CHANGELOG_MSG_VOL_MISCONFIGURED,
-                        "dangling volume. please check volfile");
-                goto error_return;
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_changelog_mt_priv_t);
-        if (!priv)
-                goto error_return;
-
-        this->local_pool = mem_pool_new (changelog_local_t, 64);
-        if (!this->local_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        CHANGELOG_MSG_NO_MEMORY,
-                        "failed to create local memory pool");
-                goto cleanup_priv;
-        }
-
-        LOCK_INIT (&priv->lock);
-        LOCK_INIT (&priv->c_snap_lock);
-
-        ret = changelog_init_options (this, priv);
-        if (ret)
-                goto cleanup_mempool;
-
-        /* snap dependency changes */
-        priv->dm.black_fop_cnt = 0;
-        priv->dm.white_fop_cnt = 0;
-        priv->dm.drain_wait_black = _gf_false;
-        priv->dm.drain_wait_white = _gf_false;
-        priv->current_color = FOP_COLOR_BLACK;
-        priv->explicit_rollover = _gf_false;
-
-        priv->cr.notify = _gf_false;
-        /* Mutex is not needed as threads are not spawned yet */
-        priv->bn.bnotify = _gf_false;
-        priv->bn.bnotify_error = _gf_false;
-        ret = changelog_barrier_pthread_init (this, priv);
-        if (ret)
-                goto cleanup_options;
-        LOCK_INIT (&priv->bflags.lock);
-        priv->bflags.barrier_ext = _gf_false;
-
-        /* Changelog barrier init */
-        INIT_LIST_HEAD (&priv->queue);
-        priv->barrier_enabled = _gf_false;
-
+init(xlator_t *this)
+{
+    int ret = -1;
+    changelog_priv_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("changelog", this, error_return);
+
+    if (!this->children || this->children->next) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_CHILD_MISCONFIGURED,
+                NULL);
+        goto error_return;
+    }
+
+    if (!this->parents) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_VOL_MISCONFIGURED,
+                NULL);
+        goto error_return;
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_changelog_mt_priv_t);
+    if (!priv)
+        goto error_return;
+
+    this->local_pool = mem_pool_new(changelog_local_t, 64);
+    if (!this->local_pool) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, CHANGELOG_MSG_NO_MEMORY,
+                NULL);
+        goto cleanup_priv;
+    }
+
+    LOCK_INIT(&priv->lock);
+    LOCK_INIT(&priv->c_snap_lock);
+    GF_ATOMIC_INIT(priv->listnercnt, 0);
+    GF_ATOMIC_INIT(priv->clntcnt, 0);
+    GF_ATOMIC_INIT(priv->xprtcnt, 0);
+    INIT_LIST_HEAD(&priv->xprt_list);
+    priv->htime_fd = -1;
+
+    ret = changelog_init_options(this, priv);
+    if (ret)
+        goto cleanup_mempool;
+
+    /* snap dependency changes */
+    priv->dm.black_fop_cnt = 0;
+    priv->dm.white_fop_cnt = 0;
+    priv->dm.drain_wait_black = _gf_false;
+    priv->dm.drain_wait_white = _gf_false;
+    priv->current_color = FOP_COLOR_BLACK;
+    priv->explicit_rollover = _gf_false;
+
+    priv->cr.notify = _gf_false;
+    /* Mutex is not needed as threads are not spawned yet */
+    priv->bn.bnotify = _gf_false;
+    priv->bn.bnotify_error = _gf_false;
+    ret = changelog_barrier_pthread_init(this, priv);
+    if (ret)
+        goto cleanup_options;
+    LOCK_INIT(&priv->bflags.lock);
+    priv->bflags.barrier_ext = _gf_false;
+
+    /* Changelog barrier init */
+    INIT_LIST_HEAD(&priv->queue);
+    priv->barrier_enabled = _gf_false;
+
+    if (priv->rpc_active || priv->active) {
         /* RPC ball rolling.. */
-        ret = changelog_init_rpc (this, priv);
-        if (ret)
-                goto cleanup_barrier;
-
-        ret = changelog_init (this, priv);
+        ret = changelog_init_rpc(this, priv);
         if (ret)
-                goto cleanup_rpc;
-
-        gf_msg_debug (this->name, 0, "changelog translator loaded");
-
-        this->private = priv;
-        return 0;
-
- cleanup_rpc:
-        changelog_cleanup_rpc (this, priv);
- cleanup_barrier:
-        changelog_barrier_pthread_destroy (priv);
- cleanup_options:
-        changelog_freeup_options (this, priv);
- cleanup_mempool:
-        mem_pool_destroy (this->local_pool);
- cleanup_priv:
-        GF_FREE (priv);
- error_return:
-        this->private = NULL;
-        return -1;
+            goto cleanup_barrier;
+        priv->rpc_active = _gf_true;
+    }
+
+    ret = changelog_init(this, priv);
+    if (ret)
+        goto cleanup_rpc;
+
+    gf_msg_debug(this->name, 0, "changelog translator loaded");
+
+    this->private = priv;
+    return 0;
+
+cleanup_rpc:
+    if (priv->rpc_active) {
+        changelog_cleanup_rpc(this, priv);
+    }
+cleanup_barrier:
+    changelog_barrier_pthread_destroy(priv);
+cleanup_options:
+    changelog_freeup_options(this, priv);
+cleanup_mempool:
+    mem_pool_destroy(this->local_pool);
+    this->local_pool = NULL;
+cleanup_priv:
+    GF_FREE(priv);
+error_return:
+    this->private = NULL;
+    return -1;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        changelog_priv_t *priv = NULL;
-        struct list_head  queue = {0, };
+    changelog_priv_t *priv = NULL;
+    struct list_head queue = {
+        0,
+    };
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv) {
-                /* terminate RPC server/threads */
-                changelog_cleanup_rpc (this, priv);
+    if (priv) {
+        if (priv->active || priv->rpc_active) {
+            /* terminate RPC server/threads */
+            changelog_cleanup_rpc(this, priv);
+            GF_FREE(priv->ev_dispatcher);
+        }
+        /* call barrier_disable to cancel timer */
+        if (priv->barrier_enabled)
+            __chlog_barrier_disable(this, &queue);
 
-                /* call barrier_disable to cancel timer */
-                if (priv->barrier_enabled)
-                        __chlog_barrier_disable (this, &queue);
+        /* cleanup barrier related objects */
+        changelog_barrier_pthread_destroy(priv);
 
-                /* cleanup barrier related objects */
-                changelog_barrier_pthread_destroy (priv);
+        /* cleanup helper threads */
+        changelog_cleanup_helper_threads(this, priv);
 
-                /* cleanup helper threads */
-                changelog_cleanup_helper_threads (this, priv);
+        /* cleanup allocated options */
+        changelog_freeup_options(this, priv);
 
-                /* cleanup allocated options */
-                changelog_freeup_options (this, priv);
+        /* deallocate mempool */
+        mem_pool_destroy(this->local_pool);
 
-                /* deallocate mempool */
-                mem_pool_destroy (this->local_pool);
-                /* finally, dealloac private variable */
-                GF_FREE (priv);
+        if (priv->htime_fd != -1) {
+            sys_close(priv->htime_fd);
         }
 
-        this->private = NULL;
-        this->local_pool = NULL;
+        /* finally, dealloac private variable */
+        GF_FREE(priv);
+    }
 
-        return;
+    this->private = NULL;
+    this->local_pool = NULL;
+
+    return;
 }
 
 struct xlator_fops fops = {
-        .open         = changelog_open,
-        .mknod        = changelog_mknod,
-        .mkdir        = changelog_mkdir,
-        .create       = changelog_create,
-        .symlink      = changelog_symlink,
-        .writev       = changelog_writev,
-        .truncate     = changelog_truncate,
-        .ftruncate    = changelog_ftruncate,
-        .link         = changelog_link,
-        .rename       = changelog_rename,
-        .unlink       = changelog_unlink,
-        .rmdir        = changelog_rmdir,
-        .setattr      = changelog_setattr,
-        .fsetattr     = changelog_fsetattr,
-        .setxattr     = changelog_setxattr,
-        .fsetxattr    = changelog_fsetxattr,
-        .removexattr  = changelog_removexattr,
-        .fremovexattr = changelog_fremovexattr,
-        .ipc          = changelog_ipc,
-        .xattrop      = changelog_xattrop,
-        .fxattrop     = changelog_fxattrop,
+    .open = changelog_open,
+    .mknod = changelog_mknod,
+    .mkdir = changelog_mkdir,
+    .create = changelog_create,
+    .symlink = changelog_symlink,
+    .writev = changelog_writev,
+    .truncate = changelog_truncate,
+    .ftruncate = changelog_ftruncate,
+    .link = changelog_link,
+    .rename = changelog_rename,
+    .unlink = changelog_unlink,
+    .rmdir = changelog_rmdir,
+    .setattr = changelog_setattr,
+    .fsetattr = changelog_fsetattr,
+    .setxattr = changelog_setxattr,
+    .fsetxattr = changelog_fsetxattr,
+    .removexattr = changelog_removexattr,
+    .fremovexattr = changelog_fremovexattr,
+    .ipc = changelog_ipc,
+    .xattrop = changelog_xattrop,
+    .fxattrop = changelog_fxattrop,
 };
 
 struct xlator_cbks cbks = {
-        .forget = changelog_forget,
-        .release = changelog_release,
+    .forget = changelog_forget,
+    .release = changelog_release,
 };
 
 struct volume_options options[] = {
-        {.key = {"changelog"},
-         .type = GF_OPTION_TYPE_BOOL,
-         .default_value = "off",
-         .description = "enable/disable change-logging",
-         .op_version = {3},
-         .flags = OPT_FLAG_SETTABLE,
-         .level = OPT_STATUS_BASIC,
-         .tags = {"journal", "georep", "glusterfind"}
-        },
-        {.key = {"changelog-brick"},
-         .type = GF_OPTION_TYPE_PATH,
-         .description = "brick path to generate unique socket file name."
-                       " should be the export directory of the volume strictly.",
-         .default_value = "{{ brick.path }}",
-         .op_version = {3},
-         .tags = {"journal"}
-        },
-        {.key = {"changelog-dir"},
-         .type = GF_OPTION_TYPE_PATH,
-         .description = "directory for the changelog files",
-         .default_value = "{{ brick.path }}/.glusterfs/changelogs",
-         .op_version = {3},
-         .flags = OPT_FLAG_SETTABLE,
-         .level = OPT_STATUS_ADVANCED,
-         .tags = {"journal", "georep", "glusterfind"}
-        },
-        {.key = {"op-mode"},
-         .type = GF_OPTION_TYPE_STR,
-         .default_value = "realtime",
-         .value = {"realtime"},
-         .description = "operation mode - futuristic operation modes",
-         .op_version = {3},
-         .tags = {"journal"}
-        },
-        {.key = {"encoding"},
-         .type = GF_OPTION_TYPE_STR,
-         .default_value = "ascii",
-         .value = {"binary", "ascii"},
-         .description = "encoding type for changelogs",
-         .op_version = {3},
-         .flags = OPT_FLAG_SETTABLE,
-         .level = OPT_STATUS_ADVANCED,
-         .tags = {"journal"}
-        },
-        {.key = {"rollover-time"},
-         .default_value = "15",
-         .type = GF_OPTION_TYPE_TIME,
-         .description = "time to switch to a new changelog file (in seconds)",
-         .op_version = {3},
-         .flags = OPT_FLAG_SETTABLE,
-         .level = OPT_STATUS_ADVANCED,
-         .tags = {"journal", "georep", "glusterfind"}
-        },
-        {.key = {"fsync-interval"},
-         .type = GF_OPTION_TYPE_TIME,
-         .default_value = "5",
-         .description = "do not open CHANGELOG file with O_SYNC mode."
-                        " instead perform fsync() at specified intervals",
-         .op_version = {3},
-         .flags = OPT_FLAG_SETTABLE,
-         .level = OPT_STATUS_ADVANCED,
-         .tags = {"journal"}
-        },
-        { .key = {"changelog-barrier-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .default_value = BARRIER_TIMEOUT,
-          .description = "After 'timeout' seconds since the time 'barrier' "
-                         "option was set to \"on\", unlink/rmdir/rename  "
-                         "operations are no longer blocked and previously "
-                         "blocked fops are allowed to go through",
-         .op_version = {3},
-         .flags = OPT_FLAG_SETTABLE,
-         .level = OPT_STATUS_ADVANCED,
-         .tags = {"journal"}
-        },
-        {.key = {"capture-del-path"},
-         .type = GF_OPTION_TYPE_BOOL,
-         .default_value = "off",
-         .description = "enable/disable capturing paths of deleted entries",
-         .op_version = {3},
-         .flags = OPT_FLAG_SETTABLE,
-         .level = OPT_STATUS_BASIC,
-         .tags = {"journal", "glusterfind"}
-        },
-        {.key = {NULL}
-        },
+    {.key = {"changelog"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "enable/disable change-logging",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_BASIC,
+     .tags = {"journal", "georep", "glusterfind"}},
+    {.key = {"changelog-notification"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "enable/disable changelog live notification",
+     .op_version = {3},
+     .level = OPT_STATUS_BASIC,
+     .tags = {"bitrot", "georep"}},
+    {.key = {"changelog-brick"},
+     .type = GF_OPTION_TYPE_PATH,
+     .description = "brick path to generate unique socket file name."
+                    " should be the export directory of the volume strictly.",
+     .default_value = "{{ brick.path }}",
+     .op_version = {3},
+     .tags = {"journal"}},
+    {.key = {"changelog-dir"},
+     .type = GF_OPTION_TYPE_PATH,
+     .description = "directory for the changelog files",
+     .default_value = "{{ brick.path }}/.glusterfs/changelogs",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal", "georep", "glusterfind"}},
+    {.key = {"op-mode"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "realtime",
+     .value = {"realtime"},
+     .description = "operation mode - futuristic operation modes",
+     .op_version = {3},
+     .tags = {"journal"}},
+    {.key = {"encoding"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "ascii",
+     .value = {"binary", "ascii"},
+     .description = "encoding type for changelogs",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal"}},
+    {.key = {"rollover-time"},
+     .default_value = "15",
+     .type = GF_OPTION_TYPE_TIME,
+     .description = "time to switch to a new changelog file (in seconds)",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal", "georep", "glusterfind"}},
+    {.key = {"fsync-interval"},
+     .type = GF_OPTION_TYPE_TIME,
+     .default_value = "5",
+     .description = "do not open CHANGELOG file with O_SYNC mode."
+                    " instead perform fsync() at specified intervals",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal"}},
+    {.key = {"changelog-barrier-timeout"},
+     .type = GF_OPTION_TYPE_TIME,
+     .default_value = BARRIER_TIMEOUT,
+     .description = "After 'timeout' seconds since the time 'barrier' "
+                    "option was set to \"on\", unlink/rmdir/rename  "
+                    "operations are no longer blocked and previously "
+                    "blocked fops are allowed to go through",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal"}},
+    {.key = {"capture-del-path"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "enable/disable capturing paths of deleted entries",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_BASIC,
+     .tags = {"journal", "glusterfind"}},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "changelog",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/changetimerecorder/Makefile.am b/xlators/features/changetimerecorder/Makefile.am
deleted file mode 100644
index a985f42a877..00000000000
--- a/xlators/features/changetimerecorder/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES =
diff --git a/xlators/features/changetimerecorder/src/Makefile.am b/xlators/features/changetimerecorder/src/Makefile.am
deleted file mode 100644
index 620017e3309..00000000000
--- a/xlators/features/changetimerecorder/src/Makefile.am
+++ /dev/null
@@ -1,26 +0,0 @@
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-
-# changetimerecorder can only get build when libgfdb is enabled
-if BUILD_GFDB
-   xlator_LTLIBRARIES = changetimerecorder.la
-endif
-
-changetimerecorder_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
-
-changetimerecorder_la_SOURCES = changetimerecorder.c \
-	ctr-helper.c ctr-xlator-ctx.c
-
-changetimerecorder_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la\
-	$(top_builddir)/libglusterfs/src/gfdb/libgfdb.la
-
-noinst_HEADERS = ctr-messages.h changetimerecorder.h ctr_mem_types.h \
-		ctr-helper.h ctr-xlator-ctx.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/libglusterfs/src/gfdb \
-	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
-	-DDATADIR=\"$(localstatedir)\"
-
-AM_CFLAGS = -Wall $(GF_CFLAGS) $(SQLITE_CFLAGS)
-
-CLEANFILES =
diff --git a/xlators/features/changetimerecorder/src/changetimerecorder.c b/xlators/features/changetimerecorder/src/changetimerecorder.c
deleted file mode 100644
index 40bfcf4a32a..00000000000
--- a/xlators/features/changetimerecorder/src/changetimerecorder.c
+++ /dev/null
@@ -1,2518 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include <ctype.h>
-#include <sys/uio.h>
-
-#include "gfdb_sqlite3.h"
-#include "ctr-helper.h"
-#include "ctr-messages.h"
-#include "syscall.h"
-
-#include "changetimerecorder.h"
-#include "tier-ctr-interface.h"
-
-/*******************************inode forget***********************************/
-int
-ctr_forget (xlator_t *this, inode_t *inode)
-{
-        fini_ctr_xlator_ctx (this, inode);
-        return 0;
-}
-
-/************************** Look up heal **************************************/
-/*
-Problem: The CTR xlator records file meta (heat/hardlinks)
-into the data. This works fine for files which are created
-after ctr xlator is switched ON. But for files which were
-created before CTR xlator is ON, CTR xlator is not able to
-record either of the meta i.e heat or hardlinks. Thus making
-those files immune to promotions/demotions.
-
-Solution: The solution that is implemented in this patch is
-do ctr-db heal of all those pre-existent files, using named lookup.
-For this purpose we use the inode-xlator context variable option
-in gluster.
-The inode-xlator context variable for ctr xlator will have the
-following,
-    a. A Lock for the context variable
-    b. A hardlink list: This list represents the successful looked
-       up hardlinks.
-These are the scenarios when the hardlink list is updated:
-1) Named-Lookup: Whenever a named lookup happens on a file, in the
-   wind path we copy all required hardlink and inode information to
-   ctr_db_record structure, which resides in the frame->local variable.
-   We dont update the database in wind. During the unwind, we read the
-   information from the ctr_db_record and ,
-   Check if the inode context variable is created, if not we create it.
-   Check if the hard link is there in the hardlink list.
-      If its not there we add it to the list and send a update to the
-      database using libgfdb.
-      Please note: The database transaction can fail(and we ignore) as there
-      already might be a record in the db. This update to the db is to heal
-      if its not there.
-      If its there in the list we ignore it.
-2) Inode Forget: Whenever an inode forget hits we clear the hardlink list in
-   the inode context variable and delete the inode context variable.
-   Please note: An inode forget may happen for two reason,
-   a. when the inode is delete.
-   b. the in-memory inode is evicted from the inode table due to cache limits.
-3) create: whenever a create happens we create the inode context variable and
-   add the hardlink. The database updation is done as usual by ctr.
-4) link: whenever a hardlink is created for the inode, we create the inode
- context variable, if not present, and add the hardlink to the list.
-5) unlink: whenever a unlink happens we delete the hardlink from the list.
-6) mknod: same as create.
-7) rename: whenever a rename happens we update the hardlink in list. if the
-   hardlink was not present for updation, we add the hardlink to the list.
-
-What is pending:
-1) This solution will only work for named lookups.
-2) We dont track afr-self-heal/dht-rebalancer traffic for healing.
-
-*/
-
-
-/* This function doesnot write anything to the db,
- * just created the local variable
- * for the frame and sets values for the ctr_db_record */
-static int
-ctr_lookup_wind(call_frame_t                    *frame,
-                xlator_t                        *this,
-                gf_ctr_inode_context_t          *ctr_inode_cx)
-{
-        int ret                         = -1;
-        gf_ctr_private_t *_priv         = NULL;
-        gf_ctr_local_t *ctr_local       = NULL;
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-        GF_ASSERT(this);
-        IS_CTR_INODE_CX_SANE(ctr_inode_cx);
-
-        _priv = this->private;
-        GF_ASSERT (_priv);
-
-        if (_priv->ctr_record_wind && ctr_inode_cx->ia_type != IA_IFDIR) {
-
-                frame->local = init_ctr_local_t (this);
-                if (!frame->local) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_CREATE_CTR_LOCAL_ERROR_WIND,
-                                "WIND: Error while creating ctr local");
-                        goto out;
-                };
-                ctr_local = frame->local;
-                /*Definately no internal fops will reach here*/
-                ctr_local->is_internal_fop = _gf_false;
-                /*Dont record counters*/
-                CTR_DB_REC(ctr_local).do_record_counters = _gf_false;
-                /*Don't record time at all*/
-                CTR_DB_REC(ctr_local).do_record_times = _gf_false;
-
-                /* Copy gfid into db record*/
-                gf_uuid_copy (CTR_DB_REC(ctr_local).gfid,
-                                *(ctr_inode_cx->gfid));
-
-                /* Set fop_path and fop_type, required by libgfdb to make
-                * decision while inserting the record */
-                CTR_DB_REC(ctr_local).gfdb_fop_path = ctr_inode_cx->fop_path;
-                CTR_DB_REC(ctr_local).gfdb_fop_type = ctr_inode_cx->fop_type;
-
-                /* Copy hard link info*/
-                gf_uuid_copy (CTR_DB_REC(ctr_local).pargfid,
-                        *((NEW_LINK_CX(ctr_inode_cx))->pargfid));
-                strncpy (CTR_DB_REC(ctr_local).file_name,
-                         NEW_LINK_CX(ctr_inode_cx)->basename,
-                         sizeof(CTR_DB_REC(ctr_local).file_name));
-
-                /* Since we are in lookup we can ignore errors while
-                 * Inserting in the DB, because there may be many
-                 * to write to the DB attempts for healing.
-                 * We dont want to log all failed attempts and
-                 * bloat the log*/
-                 ctr_local->gfdb_db_record.ignore_errors = _gf_true;
-        }
-
-        ret = 0;
-
-out:
-
-        if (ret) {
-                free_ctr_local (ctr_local);
-                frame->local = NULL;
-        }
-
-        return ret;
-}
-
-
-/* This function inserts the ctr_db_record populated by ctr_lookup_wind
- * in to the db. It also destroys the frame->local created by ctr_lookup_wind */
-static int
-ctr_lookup_unwind (call_frame_t          *frame,
-                   xlator_t              *this)
-{
-        int ret = -1;
-        gf_ctr_private_t *_priv         = NULL;
-        gf_ctr_local_t *ctr_local       = NULL;
-
-        GF_ASSERT(frame);
-        GF_ASSERT(this);
-
-        _priv = this->private;
-        GF_ASSERT (_priv);
-
-        GF_ASSERT(_priv->_db_conn);
-
-        ctr_local = frame->local;
-
-        if (ctr_local && (ctr_local->ia_inode_type != IA_IFDIR)) {
-
-                ret = insert_record(_priv->_db_conn,
-                                &ctr_local->gfdb_db_record);
-                if (ret == -1) {
-                        gf_msg (this->name,
-                                _gfdb_log_level (GF_LOG_ERROR,
-                                        ctr_local->
-                                        gfdb_db_record.ignore_errors),
-                                0, CTR_MSG_FILL_CTR_LOCAL_ERROR_UNWIND,
-                               "UNWIND: Error filling ctr local");
-                        goto out;
-                }
-        }
-        ret = 0;
-out:
-        free_ctr_local (ctr_local);
-        frame->local = NULL;
-        return ret;
-}
-
-/******************************************************************************
- *
- *                        FOPS HANDLING BELOW
- *
- * ***************************************************************************/
-
-/****************************LOOKUP********************************************/
-
-
-int32_t
-ctr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, dict_t *dict, struct iatt *postparent)
-{
-        int ret                                 = -1;
-        ctr_xlator_ctx_t *ctr_xlator_ctx        = NULL;
-        gf_ctr_local_t *ctr_local               = NULL;
-        ctr_heal_ret_val_t  ret_val             = CTR_CTX_ERROR;
-        gf_boolean_t    _is_heal_needed         = _gf_false;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        /* if the lookup failed lookup dont do anything*/
-        if (op_ret == -1) {
-                gf_msg_trace (this->name, 0, "lookup failed with %s",
-                              strerror (op_errno));
-                goto out;
-        }
-
-        /* Ignore directory lookups */
-        if (inode->ia_type == IA_IFDIR) {
-                goto out;
-        }
-
-        /* if frame local was not set by the ctr_lookup()
-         * so dont so anything*/
-        if (!frame->local) {
-                goto out;
-        }
-
-        /* if the lookup is for dht link donot record*/
-        if (dht_is_linkfile (buf, dict)) {
-                gf_msg_trace (this->name, 0, "Ignoring Lookup "
-                                "for dht link file");
-                goto out;
-        }
-
-        ctr_local = frame->local;
-        /*Assign the proper inode type*/
-        ctr_local->ia_inode_type = inode->ia_type;
-
-        /* Copy gfid directly from inode */
-        gf_uuid_copy (CTR_DB_REC(ctr_local).gfid, inode->gfid);
-
-        /* Checking if gfid and parent gfid is valid */
-        if (gf_uuid_is_null(CTR_DB_REC(ctr_local).gfid) ||
-                gf_uuid_is_null(CTR_DB_REC(ctr_local).pargfid)) {
-                gf_msg_trace (this->name, 0,
-                        "Invalid GFID");
-                goto out;
-        }
-
-        /* if its a first entry
-         * then mark the ctr_record for create
-         * A create will attempt a file and a hard link created in the db*/
-        ctr_xlator_ctx = get_ctr_xlator_ctx (this, inode);
-        if (!ctr_xlator_ctx) {
-                 /* This marks inode heal */
-                CTR_DB_REC(ctr_local).gfdb_fop_type = GFDB_FOP_CREATE_WRITE;
-                _is_heal_needed = _gf_true;
-        }
-
-        /* Copy the correct gfid from resolved inode */
-        gf_uuid_copy (CTR_DB_REC(ctr_local).gfid, inode->gfid);
-
-        /* Add hard link to the list */
-        ret_val = add_hard_link_ctx (frame, this, inode);
-        if (ret_val == CTR_CTX_ERROR) {
-                gf_msg_trace (this->name, 0,
-                        "Failed adding hardlink to list");
-                goto out;
-        }
-        /* If inode needs healing then heal the hardlink also */
-        else if (ret_val & CTR_TRY_INODE_HEAL) {
-                /* This marks inode heal */
-                CTR_DB_REC(ctr_local).gfdb_fop_type = GFDB_FOP_CREATE_WRITE;
-                _is_heal_needed = _gf_true;
-        }
-        /* If hardlink needs healing */
-        else if (ret_val & CTR_TRY_HARDLINK_HEAL) {
-                _is_heal_needed = _gf_true;
-        }
-
-        /* If lookup heal needed */
-        if (!_is_heal_needed)
-                goto out;
-
-        /* FINALLY HEAL : Inserts the ctr_db_record populated by ctr_lookup_wind
-        * in to the db. It also destroys the frame->local
-        * created by ctr_lookup_wind */
-        ret = ctr_lookup_unwind(frame, this);
-        if (ret) {
-                gf_msg_trace (this->name, 0,
-                        "Failed healing/inserting link");
-        }
-
-
-out:
-        free_ctr_local ((gf_ctr_local_t *)frame->local);
-        frame->local = NULL;
-
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             dict, postparent);
-
-        return 0;
-}
-
-
-
-int32_t
-ctr_lookup (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xdata)
-{
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx       = &ctr_inode_cx;
-        gf_ctr_link_context_t  ctr_link_cx;
-        gf_ctr_link_context_t  *_link_cx        = &ctr_link_cx;
-        int ret                                 = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-
-        /* Dont handle nameless lookups*/
-        if (!loc->parent || !loc->name)
-                goto out;
-
-        /*fill ctr link context*/
-        FILL_CTR_LINK_CX(_link_cx, loc->parent->gfid, loc->name, out);
-
-         /* Fill ctr inode context*/
-         /* IA_IFREG : We assume its a file in the wind
-          * but in the unwind we are sure what the inode is a file
-          * or directory
-          * gfid: we are just filling loc->gfid which is not correct.
-          * In unwind we fill the correct gfid for successful lookup*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, IA_IFREG,
-                loc->gfid, _link_cx, NULL,
-                GFDB_FOP_DENTRY_WRITE, GFDB_FOP_WIND);
-
-        /* Create the frame->local and populate ctr_db_record
-         * No writing to the db yet */
-        ret = ctr_lookup_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_LINK_WIND_FAILED,
-                        "Failed to insert link wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
-        return 0;
-}
-
-
-
-
-/****************************WRITEV********************************************/
-int32_t
-ctr_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf,
-                    dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO (this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind (frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_WRITEV_UNWIND_FAILED,
-                        "Failed to insert writev unwind");
-        }
-
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
-                        postbuf, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int32_t count, off_t off,
-            uint32_t flags,
-            struct iobref *iobref, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-        /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                                fd->inode->gfid, NULL, NULL,
-                                GFDB_FOP_INODE_WRITE, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_WRITEV_WIND_FAILED,
-                        "Failed to insert writev wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
-                    off, flags, iobref, xdata);
-
-        return 0;
-}
-
-/******************************setattr*****************************************/
-
-int32_t
-ctr_setattr_cbk (call_frame_t *frame,
-                       void *cookie, xlator_t *this, int32_t op_ret,
-                       int32_t op_errno, struct iatt *preop_stbuf,
-                       struct iatt *postop_stbuf, dict_t *xdata)
-{
-
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_SETATTR_UNWIND_FAILED,
-                        "Failed to insert setattr unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop_stbuf,
-                       postop_stbuf, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_setattr (call_frame_t *frame,
-                   xlator_t *this, loc_t *loc,
-                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
-{
-
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-        CTR_RECORD_METADATA_HEAT_IS_DISABLED_THEN_GOTO (this, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_SETATTR_WIND_FAILED,
-                        "Failed to insert setattr wind");
-        }
-out:
-
-        STACK_WIND (frame, ctr_setattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr, loc, stbuf,
-                    valid, xdata);
-
-        return 0;
-}
-
-/*************************** fsetattr ***************************************/
-int32_t
-ctr_fsetattr_cbk (call_frame_t *frame,
-                        void *cookie, xlator_t *this, int32_t op_ret,
-                        int32_t op_errno, struct iatt *preop_stbuf,
-                        struct iatt *postop_stbuf, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_SETATTR_UNWIND_FAILED,
-                        "Failed to insert fsetattr unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno,
-                                preop_stbuf, postop_stbuf, xdata);
-
-        return 0;
-}
-
-
-int32_t
-ctr_fsetattr (call_frame_t *frame,
-                    xlator_t *this, fd_t *fd,
-                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-        CTR_RECORD_METADATA_HEAT_IS_DISABLED_THEN_GOTO (this, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_SETATTR_WIND_FAILED,
-                        "Failed to insert fsetattr wind");
-        }
-out:
-        STACK_WIND (frame, ctr_fsetattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetattr,
-                    fd, stbuf, valid, xdata);
-
-        return 0;
-}
-/****************************fremovexattr************************************/
-
-int32_t
-ctr_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FREMOVEXATTR_UNWIND_FAILED,
-                        "Failed to insert fremovexattr unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      const char *name, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-        CTR_RECORD_METADATA_HEAT_IS_DISABLED_THEN_GOTO (this, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FREMOVEXATTR_WIND_FAILED,
-                        "Failed to insert fremovexattr wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_fremovexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
-}
-
-/****************************removexattr*************************************/
-
-int32_t
-ctr_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_REMOVEXATTR_UNWIND_FAILED,
-                        "Failed to insert removexattr unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     const char *name, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-        CTR_RECORD_METADATA_HEAT_IS_DISABLED_THEN_GOTO (this, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_REMOVEXATTR_WIND_FAILED,
-                        "Failed to insert removexattr wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_removexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
-}
-
-/****************************truncate****************************************/
-
-int32_t
-ctr_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_TRUNCATE_UNWIND_FAILED,
-                        "Failed to insert truncate unwind");
-        }
-
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                      postbuf, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  off_t offset, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_TRUNCATE_WIND_FAILED,
-                        "Failed to insert truncate wind");
-        }
-out:
-        STACK_WIND (frame, ctr_truncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
-}
-
-/****************************ftruncate***************************************/
-
-int32_t
-ctr_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FTRUNCATE_UNWIND_FAILED,
-                        "Failed to insert ftruncate unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                      postbuf, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   off_t offset, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FTRUNCATE_WIND_FAILED,
-                        "Failed to insert ftruncate wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_ftruncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ftruncate,
-                    fd, offset, xdata);
-        return 0;
-}
-
-/****************************rename******************************************/
-int32_t
-ctr_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
-{
-        int ret                         = -1;
-        uint32_t remaining_links        = -1;
-        gf_ctr_local_t *ctr_local       = NULL;
-        gfdb_fop_type_t fop_type        = GFDB_FOP_INVALID_OP;
-        gfdb_fop_path_t fop_path        = GFDB_FOP_INVALID;
-
-        GF_ASSERT(frame);
-        GF_ASSERT(this);
-
-        CTR_IS_DISABLED_THEN_GOTO (this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind (frame, this,
-                        GFDB_FOP_DENTRY_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_RENAME_UNWIND_FAILED,
-                        "Failed to insert rename unwind");
-                goto out;
-        }
-
-        if (!xdata)
-                goto out;
-        /*
-         *
-         * Extracting GF_RESPONSE_LINK_COUNT_XDATA from POSIX Xlator
-         * This is only set when we are overwriting hardlinks.
-         *
-         * */
-        ret = dict_get_uint32 (xdata , GF_RESPONSE_LINK_COUNT_XDATA,
-                                &remaining_links);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_GET_CTR_RESPONSE_LINK_COUNT_XDATA_FAILED,
-                        "Failed to getting GF_RESPONSE_LINK_COUNT_XDATA");
-                remaining_links = -1;
-                goto out;
-        }
-
-        ctr_local = frame->local;
-        if (!ctr_local) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, CTR_MSG_NULL_LOCAL,
-                        "ctr_local is NULL.");
-                goto out;
-        }
-
-        /* This is not the only link */
-        if (remaining_links > 1) {
-                fop_type = GFDB_FOP_DENTRY_WRITE;
-                fop_path = GFDB_FOP_UNDEL;
-        }
-        /* Last link that was deleted */
-        else if (remaining_links == 1) {
-                fop_type = GFDB_FOP_DENTRY_WRITE;
-                fop_path = GFDB_FOP_UNDEL_ALL;
-        } else {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_RENAME_UNWIND_FAILED,
-                        "Invalid link count from posix");
-                goto out;
-        }
-
-        ret = ctr_delete_hard_link_from_db (this,
-                                    CTR_DB_REC(ctr_local).old_gfid,
-                                    CTR_DB_REC(ctr_local).pargfid,
-                                    CTR_DB_REC(ctr_local).file_name,
-                                    fop_type, fop_path);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_UNLINK_UNWIND_FAILED,
-                        "Failed to delete records of %s",
-                        CTR_DB_REC(ctr_local).old_file_name);
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
-                             preoldparent, postoldparent, prenewparent,
-                             postnewparent,
-                             xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                loc_t *newloc, dict_t *xdata)
-{
-        int ret                                         = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx               = &ctr_inode_cx;
-        gf_ctr_link_context_t new_link_cx, old_link_cx;
-        gf_ctr_link_context_t *_nlink_cx                = &new_link_cx;
-        gf_ctr_link_context_t *_olink_cx                = &old_link_cx;
-        int is_dict_created                             = 0;
-        ctr_xlator_ctx_t *ctr_xlator_ctx                = NULL;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-        /*Fill old link context*/
-        FILL_CTR_LINK_CX(_olink_cx, oldloc->pargfid, oldloc->name, out);
-
-        /*Fill new link context*/
-        FILL_CTR_LINK_CX(_nlink_cx, newloc->pargfid, newloc->name, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, oldloc->inode->ia_type,
-                oldloc->inode->gfid, _nlink_cx, _olink_cx,
-                GFDB_FOP_DENTRY_WRITE, GFDB_FOP_WIND);
-
-
-        /* If the rename is a overwrite of hardlink
-         * rename ("file1", "file2")
-         * file1 is hardlink for gfid say 00000000-0000-0000-0000-00000000000A
-         * file2 is hardlink for gfid say 00000000-0000-0000-0000-00000000000B
-         * so we are saving file2 gfid in old_gfid so that we delete entries
-         * from the db during rename callback if the fop is successful
-         * */
-        if (newloc->inode) {
-                /* This is the GFID from where the newloc hardlink will be
-                 * unlinked */
-                _inode_cx->old_gfid = &newloc->inode->gfid;
-        }
-
-        /* Is a metatdata fop */
-        _inode_cx->is_metadata_fop = _gf_true;
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_RENAME_WIND_FAILED,
-                        "Failed to insert rename wind");
-        } else {
-                /* We are doing updation of hard link in inode context in wind
-                 * As we dont get the "inode" in the call back for rename */
-                ret = update_hard_link_ctx (frame, this, oldloc->inode);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_UPDATE_HARDLINK_FAILED, "Failed "
-                                "updating hard link in ctr inode context");
-                        goto out;
-                }
-
-                /* If the newloc has an inode. i.e aquiring hardlink of an
-                 * exisitng file i.e overwritting a file.
-                 * */
-                if (newloc->inode) {
-
-                        /* Getting the ctr inode context variable for
-                         * inode whose hardlink will be aquired during
-                         * the rename
-                         * */
-                        ctr_xlator_ctx = get_ctr_xlator_ctx (this,
-                                                                newloc->inode);
-                        if (!ctr_xlator_ctx) {
-                                /* Since there is no ctr inode context
-                                 * so nothing more to do */
-                                ret = 0;
-                                goto out;
-                        }
-
-                        /* Deleting hardlink from context variable */
-                        ret = ctr_delete_hard_link (this, ctr_xlator_ctx,
-                                                newloc->pargfid, newloc->name);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CTR_MSG_DELETE_HARDLINK_FAILED,
-                                        "Failed to delete hard link");
-                                goto out;
-                        }
-
-                        /* Requesting for number of hardlinks on the newloc
-                         * inode from POSIX.
-                         * */
-                        is_dict_created = set_posix_link_request (this, &xdata);
-                        if (is_dict_created == -1) {
-                                ret = -1;
-                                goto out;
-                        }
-                }
-        }
-
-out:
-        STACK_WIND (frame, ctr_rename_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->rename,
-                    oldloc, newloc, xdata);
-
-        if (is_dict_created == 1) {
-                dict_unref (xdata);
-        }
-
-        return 0;
-}
-
-/****************************unlink******************************************/
-int32_t
-ctr_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
-{
-        int ret                                 = -1;
-        uint32_t remaining_links                = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        if (!xdata)
-                goto out;
-
-        /*
-         *
-         * Extracting GF_RESPONSE_LINK_COUNT_XDATA from POSIX Xlator
-         *
-         * */
-        ret = dict_get_uint32 (xdata , GF_RESPONSE_LINK_COUNT_XDATA,
-                                &remaining_links);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_GET_CTR_RESPONSE_LINK_COUNT_XDATA_FAILED,
-                        "Failed to getting GF_RESPONSE_LINK_COUNT_XDATA");
-                remaining_links = -1;
-        }
-
-        /*This is not the only link*/
-        if (remaining_links != 1) {
-
-                ret = ctr_insert_unwind(frame, this, GFDB_FOP_DENTRY_WRITE,
-                                        GFDB_FOP_UNDEL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_INSERT_UNLINK_UNWIND_FAILED,
-                                "Failed to insert unlink unwind");
-                }
-        }
-        /*Last link that was deleted*/
-        else if (remaining_links == 1) {
-
-                ret = ctr_insert_unwind(frame, this, GFDB_FOP_DENTRY_WRITE,
-                                        GFDB_FOP_UNDEL_ALL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_INSERT_UNLINK_UNWIND_FAILED,
-                                "Failed to insert unlink unwind");
-                }
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_unlink (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int xflag, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-        gf_ctr_link_context_t ctr_link_cx;
-        gf_ctr_link_context_t *_link_cx = &ctr_link_cx;
-        gf_boolean_t is_xdata_created = _gf_false;
-        struct iatt dummy_stat        = {0};
-
-        GF_ASSERT (frame);
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        /*Fill link context*/
-        FILL_CTR_LINK_CX(_link_cx, loc->pargfid, loc->name, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, _link_cx, NULL,
-                GFDB_FOP_DENTRY_WRITE, GFDB_FOP_WDEL);
-
-        /*Internal FOP*/
-        _inode_cx->is_internal_fop = is_internal_fop (frame, xdata);
-
-        /* Is a metadata FOP */
-        _inode_cx->is_metadata_fop = _gf_true;
-
-        /* If its a internal FOP and dht link file donot record*/
-        if (_inode_cx->is_internal_fop &&
-                        dht_is_linkfile (&dummy_stat, xdata)) {
-                goto out;
-        }
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_UNLINK_UNWIND_FAILED,
-                        "Failed to insert unlink wind");
-        } else {
-                /* We are doing delete of hard link in inode context in wind
-                 * As we dont get the "inode" in the call back for rename */
-                ret = delete_hard_link_ctx (frame, this, loc->inode);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_DELETE_HARDLINK_FAILED, "Failed "
-                                "deleting hard link from ctr inode context");
-                }
-        }
-
-        /*
-         *
-         * Sending GF_REQUEST_LINK_COUNT_XDATA
-         * to POSIX Xlator to send link count in unwind path
-         *
-         * */
-        /*create xdata if NULL*/
-        if (!xdata) {
-                xdata = dict_new();
-                is_xdata_created = (xdata) ? _gf_true : _gf_false;
-        }
-        if (!xdata) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_XDATA_NULL, "xdata is NULL :Cannot send "
-                        "GF_REQUEST_LINK_COUNT_XDATA to posix");
-                goto out;
-        }
-
-        ret = dict_set_int32 (xdata, GF_REQUEST_LINK_COUNT_XDATA, 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_SET_CTR_RESPONSE_LINK_COUNT_XDATA_FAILED,
-                        "Failed setting GF_REQUEST_LINK_COUNT_XDATA");
-                if (is_xdata_created) {
-                        dict_unref (xdata);
-                }
-                goto out;
-        }
-
-out:
-        STACK_WIND (frame, ctr_unlink_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->unlink,
-                    loc, xflag, xdata);
-
-        if (is_xdata_created)
-                dict_unref (xdata);
-
-        return 0;
-}
-
-/****************************fsync******************************************/
-int32_t
-ctr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_INODE_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FSYNC_UNWIND_FAILED,
-                        "Failed to insert fsync unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                      xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               int32_t flags, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL,
-                GFDB_FOP_INODE_WRITE, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FSYNC_WIND_FAILED,
-                        "Failed to insert fsync wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_fsync_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsync,
-                    fd, flags, xdata);
-        return 0;
-}
-
-/****************************setxattr****************************************/
-
-int
-ctr_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_INODE_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FSYNC_UNWIND_FAILED,
-                        "Failed to insert setxattr unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-
-        return 0;
-}
-
-int
-ctr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr, int flags, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-        CTR_RECORD_METADATA_HEAT_IS_DISABLED_THEN_GOTO (this, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, NULL, NULL,
-                GFDB_FOP_INODE_WRITE, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_SETATTR_WIND_FAILED,
-                        "Failed to insert setxattr wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_setxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setxattr,
-                    loc, xattr, flags, xdata);
-        return 0;
-}
-/**************************** fsetxattr *************************************/
-int32_t
-ctr_fsetxattr_cbk (call_frame_t *frame,
-                         void *cookie, xlator_t *this, int32_t op_ret,
-                         int32_t op_errno, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_INODE_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_FSYNC_UNWIND_FAILED,
-                        "Failed to insert fsetxattr unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-               int32_t flags, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-        CTR_RECORD_METADATA_HEAT_IS_DISABLED_THEN_GOTO (this, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL,
-                GFDB_FOP_INODE_WRITE, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_SETATTR_WIND_FAILED,
-                        "Failed to insert fsetxattr wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_fsetxattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-        return 0;
-}
-/****************************mknod*******************************************/
-
-
-int32_t
-ctr_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
-{
-        int ret = -1;
-        ctr_heal_ret_val_t ret_val = CTR_CTX_ERROR;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        /* Add hard link to the list */
-        ret_val = add_hard_link_ctx (frame, this, inode);
-        if (ret_val == CTR_CTX_ERROR) {
-                gf_msg_trace (this->name, 0, "Failed adding hard link");
-        }
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_CREATE_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_MKNOD_UNWIND_FAILED,
-                        "Failed to insert mknod unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                preparent, postparent, xdata);
-
-        return 0;
-}
-
-
-int
-ctr_mknod (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-        gf_ctr_link_context_t  ctr_link_cx;
-        gf_ctr_link_context_t  *_link_cx = &ctr_link_cx;
-        uuid_t gfid                             = {0,};
-        uuid_t *ptr_gfid                        = &gfid;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-
-        /*get gfid from xdata dict*/
-        ret = dict_get_gfuuid (xdata, "gfid-req", &gfid);
-        if (ret) {
-                gf_msg_debug (this->name, 0, "failed to get gfid from dict");
-                goto out;
-        }
-
-        /*fill ctr link context*/
-        FILL_CTR_LINK_CX (_link_cx, loc->pargfid, loc->name, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT (_inode_cx, loc->inode->ia_type,
-                *ptr_gfid, _link_cx, NULL,
-                GFDB_FOP_CREATE_WRITE, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_MKNOD_WIND_FAILED,
-                        "Failed to insert mknod wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_mknod_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->mknod,
-                    loc, mode, rdev, umask, xdata);
-        return 0;
-}
-
-/****************************create******************************************/
-int
-ctr_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno,
-                fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                struct iatt *preparent, struct iatt *postparent,
-                dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = add_hard_link_ctx (frame, this, inode);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_ADD_HARDLINK_FAILED,
-                        "Failed adding hard link");
-        }
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_CREATE_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_CREATE_UNWIND_FAILED,
-                        "Failed to insert create unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode,
-                             stbuf,
-                        preparent, postparent, xdata);
-
-        return 0;
-}
-
-int
-ctr_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-        gf_ctr_link_context_t  ctr_link_cx;
-        gf_ctr_link_context_t  *_link_cx = &ctr_link_cx;
-        uuid_t            gfid                  = {0,};
-        uuid_t            *ptr_gfid             = &gfid;
-        struct iatt dummy_stat                  = {0};
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-
-        /*Get GFID from Xdata dict*/
-        ret = dict_get_gfuuid (xdata, "gfid-req", &gfid);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_GET_GFID_FROM_DICT_FAILED,
-                        "failed to get gfid from dict");
-                goto out;
-        }
-
-        /*fill ctr link context*/
-        FILL_CTR_LINK_CX(_link_cx, loc->pargfid, loc->name, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                *ptr_gfid, _link_cx, NULL,
-                GFDB_FOP_CREATE_WRITE, GFDB_FOP_WIND);
-
-        /*Internal FOP*/
-        _inode_cx->is_internal_fop = is_internal_fop (frame, xdata);
-
-        /* If its a internal FOP and dht link file donot record*/
-        if (_inode_cx->is_internal_fop &&
-                        dht_is_linkfile (&dummy_stat, xdata)) {
-                goto out;
-        }
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, &ctr_inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_CREATE_WIND_FAILED,
-                        "Failed to insert create wind");
-        }
-out:
-        STACK_WIND (frame, ctr_create_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
-}
-
-/****************************link********************************************/
-
-int
-ctr_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno,
-              inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        /* Add hard link to the list */
-        ret = add_hard_link_ctx (frame, this, inode);
-        if (ret) {
-                gf_msg_trace (this->name, 0, "Failed adding hard link");
-        }
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_DENTRY_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_CREATE_UNWIND_FAILED,
-                        "Failed to insert create unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, stbuf,
-                       preparent, postparent, xdata);
-        return 0;
-}
-
-int
-ctr_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-        gf_ctr_link_context_t  ctr_link_cx;
-        gf_ctr_link_context_t  *_link_cx = &ctr_link_cx;
-        struct iatt dummy_stat          = {0};
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-
-        /*fill ctr link context*/
-        FILL_CTR_LINK_CX(_link_cx, newloc->pargfid, newloc->name, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, oldloc->inode->ia_type,
-                oldloc->inode->gfid, _link_cx, NULL,
-                GFDB_FOP_DENTRY_WRITE, GFDB_FOP_WIND);
-
-        /*Internal FOP*/
-        _inode_cx->is_internal_fop = is_internal_fop (frame, xdata);
-
-        /* Is a metadata fop */
-        _inode_cx->is_metadata_fop = _gf_true;
-
-        /* If its a internal FOP and dht link file donot record*/
-        if (_inode_cx->is_internal_fop &&
-                        dht_is_linkfile (&dummy_stat, xdata)) {
-                goto out;
-        }
-
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_LINK_WIND_FAILED,
-                        "Failed to insert link wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_link_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->link,
-                    oldloc, newloc, xdata);
-        return 0;
-}
-
-/******************************readv*****************************************/
-int ctr_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno,
-               struct iovec *vector, int count, struct iatt *stbuf,
-               struct iobref *iobref, dict_t *xdata) {
-
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_FOP_FAILED_THEN_GOTO (this, op_ret, op_errno, out);
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_INODE_READ,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_CREATE_UNWIND_FAILED,
-                        "Failed to insert create unwind");
-        }
-
-out:
-        ctr_free_frame_local (frame);
-
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                                stbuf, iobref, xdata);
-        return 0;
-}
-
-
-int
-ctr_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t off, uint32_t flags, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL,
-                GFDB_FOP_INODE_READ, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_READV_WIND_FAILED,
-                        "Failed to insert readv wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_readv_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->readv,
-                    fd, size, off, flags, xdata);
-        return 0;
-}
-
-/*******************************ctr_ipc****************************************/
-
-/*This is the call back function per record/file from data base*/
-static int
-ctr_db_query_callback (gfdb_query_record_t *gfdb_query_record,
-                        void *args) {
-        int ret = -1;
-        ctr_query_cbk_args_t *query_cbk_args = args;
-
-        GF_VALIDATE_OR_GOTO ("ctr", query_cbk_args, out);
-
-        ret = gfdb_write_query_record (query_cbk_args->query_fd,
-                                       gfdb_query_record);
-        if (ret) {
-                gf_msg ("ctr", GF_LOG_ERROR, 0,
-                        CTR_MSG_FATAL_ERROR,
-                        "Failed to write to query file");
-                goto out;
-        }
-
-        query_cbk_args->count++;
-
-        ret = 0;
-out:
-        return ret;
-}
-
-/* This function does all the db queries related to tiering and
- * generates/populates new/existing query file
- * inputs:
- * xlator_t *this : CTR Translator
- * void *conn_node : Database connection
- * char *query_file: the query file that needs to be updated
- * gfdb_ipc_ctr_params_t *ipc_ctr_params: the query parameters
- * Return:
- * On success 0
- * On failure -1
- * */
-int
-ctr_db_query (xlator_t *this,
-              void *conn_node,
-              char *query_file,
-              gfdb_ipc_ctr_params_t *ipc_ctr_params)
-{
-        int ret = -1;
-        ctr_query_cbk_args_t query_cbk_args = {0};
-
-        GF_VALIDATE_OR_GOTO ("ctr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, conn_node, out);
-        GF_VALIDATE_OR_GOTO (this->name, query_file, out);
-        GF_VALIDATE_OR_GOTO (this->name, ipc_ctr_params, out);
-
-        /*Query for eligible files from db*/
-        query_cbk_args.query_fd = open (query_file,
-                        O_WRONLY | O_CREAT | O_APPEND,
-                        S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (query_cbk_args.query_fd < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        CTR_MSG_FATAL_ERROR,
-                        "Failed to open query file %s", query_file);
-                goto out;
-        }
-        if (!ipc_ctr_params->is_promote) {
-                if (ipc_ctr_params->emergency_demote) {
-                        /* emergency demotion mode */
-                        ret = find_all (conn_node,
-                                ctr_db_query_callback,
-                                (void *)&query_cbk_args,
-                                ipc_ctr_params->query_limit);
-                } else {
-                        if (ipc_ctr_params->write_freq_threshold == 0 &&
-                                ipc_ctr_params->read_freq_threshold == 0) {
-                                ret = find_unchanged_for_time (
-                                        conn_node,
-                                        ctr_db_query_callback,
-                                        (void *)&query_cbk_args,
-                                        &ipc_ctr_params->time_stamp);
-                        } else {
-                                ret = find_unchanged_for_time_freq (
-                                        conn_node,
-                                        ctr_db_query_callback,
-                                        (void *)&query_cbk_args,
-                                        &ipc_ctr_params->time_stamp,
-                                        ipc_ctr_params->write_freq_threshold,
-                                        ipc_ctr_params->read_freq_threshold,
-                                        _gf_false);
-                        }
-                }
-        } else {
-                if (ipc_ctr_params->write_freq_threshold == 0 &&
-                        ipc_ctr_params->read_freq_threshold == 0) {
-                        ret = find_recently_changed_files (
-                                conn_node,
-                                ctr_db_query_callback,
-                                (void *)&query_cbk_args,
-                                &ipc_ctr_params->time_stamp);
-                } else {
-                        ret = find_recently_changed_files_freq (
-                                conn_node,
-                                ctr_db_query_callback,
-                                (void *)&query_cbk_args,
-                                &ipc_ctr_params->time_stamp,
-                                ipc_ctr_params->write_freq_threshold,
-                                ipc_ctr_params->read_freq_threshold,
-                                _gf_false);
-                }
-        }
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_FATAL_ERROR,
-                        "FATAL: query from db failed");
-                        goto out;
-        }
-
-        ret = clear_files_heat (conn_node);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_FATAL_ERROR,
-                        "FATAL: Failed to clear db entries");
-                        goto out;
-        }
-
-        ret = 0;
-out:
-
-        if (!ret)
-                ret = query_cbk_args.count;
-
-        if (query_cbk_args.query_fd >= 0) {
-                sys_close (query_cbk_args.query_fd);
-                query_cbk_args.query_fd = -1;
-        }
-
-        return ret;
-}
-
-void *
-ctr_compact_thread (void *args)
-{
-        int ret = -1;
-        void *db_conn = NULL;
-
-        xlator_t *this = NULL;
-        gf_ctr_private_t *priv = NULL;
-        gf_boolean_t compact_active = _gf_false;
-        gf_boolean_t compact_mode_switched = _gf_false;
-
-        this = (xlator_t *)args;
-
-        GF_VALIDATE_OR_GOTO("ctr", this, out);
-
-        priv = this->private;
-
-        db_conn = priv->_db_conn;
-        compact_active = priv->compact_active;
-        compact_mode_switched = priv->compact_mode_switched;
-
-        gf_msg ("ctr-compact", GF_LOG_INFO, 0, CTR_MSG_SET,
-                "Starting compaction");
-
-        ret = compact_db(db_conn, compact_active,
-                         compact_mode_switched);
-
-        if (ret) {
-                gf_msg ("ctr-compact", GF_LOG_ERROR, 0, CTR_MSG_SET,
-                        "Failed to perform the compaction");
-        }
-
-        ret = pthread_mutex_lock (&priv->compact_lock);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                        "Failed to acquire lock");
-                goto out;
-        }
-
-        /* We are done compaction on this brick. Set all flags to false */
-        priv->compact_active = _gf_false;
-        priv->compact_mode_switched = _gf_false;
-
-        ret = pthread_mutex_unlock (&priv->compact_lock);
-
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                        "Failed to release lock");
-                goto out;
-        }
-
-out:
-        return NULL;
-}
-
-int
-ctr_ipc_helper (xlator_t *this, dict_t *in_dict,
-                dict_t *out_dict)
-{
-        int ret = -1;
-        char *ctr_ipc_ops = NULL;
-        gf_ctr_private_t *priv = NULL;
-        char *db_version = NULL;
-        char *db_param_key = NULL;
-        char *db_param = NULL;
-        char *query_file = NULL;
-        gfdb_ipc_ctr_params_t *ipc_ctr_params = NULL;
-        int result = 0;
-        pthread_t compact_thread;
-
-        GF_VALIDATE_OR_GOTO ("ctr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv->_db_conn, out);
-        GF_VALIDATE_OR_GOTO (this->name, in_dict, out);
-        GF_VALIDATE_OR_GOTO (this->name, out_dict, out);
-
-        GET_DB_PARAM_FROM_DICT(this->name, in_dict, GFDB_IPC_CTR_KEY,
-                                ctr_ipc_ops, out);
-
-        /*if its a db clear operation */
-        if (strncmp (ctr_ipc_ops, GFDB_IPC_CTR_CLEAR_OPS,
-                        strlen (GFDB_IPC_CTR_CLEAR_OPS)) == 0) {
-
-                ret = clear_files_heat (priv->_db_conn);
-                if (ret)
-                        goto out;
-
-        } /* if its a query operation, in  which case its query + clear db*/
-        else if (strncmp (ctr_ipc_ops, GFDB_IPC_CTR_QUERY_OPS,
-                                strlen (GFDB_IPC_CTR_QUERY_OPS)) == 0) {
-
-                ret = dict_get_str (in_dict, GFDB_IPC_CTR_GET_QFILE_PATH,
-                                                                &query_file);
-                if (ret) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                                        "Failed extracting query file path");
-                        goto out;
-                }
-
-                ret = dict_get_bin (in_dict, GFDB_IPC_CTR_GET_QUERY_PARAMS,
-                                   (void *)&ipc_ctr_params);
-                if (ret) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                                        "Failed extracting query parameters");
-                        goto out;
-                }
-
-                ret = ctr_db_query (this, priv->_db_conn, query_file,
-                                ipc_ctr_params);
-
-                ret = dict_set_int32 (out_dict,
-                                      GFDB_IPC_CTR_RET_QUERY_COUNT, ret);
-                if (ret) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                                        "Failed setting query reply");
-                        goto out;
-                }
-
-        } /* if its a query for db version */
-        else if (strncmp (ctr_ipc_ops, GFDB_IPC_CTR_GET_DB_VERSION_OPS,
-                        strlen (GFDB_IPC_CTR_GET_DB_VERSION_OPS)) == 0) {
-
-                ret = get_db_version (priv->_db_conn, &db_version);
-                if (ret == -1 || !db_version) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                                        "Failed extracting db version ");
-                        goto out;
-                }
-
-                SET_DB_PARAM_TO_DICT(this->name, out_dict,
-                                        GFDB_IPC_CTR_RET_DB_VERSION,
-                                        db_version, ret, error);
-
-        } /* if its a query for a db setting */
-        else if (strncmp (ctr_ipc_ops, GFDB_IPC_CTR_GET_DB_PARAM_OPS,
-                                strlen (GFDB_IPC_CTR_GET_DB_PARAM_OPS)) == 0) {
-
-                ret = dict_get_str (in_dict, GFDB_IPC_CTR_GET_DB_KEY,
-                                &db_param_key);
-                if (ret) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                                        "Failed extracting db param key");
-                        goto out;
-                }
-
-                ret = get_db_params (priv->_db_conn, db_param_key, &db_param);
-                if (ret == -1 || !db_param) {
-                        goto out;
-                }
-
-                SET_DB_PARAM_TO_DICT(this->name, out_dict,
-                                        db_param_key,
-                                        db_param, ret, error);
-        } /* if its an attempt to compact the database */
-        else if (strncmp (ctr_ipc_ops, GFDB_IPC_CTR_SET_COMPACT_PRAGMA,
-                          strlen (GFDB_IPC_CTR_SET_COMPACT_PRAGMA)) == 0) {
-
-                ret = pthread_mutex_lock (&priv->compact_lock);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                                "Failed to acquire lock for compaction");
-                        goto out;
-                }
-
-                if ((priv->compact_active || priv->compact_mode_switched)) {
-                        /* Compaction in progress. LEAVE */
-                        gf_msg (this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                               "Compaction already in progress.");
-                        pthread_mutex_unlock (&priv->compact_lock);
-                        goto out;
-                }
-                /* At this point, we should be the only one on the brick */
-                /* compacting */
-
-                /* Grab the arguments from the dictionary */
-                ret = dict_get_int32 (in_dict, "compact_active", &result);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                               "Failed to get compaction type");
-                        goto out;
-                }
-
-                if (result) {
-                        priv->compact_active = _gf_true;
-                }
-
-                ret = dict_get_int32 (in_dict, "compact_mode_switched"
-                                     , &result);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                               "Failed to see if compaction switched");
-                        goto out;
-                }
-
-                if (result) {
-                        priv->compact_mode_switched = _gf_true;
-                        gf_msg ("ctr-compact", GF_LOG_TRACE, 0, CTR_MSG_SET,
-                                "Pre-thread: Compact mode switch is true");
-                } else {
-                        gf_msg ("ctr-compact", GF_LOG_TRACE, 0, CTR_MSG_SET,
-                                "Pre-thread: Compact mode switch is false");
-                }
-
-                ret = pthread_mutex_unlock (&priv->compact_lock);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                                "Failed to release lock for compaction");
-                        goto out;
-                }
-
-                ret = gf_thread_create (&compact_thread, NULL,
-                                        ctr_compact_thread, (void *)this,
-                                        "ctrcomp");
-
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                                "Failed to spawn compaction thread");
-                        goto out;
-                }
-
-                goto out;
-        } /* default case */
-        else {
-                goto out;
-        }
-
-        ret = 0;
-        goto out;
-error:
-        GF_FREE (db_param_key);
-        GF_FREE (db_param);
-        GF_FREE (db_version);
-out:
-        return ret;
-}
-
-
-/* IPC Call from tier migrator to clear the heat on the DB */
-int32_t
-ctr_ipc (call_frame_t *frame, xlator_t *this, int32_t op,
-        dict_t *in_dict)
-{
-        int ret                         = -1;
-        gf_ctr_private_t *priv         = NULL;
-        dict_t *out_dict                = NULL;
-
-        GF_ASSERT(this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT(priv->_db_conn);
-        GF_VALIDATE_OR_GOTO (this->name, in_dict, wind);
-
-
-        if (op != GF_IPC_TARGET_CTR)
-                goto wind;
-
-        out_dict = dict_new();
-        if (!out_dict) {
-                goto out;
-        }
-
-        ret = ctr_ipc_helper (this, in_dict, out_dict);
-        if (ret) {
-                gf_msg(this->name, GF_LOG_ERROR, 0, CTR_MSG_SET,
-                "Failed in ctr_ipc_helper");
-        }
-out:
-
-        STACK_UNWIND_STRICT (ipc, frame, ret, 0, out_dict);
-
-        if (out_dict)
-                dict_unref(out_dict);
-
-        return 0;
-
- wind:
-        STACK_WIND (frame, default_ipc_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ipc, op, in_dict);
-
-
-
-        return 0;
-}
-
-
-/******************************************************************************/
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        char *temp_str = NULL;
-        int ret = 0;
-        gf_ctr_private_t *priv = NULL;
-
-        priv = this->private;
-        if (dict_get_str(options, "changetimerecorder.frequency",
-                         &temp_str)) {
-                gf_msg(this->name, GF_LOG_TRACE, 0, CTR_MSG_SET, "set");
-        }
-
-        GF_OPTION_RECONF ("ctr-enabled", priv->enabled, options,
-                          bool, out);
-
-        GF_OPTION_RECONF ("record-counters", priv->ctr_record_counter, options,
-                          bool, out);
-
-        GF_OPTION_RECONF ("ctr-record-metadata-heat",
-                          priv->ctr_record_metadata_heat, options,
-                          bool, out);
-
-        GF_OPTION_RECONF ("ctr_link_consistency", priv->ctr_link_consistency,
-                        options, bool, out);
-
-        GF_OPTION_RECONF ("ctr_lookupheal_inode_timeout",
-                                priv->ctr_lookupheal_inode_timeout,
-                                options, uint64, out);
-
-        GF_OPTION_RECONF ("ctr_lookupheal_link_timeout",
-                                priv->ctr_lookupheal_link_timeout,
-                                options, uint64, out);
-
-        GF_OPTION_RECONF ("record-exit", priv->ctr_record_unwind, options,
-                          bool, out);
-
-        GF_OPTION_RECONF ("record-entry", priv->ctr_record_wind, options,
-                          bool, out);
-
-
-
-
-        /* If database is sqlite */
-        if (priv->gfdb_db_type == GFDB_SQLITE3) {
-
-                /* AUTOCHECKPOINT */
-                if (dict_get_str (options, GFDB_SQL_PARAM_WAL_AUTOCHECK,
-                                &temp_str) == 0) {
-                        ret = set_db_params (priv->_db_conn,
-                                        "wal_autocheckpoint", temp_str);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CTR_MSG_SET_VALUE_TO_SQL_PARAM_FAILED,
-                                        "Failed  to set %s",
-                                        GFDB_SQL_PARAM_WAL_AUTOCHECK);
-                        }
-                }
-
-                /* CACHE_SIZE */
-                if (dict_get_str (options, GFDB_SQL_PARAM_CACHE_SIZE, &temp_str)
-                                == 0) {
-                        ret = set_db_params (priv->_db_conn, "cache_size",
-                                        temp_str);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        CTR_MSG_SET_VALUE_TO_SQL_PARAM_FAILED,
-                                        "Failed  to set %s",
-                                        GFDB_SQL_PARAM_CACHE_SIZE);
-                        }
-                }
-        }
-
-        ret = 0;
-
-out:
-
-        return ret;
-}
-
-/****************************init********************************************/
-
-int32_t
-init (xlator_t *this)
-{
-        gf_ctr_private_t *priv = NULL;
-        int ret_db              = -1;
-        dict_t *params_dict      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("ctr", this, error);
-
-        if (!this->children || this->children->next) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_FATAL_ERROR,
-                        "FATAL: ctr should have exactly one child");
-                goto error;
-        }
-
-        if (!this->parents) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        CTR_MSG_DANGLING_VOLUME,
-                        "dangling volume. check volfile ");
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_ctr_mt_private_t);
-        if (!priv) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        CTR_MSG_CALLOC_FAILED,
-                        "Calloc did not work!!!");
-                goto error;
-        }
-
-        /*Default values for the translator*/
-        priv->ctr_record_wind          = _gf_true;
-        priv->ctr_record_unwind        = _gf_false;
-        priv->ctr_hot_brick            = _gf_false;
-        priv->gfdb_db_type             = GFDB_SQLITE3;
-        priv->gfdb_sync_type           = GFDB_DB_SYNC;
-        priv->enabled                  = _gf_true;
-        priv->_db_conn                 = NULL;
-        priv->ctr_lookupheal_link_timeout =
-                                CTR_DEFAULT_HARDLINK_EXP_PERIOD;
-        priv->ctr_lookupheal_inode_timeout =
-                                CTR_DEFAULT_INODE_EXP_PERIOD;
-
-        /* For compaction */
-        priv->compact_active = _gf_false;
-        priv->compact_mode_switched = _gf_false;
-        ret_db = pthread_mutex_init (&priv->compact_lock, NULL);
-
-        if (ret_db) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_FATAL_ERROR,
-                        "FATAL: Failed initializing compaction mutex");
-                goto error;
-        }
-
-        /*Extract ctr xlator options*/
-        ret_db = extract_ctr_options (this, priv);
-        if (ret_db) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_EXTRACT_CTR_XLATOR_OPTIONS_FAILED,
-                        "Failed extracting ctr xlator options");
-                goto error;
-        }
-
-        params_dict = dict_new ();
-        if (!params_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INIT_DB_PARAMS_FAILED,
-                        "DB Params cannot initialized!");
-                goto error;
-        }
-
-        /*Extract db params options*/
-        ret_db = extract_db_params(this, params_dict, priv->gfdb_db_type);
-        if (ret_db) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_EXTRACT_DB_PARAM_OPTIONS_FAILED,
-                        "Failed extracting db params options");
-                goto error;
-        }
-
-        /*Create a memory pool for ctr xlator*/
-        this->local_pool = mem_pool_new (gf_ctr_local_t, 64);
-        if (!this->local_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_CREATE_LOCAL_MEMORY_POOL_FAILED,
-                        "failed to create local memory pool");
-                goto error;
-        }
-
-        /*Initialize Database Connection*/
-        priv->_db_conn = init_db(params_dict, priv->gfdb_db_type);
-        if (!priv->_db_conn) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                       CTR_MSG_FATAL_ERROR,
-                       "FATAL: Failed initializing data base");
-                        goto error;
-        }
-
-
-        ret_db = 0;
-        goto out;
-
-/*Error handling */
-error:
-
-        if (this)
-                mem_pool_destroy (this->local_pool);
-
-        if (priv) {
-                GF_FREE (priv->ctr_db_path);
-        }
-        GF_FREE (priv);
-
-        if (params_dict)
-                dict_unref (params_dict);
-
-        return -1;
-
-out:
-
-        if (params_dict)
-                dict_unref (params_dict);
-
-        this->private = (void *)priv;
-        return 0;
-}
-
-int
-notify (xlator_t *this, int event, void *data, ...)
-{
-
-       gf_ctr_private_t *priv = NULL;
-       int               ret  = 0;
-
-       priv = this->private;
-
-       if (!priv)
-               goto out;
-
-       ret = default_notify (this, event, data);
-
-out:
-      return ret;
-
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("ctr", this, out);
-
-        ret = xlator_mem_acct_init (this, gf_ctr_mt_end + 1);
-
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_MEM_ACC_INIT_FAILED, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
-out:
-        return ret;
-}
-
-
-void
-fini (xlator_t *this)
-{
-        gf_ctr_private_t *priv = NULL;
-
-        priv = this->private;
-
-        if (priv) {
-                if (fini_db (priv->_db_conn)) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                CTR_MSG_CLOSE_DB_CONN_FAILED, "Failed closing "
-                                "db connection");
-                }
-
-                if (priv->_db_conn)
-                        priv->_db_conn = NULL;
-
-                GF_FREE (priv->ctr_db_path);
-                if (pthread_mutex_destroy (&priv->compact_lock)) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                CTR_MSG_CLOSE_DB_CONN_FAILED, "Failed to "
-                                "destroy the compaction mutex");
-                }
-        }
-        GF_FREE (priv);
-        mem_pool_destroy (this->local_pool);
-        this->local_pool = NULL;
-
-        return;
-}
-
-struct xlator_fops fops = {
-        /*lookup*/
-        .lookup         = ctr_lookup,
-        /*write fops */
-        .mknod          = ctr_mknod,
-        .create         = ctr_create,
-        .truncate       = ctr_truncate,
-        .ftruncate      = ctr_ftruncate,
-        .setxattr       = ctr_setxattr,
-        .fsetxattr      = ctr_fsetxattr,
-        .removexattr    = ctr_removexattr,
-        .fremovexattr   = ctr_fremovexattr,
-        .unlink         = ctr_unlink,
-        .link           = ctr_link,
-        .rename         = ctr_rename,
-        .writev         = ctr_writev,
-        .setattr        = ctr_setattr,
-        .fsetattr       = ctr_fsetattr,
-        /*read fops*/
-        .readv          = ctr_readv,
-        /* IPC call*/
-        .ipc            = ctr_ipc
-};
-
-struct xlator_cbks cbks = {
-        .forget = ctr_forget
-};
-
-struct volume_options options[] = {
-        { .key  = {"ctr-enabled",},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off",
-          .description = "Enables the CTR",
-          .flags       = OPT_FLAG_SETTABLE
-        },
-        { .key  = {"record-entry"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "on"
-        },
-        { .key  = {"record-exit"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off"
-        },
-        { .key  = {"record-counters"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off",
-          .op_version  = {GD_OP_VERSION_3_7_0},
-          .flags       = OPT_FLAG_SETTABLE,
-          .tags = {}
-        },
-        { .key  = {"ctr-record-metadata-heat"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off",
-          .flags       = OPT_FLAG_SETTABLE,
-          .op_version  = {GD_OP_VERSION_3_7_0},
-          .tags = {}
-        },
-        { .key  = {"ctr_link_consistency"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off",
-          .flags       = OPT_FLAG_SETTABLE,
-          .op_version  = {GD_OP_VERSION_3_7_0},
-          .tags = {}
-        },
-        { .key  = {"ctr_lookupheal_link_timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "300",
-          .flags       = OPT_FLAG_SETTABLE,
-          .op_version  = {GD_OP_VERSION_3_7_2},
-          .tags = {}
-        },
-        { .key  = {"ctr_lookupheal_inode_timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "300",
-          .flags       = OPT_FLAG_SETTABLE,
-          .op_version  = {GD_OP_VERSION_3_7_2},
-          .tags = {}
-        },
-        { .key  = {"hot-brick"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off"
-        },
-        { .key  = {"db-type"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"hashfile", "rocksdb", "changelog", "sqlite3",
-                    "hyperdex"},
-          .default_value = "sqlite3",
-          .op_version  = {GD_OP_VERSION_3_7_0},
-          .flags       = OPT_FLAG_SETTABLE,
-          .tags = {}
-        },
-        { .key  = {"db-sync"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"sync", "async"},
-          .default_value = "sync"
-        },
-        { .key  = {"db-path"},
-          .type = GF_OPTION_TYPE_PATH
-        },
-        { .key  = {"db-name"},
-          .type = GF_OPTION_TYPE_STR
-        },
-        { .key  = {GFDB_SQL_PARAM_SYNC},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"off", "normal", "full"},
-          .default_value = "normal"
-        },
-        { .key  = {GFDB_SQL_PARAM_JOURNAL_MODE},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"delete", "truncate", "persist", "memory", "wal", "off"},
-          .default_value = "wal",
-          .flags       = OPT_FLAG_SETTABLE,
-          .op_version  = {GD_OP_VERSION_3_7_0},
-          .tags = {}
-        },
-        { .key  = {GFDB_SQL_PARAM_AUTO_VACUUM},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"off", "full", "incr"},
-          .default_value = "off",
-          .flags       = OPT_FLAG_SETTABLE,
-          .op_version  = {GD_OP_VERSION_3_7_0},
-          .tags = {}
-        },
-        { .key  = {GFDB_SQL_PARAM_WAL_AUTOCHECK},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "25000",
-          .flags       = OPT_FLAG_SETTABLE,
-          .op_version  = {GD_OP_VERSION_3_7_0},
-          .tags = {}
-        },
-        { .key  = {GFDB_SQL_PARAM_CACHE_SIZE},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "12500",
-          .flags       = OPT_FLAG_SETTABLE,
-          .op_version  = {GD_OP_VERSION_3_7_0},
-          .tags = {}
-        },
-        { .key  = {GFDB_SQL_PARAM_PAGE_SIZE},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "4096",
-          .flags       = OPT_FLAG_SETTABLE,
-          .op_version  = {GD_OP_VERSION_3_7_0},
-          .tags = {}
-        },
-        { .key  = {NULL} },
-};
diff --git a/xlators/features/changetimerecorder/src/changetimerecorder.h b/xlators/features/changetimerecorder/src/changetimerecorder.h
deleted file mode 100644
index 2a8bbd18c5b..00000000000
--- a/xlators/features/changetimerecorder/src/changetimerecorder.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
-   Copyright (c) 2006-2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __CTR_H
-#define __CTR_H
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "common-utils.h"
-#include "ctr_mem_types.h"
-#include "ctr-helper.h"
-
-#endif /* __CTR_H */
diff --git a/xlators/features/changetimerecorder/src/ctr-helper.c b/xlators/features/changetimerecorder/src/ctr-helper.c
deleted file mode 100644
index 822526ca833..00000000000
--- a/xlators/features/changetimerecorder/src/ctr-helper.c
+++ /dev/null
@@ -1,307 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#include "gfdb_sqlite3.h"
-#include "ctr-helper.h"
-#include "ctr-messages.h"
-
-/*******************************************************************************
- *
- *                      Fill unwind into db record
- *
- ******************************************************************************/
-int
-fill_db_record_for_unwind(xlator_t              *this,
-                          gf_ctr_local_t        *ctr_local,
-                          gfdb_fop_type_t       fop_type,
-                          gfdb_fop_path_t       fop_path)
-{
-        int ret                         = -1;
-        gfdb_time_t *ctr_uwtime         = NULL;
-        gf_ctr_private_t *_priv                 = NULL;
-
-        GF_ASSERT (this);
-        _priv = this->private;
-        GF_ASSERT (_priv);
-
-        GF_ASSERT(ctr_local);
-
-        /*If not unwind path error*/
-        if (!isunwindpath(fop_path)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, CTR_MSG_WRONG_FOP_PATH,
-                        "Wrong fop_path. Should be unwind");
-                goto out;
-        }
-
-        ctr_uwtime = &CTR_DB_REC(ctr_local).gfdb_unwind_change_time;
-        CTR_DB_REC(ctr_local).gfdb_fop_path = fop_path;
-        CTR_DB_REC(ctr_local).gfdb_fop_type = fop_type;
-
-        ret = gettimeofday (ctr_uwtime, NULL);
-        if (ret == -1) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CTR_MSG_FILL_UNWIND_TIME_REC_ERROR, "Error "
-                                "filling unwind time record %s",
-                                strerror(errno));
-                        goto out;
-                }
-
-        /* Special case i.e if its a tier rebalance
-         * + cold tier brick
-         * + its a create/mknod FOP
-         * we record unwind time as zero */
-        if (ctr_local->client_pid == GF_CLIENT_PID_TIER_DEFRAG
-                && (!_priv->ctr_hot_brick)
-                && isdentrycreatefop(fop_type)) {
-                memset(ctr_uwtime, 0, sizeof(*ctr_uwtime));
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-
-/*******************************************************************************
- *
- *                      Fill wind into db record
- *
- ******************************************************************************/
-int
-fill_db_record_for_wind (xlator_t               *this,
-                        gf_ctr_local_t          *ctr_local,
-                        gf_ctr_inode_context_t  *ctr_inode_cx)
-{
-        int ret                                 = -1;
-        gfdb_time_t *ctr_wtime                  = NULL;
-        gf_ctr_private_t *_priv                 = NULL;
-
-        GF_ASSERT (this);
-        _priv = this->private;
-        GF_ASSERT (_priv);
-        GF_ASSERT (ctr_local);
-        IS_CTR_INODE_CX_SANE (ctr_inode_cx);
-
-        /*if not wind path error!*/
-        if (!iswindpath(ctr_inode_cx->fop_path)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_WRONG_FOP_PATH,
-                        "Wrong fop_path. Should be wind");
-                goto out;
-        }
-
-        ctr_wtime = &CTR_DB_REC(ctr_local).gfdb_wind_change_time;
-        CTR_DB_REC(ctr_local).gfdb_fop_path = ctr_inode_cx->fop_path;
-        CTR_DB_REC(ctr_local).gfdb_fop_type = ctr_inode_cx->fop_type;
-        CTR_DB_REC(ctr_local).link_consistency = _priv->ctr_link_consistency;
-
-        ret = gettimeofday (ctr_wtime, NULL);
-        if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                CTR_MSG_FILL_UNWIND_TIME_REC_ERROR,
-                                "Error filling wind time record %s",
-                                strerror(errno));
-                        goto out;
-        }
-
-        /* Special case i.e if its a tier rebalance
-         * + cold tier brick
-         * + its a create/mknod FOP
-         * we record wind time as zero */
-        if (ctr_local->client_pid == GF_CLIENT_PID_TIER_DEFRAG
-                && (!_priv->ctr_hot_brick)
-                && isdentrycreatefop(ctr_inode_cx->fop_type)) {
-                memset(ctr_wtime, 0, sizeof(*ctr_wtime));
-        }
-
-        /* Copy gfid into db record */
-        gf_uuid_copy (CTR_DB_REC(ctr_local).gfid, *(ctr_inode_cx->gfid));
-
-        /* Copy older gfid if any */
-        if (ctr_inode_cx->old_gfid &&
-                (!gf_uuid_is_null (*(ctr_inode_cx->old_gfid)))) {
-                gf_uuid_copy (CTR_DB_REC(ctr_local).old_gfid,
-                                *(ctr_inode_cx->old_gfid));
-        }
-
-        /*Hard Links*/
-        if (isdentryfop(ctr_inode_cx->fop_type)) {
-                /*new link fop*/
-                if (NEW_LINK_CX(ctr_inode_cx)) {
-                        gf_uuid_copy (CTR_DB_REC(ctr_local).pargfid,
-                                *((NEW_LINK_CX(ctr_inode_cx))->pargfid));
-                        strcpy (CTR_DB_REC(ctr_local).file_name,
-                                NEW_LINK_CX(ctr_inode_cx)->basename);
-                }
-                /*rename fop*/
-                if (OLD_LINK_CX(ctr_inode_cx)) {
-                        gf_uuid_copy (CTR_DB_REC(ctr_local).old_pargfid,
-                                *((OLD_LINK_CX(ctr_inode_cx))->pargfid));
-                        strcpy (CTR_DB_REC(ctr_local).old_file_name,
-                                OLD_LINK_CX(ctr_inode_cx)->basename);
-                }
-        }
-
-        ret = 0;
-out:
-        /*On error roll back and clean the record*/
-        if (ret == -1) {
-                CLEAR_CTR_DB_RECORD (ctr_local);
-        }
-        return ret;
-}
-
-
-/******************************************************************************
- *
- *                      CTR xlator init related functions
- *
- *
- * ****************************************************************************/
-static int
-extract_sql_params(xlator_t *this, dict_t *params_dict)
-{
-        int ret                         = -1;
-        char *db_path                   = NULL;
-        char *db_name                   = NULL;
-        char *db_full_path              = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (params_dict);
-
-        /*Extract the path of the db*/
-        db_path = NULL;
-        GET_DB_PARAM_FROM_DICT_DEFAULT(this->name, this->options, "db-path",
-                                        db_path, "/var/run/gluster/");
-
-        /*Extract the name of the db*/
-        db_name = NULL;
-        GET_DB_PARAM_FROM_DICT_DEFAULT(this->name, this->options, "db-name",
-                                        db_name, "gf_ctr_db.db");
-
-        /*Construct full path of the db*/
-        ret = gf_asprintf(&db_full_path, "%s/%s", db_path, db_name);
-        if (ret < 0) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        CTR_MSG_CONSTRUCT_DB_PATH_FAILED,
-                        "Construction of full db path failed!");
-                goto out;
-        }
-
-        /*Setting the SQL DB Path*/
-        SET_DB_PARAM_TO_DICT(this->name, params_dict, GFDB_SQL_PARAM_DBPATH,
-                                db_full_path, ret, out);
-
-        /*Extact rest of the sql params*/
-        ret = gfdb_set_sql_params(this->name, this->options, params_dict);
-        if (ret) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        CTR_MSG_SET_VALUE_TO_SQL_PARAM_FAILED,
-                        "Failed setting values to sql param dict!");
-        }
-
-        ret = 0;
-
-out:
-        if (ret)
-                GF_FREE (db_full_path);
-        return ret;
-}
-
-
-
-int extract_db_params(xlator_t *this, dict_t *params_dict,
-                                                gfdb_db_type_t db_type) {
-
-        int ret = -1;
-
-        GF_ASSERT (this);
-        GF_ASSERT (params_dict);
-
-        switch (db_type) {
-        case GFDB_SQLITE3:
-                ret = extract_sql_params(this, params_dict);
-                if (ret)
-                        goto out;
-                break;
-        case GFDB_ROCKS_DB:
-        case GFDB_HYPERDEX:
-        case GFDB_HASH_FILE_STORE:
-        case GFDB_INVALID_DB:
-        case GFDB_DB_END:
-                goto out;
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-int extract_ctr_options (xlator_t *this, gf_ctr_private_t *_priv) {
-        int ret         = -1;
-        char *_val_str  = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (_priv);
-
-        /*Checking if the CTR Translator is enabled. By default its disabled*/
-        _priv->enabled = _gf_false;
-        GF_OPTION_INIT ("ctr-enabled", _priv->enabled, bool, out);
-        if (!_priv->enabled) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_INFO, 0,
-                        CTR_MSG_XLATOR_DISABLED,
-                        "CTR Xlator is disabled.");
-                ret = 0;
-                goto out;
-        }
-
-        /*Extract db type*/
-        GF_OPTION_INIT ("db-type", _val_str, str, out);
-        _priv->gfdb_db_type = gf_string2gfdbdbtype(_val_str);
-
-        /*Extract flag for record on wind*/
-        GF_OPTION_INIT ("record-entry", _priv->ctr_record_wind, bool, out);
-
-        /*Extract flag for record on unwind*/
-        GF_OPTION_INIT ("record-exit", _priv->ctr_record_unwind, bool, out);
-
-        /*Extract flag for record on counters*/
-        GF_OPTION_INIT ("record-counters", _priv->ctr_record_counter, bool,
-                        out);
-
-        /* Extract flag for record metadata heat */
-        GF_OPTION_INIT ("ctr-record-metadata-heat",
-                        _priv->ctr_record_metadata_heat, bool,
-                        out);
-
-        /*Extract flag for link consistency*/
-        GF_OPTION_INIT ("ctr_link_consistency", _priv->ctr_link_consistency,
-                        bool, out);
-
-        /*Extract ctr_lookupheal_inode_timeout */
-        GF_OPTION_INIT ("ctr_lookupheal_inode_timeout",
-                        _priv->ctr_lookupheal_inode_timeout,
-                        uint64, out);
-
-        /*Extract ctr_lookupheal_link_timeout*/
-        GF_OPTION_INIT ("ctr_lookupheal_link_timeout",
-                        _priv->ctr_lookupheal_link_timeout,
-                        uint64, out);
-
-        /*Extract flag for hot tier brick*/
-        GF_OPTION_INIT ("hot-brick", _priv->ctr_hot_brick, bool, out);
-
-        /*Extract flag for sync mode*/
-        GF_OPTION_INIT ("db-sync", _val_str, str, out);
-        _priv->gfdb_sync_type = gf_string2gfdbdbsync(_val_str);
-
-        ret = 0;
-
-out:
-        return ret;
-}
diff --git a/xlators/features/changetimerecorder/src/ctr-helper.h b/xlators/features/changetimerecorder/src/ctr-helper.h
deleted file mode 100644
index 4fd4f745f4d..00000000000
--- a/xlators/features/changetimerecorder/src/ctr-helper.h
+++ /dev/null
@@ -1,927 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __CTR_HELPER_H
-#define __CTR_HELPER_H
-
-
-#include "xlator.h"
-#include "ctr_mem_types.h"
-#include "iatt.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "defaults.h"
-#include "logging.h"
-#include "common-utils.h"
-#include <time.h>
-#include <sys/time.h>
-#include <pthread.h>
-
-#include "gfdb_data_store.h"
-#include "ctr-xlator-ctx.h"
-#include "ctr-messages.h"
-
-#define CTR_DEFAULT_HARDLINK_EXP_PERIOD 300  /* Five mins */
-#define CTR_DEFAULT_INODE_EXP_PERIOD    300 /* Five mins */
-
-
-typedef struct ctr_query_cbk_args {
-        int query_fd;
-        int count;
-} ctr_query_cbk_args_t;
-
-
-/*CTR Xlator Private structure*/
-typedef struct gf_ctr_private {
-        gf_boolean_t                    enabled;
-        char                            *ctr_db_path;
-        gf_boolean_t                    ctr_hot_brick;
-        gf_boolean_t                    ctr_record_wind;
-        gf_boolean_t                    ctr_record_unwind;
-        gf_boolean_t                    ctr_record_counter;
-        gf_boolean_t                    ctr_record_metadata_heat;
-        gf_boolean_t                    ctr_link_consistency;
-        gfdb_db_type_t                  gfdb_db_type;
-        gfdb_sync_type_t                gfdb_sync_type;
-        gfdb_conn_node_t                *_db_conn;
-        uint64_t                        ctr_lookupheal_link_timeout;
-        uint64_t                        ctr_lookupheal_inode_timeout;
-        gf_boolean_t                    compact_active;
-        gf_boolean_t                    compact_mode_switched;
-        pthread_mutex_t                 compact_lock;
-} gf_ctr_private_t;
-
-
-/*
- * gf_ctr_local_t is the ctr xlator local data structure that is stored in
- * the call_frame of each FOP.
- *
- * gfdb_db_record: The gf_ctr_local contains a gfdb_db_record object, which is
- * used by the insert_record() api from the libgfdb. The gfdb_db_record object
- * will contain all the inode and hardlink(only for dentry fops: create,
- * mknod,link, unlink, rename).The ctr_local is keep alive till the unwind
- * call and will be release during the unwind. The same gfdb_db_record will
- * used for the unwind insert_record() api, to record unwind in the database.
- *
- * ia_inode_type in gf_ctr_local will tell the type of the inode. This is
- * important for during the unwind path. As we will not have the inode during
- * the unwind path. We would have include this in the gfdb_db_record itself
- * but currently we record only file inode information.
- *
- * is_internal_fop in gf_ctr_local will tell us if this is a internal fop and
- * take special/no action. We dont record change/acces times or increement heat
- * counter for internal fops from rebalancer.
- * */
-typedef struct gf_ctr_local {
-        gfdb_db_record_t        gfdb_db_record;
-        ia_type_t               ia_inode_type;
-        gf_boolean_t            is_internal_fop;
-        gf_special_pid_t        client_pid;
-} gf_ctr_local_t;
-/*
- * Easy access of gfdb_db_record of ctr_local
- * */
-#define CTR_DB_REC(ctr_local)\
-        (ctr_local->gfdb_db_record)
-
-/*Clear db record*/
-#define CLEAR_CTR_DB_RECORD(ctr_local)\
-do {\
-        ctr_local->gfdb_db_record.gfdb_fop_path = GFDB_FOP_INVALID;\
-        memset(&(ctr_local->gfdb_db_record.gfdb_wind_change_time),\
-                0, sizeof(gfdb_time_t));\
-        memset(&(ctr_local->gfdb_db_record.gfdb_unwind_change_time),\
-                0, sizeof(gfdb_time_t));\
-        gf_uuid_clear (ctr_local->gfdb_db_record.gfid);\
-        gf_uuid_clear (ctr_local->gfdb_db_record.pargfid);\
-        memset(ctr_local->gfdb_db_record.file_name, 0, GF_NAME_MAX + 1);\
-        memset(ctr_local->gfdb_db_record.old_file_name, 0, GF_NAME_MAX + 1);\
-        ctr_local->gfdb_db_record.gfdb_fop_type = GFDB_FOP_INVALID_OP;\
-        ctr_local->ia_inode_type = IA_INVAL;\
-} while (0)
-
-
-static gf_ctr_local_t *
-init_ctr_local_t (xlator_t *this) {
-
-        gf_ctr_local_t  *ctr_local     = NULL;
-
-        GF_ASSERT(this);
-
-        ctr_local = mem_get0 (this->local_pool);
-        if (!ctr_local) {
-                gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
-                        CTR_MSG_CREATE_CTR_LOCAL_ERROR_WIND,
-                        "Error while creating ctr local");
-                goto out;
-        }
-
-        CLEAR_CTR_DB_RECORD (ctr_local);
-out:
-        return ctr_local;
-}
-
-static void
-free_ctr_local (gf_ctr_local_t *ctr_local)
-{
-        if (ctr_local)
-                mem_put (ctr_local);
-}
-
-
-
-/******************************************************************************
- *
- *
- *                      Context Carrier Structures
- *
- *
- * ****************************************************************************/
-
-/*
- * Context Carrier structures are used to carry relavent information about
- * inodes and links from the fops calls to the ctr_insert_wind.
- * These structure just have pointers to the original data and donot
- * do a deep copy of any data. This info is deep copied to
- * ctr_local->gfdb_db_record and passed to insert_record() api of libgfdb. This
- * info remains persistent for the unwind in  ctr_local->gfdb_db_record
- * and once used will be destroyed.
- *
- * gf_ctr_link_context_t : Context structure for hard links
- * gf_ctr_inode_context_t : Context structure for inodes
- *
- * */
-
- /*Context Carrier Structure for hard links*/
-typedef struct gf_ctr_link_context {
-        uuid_t                  *pargfid;
-        const char              *basename;
-} gf_ctr_link_context_t;
-
- /*Context Carrier Structure for inodes*/
-typedef struct gf_ctr_inode_context {
-        ia_type_t               ia_type;
-        uuid_t                  *gfid;
-        uuid_t                  *old_gfid;
-        gf_ctr_link_context_t   *new_link_cx;
-        gf_ctr_link_context_t   *old_link_cx;
-        gfdb_fop_type_t         fop_type;
-        gfdb_fop_path_t         fop_path;
-        gf_boolean_t            is_internal_fop;
-        /* Indicating metadata fops */
-        gf_boolean_t            is_metadata_fop;
-} gf_ctr_inode_context_t;
-
-
-/*******************Util Macros for Context Carrier Structures*****************/
-
-/*Checks if ctr_link_cx is sane!*/
-#define IS_CTR_LINK_CX_SANE(ctr_link_cx)\
-do {\
-        if (ctr_link_cx) {\
-                if (ctr_link_cx->pargfid)\
-                        GF_ASSERT (*(ctr_link_cx->pargfid));\
-                GF_ASSERT (ctr_link_cx->basename);\
-        };\
-} while (0)
-
-/*Clear and fill the ctr_link_context with values*/
-#define FILL_CTR_LINK_CX(ctr_link_cx, _pargfid, _basename, label)\
-do {\
-        GF_VALIDATE_OR_GOTO ("ctr", ctr_link_cx, label);\
-        GF_VALIDATE_OR_GOTO ("ctr", _pargfid, label);\
-        GF_VALIDATE_OR_GOTO ("ctr", _basename, label);\
-        memset (ctr_link_cx, 0, sizeof (*ctr_link_cx));\
-        ctr_link_cx->pargfid = &_pargfid;\
-        ctr_link_cx->basename = _basename;\
-} while (0)
-
-#define NEW_LINK_CX(ctr_inode_cx)\
-        ctr_inode_cx->new_link_cx\
-
-#define OLD_LINK_CX(ctr_inode_cx)\
-        ctr_inode_cx->old_link_cx\
-
-/*Checks if ctr_inode_cx is sane!*/
-#define IS_CTR_INODE_CX_SANE(ctr_inode_cx)\
-do {\
-        GF_ASSERT (ctr_inode_cx);\
-        GF_ASSERT (ctr_inode_cx->gfid);\
-        GF_ASSERT (*(ctr_inode_cx->gfid));\
-        GF_ASSERT (ctr_inode_cx->fop_type != GFDB_FOP_INVALID_OP);\
-        GF_ASSERT (ctr_inode_cx->fop_path != GFDB_FOP_INVALID);\
-        IS_CTR_LINK_CX_SANE (NEW_LINK_CX(ctr_inode_cx));\
-        IS_CTR_LINK_CX_SANE (OLD_LINK_CX(ctr_inode_cx));\
-} while (0)
-
-/*Clear and fill the ctr_inode_context with values*/
-#define FILL_CTR_INODE_CONTEXT(ctr_inode_cx,\
-                                _ia_type,\
-                                _gfid,\
-                                _new_link_cx,\
-                                _old_link_cx,\
-                                _fop_type,\
-                                _fop_path)\
-do {\
-        GF_ASSERT (ctr_inode_cx);\
-        GF_ASSERT (_gfid);\
-        GF_ASSERT (_fop_type != GFDB_FOP_INVALID_OP);\
-        GF_ASSERT (_fop_path != GFDB_FOP_INVALID);\
-        memset(ctr_inode_cx, 0, sizeof(*ctr_inode_cx));\
-        ctr_inode_cx->ia_type = _ia_type;\
-        ctr_inode_cx->gfid = &_gfid;\
-        IS_CTR_LINK_CX_SANE(NEW_LINK_CX(ctr_inode_cx));\
-        if (_new_link_cx)\
-                NEW_LINK_CX(ctr_inode_cx) = _new_link_cx;\
-        IS_CTR_LINK_CX_SANE(OLD_LINK_CX(ctr_inode_cx));\
-        if (_old_link_cx)\
-                OLD_LINK_CX(ctr_inode_cx) = _old_link_cx;\
-        ctr_inode_cx->fop_type = _fop_type;\
-        ctr_inode_cx->fop_path = _fop_path;\
-} while (0)
-
-
-/******************************************************************************
- *
- *                      Util functions or macros used by
- *                      insert wind and insert unwind
- *
- * ****************************************************************************/
-/* Free ctr frame local */
-static inline void
-ctr_free_frame_local (call_frame_t *frame) {
-        if (frame) {
-                free_ctr_local ((gf_ctr_local_t *) frame->local);
-                frame->local = NULL;
-        }
-}
-
-/* Setting GF_REQUEST_LINK_COUNT_XDATA in dict
- * that has to be sent to POSIX Xlator to send
- * link count in unwind path.
- * return 0 for success with not creation of dict
- * return 1 for success with creation of dict
- * return -1 for failure.
- * */
-static inline int
-set_posix_link_request (xlator_t        *this,
-                        dict_t          **xdata)
-{
-        int ret                         = -1;
-        gf_boolean_t is_created         = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("ctr", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, xdata, out);
-
-        /*create xdata if NULL*/
-        if (!*xdata) {
-                *xdata = dict_new();
-                is_created = _gf_true;
-                ret = 1;
-        } else {
-                ret = 0;
-        }
-
-        if (!*xdata) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, CTR_MSG_XDATA_NULL,
-                        "xdata is NULL :Cannot send "
-                        "GF_REQUEST_LINK_COUNT_XDATA to posix");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_int32 (*xdata, GF_REQUEST_LINK_COUNT_XDATA, 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_SET_CTR_RESPONSE_LINK_COUNT_XDATA_FAILED,
-                        "Failed setting GF_REQUEST_LINK_COUNT_XDATA");
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
-out:
-        if (ret == -1) {
-                if (*xdata && is_created) {
-                        dict_unref (*xdata);
-                }
-        }
-        return ret;
-}
-
-
-/*
- * If a bitrot fop
- * */
-#define BITROT_FOP(frame)\
-        (frame->root->pid == GF_CLIENT_PID_BITD ||\
-        frame->root->pid == GF_CLIENT_PID_SCRUB)
-
-
-/*
- * If a rebalancer fop
- * */
-#define REBALANCE_FOP(frame)\
-        (frame->root->pid == GF_CLIENT_PID_DEFRAG)
-
-/*
- * If its a tiering rebalancer fop
- * */
-#define TIER_REBALANCE_FOP(frame)\
-        (frame->root->pid == GF_CLIENT_PID_TIER_DEFRAG)
-
-/*
- * If its a AFR SELF HEAL
- * */
- #define AFR_SELF_HEAL_FOP(frame)\
-        (frame->root->pid == GF_CLIENT_PID_SELF_HEALD)
-
-/*
- * if a rebalancer fop goto
- * */
-#define CTR_IF_REBALANCE_FOP_THEN_GOTO(frame, label)\
-do {\
-        if (REBALANCE_FOP (frame))\
-                goto label;\
-} while (0)
-
-/*
- * Internal fop
- *
- * */
-static inline gf_boolean_t
-is_internal_fop (call_frame_t *frame,
-                              dict_t       *xdata)
-{
-        gf_boolean_t ret = _gf_false;
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-
-        if (AFR_SELF_HEAL_FOP (frame)) {
-                ret = _gf_true;
-        }
-        if (BITROT_FOP (frame)) {
-                ret = _gf_true;
-        }
-        if (REBALANCE_FOP (frame) || TIER_REBALANCE_FOP (frame)) {
-                ret = _gf_true;
-                if (xdata && dict_get (xdata, CTR_ATTACH_TIER_LOOKUP)) {
-                        ret = _gf_false;
-                }
-        }
-        if (xdata && dict_get (xdata, GLUSTERFS_INTERNAL_FOP_KEY)) {
-                ret = _gf_true;
-        }
-
-        return ret;
-}
-
-#define CTR_IF_INTERNAL_FOP_THEN_GOTO(frame, dict, label)\
-do {\
-        if (is_internal_fop (frame, dict)) \
-                        goto label; \
-} while (0)
-
-/* if fop has failed exit */
-#define CTR_IF_FOP_FAILED_THEN_GOTO(this, op_ret, op_errno, label)\
-do {\
-        if (op_ret == -1) {\
-                gf_msg_trace (this->name, 0, "Failed fop with %s",\
-                              strerror (op_errno));\
-                goto label;\
-        };\
-} while (0)
-
-/*
- * IS CTR Xlator is disabled then goto to label
- * */
- #define CTR_IS_DISABLED_THEN_GOTO(this, label)\
- do {\
-        gf_ctr_private_t *_priv = NULL;\
-        GF_ASSERT (this);\
-        GF_ASSERT (this->private);\
-        _priv = this->private;\
-        if (!_priv->enabled)\
-                goto label;\
- } while (0)
-
-/*
- * IS CTR record metadata heat is disabled then goto to label
- * */
- #define CTR_RECORD_METADATA_HEAT_IS_DISABLED_THEN_GOTO(this, label)\
- do {\
-        gf_ctr_private_t *_priv = NULL;\
-        GF_ASSERT (this);\
-        GF_ASSERT (this->private);\
-        _priv = this->private;\
-        if (!_priv->ctr_record_metadata_heat)\
-                goto label;\
- } while (0)
-
-int
-fill_db_record_for_unwind (xlator_t              *this,
-                          gf_ctr_local_t        *ctr_local,
-                          gfdb_fop_type_t       fop_type,
-                          gfdb_fop_path_t       fop_path);
-
-int
-fill_db_record_for_wind (xlator_t                *this,
-                        gf_ctr_local_t          *ctr_local,
-                        gf_ctr_inode_context_t  *ctr_inode_cx);
-
-/*******************************************************************************
- *                              CTR INSERT WIND
- * *****************************************************************************
- * Function used to insert/update record into the database during a wind fop
- * This function creates ctr_local structure into the frame of the fop
- * call.
- * ****************************************************************************/
-
-static inline int
-ctr_insert_wind (call_frame_t                    *frame,
-                xlator_t                        *this,
-                gf_ctr_inode_context_t          *ctr_inode_cx)
-{
-        int ret                         = -1;
-        gf_ctr_private_t *_priv         = NULL;
-        gf_ctr_local_t *ctr_local       = NULL;
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-        GF_ASSERT(this);
-        IS_CTR_INODE_CX_SANE(ctr_inode_cx);
-
-        _priv = this->private;
-        GF_ASSERT (_priv);
-
-        GF_ASSERT(_priv->_db_conn);
-
-        /*If record_wind option of CTR is on record wind for
-         * regular files only*/
-        if (_priv->ctr_record_wind && ctr_inode_cx->ia_type != IA_IFDIR) {
-                frame->local = init_ctr_local_t (this);
-                if (!frame->local) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_CREATE_CTR_LOCAL_ERROR_WIND,
-                                "WIND: Error while creating ctr local");
-                        goto out;
-                };
-                ctr_local = frame->local;
-                ctr_local->client_pid = frame->root->pid;
-                ctr_local->is_internal_fop = ctr_inode_cx->is_internal_fop;
-
-                /* Decide whether to record counters or not */
-                CTR_DB_REC(ctr_local).do_record_counters = _gf_false;
-                /* If record counter is enabled */
-                if (_priv->ctr_record_counter) {
-                        /* If not a internal fop */
-                        if (!(ctr_local->is_internal_fop)) {
-                                /* If its a metadata fop AND
-                                 * record metadata heat
-                                 * OR
-                                 * its NOT a metadata fop */
-                                if ((ctr_inode_cx->is_metadata_fop
-                                        && _priv->ctr_record_metadata_heat)
-                                        ||
-                                        (!ctr_inode_cx->is_metadata_fop)) {
-                                        CTR_DB_REC(ctr_local).do_record_counters
-                                                = _gf_true;
-                                }
-                        }
-                }
-
-                /* Decide whether to record times or not
-                 * For non internal FOPS record times as usual*/
-                CTR_DB_REC(ctr_local).do_record_times = _gf_false;
-                if (!ctr_local->is_internal_fop) {
-                        /* If its a metadata fop AND
-                        * record metadata heat
-                        * OR
-                        * its NOT a metadata fop */
-                        if ((ctr_inode_cx->is_metadata_fop &&
-                                _priv->ctr_record_metadata_heat)
-                                ||
-                                (!ctr_inode_cx->is_metadata_fop)) {
-                                CTR_DB_REC(ctr_local).do_record_times =
-                                        (_priv->ctr_record_wind
-                                        || _priv->ctr_record_unwind);
-                        }
-                }
-                /* when its a internal FOPS*/
-                else {
-                        /* Record times only for create
-                         * i.e when the inode is created */
-                        CTR_DB_REC(ctr_local).do_record_times =
-                                (isdentrycreatefop(ctr_inode_cx->fop_type)) ?
-                                        _gf_true : _gf_false;
-                }
-
-                /*Fill the db record for insertion*/
-                ret = fill_db_record_for_wind (this, ctr_local, ctr_inode_cx);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_FILL_CTR_LOCAL_ERROR_WIND,
-                                "WIND: Error filling  ctr local");
-                        goto out;
-                }
-
-                /*Insert the db record*/
-                ret = insert_record (_priv->_db_conn,
-                                &ctr_local->gfdb_db_record);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_INSERT_RECORD_WIND_FAILED,
-                                "WIND: Inserting of record failed!");
-                        goto out;
-                }
-        }
-        ret = 0;
-out:
-
-        if (ret) {
-                free_ctr_local (ctr_local);
-                frame->local = NULL;
-        }
-
-        return ret;
-}
-
-
-
-
-/*******************************************************************************
- *                             CTR INSERT UNWIND
- * *****************************************************************************
- * Function used to insert/update record into the database during a unwind fop
- * This function destroys ctr_local structure into the frame of the fop
- * call at the end.
- * ****************************************************************************/
-static inline int
-ctr_insert_unwind (call_frame_t          *frame,
-                  xlator_t              *this,
-                  gfdb_fop_type_t       fop_type,
-                  gfdb_fop_path_t       fop_path)
-{
-        int ret = -1;
-        gf_ctr_private_t *_priv         = NULL;
-        gf_ctr_local_t *ctr_local       = NULL;
-
-        GF_ASSERT(frame);
-        GF_ASSERT(this);
-
-        _priv = this->private;
-        GF_ASSERT (_priv);
-
-        GF_ASSERT(_priv->_db_conn);
-
-        ctr_local = frame->local;
-
-        if (ctr_local
-            && (_priv->ctr_record_unwind || isdentryfop(fop_type))
-            && (ctr_local->ia_inode_type != IA_IFDIR)) {
-
-                CTR_DB_REC(ctr_local).do_record_uwind_time =
-                                                _priv->ctr_record_unwind;
-
-                ret = fill_db_record_for_unwind(this, ctr_local, fop_type,
-                                               fop_path);
-                if (ret == -1) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0,
-                               CTR_MSG_FILL_CTR_LOCAL_ERROR_UNWIND,
-                               "UNWIND: Error filling ctr local");
-                        goto out;
-                }
-
-                ret = insert_record(_priv->_db_conn,
-                                        &ctr_local->gfdb_db_record);
-                if (ret == -1) {
-                        gf_msg(this->name, GF_LOG_ERROR, 0,
-                               CTR_MSG_FILL_CTR_LOCAL_ERROR_UNWIND,
-                               "UNWIND: Error filling ctr local");
-                        goto out;
-                }
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-/******************************************************************************
- *                          Delete file/flink record/s from db
- * ****************************************************************************/
-static inline int
-ctr_delete_hard_link_from_db (xlator_t               *this,
-                              uuid_t                 gfid,
-                              uuid_t                 pargfid,
-                              char                   *basename,
-                              gfdb_fop_type_t        fop_type,
-                              gfdb_fop_path_t        fop_path)
-{
-        int                     ret                = -1;
-        gfdb_db_record_t        gfdb_db_record;
-        gf_ctr_private_t        *_priv             = NULL;
-
-        _priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, _priv, out);
-        GF_VALIDATE_OR_GOTO (this->name, (!gf_uuid_is_null (gfid)), out);
-        GF_VALIDATE_OR_GOTO (this->name, (!gf_uuid_is_null (pargfid)), out);
-        GF_VALIDATE_OR_GOTO (this->name, (fop_type == GFDB_FOP_DENTRY_WRITE),
-                             out);
-        GF_VALIDATE_OR_GOTO (this->name,
-                             (fop_path == GFDB_FOP_UNDEL || GFDB_FOP_UNDEL_ALL),
-                             out);
-
-        /* Set gfdb_db_record to 0 */
-        memset (&gfdb_db_record, 0, sizeof(gfdb_db_record));
-
-        /* Copy gfid into db record */
-        gf_uuid_copy (gfdb_db_record.gfid, gfid);
-
-        /* Copy pargid into db record */
-        gf_uuid_copy (gfdb_db_record.pargfid, pargfid);
-
-        /* Copy basename */
-        strncpy (gfdb_db_record.file_name, basename, GF_NAME_MAX - 1);
-
-        gfdb_db_record.gfdb_fop_path = fop_path;
-        gfdb_db_record.gfdb_fop_type = fop_type;
-
-        /*send delete request to db*/
-        ret = insert_record (_priv->_db_conn, &gfdb_db_record);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_INSERT_RECORD_WIND_FAILED,
-                        "Failed to delete record. %s", basename);
-                goto out;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-/******************************* Hard link function ***************************/
-
-static inline gf_boolean_t
-__is_inode_expired (ctr_xlator_ctx_t *ctr_xlator_ctx,
-                    gf_ctr_private_t *_priv,
-                    gfdb_time_t *current_time)
-{
-        gf_boolean_t    ret       = _gf_false;
-        uint64_t        time_diff = 0;
-
-        GF_ASSERT (ctr_xlator_ctx);
-        GF_ASSERT (_priv);
-        GF_ASSERT (current_time);
-
-        time_diff = current_time->tv_sec -
-                        ctr_xlator_ctx->inode_heal_period;
-
-        ret = (time_diff >= _priv->ctr_lookupheal_inode_timeout) ?
-                        _gf_true : _gf_false;
-        return ret;
-}
-
-static inline gf_boolean_t
-__is_hardlink_expired (ctr_hard_link_t *ctr_hard_link,
-                       gf_ctr_private_t *_priv,
-                       gfdb_time_t *current_time)
-{
-        gf_boolean_t    ret       = _gf_false;
-        uint64_t        time_diff = 0;
-
-        GF_ASSERT (ctr_hard_link);
-        GF_ASSERT (_priv);
-        GF_ASSERT (current_time);
-
-        time_diff = current_time->tv_sec -
-                        ctr_hard_link->hardlink_heal_period;
-
-        ret = ret || (time_diff >= _priv->ctr_lookupheal_link_timeout) ?
-                        _gf_true : _gf_false;
-
-        return ret;
-}
-
-
-/* Return values of heal*/
-typedef enum ctr_heal_ret_val {
-        CTR_CTX_ERROR = -1,
-        /* No healing required */
-        CTR_TRY_NO_HEAL = 0,
-        /* Try healing hard link */
-        CTR_TRY_HARDLINK_HEAL = 1,
-        /* Try healing inode */
-        CTR_TRY_INODE_HEAL = 2,
-} ctr_heal_ret_val_t;
-
-
-
-/**
- * @brief Function to add hard link to the inode context variable.
- *        The inode context maintainences a in-memory list. This is used
- *        smart healing of database.
- * @param frame of the FOP
- * @param this is the Xlator instant
- * @param inode
- * @return Return ctr_heal_ret_val_t
- */
-
-static inline ctr_heal_ret_val_t
-add_hard_link_ctx (call_frame_t *frame,
-                   xlator_t     *this,
-                   inode_t      *inode)
-{
-        ctr_heal_ret_val_t ret_val = CTR_TRY_NO_HEAL;
-        int ret = -1;
-        gf_ctr_local_t   *ctr_local       = NULL;
-        ctr_xlator_ctx_t *ctr_xlator_ctx  = NULL;
-        ctr_hard_link_t  *ctr_hard_link   = NULL;
-        gf_ctr_private_t *_priv           = NULL;
-        gfdb_time_t       current_time    = {0};
-
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (inode);
-        GF_ASSERT (this->private);
-
-        _priv = this->private;
-
-        ctr_local = frame->local;
-        if (!ctr_local) {
-                goto out;
-        }
-
-        ctr_xlator_ctx  = init_ctr_xlator_ctx (this, inode);
-        if (!ctr_xlator_ctx) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_ACCESS_CTR_INODE_CONTEXT_FAILED,
-                        "Failed accessing ctr inode context");
-                goto out;
-        }
-
-        LOCK (&ctr_xlator_ctx->lock);
-
-        /* Check if the hard link already exists
-         * in the ctr inode context*/
-        ctr_hard_link = ctr_search_hard_link_ctx (this,
-                                ctr_xlator_ctx,
-                                CTR_DB_REC(ctr_local).pargfid,
-                                CTR_DB_REC(ctr_local).file_name);
-        /* if there then ignore */
-        if (ctr_hard_link) {
-
-                ret = gettimeofday (&current_time, NULL);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get current time");
-                        ret_val = CTR_CTX_ERROR;
-                        goto unlock;
-                }
-
-                if (__is_hardlink_expired (ctr_hard_link,
-                                           _priv, &current_time)) {
-                        ctr_hard_link->hardlink_heal_period =
-                                        current_time.tv_sec;
-                        ret_val = ret_val | CTR_TRY_HARDLINK_HEAL;
-                }
-
-                if (__is_inode_expired (ctr_xlator_ctx,
-                                           _priv, &current_time)) {
-                        ctr_xlator_ctx->inode_heal_period =
-                                                current_time.tv_sec;
-                        ret_val = ret_val | CTR_TRY_INODE_HEAL;
-                }
-
-                goto unlock;
-        }
-
-        /* Add the hard link to the list*/
-        ret = ctr_add_hard_link (this, ctr_xlator_ctx,
-                        CTR_DB_REC(ctr_local).pargfid,
-                        CTR_DB_REC(ctr_local).file_name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_ADD_HARDLINK_TO_CTR_INODE_CONTEXT_FAILED,
-                        "Failed to add hardlink to the ctr inode context");
-                ret_val = CTR_CTX_ERROR;
-                goto unlock;
-        }
-
-        ret_val = CTR_TRY_NO_HEAL;
-unlock:
-        UNLOCK (&ctr_xlator_ctx->lock);
-out:
-        return ret_val;
-}
-
-static inline int
-delete_hard_link_ctx (call_frame_t *frame,
-                      xlator_t     *this,
-                      inode_t      *inode)
-{
-        int ret = -1;
-        ctr_xlator_ctx_t *ctr_xlator_ctx  = NULL;
-        gf_ctr_local_t   *ctr_local       = NULL;
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (inode);
-
-        ctr_local = frame->local;
-        if (!ctr_local) {
-                goto out;
-        }
-
-        ctr_xlator_ctx = get_ctr_xlator_ctx (this, inode);
-        if (!ctr_xlator_ctx) {
-                /* Since there is no ctr inode context so nothing more to do */
-                ret = 0;
-                goto out;
-        }
-
-        ret = ctr_delete_hard_link (this, ctr_xlator_ctx,
-                        CTR_DB_REC(ctr_local).pargfid,
-                        CTR_DB_REC(ctr_local).file_name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_DELETE_HARDLINK_FAILED,
-                        "Failed to delete hard link");
-                goto out;
-        }
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-static inline int
-update_hard_link_ctx (call_frame_t *frame,
-                      xlator_t     *this,
-                      inode_t      *inode)
-{
-        int ret = -1;
-        ctr_xlator_ctx_t *ctr_xlator_ctx  = NULL;
-        gf_ctr_local_t   *ctr_local       = NULL;
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (inode);
-
-        ctr_local = frame->local;
-        if (!ctr_local) {
-                goto out;
-        }
-
-        ctr_xlator_ctx  = init_ctr_xlator_ctx (this, inode);
-        if (!ctr_xlator_ctx) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_ACCESS_CTR_INODE_CONTEXT_FAILED,
-                        "Failed accessing ctr inode context");
-                goto out;
-        }
-
-        ret = ctr_update_hard_link (this, ctr_xlator_ctx,
-                        CTR_DB_REC(ctr_local).pargfid,
-                        CTR_DB_REC(ctr_local).file_name,
-                        CTR_DB_REC(ctr_local).old_pargfid,
-                        CTR_DB_REC(ctr_local).old_file_name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_DELETE_HARDLINK_FAILED,
-                        "Failed to delete hard link");
-                goto out;
-        }
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-
-/******************************************************************************
- *
- *                      CTR xlator init related functions
- *
- *
- * ****************************************************************************/
-int
-extract_db_params (xlator_t              *this,
-                  dict_t                *params_dict,
-                  gfdb_db_type_t        db_type);
-
-int
-extract_ctr_options (xlator_t            *this,
-                    gf_ctr_private_t    *_priv);
-
-#endif
diff --git a/xlators/features/changetimerecorder/src/ctr-messages.h b/xlators/features/changetimerecorder/src/ctr-messages.h
deleted file mode 100644
index bc9a9e0f3ab..00000000000
--- a/xlators/features/changetimerecorder/src/ctr-messages.h
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
- This file is part of GlusterFS.
-
- This file is licensed to you under your choice of the GNU Lesser
- General Public License, version 3 or any later version (LGPLv3 or
- later), or the GNU General Public License, version 2 (GPLv2), in all
- cases as published by the Free Software Foundation.
- */
-
-#ifndef _CTR_MESSAGES_H_
-#define _CTR_MESSAGES_H_
-
-#include "glfs-message-id.h"
-
-/* To add new message IDs, append new identifiers at the end of the list.
- *
- * Never remove a message ID. If it's not used anymore, you can rename it or
- * leave it as it is, but not delete it. This is to prevent reutilization of
- * IDs by other messages.
- *
- * The component name must match one of the entries defined in
- * glfs-message-id.h.
- */
-
-GLFS_MSGID(CTR,
-        CTR_MSG_CREATE_CTR_LOCAL_ERROR_WIND,
-        CTR_MSG_FILL_CTR_LOCAL_ERROR_UNWIND,
-        CTR_MSG_FILL_CTR_LOCAL_ERROR_WIND,
-        CTR_MSG_INSERT_LINK_WIND_FAILED,
-        CTR_MSG_INSERT_WRITEV_WIND_FAILED,
-        CTR_MSG_INSERT_WRITEV_UNWIND_FAILED,
-        CTR_MSG_INSERT_SETATTR_WIND_FAILED,
-        CTR_MSG_INSERT_SETATTR_UNWIND_FAILED,
-        CTR_MSG_INSERT_FREMOVEXATTR_UNWIND_FAILED,
-        CTR_MSG_INSERT_FREMOVEXATTR_WIND_FAILED,
-        CTR_MSG_INSERT_REMOVEXATTR_WIND_FAILED,
-        CTR_MSG_INSERT_REMOVEXATTR_UNWIND_FAILED,
-        CTR_MSG_INSERT_TRUNCATE_WIND_FAILED,
-        CTR_MSG_INSERT_TRUNCATE_UNWIND_FAILED,
-        CTR_MSG_INSERT_FTRUNCATE_UNWIND_FAILED,
-        CTR_MSG_INSERT_FTRUNCATE_WIND_FAILED,
-        CTR_MSG_INSERT_RENAME_WIND_FAILED,
-        CTR_MSG_INSERT_RENAME_UNWIND_FAILED,
-        CTR_MSG_ACCESS_CTR_INODE_CONTEXT_FAILED,
-        CTR_MSG_ADD_HARDLINK_FAILED,
-        CTR_MSG_DELETE_HARDLINK_FAILED,
-        CTR_MSG_UPDATE_HARDLINK_FAILED,
-        CTR_MSG_GET_CTR_RESPONSE_LINK_COUNT_XDATA_FAILED,
-        CTR_MSG_SET_CTR_RESPONSE_LINK_COUNT_XDATA_FAILED,
-        CTR_MSG_INSERT_UNLINK_UNWIND_FAILED,
-        CTR_MSG_INSERT_UNLINK_WIND_FAILED,
-        CTR_MSG_XDATA_NULL,
-        CTR_MSG_INSERT_FSYNC_WIND_FAILED,
-        CTR_MSG_INSERT_FSYNC_UNWIND_FAILED,
-        CTR_MSG_INSERT_MKNOD_UNWIND_FAILED,
-        CTR_MSG_INSERT_MKNOD_WIND_FAILED,
-        CTR_MSG_INSERT_CREATE_WIND_FAILED,
-        CTR_MSG_INSERT_CREATE_UNWIND_FAILED,
-        CTR_MSG_INSERT_RECORD_WIND_FAILED,
-        CTR_MSG_INSERT_READV_WIND_FAILED,
-        CTR_MSG_GET_GFID_FROM_DICT_FAILED,
-        CTR_MSG_SET,
-        CTR_MSG_FATAL_ERROR,
-        CTR_MSG_DANGLING_VOLUME,
-        CTR_MSG_CALLOC_FAILED,
-        CTR_MSG_EXTRACT_CTR_XLATOR_OPTIONS_FAILED,
-        CTR_MSG_INIT_DB_PARAMS_FAILED,
-        CTR_MSG_CREATE_LOCAL_MEMORY_POOL_FAILED,
-        CTR_MSG_MEM_ACC_INIT_FAILED,
-        CTR_MSG_CLOSE_DB_CONN_FAILED,
-        CTR_MSG_FILL_UNWIND_TIME_REC_ERROR,
-        CTR_MSG_WRONG_FOP_PATH,
-        CTR_MSG_CONSTRUCT_DB_PATH_FAILED,
-        CTR_MSG_SET_VALUE_TO_SQL_PARAM_FAILED,
-        CTR_MSG_XLATOR_DISABLED,
-        CTR_MSG_HARDLINK_MISSING_IN_LIST,
-        CTR_MSG_ADD_HARDLINK_TO_LIST_FAILED,
-        CTR_MSG_INIT_LOCK_FAILED,
-        CTR_MSG_COPY_FAILED,
-        CTR_MSG_EXTRACT_DB_PARAM_OPTIONS_FAILED,
-        CTR_MSG_ADD_HARDLINK_TO_CTR_INODE_CONTEXT_FAILED,
-        CTR_MSG_NULL_LOCAL
-);
-
-#endif /* !_CTR_MESSAGES_H_ */
diff --git a/xlators/features/changetimerecorder/src/ctr-xlator-ctx.c b/xlators/features/changetimerecorder/src/ctr-xlator-ctx.c
deleted file mode 100644
index 7700ad40ba6..00000000000
--- a/xlators/features/changetimerecorder/src/ctr-xlator-ctx.c
+++ /dev/null
@@ -1,409 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#include "ctr-xlator-ctx.h"
-#include "ctr-messages.h"
-#include <time.h>
-#include <sys/time.h>
-
-#define IS_THE_ONLY_HARDLINK(ctr_hard_link)\
-        (ctr_hard_link->list.next == ctr_hard_link->list.prev)
-
-
-static void
-fini_ctr_hard_link (ctr_hard_link_t **ctr_hard_link) {
-
-        GF_ASSERT (ctr_hard_link);
-
-        if (*ctr_hard_link)
-                return;
-        GF_FREE ((*ctr_hard_link)->base_name);
-        GF_FREE (*ctr_hard_link);
-        *ctr_hard_link = NULL;
-}
-
-
-/* Please lock the ctr_xlator_ctx before using this function */
-ctr_hard_link_t *
-ctr_search_hard_link_ctx (xlator_t                  *this,
-                            ctr_xlator_ctx_t        *ctr_xlator_ctx,
-                            uuid_t                  pgfid,
-                            const char              *base_name)
-{
-        ctr_hard_link_t *_hard_link             = NULL;
-        ctr_hard_link_t *searched_hardlink      = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (ctr_xlator_ctx);
-
-        if (pgfid == NULL || base_name == NULL)
-                goto out;
-
-        /*linear search*/
-        list_for_each_entry (_hard_link, &ctr_xlator_ctx->hardlink_list, list) {
-                if (gf_uuid_compare (_hard_link->pgfid, pgfid) == 0
-                        && _hard_link->base_name
-                        && strcmp(_hard_link->base_name, base_name) == 0) {
-                        searched_hardlink = _hard_link;
-                        break;
-                }
-        }
-
-out:
-        return searched_hardlink;
-}
-
-
-
-
-/* Please lock the ctr_xlator_ctx before using this function */
-int
-ctr_add_hard_link (xlator_t             *this,
-               ctr_xlator_ctx_t         *ctr_xlator_ctx,
-               uuid_t                   pgfid,
-               const char               *base_name)
-{
-        int ret                                 = -1;
-        ctr_hard_link_t *ctr_hard_link          = NULL;
-        struct timeval  current_time                   = {0};
-
-        GF_ASSERT (this);
-        GF_ASSERT (ctr_xlator_ctx);
-
-        if (pgfid == NULL || base_name == NULL)
-                goto out;
-
-        ctr_hard_link = GF_CALLOC (1, sizeof (*ctr_hard_link),
-                                        gf_ctr_mt_hard_link_t);
-        if (!ctr_hard_link) {
-                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
-                        CTR_MSG_CALLOC_FAILED, "Failed allocating "
-                        "ctr_hard_link");
-                goto out;
-        }
-
-        /*Initialize the ctr_hard_link object and
-         * Assign the values : parent GFID and basename*/
-        INIT_LIST_HEAD (&ctr_hard_link->list);
-        gf_uuid_copy (ctr_hard_link->pgfid, pgfid);
-        ret = gf_asprintf(&ctr_hard_link->base_name, "%s", base_name);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_COPY_FAILED, "Failed copying basename"
-                        "to ctr_hard_link");
-                goto error;
-        }
-
-        ret = gettimeofday (&current_time, NULL);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get current time");
-                goto error;
-        }
-
-        /*Add the hard link to the list*/
-        list_add_tail (&ctr_hard_link->list,
-                        &ctr_xlator_ctx->hardlink_list);
-
-        ctr_hard_link->hardlink_heal_period = current_time.tv_sec;
-
-        /*aal izz well!*/
-        ret = 0;
-        goto out;
-error:
-        GF_FREE (ctr_hard_link);
-out:
-        return ret;
-}
-
-static void
-__delete_hard_link_from_list (ctr_hard_link_t  **ctr_hard_link)
-{
-        GF_ASSERT (ctr_hard_link);
-        GF_ASSERT (*ctr_hard_link);
-
-        /*Remove hard link from list*/
-        list_del(&(*ctr_hard_link)->list);
-        fini_ctr_hard_link (ctr_hard_link);
-}
-
-
-int
-ctr_delete_hard_link (xlator_t                *this,
-                  ctr_xlator_ctx_t      *ctr_xlator_ctx,
-                  uuid_t                pgfid,
-                  const char            *base_name)
-{
-        int ret                         = -1;
-        ctr_hard_link_t *ctr_hard_link     = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (ctr_xlator_ctx);
-
-
-        LOCK (&ctr_xlator_ctx->lock);
-
-        /*Check if the hard link is present */
-        ctr_hard_link = ctr_search_hard_link_ctx (this, ctr_xlator_ctx,
-                                                        pgfid, base_name);
-        if (!ctr_hard_link) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_HARDLINK_MISSING_IN_LIST,
-                        "Hard link doesn't exist in the list");
-                goto out;
-        }
-
-        __delete_hard_link_from_list (&ctr_hard_link);
-        ctr_hard_link = NULL;
-
-        ret = 0;
-out:
-        UNLOCK (&ctr_xlator_ctx->lock);
-
-        return ret;
-}
-
-
-
-
-int
-ctr_update_hard_link (xlator_t                *this,
-                  ctr_xlator_ctx_t      *ctr_xlator_ctx,
-                  uuid_t                pgfid,
-                  const char            *base_name,
-                  uuid_t                old_pgfid,
-                  const char            *old_base_name)
-{
-        int ret                            = -1;
-        ctr_hard_link_t *ctr_hard_link     = NULL;
-        struct timeval current_time        = {0};
-
-        GF_ASSERT (this);
-        GF_ASSERT (ctr_xlator_ctx);
-
-
-        LOCK (&ctr_xlator_ctx->lock);
-
-        /*Check if the hard link is present */
-        ctr_hard_link = ctr_search_hard_link_ctx (this, ctr_xlator_ctx,
-                                                old_pgfid, old_base_name);
-        if (!ctr_hard_link) {
-                gf_msg_trace (this->name, 0, "Hard link doesn't exist"
-                              " in the list");
-                /* Since the hard link is not present in the list
-                 * we add it to the list */
-                ret = ctr_add_hard_link (this, ctr_xlator_ctx,
-                                        pgfid, base_name);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                CTR_MSG_ADD_HARDLINK_TO_LIST_FAILED,
-                                "Failed adding hard link to the list");
-                        goto out;
-                }
-                ret = 0;
-                goto out;
-        }
-
-        /* update the hard link */
-        gf_uuid_copy (ctr_hard_link->pgfid, pgfid);
-        GF_FREE (ctr_hard_link->base_name);
-        ret = gf_asprintf(&ctr_hard_link->base_name, "%s", base_name);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        CTR_MSG_COPY_FAILED, "Failed copying basename"
-                                        "to ctr_hard_link");
-                /* delete the corrupted entry */
-                __delete_hard_link_from_list (&ctr_hard_link);
-                ctr_hard_link = NULL;
-                goto out;
-        }
-
-        ret = gettimeofday (&current_time, NULL);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get current time");
-                ctr_hard_link->hardlink_heal_period = 0;
-        } else {
-                ctr_hard_link->hardlink_heal_period = current_time.tv_sec;
-        }
-
-        ret = 0;
-
-out:
-        UNLOCK (&ctr_xlator_ctx->lock);
-
-        return ret;
-}
-
-
-
-
-/* Delete all hardlinks */
-static int
-ctr_delete_all_hard_link (xlator_t *this,
-                          ctr_xlator_ctx_t *ctr_xlator_ctx)
-{
-        int ret = -1;
-        ctr_hard_link_t *ctr_hard_link     = NULL;
-        ctr_hard_link_t *tmp               = NULL;
-
-        GF_ASSERT (ctr_xlator_ctx);
-
-        LOCK (&ctr_xlator_ctx->lock);
-
-        list_for_each_entry_safe(ctr_hard_link, tmp,
-                        &ctr_xlator_ctx->hardlink_list, list)
-        {
-                /*Remove hard link from list*/
-                __delete_hard_link_from_list (&ctr_hard_link);
-                ctr_hard_link = NULL;
-
-        }
-
-
-        UNLOCK (&ctr_xlator_ctx->lock);
-
-        ret = 0;
-
-        return ret;
-}
-
-
-/* Please lock the inode before using this function */
-static ctr_xlator_ctx_t *
-__get_ctr_xlator_ctx (xlator_t  *this,
-                      inode_t   *inode)
-{
-        int ret                                 = 0;
-        uint64_t _addr                          = 0;
-        ctr_xlator_ctx_t *ctr_xlator_ctx        = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (inode);
-
-        ret = __inode_ctx_get (inode, this, &_addr);
-        if (ret < 0)
-                _addr = 0;
-        if (_addr != 0) {
-                ctr_xlator_ctx = (ctr_xlator_ctx_t *) (long)_addr;
-        }
-
-        return ctr_xlator_ctx;
-}
-
-
-ctr_xlator_ctx_t *
-init_ctr_xlator_ctx (xlator_t *this,
-                     inode_t *inode)
-{
-        int ret                                 = -1;
-        uint64_t _addr                          = 0;
-        ctr_xlator_ctx_t *ctr_xlator_ctx        = NULL;
-        struct timeval current_time             = {0};
-
-        GF_ASSERT (this);
-        GF_ASSERT (inode);
-
-        LOCK (&inode->lock);
-        {
-                ctr_xlator_ctx = __get_ctr_xlator_ctx (this, inode);
-                if (ctr_xlator_ctx) {
-                        ret = 0;
-                        goto out;
-                }
-                ctr_xlator_ctx = GF_CALLOC (1, sizeof (*ctr_xlator_ctx),
-                                                        gf_ctr_mt_xlator_ctx);
-                if (!ctr_xlator_ctx)
-                        goto out;
-
-                ret = LOCK_INIT (&ctr_xlator_ctx->lock);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, ret,
-                                CTR_MSG_INIT_LOCK_FAILED,
-                                "Failed init lock %s", strerror(ret));
-                        goto out;
-                }
-                _addr = (uint64_t) ctr_xlator_ctx;
-
-                ret = __inode_ctx_set (inode, this, &_addr);
-                if (ret) {
-                        goto out;
-                }
-
-                INIT_LIST_HEAD (&ctr_xlator_ctx->hardlink_list);
-
-                ret = gettimeofday (&current_time, NULL);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get current time");
-                        goto out;
-                }
-
-                ctr_xlator_ctx->inode_heal_period = current_time.tv_sec;
-        }
-        ret = 0;
-out:
-        if (ret) {
-                GF_FREE (ctr_xlator_ctx);
-                ctr_xlator_ctx = NULL;
-        }
-
-        UNLOCK (&inode->lock);
-
-        return ctr_xlator_ctx;
-}
-
-
-
-
-void
-fini_ctr_xlator_ctx (xlator_t *this,
-                     inode_t *inode)
-{
-        int ret                                 = 0;
-        uint64_t _addr                          = 0;
-        ctr_xlator_ctx_t *ctr_xlator_ctx        = NULL;
-
-
-        inode_ctx_del (inode, this, &_addr);
-        if (!_addr)
-                return;
-
-        ctr_xlator_ctx = (ctr_xlator_ctx_t *) (long) _addr;
-
-        ret = ctr_delete_all_hard_link (this, ctr_xlator_ctx);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING , 0,
-                        CTR_MSG_DELETE_HARDLINK_FAILED, "Failed deleting all "
-                        "hard links from inode context");
-        }
-
-        LOCK_DESTROY (&ctr_xlator_ctx->lock);
-
-        GF_FREE (ctr_xlator_ctx);
-
-}
-
-
-
-
-ctr_xlator_ctx_t *
-get_ctr_xlator_ctx (xlator_t *this,
-                    inode_t *inode)
-{
-        ctr_xlator_ctx_t *ctr_xlator_ctx = NULL;
-
-        LOCK (&inode->lock);
-        ctr_xlator_ctx = __get_ctr_xlator_ctx (this, inode);
-        UNLOCK (&inode->lock);
-
-        return ctr_xlator_ctx;
-}
-
diff --git a/xlators/features/changetimerecorder/src/ctr-xlator-ctx.h b/xlators/features/changetimerecorder/src/ctr-xlator-ctx.h
deleted file mode 100644
index 7f1c6cb1712..00000000000
--- a/xlators/features/changetimerecorder/src/ctr-xlator-ctx.h
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __CTR_XLATOR_CTX_H
-#define __CTR_XLATOR_CTX_H
-
-#include "xlator.h"
-#include "ctr_mem_types.h"
-#include "iatt.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "locking.h"
-#include "common-utils.h"
-#include <time.h>
-#include <sys/time.h>
-
-typedef struct ctr_hard_link {
-        uuid_t                  pgfid;
-        char                    *base_name;
-        /* Hardlink expiry : Defines the expiry period after which a
-         * database heal is attempted. */
-        uint64_t                  hardlink_heal_period;
-        struct list_head        list;
-} ctr_hard_link_t;
-
-typedef struct ctr_xlator_ctx {
-        /* This represents the looked up hardlinks
-         * NOTE: This doesn't represent all physical hardlinks of the inode*/
-        struct list_head        hardlink_list;
-        uint64_t                inode_heal_period;
-        gf_lock_t               lock;
-} ctr_xlator_ctx_t;
-
-
-ctr_hard_link_t *
-ctr_search_hard_link_ctx (xlator_t                  *this,
-                          ctr_xlator_ctx_t        *ctr_xlator_ctx,
-                          uuid_t                  pgfid,
-                          const char              *base_name);
-
-
-int
-ctr_add_hard_link (xlator_t          *this,
-               ctr_xlator_ctx_t         *ctr_xlator_ctx,
-               uuid_t                   pgfid,
-               const char               *base_name);
-
-
-
-int
-ctr_delete_hard_link (xlator_t                *this,
-                  ctr_xlator_ctx_t      *ctr_xlator_ctx,
-                  uuid_t                pgfid,
-                  const char            *base_name);
-
-
-int
-ctr_update_hard_link (xlator_t                *this,
-                  ctr_xlator_ctx_t      *ctr_xlator_ctx,
-                  uuid_t                pgfid,
-                  const char            *base_name,
-                  uuid_t                old_pgfid,
-                  const char            *old_base_name);
-
-
-ctr_xlator_ctx_t *
-get_ctr_xlator_ctx (xlator_t *this,
-                    inode_t *inode);
-
-
-
-
-ctr_xlator_ctx_t *
-init_ctr_xlator_ctx (xlator_t *this,
-                     inode_t *inode);
-
-
-void
-fini_ctr_xlator_ctx (xlator_t *this,
-                     inode_t *inode);
-
-#endif
diff --git a/xlators/features/changetimerecorder/src/ctr_mem_types.h b/xlators/features/changetimerecorder/src/ctr_mem_types.h
deleted file mode 100644
index f408c028e24..00000000000
--- a/xlators/features/changetimerecorder/src/ctr_mem_types.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
-  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef __CTR_MEM_TYPES_H__
-#define __CTR_MEM_TYPES_H__
-
-#include "gfdb_mem-types.h"
-
-enum gf_ctr_mem_types_ {
-        gf_ctr_mt_private_t = gfdb_mt_end + 1,
-        gf_ctr_mt_xlator_ctx,
-        gf_ctr_mt_hard_link_t,
-        gf_ctr_mt_end
-};
-#endif
-
diff --git a/xlators/features/cloudsync/src/Makefile.am b/xlators/features/cloudsync/src/Makefile.am
index 4e8319c9d34..e2a277e372b 100644
--- a/xlators/features/cloudsync/src/Makefile.am
+++ b/xlators/features/cloudsync/src/Makefile.am
@@ -1,3 +1,5 @@
+SUBDIRS = cloudsync-plugins
+
 xlator_LTLIBRARIES = cloudsync.la
 
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
@@ -10,9 +12,9 @@ CLOUDSYNC_BLD = $(top_builddir)/xlators/features/cloudsync/src
 cloudsynccommon_sources = $(CLOUDSYNC_SRC)/cloudsync-common.c
 
 noinst_HEADERS = $(CLOUDSYNC_BLD)/cloudsync.h \
-	$(CLOUDSYNC_BLD)/cloudsync-mem-types.h \
-	$(CLOUDSYNC_BLD)/cloudsync-messages.h \
-	$(CLOUDSYNC_BLD)/cloudsync-common.h
+		 $(CLOUDSYNC_BLD)/cloudsync-mem-types.h \
+		 $(CLOUDSYNC_BLD)/cloudsync-messages.h \
+		 $(CLOUDSYNC_BLD)/cloudsync-common.h
 
 cloudsync_la_SOURCES = $(cloudsync_sources) $(cloudsynccommon_sources)
 
@@ -21,10 +23,10 @@ BUILT_SOURCES = cloudsync-autogen-fops.h
 
 cloudsync_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
-cloudsync_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+cloudsync_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(LIB_DL)
 
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	      -DCS_PLUGINDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/cloudsync-plugins\"
 AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
 
 noinst_PYTHON = cloudsync-fops-c.py cloudsync-fops-h.py
@@ -42,4 +44,3 @@ CLEANFILES = $(nodist_cloudsync_la_SOURCES)
 
 uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/cloudsync.so
-
diff --git a/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.c b/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.c
index 6bb68cd170c..ee63f983980 100644
--- a/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.c
+++ b/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.c
@@ -20,11 +20,11 @@
 
 #include <dlfcn.h>
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 #include "cloudsync.h"
 #include "cloudsync-common.h"
-#include "call-stub.h"
+#include <glusterfs/call-stub.h>
 
 #pragma generate
diff --git a/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.h b/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.h
index 2db2a9c88c7..d922c77d8aa 100644
--- a/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.h
+++ b/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.h
@@ -15,7 +15,7 @@
 #ifndef _CLOUDSYNC_AUTOGEN_FOPS_H
 #define _CLOUDSYNC_AUTOGEN_FOPS_H
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "cloudsync.h"
 #include "cloudsync-common.h"
 
diff --git a/xlators/features/cloudsync/src/cloudsync-common.c b/xlators/features/cloudsync/src/cloudsync-common.c
index d0d00decca3..445a31b90e7 100644
--- a/xlators/features/cloudsync/src/cloudsync-common.c
+++ b/xlators/features/cloudsync/src/cloudsync-common.c
@@ -11,34 +11,50 @@
 #include "cloudsync-common.h"
 
 void
-cs_local_wipe (xlator_t *this, cs_local_t *local)
+cs_xattrinfo_wipe(cs_local_t *local)
 {
-        if (!local)
-                return;
+    if (local->xattrinfo.lxattr) {
+        if (local->xattrinfo.lxattr->file_path)
+            GF_FREE(local->xattrinfo.lxattr->file_path);
 
-        loc_wipe (&local->loc);
+        if (local->xattrinfo.lxattr->volname)
+            GF_FREE(local->xattrinfo.lxattr->volname);
 
-        if (local->fd) {
-                fd_unref (local->fd);
-                local->fd = NULL;
-        }
+        GF_FREE(local->xattrinfo.lxattr);
+    }
+}
+
+void
+cs_local_wipe(xlator_t *this, cs_local_t *local)
+{
+    if (!local)
+        return;
+
+    loc_wipe(&local->loc);
+
+    if (local->fd) {
+        fd_unref(local->fd);
+        local->fd = NULL;
+    }
+
+    if (local->stub) {
+        call_stub_destroy(local->stub);
+        local->stub = NULL;
+    }
 
-        if (local->stub) {
-                call_stub_destroy (local->stub);
-                local->stub = NULL;
-        }
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
 
-        if (local->xattr_req)
-                dict_unref (local->xattr_req);
+    if (local->xattr_rsp)
+        dict_unref(local->xattr_rsp);
 
-        if (local->xattr_rsp)
-                dict_unref (local->xattr_rsp);
+    if (local->dlfd)
+        fd_unref(local->dlfd);
 
-        if (local->dlfd)
-                fd_unref (local->dlfd);
+    if (local->remotepath)
+        GF_FREE(local->remotepath);
 
-        if (local->remotepath)
-                GF_FREE (local->remotepath);
+    cs_xattrinfo_wipe(local);
 
-        mem_put (local);
+    mem_put(local);
 }
diff --git a/xlators/features/cloudsync/src/cloudsync-common.h b/xlators/features/cloudsync/src/cloudsync-common.h
index 3298ab0a6f2..11d233460a4 100644
--- a/xlators/features/cloudsync/src/cloudsync-common.h
+++ b/xlators/features/cloudsync/src/cloudsync-common.h
@@ -10,88 +10,125 @@
 #ifndef _CLOUDSYNC_COMMON_H
 #define _CLOUDSYNC_COMMON_H
 
-#include "glusterfs.h"
-#include "call-stub.h"
-#include "xlator.h"
-#include "syncop.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/compat-errno.h>
 #include "cloudsync-mem-types.h"
 #include "cloudsync-messages.h"
 
+typedef struct cs_loc_xattr {
+    char *file_path;
+    uuid_t uuid;
+    uuid_t gfid;
+    char *volname;
+} cs_loc_xattr_t;
+
+typedef struct cs_size_xattr {
+    uint64_t size;
+    uint64_t blksize;
+    uint64_t blocks;
+} cs_size_xattr_t;
+
 typedef struct cs_local {
-        loc_t           loc;
-        fd_t            *fd;
-        call_stub_t     *stub;
-        call_frame_t    *main_frame;
-        int              op_errno;
-        int              op_ret;
-        fd_t            *dlfd;
-        off_t            dloffset;
-        struct iatt      stbuf;
-        dict_t          *xattr_rsp;
-        dict_t          *xattr_req;
-        glusterfs_fop_t  fop;
-        gf_boolean_t     locked;
-        int              call_cnt;
-        inode_t         *inode;
-        char            *remotepath;
+    loc_t loc;
+    fd_t *fd;
+    call_stub_t *stub;
+    call_frame_t *main_frame;
+    int op_errno;
+    int op_ret;
+    fd_t *dlfd;
+    off_t dloffset;
+    struct iatt stbuf;
+    dict_t *xattr_rsp;
+    dict_t *xattr_req;
+    glusterfs_fop_t fop;
+    gf_boolean_t locked;
+    int call_cnt;
+    inode_t *inode;
+    char *remotepath;
+
+    struct {
+        /* offset, flags and size are the information needed
+         * by read fop for remote read operation. These will be
+         * populated in cloudsync read fop, before being passed
+         * on to the plugin performing remote read.
+         */
+        off_t offset;
+        uint32_t flags;
+        size_t size;
+        cs_loc_xattr_t *lxattr;
+    } xattrinfo;
+
 } cs_local_t;
 
-typedef int (*fop_download_t) (call_frame_t *frame, void *config);
+typedef int (*fop_download_t)(call_frame_t *frame, void *config);
+
+typedef int (*fop_remote_read_t)(call_frame_t *, void *);
 
-typedef void *(*store_init) (xlator_t *this);
+typedef void *(*store_init)(xlator_t *this);
 
-typedef int (*store_reconfigure) (xlator_t *this, dict_t *options);
+typedef int (*store_reconfigure)(xlator_t *this, dict_t *options);
 
-typedef void (*store_fini) (void *config);
+typedef void (*store_fini)(void *config);
 
 struct cs_remote_stores {
-        char            *name; /* store name */
-        void            *config; /* store related information */
-        fop_download_t   dlfop; /* store specific download function */
-        store_init       init; /* store init to initialize store config */
-        store_reconfigure reconfigure; /* reconfigure store config */
-        store_fini       fini;
-        void            *handle; /* shared library handle*/
+    char *name;                    /* store name */
+    void *config;                  /* store related information */
+    fop_download_t dlfop;          /* store specific download function */
+    fop_remote_read_t rdfop;       /* store specific read function */
+    store_init init;               /* store init to initialize store config */
+    store_reconfigure reconfigure; /* reconfigure store config */
+    store_fini fini;
+    void *handle; /* shared library handle*/
 };
 
 typedef struct cs_private {
-        xlator_t *this;
-        struct cs_remote_stores *stores;
-        gf_boolean_t abortdl;
-        pthread_spinlock_t  lock;
+    xlator_t *this;
+    struct cs_remote_stores *stores;
+    gf_boolean_t abortdl;
+    pthread_spinlock_t lock;
+    gf_boolean_t remote_read;
 } cs_private_t;
 
 void
-cs_local_wipe (xlator_t *this, cs_local_t *local);
-
-#define CS_STACK_UNWIND(fop, frame, params ...) do {    \
-        cs_local_t *__local  = NULL;                    \
-        xlator_t    *__xl    = NULL;                    \
-        if (frame) {                                    \
-                __xl         = frame->this;             \
-                __local      = frame->local;            \
-                frame->local = NULL;                    \
-        }                                               \
-        STACK_UNWIND_STRICT (fop, frame, params);       \
-        cs_local_wipe (__xl, __local);                 \
-} while (0)
-
-#define CS_STACK_DESTROY(frame) do {                   \
-        cs_local_t *__local  = NULL;            \
-        xlator_t    *__xl    = NULL;            \
-        __xl                 = frame->this;     \
-        __local              = frame->local;    \
-        frame->local         = NULL;            \
-        STACK_DESTROY (frame->root);            \
-        cs_local_wipe (__xl, __local);         \
-} while (0)
+cs_local_wipe(xlator_t *this, cs_local_t *local);
+
+void
+cs_xattrinfo_wipe(cs_local_t *local);
+
+#define CS_STACK_UNWIND(fop, frame, params...)                                 \
+    do {                                                                       \
+        cs_local_t *__local = NULL;                                            \
+        xlator_t *__xl = NULL;                                                 \
+        if (frame) {                                                           \
+            __xl = frame->this;                                                \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        cs_local_wipe(__xl, __local);                                          \
+    } while (0)
+
+#define CS_STACK_DESTROY(frame)                                                \
+    do {                                                                       \
+        cs_local_t *__local = NULL;                                            \
+        xlator_t *__xl = NULL;                                                 \
+        __xl = frame->this;                                                    \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        cs_local_wipe(__xl, __local);                                          \
+    } while (0)
 
 typedef struct store_methods {
-        int (*fop_download) (call_frame_t *frame, void *config);
-        /* return type should be the store config */
-        void *(*fop_init) (xlator_t *this);
-        int (*fop_reconfigure) (xlator_t *this, dict_t *options);
-        void (*fop_fini) (void *config);
+    int (*fop_download)(call_frame_t *frame, void *config);
+    int (*fop_remote_read)(call_frame_t *, void *);
+    /* return type should be the store config */
+    void *(*fop_init)(xlator_t *this);
+    int (*fop_reconfigure)(xlator_t *this, dict_t *options);
+    void (*fop_fini)(void *config);
 } store_methods_t;
 
 #endif /* _CLOUDSYNC_COMMON_H */
diff --git a/xlators/features/cloudsync/src/cloudsync-fops-c.py b/xlators/features/cloudsync/src/cloudsync-fops-c.py
index fda56752871..c27df97ae58 100644..100755
--- a/xlators/features/cloudsync/src/cloudsync-fops-c.py
+++ b/xlators/features/cloudsync/src/cloudsync-fops-c.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 from __future__ import print_function
 import os
@@ -14,7 +14,7 @@ int32_t
 cs_@NAME@ (call_frame_t *frame, xlator_t *this,
            @LONG_ARGS@)
 {
-        int                         op_errno        = -1;
+        int                         op_errno        = EINVAL ;
         cs_local_t                 *local           = NULL;
         int                         ret             = 0;
         cs_inode_ctx_t             *ctx             = NULL;
@@ -35,11 +35,19 @@ cs_@NAME@ (call_frame_t *frame, xlator_t *this,
         __cs_inode_ctx_get (this, fd->inode, &ctx);
 
         if (ctx)
-                state = __cs_get_file_state (this, fd->inode, ctx);
+                state = __cs_get_file_state (fd->inode, ctx);
         else
                 state = GF_CS_LOCAL;
 
-        local->xattr_req = xdata ? dict_ref (xdata) : (xdata = dict_new ());
+        xdata = xdata ? dict_ref (xdata) : dict_new ();
+
+        if (!xdata) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->xattr_req = xdata;
 
         ret = dict_set_uint32 (local->xattr_req, GF_CS_OBJECT_STATUS, 1);
         if (ret) {
@@ -137,15 +145,15 @@ cs_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         } else {
                                 __cs_inode_ctx_update (this, fd->inode, val);
                                 gf_msg (this->name, GF_LOG_INFO, 0, 0,
-                                        " state = %ld", val);
+                                        " state = %" PRIu64, val);
 
                                 if (local->call_cnt == 1 &&
                                     (val == GF_CS_REMOTE ||
                                      val == GF_CS_DOWNLOADING))  {
                                         gf_msg (this->name, GF_LOG_INFO, 0,
                                                 0, " will repair and download "
-                                                "the file, current state : %ld",
-                                                val);
+                                                "the file, current state : %"
+                                                PRIu64, val);
                                         goto repair;
                                 } else {
                                         gf_msg (this->name, GF_LOG_ERROR, 0, 0,
@@ -187,19 +195,29 @@ int32_t
 cs_@NAME@ (call_frame_t *frame, xlator_t *this,
            @LONG_ARGS@)
 {
+        int              op_errno = EINVAL;
         cs_local_t      *local = NULL;
         int              ret   = 0;
 
         local = cs_local_init (this, frame, loc, NULL, GF_FOP_@UPNAME@);
         if (!local) {
                 gf_msg (this->name, GF_LOG_ERROR, 0, 0, "local is NULL");
+                op_errno = ENOMEM;
                 goto err;
         }
 
         if (loc->inode->ia_type == IA_IFDIR)
                 goto wind;
 
-        local->xattr_req = xdata ? dict_ref (xdata) : dict_new ();
+        xdata = xdata ? dict_ref (xdata) : dict_new ();
+
+        if (!xdata) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->xattr_req = xdata;
 
         ret = dict_set_uint32 (local->xattr_req, GF_CS_OBJECT_STATUS, 1);
         if (ret) {
@@ -215,7 +233,7 @@ wind:
 
         return 0;
 err:
-        CS_STACK_UNWIND (@NAME@, frame, -1, errno, @CBK_ERROR_ARGS@);
+        CS_STACK_UNWIND (@NAME@, frame, -1, op_errno, @CBK_ERROR_ARGS@);
 
         return 0;
 }
@@ -274,7 +292,7 @@ fd_ops = ['readv', 'writev', 'flush', 'fsync', 'fsyncdir', 'ftruncate',
 # These are the current actual lists used to generate the code
 
 # The following list contains fops which are fd based that modifies data
-fd_data_modify_op_fop_template = ['readv', 'writev', 'flush', 'fsync',
+fd_data_modify_op_fop_template = ['writev', 'flush', 'fsync',
                                   'ftruncate', 'rchecksum', 'fallocate',
                                   'discard', 'zerofill', 'seek']
 
@@ -284,8 +302,8 @@ loc_stat_op_fop_template = ['lookup', 'stat', 'discover', 'access', 'setattr',
                             'getattr']
 
 # These fops need a separate implementation
-special_fops = ['readdirp', 'statfs', 'setxattr', 'unlink', 'getxattr',
-                'truncate', 'fstat']
+special_fops = ['statfs', 'setxattr', 'unlink', 'getxattr',
+                'truncate', 'fstat', 'readv', 'readdirp']
 
 def gen_defaults():
     for name in ops:
diff --git a/xlators/features/cloudsync/src/cloudsync-fops-h.py b/xlators/features/cloudsync/src/cloudsync-fops-h.py
index 37dfa5b10be..faa2de651a7 100644..100755
--- a/xlators/features/cloudsync/src/cloudsync-fops-h.py
+++ b/xlators/features/cloudsync/src/cloudsync-fops-h.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 from __future__ import print_function
 import os
@@ -16,7 +16,7 @@ cs_@NAME@ (call_frame_t *frame, xlator_t *this,
 """
 
 def gen_defaults():
-    for name, value in ops.iteritems():
+    for name, value in ops.items():
         if name == 'getspec':
             continue
         print(generate(OP_FOP_TEMPLATE, name, fop_subs))
diff --git a/xlators/features/cloudsync/src/cloudsync-mem-types.h b/xlators/features/cloudsync/src/cloudsync-mem-types.h
index 6ebcb16552b..220346405d0 100644
--- a/xlators/features/cloudsync/src/cloudsync-mem-types.h
+++ b/xlators/features/cloudsync/src/cloudsync-mem-types.h
@@ -8,16 +8,15 @@
  *   cases as published by the Free Software Foundation.
  */
 
-
 #ifndef __CLOUDSYNC_MEM_TYPES_H__
 #define __CLOUDSYNC_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 enum cs_mem_types_ {
-        gf_cs_mt_cs_private_t = gf_common_mt_end + 1,
-        gf_cs_mt_cs_remote_stores_t,
-        gf_cs_mt_cs_inode_ctx_t,
-        gf_cs_mt_end
+    gf_cs_mt_cs_private_t = gf_common_mt_end + 1,
+    gf_cs_mt_cs_remote_stores_t,
+    gf_cs_mt_cs_inode_ctx_t,
+    gf_cs_mt_cs_lxattr_t,
+    gf_cs_mt_end
 };
 #endif /* __CLOUDSYNC_MEM_TYPES_H__ */
-
diff --git a/xlators/features/cloudsync/src/cloudsync-messages.h b/xlators/features/cloudsync/src/cloudsync-messages.h
index ad4b7d2e0b8..fb08f72de7f 100644
--- a/xlators/features/cloudsync/src/cloudsync-messages.h
+++ b/xlators/features/cloudsync/src/cloudsync-messages.h
@@ -8,12 +8,9 @@
  *   cases as published by the Free Software Foundation.
  */
 
-
 #ifndef __CLOUDSYNC_MESSAGES_H__
 #define __CLOUDSYNC_MESSAGES_H__
 
 /*TODO: define relevant message ids */
 
-
 #endif /* __CLOUDSYNC_MESSAGES_H__ */
-
diff --git a/xlators/experimental/dht2/dht2-client/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/experimental/dht2/dht2-client/Makefile.am
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/Makefile.am
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile.am
new file mode 100644
index 00000000000..fb6b0580c6d
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile.am
@@ -0,0 +1,11 @@
+if BUILD_AMAZONS3_PLUGIN
+  AMAZONS3_DIR = cloudsyncs3
+endif
+
+if BUILD_CVLT_PLUGIN
+  CVLT_DIR = cvlt
+endif
+
+SUBDIRS = ${AMAZONS3_DIR} ${CVLT_DIR}
+
+CLEANFILES =
diff --git a/xlators/experimental/dht2/dht2-server/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/experimental/dht2/dht2-server/Makefile.am
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/Makefile.am
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile.am
new file mode 100644
index 00000000000..6509426ef87
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile.am
@@ -0,0 +1,12 @@
+csp_LTLIBRARIES = cloudsyncs3.la
+cspdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/cloudsync-plugins
+
+cloudsyncs3_la_SOURCES = libcloudsyncs3.c  $(top_srcdir)/xlators/features/cloudsync/src/cloudsync-common.c
+cloudsyncs3_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+cloudsyncs3_la_LDFLAGS = -module -export-symbols $(top_srcdir)/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym $(GF_XLATOR_LDFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src   -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src -lcurlpp -lcryptopp
+noinst_HEADERS = libcloudsyncs3.h libcloudsyncs3-mem-types.h
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS) -lcurl -lcrypto -I$(top_srcdir)/xlators/features/cloudsync/src
+CLEANFILES =
+
+EXTRA_DIST = libcloudsyncs3.sym
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3-mem-types.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3-mem-types.h
new file mode 100644
index 00000000000..7ccfcc9f4b6
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3-mem-types.h
@@ -0,0 +1,19 @@
+/*
+ *   Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __LIBAWS_MEM_TYPES_H__
+#define __LIBAWS_MEM_TYPES_H__
+
+#include <glusterfs/mem-types.h>
+enum libaws_mem_types_ {
+    gf_libaws_mt_aws_private_t = gf_common_mt_end + 1,
+    gf_libaws_mt_end
+};
+#endif /* __CLOUDSYNC_MEM_TYPES_H__ */
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c
new file mode 100644
index 00000000000..23c3599825a
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c
@@ -0,0 +1,584 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <stdlib.h>
+#include <openssl/hmac.h>
+#include <openssl/evp.h>
+#include <openssl/bio.h>
+#include <openssl/buffer.h>
+#include <openssl/crypto.h>
+#include <curl/curl.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include "libcloudsyncs3.h"
+#include "cloudsync-common.h"
+
+#define RESOURCE_SIZE 4096
+
+store_methods_t store_ops = {
+    .fop_download = aws_download_s3,
+    .fop_init = aws_init,
+    .fop_reconfigure = aws_reconfigure,
+    .fop_fini = aws_fini,
+};
+
+typedef struct aws_private {
+    char *hostname;
+    char *bucketid;
+    char *awssekey;
+    char *awskeyid;
+    gf_boolean_t abortdl;
+    pthread_spinlock_t lock;
+} aws_private_t;
+
+void *
+aws_init(xlator_t *this)
+{
+    aws_private_t *priv = NULL;
+    char *temp_str = NULL;
+    int ret = 0;
+
+    priv = GF_CALLOC(1, sizeof(aws_private_t), gf_libaws_mt_aws_private_t);
+    if (!priv) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+        return NULL;
+    }
+
+    priv->abortdl = _gf_false;
+
+    pthread_spin_init(&priv->lock, PTHREAD_PROCESS_PRIVATE);
+
+    pthread_spin_lock(&(priv->lock));
+    {
+        if (dict_get_str(this->options, "s3plugin-seckey", &temp_str) == 0) {
+            priv->awssekey = gf_strdup(temp_str);
+            if (!priv->awssekey) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws secret key failed");
+                ret = -1;
+                goto unlock;
+            }
+        }
+
+        if (dict_get_str(this->options, "s3plugin-keyid", &temp_str) == 0) {
+            priv->awskeyid = gf_strdup(temp_str);
+            if (!priv->awskeyid) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws key ID failed");
+                ret = -1;
+                goto unlock;
+            }
+        }
+
+        if (dict_get_str(this->options, "s3plugin-bucketid", &temp_str) == 0) {
+            priv->bucketid = gf_strdup(temp_str);
+            if (!priv->bucketid) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws bucketid failed");
+
+                ret = -1;
+                goto unlock;
+            }
+        }
+
+        if (dict_get_str(this->options, "s3plugin-hostname", &temp_str) == 0) {
+            priv->hostname = gf_strdup(temp_str);
+            if (!priv->hostname) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws hostname failed");
+
+                ret = -1;
+                goto unlock;
+            }
+        }
+
+        gf_msg_debug(this->name, 0,
+                     "stored key: %s id: %s "
+                     "bucketid %s hostname: %s",
+                     priv->awssekey, priv->awskeyid, priv->bucketid,
+                     priv->hostname);
+    }
+unlock:
+    pthread_spin_unlock(&(priv->lock));
+
+    if (ret == -1) {
+        GF_FREE(priv->awskeyid);
+        GF_FREE(priv->awssekey);
+        GF_FREE(priv->bucketid);
+        GF_FREE(priv->hostname);
+        GF_FREE(priv);
+        priv = NULL;
+    }
+
+    return (void *)priv;
+}
+
+int
+aws_reconfigure(xlator_t *this, dict_t *options)
+{
+    aws_private_t *priv = NULL;
+    char *temp_str = NULL;
+    int ret = 0;
+    cs_private_t *cspriv = NULL;
+
+    cspriv = this->private;
+
+    priv = cspriv->stores->config;
+
+    if (!priv) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "null priv");
+        return -1;
+    }
+
+    pthread_spin_lock(&(priv->lock));
+    {
+        if (dict_get_str(options, "s3plugin-seckey", &temp_str) == 0) {
+            priv->awssekey = gf_strdup(temp_str);
+            if (!priv->awssekey) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws secret key failed");
+                ret = -1;
+                goto out;
+            }
+        }
+
+        if (dict_get_str(options, "s3plugin-keyid", &temp_str) == 0) {
+            priv->awskeyid = gf_strdup(temp_str);
+            if (!priv->awskeyid) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws key ID failed");
+                ret = -1;
+                goto out;
+            }
+        }
+
+        if (dict_get_str(options, "s3plugin-bucketid", &temp_str) == 0) {
+            priv->bucketid = gf_strdup(temp_str);
+            if (!priv->bucketid) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws bucketid failed");
+                ret = -1;
+                goto out;
+            }
+        }
+
+        if (dict_get_str(options, "s3plugin-hostname", &temp_str) == 0) {
+            priv->hostname = gf_strdup(temp_str);
+            if (!priv->hostname) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws hostname failed");
+                ret = -1;
+                goto out;
+            }
+        }
+    }
+out:
+    pthread_spin_unlock(&(priv->lock));
+
+    gf_msg_debug(this->name, 0,
+                 "stored key: %s id: %s "
+                 "bucketid %s hostname: %s",
+                 priv->awssekey, priv->awskeyid, priv->bucketid,
+                 priv->hostname);
+
+    return ret;
+}
+
+void
+aws_fini(void *config)
+{
+    aws_private_t *priv = NULL;
+
+    priv = (aws_private_t *)priv;
+
+    if (priv) {
+        GF_FREE(priv->hostname);
+        GF_FREE(priv->bucketid);
+        GF_FREE(priv->awssekey);
+        GF_FREE(priv->awskeyid);
+
+        pthread_spin_destroy(&priv->lock);
+        GF_FREE(priv);
+    }
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+
+    ret = xlator_mem_acct_init(this, gf_libaws_mt_end + 1);
+
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "Memory accounting init failed");
+        return ret;
+    }
+out:
+    return ret;
+}
+char *
+aws_form_request(char *resource, char **date, char *reqtype, char *bucketid,
+                 char *filepath)
+{
+    char httpdate[256];
+    time_t ctime;
+    struct tm *gtime = NULL;
+    char *sign_req = NULL;
+    int signreq_len = -1;
+    int date_len = -1;
+    int res_len = -1;
+
+    ctime = gf_time();
+    gtime = gmtime(&ctime);
+
+    date_len = strftime(httpdate, sizeof(httpdate),
+                        "%a, %d %b %Y %H:%M:%S +0000", gtime);
+
+    *date = gf_strndup(httpdate, date_len);
+    if (*date == NULL) {
+        gf_msg("CS", GF_LOG_ERROR, ENOMEM, 0,
+               "memory allocation "
+               "failure for date");
+        goto out;
+    }
+
+    res_len = snprintf(resource, RESOURCE_SIZE, "%s/%s", bucketid, filepath);
+
+    gf_msg_debug("CS", 0, "resource %s", resource);
+
+    /* 6 accounts for the 4 new line chars, one forward slash and
+     * one null char */
+    signreq_len = res_len + date_len + strlen(reqtype) + 6;
+
+    sign_req = GF_MALLOC(signreq_len, gf_common_mt_char);
+    if (sign_req == NULL) {
+        gf_msg("CS", GF_LOG_ERROR, ENOMEM, 0,
+               "memory allocation "
+               "failure for sign_req");
+        goto out;
+    }
+
+    snprintf(sign_req, signreq_len, "%s\n\n%s\n%s\n/%s", reqtype, "", *date,
+             resource);
+
+out:
+    return sign_req;
+}
+
+char *
+aws_b64_encode(const unsigned char *input, int length)
+{
+    BIO *bio, *b64;
+    BUF_MEM *bptr;
+    char *buff = NULL;
+
+    b64 = BIO_new(BIO_f_base64());
+    bio = BIO_new(BIO_s_mem());
+    b64 = BIO_push(b64, bio);
+    BIO_write(b64, input, length);
+    BIO_flush(b64);
+    BIO_get_mem_ptr(b64, &bptr);
+
+    buff = GF_MALLOC(bptr->length, gf_common_mt_char);
+    memcpy(buff, bptr->data, bptr->length - 1);
+    buff[bptr->length - 1] = 0;
+
+    BIO_free_all(b64);
+
+    return buff;
+}
+
+char *
+aws_sign_request(char *const str, char *awssekey)
+{
+#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
+    HMAC_CTX ctx;
+#endif
+    HMAC_CTX *pctx = NULL;
+    ;
+
+    unsigned char md[256];
+    unsigned len;
+    char *base64 = NULL;
+
+#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
+    HMAC_CTX_init(&ctx);
+    pctx = &ctx;
+#else
+    pctx = HMAC_CTX_new();
+#endif
+    HMAC_Init_ex(pctx, awssekey, strlen(awssekey), EVP_sha1(), NULL);
+    HMAC_Update(pctx, (unsigned char *)str, strlen(str));
+    HMAC_Final(pctx, (unsigned char *)md, &len);
+
+#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
+    HMAC_CTX_cleanup(pctx);
+#else
+    HMAC_CTX_free(pctx);
+#endif
+    base64 = aws_b64_encode(md, len);
+
+    return base64;
+}
+
+int
+aws_dlwritev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata)
+{
+    aws_private_t *priv = NULL;
+
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, op_errno,
+               "write failed "
+               ". Aborting Download");
+
+        priv = this->private;
+        pthread_spin_lock(&(priv->lock));
+        {
+            priv->abortdl = _gf_true;
+        }
+        pthread_spin_unlock(&(priv->lock));
+    }
+
+    CS_STACK_DESTROY(frame);
+
+    return op_ret;
+}
+
+size_t
+aws_write_callback(void *dlbuf, size_t size, size_t nitems, void *mainframe)
+{
+    call_frame_t *frame = NULL;
+    fd_t *dlfd = NULL;
+    int ret = 0;
+    cs_local_t *local = NULL;
+    struct iovec iov = {
+        0,
+    };
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec dliov = {
+        0,
+    };
+    size_t tsize = 0;
+    xlator_t *this = NULL;
+    cs_private_t *xl_priv = NULL;
+    aws_private_t *priv = NULL;
+    call_frame_t *dlframe = NULL;
+
+    frame = (call_frame_t *)mainframe;
+    this = frame->this;
+    xl_priv = this->private;
+    priv = xl_priv->stores->config;
+
+    pthread_spin_lock(&(priv->lock));
+    {
+        /* returning size other than the size passed from curl will
+         * abort further download*/
+        if (priv->abortdl) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "aborting download");
+            pthread_spin_unlock(&(priv->lock));
+            return 0;
+        }
+    }
+    pthread_spin_unlock(&(priv->lock));
+
+    local = frame->local;
+    dlfd = local->dlfd;
+    tsize = size * nitems;
+
+    dliov.iov_base = (void *)dlbuf;
+    dliov.iov_len = tsize;
+
+    ret = iobuf_copy(this->ctx->iobuf_pool, &dliov, 1, &iobref, &iobuf, &iov);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "iobuf_copy failed");
+        goto out;
+    }
+
+    /* copy frame */
+    dlframe = copy_frame(frame);
+    if (!dlframe) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "copy_frame failed");
+        tsize = 0;
+        goto out;
+    }
+
+    STACK_WIND(dlframe, aws_dlwritev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, dlfd, &iov, 1, local->dloffset,
+               0, iobref, NULL);
+
+    local->dloffset += tsize;
+
+out:
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+
+    return tsize;
+}
+
+int
+aws_download_s3(call_frame_t *frame, void *config)
+{
+    char *buf;
+    int bufsize = -1;
+    CURL *handle = NULL;
+    struct curl_slist *slist = NULL;
+    struct curl_slist *tmp = NULL;
+    xlator_t *this = NULL;
+    int ret = 0;
+    int debug = 1;
+    CURLcode res;
+    char errbuf[CURL_ERROR_SIZE];
+    size_t len = 0;
+    long responsecode;
+    char *sign_req = NULL;
+    char *date = NULL;
+    char *const reqtype = "GET";
+    char *signature = NULL;
+    cs_local_t *local = NULL;
+    char resource[RESOURCE_SIZE] = {
+        0,
+    };
+    aws_private_t *priv = NULL;
+
+    this = frame->this;
+
+    local = frame->local;
+
+    priv = (aws_private_t *)config;
+
+    if (!priv->bucketid || !priv->hostname || !priv->awssekey ||
+        !priv->awskeyid) {
+        ret = -1;
+        goto out;
+    }
+
+    sign_req = aws_form_request(resource, &date, reqtype, priv->bucketid,
+                                local->remotepath);
+    if (!sign_req) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "null sign_req, "
+               "aborting download");
+        ret = -1;
+        goto out;
+    }
+
+    gf_msg_debug("CS", 0, "sign_req %s date %s", sign_req, date);
+
+    signature = aws_sign_request(sign_req, priv->awssekey);
+    if (!signature) {
+        gf_msg("CS", GF_LOG_ERROR, 0, 0,
+               "null signature, "
+               "aborting download");
+        ret = -1;
+        goto out;
+    }
+
+    handle = curl_easy_init();
+    this = frame->this;
+
+    /* special numbers 6, 20, 10 accounts for static characters in the
+     * below snprintf string format arguments*/
+    bufsize = strlen(date) + 6 + strlen(priv->awskeyid) + strlen(signature) +
+              20 + strlen(priv->hostname) + 10;
+
+    buf = (char *)alloca(bufsize);
+    if (!buf) {
+        gf_msg("CS", GF_LOG_ERROR, ENOMEM, 0,
+               "mem allocation "
+               "failed for buf");
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(buf, bufsize, "Date: %s", date);
+    slist = curl_slist_append(slist, buf);
+    snprintf(buf, bufsize, "Authorization: AWS %s:%s", priv->awskeyid,
+             signature);
+    slist = curl_slist_append(slist, buf);
+    snprintf(buf, bufsize, "https://%s/%s", priv->hostname, resource);
+
+    if (gf_log_get_loglevel() >= GF_LOG_DEBUG) {
+        tmp = slist;
+        while (tmp) {
+            gf_msg_debug(this->name, 0, "slist for curl - %s", tmp->data);
+            tmp = tmp->next;
+        }
+    }
+
+    curl_easy_setopt(handle, CURLOPT_HTTPHEADER, slist);
+    curl_easy_setopt(handle, CURLOPT_URL, buf);
+    curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, aws_write_callback);
+    curl_easy_setopt(handle, CURLOPT_WRITEDATA, frame);
+    curl_easy_setopt(handle, CURLOPT_VERBOSE, debug);
+    curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, errbuf);
+
+    res = curl_easy_perform(handle);
+    if (res != CURLE_OK) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "download failed. err: %s\n",
+               curl_easy_strerror(res));
+        ret = -1;
+        len = strlen(errbuf);
+        if (len) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "curl failure %s", errbuf);
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "curl error "
+                   "%s\n",
+                   curl_easy_strerror(res));
+        }
+    }
+
+    if (res == CURLE_OK) {
+        curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &responsecode);
+        gf_msg_debug(this->name, 0, "response code %ld", responsecode);
+        if (responsecode != 200) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "curl download failed");
+        }
+    }
+
+    curl_slist_free_all(slist);
+    curl_easy_cleanup(handle);
+
+out:
+    if (sign_req)
+        GF_FREE(sign_req);
+    if (date)
+        GF_FREE(date);
+    if (signature)
+        GF_FREE(signature);
+
+    return ret;
+}
+
+struct volume_options cs_options[] = {
+    {.key = {"s3plugin-seckey"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "aws secret key"},
+    {.key = {"s3plugin-keyid"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "aws key ID"
+
+    },
+    {.key = {"s3plugin-bucketid"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "aws bucketid"},
+    {.key = {"s3plugin-hostname"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "aws hostname e.g. s3.amazonaws.com"},
+    {.key = {NULL}},
+};
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.h
new file mode 100644
index 00000000000..85ae669486b
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.h
@@ -0,0 +1,50 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#ifndef _LIBAWS_H
+#define _LIBAWS_H
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/syncop.h>
+#include <curl/curl.h>
+#include "cloudsync-common.h"
+#include "libcloudsyncs3-mem-types.h"
+
+char *
+aws_b64_encode(const unsigned char *input, int length);
+
+size_t
+aws_write_callback(void *dlbuf, size_t size, size_t nitems, void *mainframe);
+
+int
+aws_download_s3(call_frame_t *frame, void *config);
+
+int
+aws_dlwritev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata);
+
+void *
+aws_init(xlator_t *this);
+
+int
+aws_reconfigure(xlator_t *this, dict_t *options);
+
+char *
+aws_form_request(char *resource, char **date, char *reqtype, char *bucketid,
+                 char *filepath);
+char *
+aws_sign_request(char *const str, char *awssekey);
+
+void
+aws_fini(void *config);
+
+#endif
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym
new file mode 100644
index 00000000000..0bc273670d5
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym
@@ -0,0 +1 @@
+store_ops
diff --git a/xlators/experimental/fdl/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/experimental/fdl/Makefile.am
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/Makefile.am
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/Makefile.am
new file mode 100644
index 00000000000..b512464f157
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/Makefile.am
@@ -0,0 +1,12 @@
+csp_LTLIBRARIES = cloudsynccvlt.la
+cspdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/cloudsync-plugins
+
+cloudsynccvlt_la_SOURCES = libcvlt.c  $(top_srcdir)/xlators/features/cloudsync/src/cloudsync-common.c
+cloudsynccvlt_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+cloudsynccvlt_la_LDFLAGS = -module -avoid-version -export-symbols $(top_srcdir)/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcloudsynccvlt.sym
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src   -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+noinst_HEADERS = archivestore.h libcvlt.h libcvlt-mem-types.h cvlt-messages.h
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS) -I$(top_srcdir)/xlators/features/cloudsync/src
+CLEANFILES =
+
+EXTRA_DIST = libcloudsynccvlt.sym
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/archivestore.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/archivestore.h
new file mode 100644
index 00000000000..7230ef77337
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/archivestore.h
@@ -0,0 +1,203 @@
+/*
+  Copyright (c) 2018 Commvault Systems, Inc. <http://www.commvault.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __ARCHIVESTORE_H__
+#define __ARCHIVESTORE_H__
+
+#include <stdlib.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <dlfcn.h>
+#include <uuid/uuid.h>
+
+#define CS_XATTR_ARCHIVE_UUID "trusted.cloudsync.uuid"
+#define CS_XATTR_PRODUCT_ID "trusted.cloudsync.product-id"
+#define CS_XATTR_STORE_ID "trusted.cloudsync.store-id"
+
+struct _archstore_methods;
+typedef struct _archstore_methods archstore_methods_t;
+
+struct _archstore_desc {
+    void *priv; /* Private field for store mgmt.   */
+                /* To be used only by archive store*/
+};
+typedef struct _archstore_desc archstore_desc_t;
+
+struct _archstore_info {
+    char *id;         /* Identifier for the archivestore */
+    uint32_t idlen;   /* Length of identifier string     */
+    char *prod;       /* Name of the data mgmt. product  */
+    uint32_t prodlen; /* Length of the product string    */
+};
+typedef struct _archstore_info archstore_info_t;
+
+struct _archstore_fileinfo {
+    uuid_t uuid;         /* uuid of the file                */
+    char *path;          /* file path                       */
+    uint32_t pathlength; /* length of file path             */
+};
+typedef struct _archstore_fileinfo archstore_fileinfo_t;
+
+struct _app_callback_info {
+    archstore_info_t *src_archstore;
+    archstore_fileinfo_t *src_archfile;
+    archstore_info_t *dest_archstore;
+    archstore_fileinfo_t *dest_archfile;
+};
+typedef struct _app_callback_info app_callback_info_t;
+
+typedef void (*app_callback_t)(archstore_desc_t *, app_callback_info_t *,
+                               void *, int64_t, int32_t);
+
+enum _archstore_scan_type { FULL = 1, INCREMENTAL = 2 };
+typedef enum _archstore_scan_type archstore_scan_type_t;
+
+typedef int32_t archstore_errno_t;
+
+/*
+ * Initialize archive store.
+ * arg1  pointer to structure containing archive store information
+ * arg2  error number if any generated during the initialization
+ * arg3  name of the log file
+ */
+typedef int32_t (*init_archstore_t)(archstore_desc_t *, archstore_errno_t *,
+                                    const char *);
+
+/*
+ * Clean up archive store.
+ * arg1  pointer to structure containing archive store information
+ * arg2  error number if any generated during the cleanup
+ */
+typedef int32_t (*term_archstore_t)(archstore_desc_t *, archstore_errno_t *);
+
+/*
+ * Read the contents of the file from archive store
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing archive store information
+ * arg3  pointer to structure containing information about file to be read
+ * arg4  offset in the file from which data should be read
+ * arg5  buffer where the data should be read
+ * arg6  number of bytes of data to be read
+ * arg7  error number if any generated during the read from file
+ * arg8  callback handler to be invoked after the data is read
+ * arg9  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*read_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                    archstore_fileinfo_t *, off_t, char *,
+                                    size_t, archstore_errno_t *, app_callback_t,
+                                    void *);
+
+/*
+ * Restore the contents of the file from archive store
+ * This is basically in-place restore
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing archive store information
+ * arg3  pointer to structure containing information about file to be restored
+ * arg4  error number if any generated during the file restore
+ * arg5  callback to be invoked after the file is restored
+ * arg6  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*recall_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                      archstore_fileinfo_t *,
+                                      archstore_errno_t *, app_callback_t,
+                                      void *);
+
+/*
+ * Restore the contents of the file from archive store to a different store
+ * This is basically out-of-place restore
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing source archive store information
+ * arg3  pointer to structure containing information about file to be restored
+ * arg4  pointer to structure containing destination archive store information
+ * arg5  pointer to structure containing information about the location to
+         which the file will be restored
+ * arg6  error number if any generated during the file restore
+ * arg7  callback to be invoked after the file is restored
+ * arg8  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*restore_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                       archstore_fileinfo_t *,
+                                       archstore_info_t *,
+                                       archstore_fileinfo_t *,
+                                       archstore_errno_t *, app_callback_t,
+                                       void *);
+
+/*
+ * Archive the contents of the file to archive store
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing source archive store information
+ * arg3  pointer to structure containing information about files to be archived
+ * arg4  pointer to structure containing destination archive store information
+ * arg5  pointer to structure containing information about files that failed
+ *       to be archived
+ * arg6  error number if any generated during the file archival
+ * arg7  callback to be invoked after the file is archived
+ * arg8  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*archive_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                       archstore_fileinfo_t *,
+                                       archstore_info_t *,
+                                       archstore_fileinfo_t *,
+                                       archstore_errno_t *, app_callback_t,
+                                       void *);
+
+/*
+ * Backup list of files provided in the input file
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing source archive store information
+ * arg3  pointer to structure containing information about files to be backed up
+ * arg4  pointer to structure containing destination archive store information
+ * arg5  pointer to structure containing information about files that failed
+ *       to be backed up
+ * arg6  error number if any generated during the file archival
+ * arg7  callback to be invoked after the file is archived
+ * arg8  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*backup_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                      archstore_fileinfo_t *,
+                                      archstore_info_t *,
+                                      archstore_fileinfo_t *,
+                                      archstore_errno_t *, app_callback_t,
+                                      void *);
+
+/*
+ * Scan the contents of a store and determine the files which need to be
+ * backed up.
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing archive store information
+ * arg3  type of scan whether full or incremental
+ * arg4  path to file that contains list of files to be backed up
+ * arg5  error number if any generated during scan operation
+ */
+typedef int32_t (*scan_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                    archstore_scan_type_t, char *,
+                                    archstore_errno_t *);
+
+struct _archstore_methods {
+    init_archstore_t init;
+    term_archstore_t fini;
+    backup_archstore_t backup;
+    archive_archstore_t archive;
+    scan_archstore_t scan;
+    restore_archstore_t restore;
+    recall_archstore_t recall;
+    read_archstore_t read;
+};
+
+typedef int (*get_archstore_methods_t)(archstore_methods_t *);
+
+/*
+ * Single function that will be invoked by applications for extracting
+ * the function pointers to all data management functions.
+ */
+int32_t
+get_archstore_methods(archstore_methods_t *);
+
+#endif /* End of __ARCHIVESTORE_H__ */
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/cvlt-messages.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/cvlt-messages.h
new file mode 100644
index 00000000000..57c9aa77da0
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/cvlt-messages.h
@@ -0,0 +1,30 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _CVLT_MESSAGES_H_
+#define _CVLT_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(CVLT, CVLT_EXTRACTION_FAILED, CVLT_FREE,
+           CVLT_RESOURCE_ALLOCATION_FAILED, CVLT_RESTORE_FAILED,
+           CVLT_READ_FAILED, CVLT_NO_MEMORY, CVLT_DLOPEN_FAILED);
+
+#endif /* !_CVLT_MESSAGES_H_ */
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcloudsynccvlt.sym b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcloudsynccvlt.sym
new file mode 100644
index 00000000000..0bc273670d5
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcloudsynccvlt.sym
@@ -0,0 +1 @@
+store_ops
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt-mem-types.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt-mem-types.h
new file mode 100644
index 00000000000..c24fab8bfe7
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt-mem-types.h
@@ -0,0 +1,19 @@
+/*
+ *   Copyright (c) 2018 Commvault Systems, Inc. <http://www.commvault.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __LIBCVLT_MEM_TYPES_H__
+#define __LIBCVLT_MEM_TYPES_H__
+
+#include <glusterfs/mem-types.h>
+enum libcvlt_mem_types_ {
+    gf_libcvlt_mt_cvlt_private_t = gf_common_mt_end + 1,
+    gf_libcvlt_mt_end
+};
+#endif /* __LIBCVLT_MEM_TYPES_H__ */
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.c b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.c
new file mode 100644
index 00000000000..5b7272bb448
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.c
@@ -0,0 +1,842 @@
+#include <stdlib.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include "libcvlt.h"
+#include "cloudsync-common.h"
+#include "cvlt-messages.h"
+
+#define LIBARCHIVE_SO "libopenarchive.so"
+#define ALIGN_SIZE 4096
+#define CVLT_TRAILER "cvltv1"
+
+store_methods_t store_ops = {
+    .fop_download = cvlt_download,
+    .fop_init = cvlt_init,
+    .fop_reconfigure = cvlt_reconfigure,
+    .fop_fini = cvlt_fini,
+    .fop_remote_read = cvlt_read,
+};
+
+static const int32_t num_req = 32;
+static const int32_t num_iatt = 32;
+static char *plugin = "cvlt_cloudSync";
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    if (!this)
+        return ret;
+
+    ret = xlator_mem_acct_init(this, gf_libcvlt_mt_end + 1);
+
+    if (ret != 0) {
+        return ret;
+    }
+
+    return ret;
+}
+
+static void
+cvlt_free_resources(archive_t *arch)
+{
+    /*
+     * We will release all the resources that were allocated by the xlator.
+     * Check whether there are any buffers which have not been released
+     * back to a mempool.
+     */
+
+    if (arch->handle) {
+        dlclose(arch->handle);
+    }
+
+    if (arch->iobuf_pool) {
+        iobuf_pool_destroy(arch->iobuf_pool);
+    }
+
+    if (arch->req_pool) {
+        mem_pool_destroy(arch->req_pool);
+        arch->req_pool = NULL;
+    }
+
+    return;
+}
+
+static int32_t
+cvlt_extract_store_fops(xlator_t *this, archive_t *arch)
+{
+    int32_t op_ret = -1;
+    get_archstore_methods_t get_archstore_methods;
+
+    /*
+     * libopenarchive.so defines methods for performing data management
+     * operations. We will extract the methods from library and these
+     * methods will be invoked for moving data between glusterfs volume
+     * and the data management product.
+     */
+
+    VALIDATE_OR_GOTO(arch, err);
+
+    arch->handle = dlopen(LIBARCHIVE_SO, RTLD_NOW);
+    if (!arch->handle) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_DLOPEN_FAILED,
+               " failed to open %s ", LIBARCHIVE_SO);
+        return op_ret;
+    }
+
+    dlerror(); /* Clear any existing error */
+
+    get_archstore_methods = dlsym(arch->handle, "get_archstore_methods");
+    if (!get_archstore_methods) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " Error extracting get_archstore_methods()");
+        dlclose(arch->handle);
+        arch->handle = NULL;
+        return op_ret;
+    }
+
+    op_ret = get_archstore_methods(&(arch->fops));
+    if (op_ret) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " Failed to extract methods in get_archstore_methods");
+        dlclose(arch->handle);
+        arch->handle = NULL;
+        return op_ret;
+    }
+
+err:
+    return op_ret;
+}
+
+static int32_t
+cvlt_alloc_resources(xlator_t *this, archive_t *arch, int num_req, int num_iatt)
+{
+    /*
+     * Initialize information about all the memory pools that will be
+     * used by this xlator.
+     */
+    arch->nreqs = 0;
+
+    arch->req_pool = NULL;
+
+    arch->handle = NULL;
+    arch->xl = this;
+
+    arch->req_pool = mem_pool_new(cvlt_request_t, num_req);
+    if (!arch->req_pool) {
+        goto err;
+    }
+
+    arch->iobuf_pool = iobuf_pool_new();
+    if (!arch->iobuf_pool) {
+        goto err;
+    }
+
+    if (cvlt_extract_store_fops(this, arch)) {
+        goto err;
+    }
+
+    return 0;
+
+err:
+
+    return -1;
+}
+
+static void
+cvlt_req_init(cvlt_request_t *req)
+{
+    sem_init(&(req->sem), 0, 0);
+
+    return;
+}
+
+static void
+cvlt_req_destroy(cvlt_request_t *req)
+{
+    if (req->iobuf) {
+        iobuf_unref(req->iobuf);
+    }
+
+    if (req->iobref) {
+        iobref_unref(req->iobref);
+    }
+
+    sem_destroy(&(req->sem));
+
+    return;
+}
+
+static cvlt_request_t *
+cvlt_alloc_req(archive_t *arch)
+{
+    cvlt_request_t *reqptr = NULL;
+
+    if (!arch) {
+        goto err;
+    }
+
+    if (arch->req_pool) {
+        reqptr = mem_get0(arch->req_pool);
+        if (reqptr) {
+            cvlt_req_init(reqptr);
+        }
+    }
+
+    if (reqptr) {
+        LOCK(&(arch->lock));
+        arch->nreqs++;
+        UNLOCK(&(arch->lock));
+    }
+
+err:
+    return reqptr;
+}
+
+static int32_t
+cvlt_free_req(archive_t *arch, cvlt_request_t *reqptr)
+{
+    if (!reqptr) {
+        goto err;
+    }
+
+    if (!arch) {
+        goto err;
+    }
+
+    if (arch->req_pool) {
+        /*
+         * Free the request resources if they exist.
+         */
+
+        cvlt_req_destroy(reqptr);
+        mem_put(reqptr);
+
+        LOCK(&(arch->lock));
+        arch->nreqs--;
+        UNLOCK(&(arch->lock));
+    }
+
+    return 0;
+
+err:
+    return -1;
+}
+
+static int32_t
+cvlt_init_xlator(xlator_t *this, archive_t *arch, int num_req, int num_iatt)
+{
+    int32_t ret = -1;
+    int32_t errnum = -1;
+    int32_t locked = 0;
+
+    /*
+     * Perform all the initializations needed for brining up the xlator.
+     */
+    if (!arch) {
+        goto err;
+    }
+
+    LOCK_INIT(&(arch->lock));
+    LOCK(&(arch->lock));
+
+    locked = 1;
+
+    ret = cvlt_alloc_resources(this, arch, num_req, num_iatt);
+
+    if (ret) {
+        goto err;
+    }
+
+    /*
+     * Now that the fops have been extracted initialize the store
+     */
+    ret = arch->fops.init(&(arch->descinfo), &errnum, plugin);
+    if (ret) {
+        goto err;
+    }
+
+    UNLOCK(&(arch->lock));
+    locked = 0;
+    ret = 0;
+
+    return ret;
+
+err:
+    if (arch) {
+        cvlt_free_resources(arch);
+
+        if (locked) {
+            UNLOCK(&(arch->lock));
+        }
+    }
+
+    return ret;
+}
+
+static int32_t
+cvlt_term_xlator(archive_t *arch)
+{
+    int32_t errnum = -1;
+
+    if (!arch) {
+        goto err;
+    }
+
+    LOCK(&(arch->lock));
+
+    /*
+     * Release the resources that have been allocated inside store
+     */
+    arch->fops.fini(&(arch->descinfo), &errnum);
+
+    cvlt_free_resources(arch);
+
+    UNLOCK(&(arch->lock));
+
+    GF_FREE(arch);
+
+    return 0;
+
+err:
+    return -1;
+}
+
+static int32_t
+cvlt_init_store_info(archive_t *priv, archstore_info_t *store_info)
+{
+    if (!store_info) {
+        return -1;
+    }
+
+    store_info->prod = priv->product_id;
+    store_info->prodlen = strlen(priv->product_id);
+
+    store_info->id = priv->store_id;
+    store_info->idlen = strlen(priv->store_id);
+
+    return 0;
+}
+
+static int32_t
+cvlt_init_file_info(cs_loc_xattr_t *xattr, archstore_fileinfo_t *file_info)
+{
+    if (!xattr || !file_info) {
+        return -1;
+    }
+
+    gf_uuid_copy(file_info->uuid, xattr->uuid);
+    file_info->path = xattr->file_path;
+    file_info->pathlength = strlen(xattr->file_path);
+
+    return 0;
+}
+
+static int32_t
+cvlt_init_gluster_store_info(cs_loc_xattr_t *xattr,
+                             archstore_info_t *store_info)
+{
+    static char *product = "glusterfs";
+
+    if (!xattr || !store_info) {
+        return -1;
+    }
+
+    store_info->prod = product;
+    store_info->prodlen = strlen(product);
+
+    store_info->id = xattr->volname;
+    store_info->idlen = strlen(xattr->volname);
+
+    return 0;
+}
+
+static int32_t
+cvlt_init_gluster_file_info(cs_loc_xattr_t *xattr,
+                            archstore_fileinfo_t *file_info)
+{
+    if (!xattr || !file_info) {
+        return -1;
+    }
+
+    gf_uuid_copy(file_info->uuid, xattr->gfid);
+    file_info->path = xattr->file_path;
+    file_info->pathlength = strlen(xattr->file_path);
+
+    return 0;
+}
+
+static void
+cvlt_copy_stat_info(struct iatt *buf, cs_size_xattr_t *xattrs)
+{
+    /*
+     * If the file was archived then the reported size will not be a
+     * correct one. We need to fix this.
+     */
+    if (buf && xattrs) {
+        buf->ia_size = xattrs->size;
+        buf->ia_blksize = xattrs->blksize;
+        buf->ia_blocks = xattrs->blocks;
+    }
+
+    return;
+}
+
+static void
+cvlt_readv_complete(archstore_desc_t *desc, app_callback_info_t *cbkinfo,
+                    void *cookie, int64_t op_ret, int32_t op_errno)
+{
+    struct iovec iov;
+    xlator_t *this = NULL;
+    struct iatt postbuf = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    cvlt_request_t *req = (cvlt_request_t *)cookie;
+    cs_local_t *local = NULL;
+    cs_private_t *cspriv = NULL;
+    archive_t *priv = NULL;
+
+    frame = req->frame;
+    this = frame->this;
+    local = frame->local;
+
+    cspriv = this->private;
+    priv = (archive_t *)cspriv->stores->config;
+
+    if (strcmp(priv->trailer, CVLT_TRAILER)) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_msg_debug(plugin, 0,
+                 " Read callback invoked offset:%" PRIu64 "bytes: %" PRIu64
+                 " op : %d ret : %" PRId64 " errno : %d",
+                 req->offset, req->bytes, req->op_type, op_ret, op_errno);
+
+    if (op_ret < 0) {
+        goto out;
+    }
+
+    req->iobref = iobref_new();
+    if (!req->iobref) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    iobref_add(req->iobref, req->iobuf);
+    iov.iov_base = iobuf_ptr(req->iobuf);
+    iov.iov_len = op_ret;
+
+    cvlt_copy_stat_info(&postbuf, &(req->szxattr));
+
+    /*
+     * Hack to notify higher layers of EOF.
+     */
+    if (!postbuf.ia_size || (req->offset + iov.iov_len >= postbuf.ia_size)) {
+        gf_msg_debug(plugin, 0, " signalling end-of-file for uuid=%s",
+                     uuid_utoa(req->file_info.uuid));
+        op_errno = ENOENT;
+    }
+
+out:
+
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, &iov, 1, &postbuf,
+                        req->iobref, local->xattr_rsp);
+
+    cvlt_free_req(priv, req);
+
+    return;
+}
+
+static void
+cvlt_download_complete(archstore_desc_t *store, app_callback_info_t *cbk_info,
+                       void *cookie, int64_t ret, int errcode)
+{
+    cvlt_request_t *req = (cvlt_request_t *)cookie;
+
+    gf_msg_debug(plugin, 0,
+                 " Download callback invoked  ret : %" PRId64 " errno : %d",
+                 ret, errcode);
+
+    req->op_ret = ret;
+    req->op_errno = errcode;
+    sem_post(&(req->sem));
+
+    return;
+}
+
+void *
+cvlt_init(xlator_t *this)
+{
+    int ret = 0;
+    archive_t *priv = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, 0,
+               "should have exactly one child");
+        ret = -1;
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, 0,
+               "dangling volume. check volfile");
+        ret = -1;
+        goto out;
+    }
+
+    priv = GF_CALLOC(1, sizeof(archive_t), gf_libcvlt_mt_cvlt_private_t);
+    if (!priv) {
+        ret = -1;
+        goto out;
+    }
+
+    priv->trailer = CVLT_TRAILER;
+    if (cvlt_init_xlator(this, priv, num_req, num_iatt)) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, 0, "xlator init failed");
+        ret = -1;
+        goto out;
+    }
+
+    GF_OPTION_INIT("cloudsync-store-id", priv->store_id, str, out);
+    GF_OPTION_INIT("cloudsync-product-id", priv->product_id, str, out);
+
+    gf_msg(plugin, GF_LOG_INFO, 0, 0,
+           "store id is : %s "
+           "product id is : %s.",
+           priv->store_id, priv->product_id);
+out:
+    if (ret == -1) {
+        cvlt_term_xlator(priv);
+        return (NULL);
+    }
+    return priv;
+}
+
+int
+cvlt_reconfigure(xlator_t *this, dict_t *options)
+{
+    cs_private_t *cspriv = NULL;
+    archive_t *priv = NULL;
+
+    cspriv = this->private;
+    priv = (archive_t *)cspriv->stores->config;
+
+    if (strcmp(priv->trailer, CVLT_TRAILER))
+        goto out;
+
+    GF_OPTION_RECONF("cloudsync-store-id", priv->store_id, options, str, out);
+
+    GF_OPTION_RECONF("cloudsync-product-id", priv->product_id, options, str,
+                     out);
+    gf_msg_debug(plugin, 0,
+                 "store id is : %s "
+                 "product id is : %s.",
+                 priv->store_id, priv->product_id);
+    return 0;
+out:
+    return -1;
+}
+
+void
+cvlt_fini(void *config)
+{
+    archive_t *priv = NULL;
+
+    priv = (archive_t *)config;
+
+    if (strcmp(priv->trailer, CVLT_TRAILER))
+        return;
+
+    cvlt_term_xlator(priv);
+    gf_msg(plugin, GF_LOG_INFO, 0, CVLT_FREE, " released xlator resources");
+    return;
+}
+
+int
+cvlt_download(call_frame_t *frame, void *config)
+{
+    archive_t *parch = NULL;
+    cs_local_t *local = frame->local;
+    cs_loc_xattr_t *locxattr = local->xattrinfo.lxattr;
+    cvlt_request_t *req = NULL;
+    archstore_info_t dest_storeinfo;
+    archstore_fileinfo_t dest_fileinfo;
+    int32_t op_ret, op_errno;
+
+    parch = (archive_t *)config;
+
+    if (strcmp(parch->trailer, CVLT_TRAILER)) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    gf_msg_debug(plugin, 0, " download invoked for uuid = %s  gfid=%s ",
+                 locxattr->uuid, uuid_utoa(locxattr->gfid));
+
+    if (!(parch->fops.restore)) {
+        op_errno = ELIBBAD;
+        goto err;
+    }
+
+    /*
+     * Download needs to be processed. Allocate a request.
+     */
+    req = cvlt_alloc_req(parch);
+
+    if (!req) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, CVLT_RESOURCE_ALLOCATION_FAILED,
+               " failed to allocated request for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /*
+     * Initialize the request object.
+     */
+    req->op_type = CVLT_RESTORE_OP;
+    req->frame = frame;
+
+    /*
+     * The file is currently residing inside a data management store.
+     * To restore the file contents we need to provide the information
+     * about data management store.
+     */
+    op_ret = cvlt_init_store_info(parch, &(req->store_info));
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract store info for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    op_ret = cvlt_init_file_info(locxattr, &(req->file_info));
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract file info for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    /*
+     * We need to perform in-place restore of the file from data management
+     * store to gusterfs volume.
+     */
+    op_ret = cvlt_init_gluster_store_info(locxattr, &dest_storeinfo);
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract destination store info for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    op_ret = cvlt_init_gluster_file_info(locxattr, &dest_fileinfo);
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract file info for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    /*
+     * Submit the restore request.
+     */
+    op_ret = parch->fops.restore(&(parch->descinfo), &(req->store_info),
+                                 &(req->file_info), &dest_storeinfo,
+                                 &dest_fileinfo, &op_errno,
+                                 cvlt_download_complete, req);
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_RESTORE_FAILED,
+               " failed to restore file gfid=%s from data management store",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    /*
+     * Wait for the restore to complete.
+     */
+    sem_wait(&(req->sem));
+
+    if (req->op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_RESTORE_FAILED,
+               " restored failed for gfid=%s", uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    if (req) {
+        cvlt_free_req(parch, req);
+    }
+
+    return 0;
+
+err:
+
+    if (req) {
+        cvlt_free_req(parch, req);
+    }
+
+    return -1;
+}
+
+int
+cvlt_read(call_frame_t *frame, void *config)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    archive_t *parch = NULL;
+    cvlt_request_t *req = NULL;
+    struct iovec iov = {
+        0,
+    };
+    struct iobref *iobref;
+    size_t size = 0;
+    off_t off = 0;
+
+    cs_local_t *local = frame->local;
+    cs_loc_xattr_t *locxattr = local->xattrinfo.lxattr;
+
+    size = local->xattrinfo.size;
+    off = local->xattrinfo.offset;
+
+    parch = (archive_t *)config;
+
+    if (strcmp(parch->trailer, CVLT_TRAILER)) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    gf_msg_debug(plugin, 0,
+                 " read invoked for gfid = %s offset = %" PRIu64
+                 " file_size = %" PRIu64,
+                 uuid_utoa(locxattr->gfid), off, local->stbuf.ia_size);
+
+    if (off >= local->stbuf.ia_size) {
+        /*
+         * Hack to notify higher layers of EOF.
+         */
+
+        op_errno = ENOENT;
+        op_ret = 0;
+
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_READ_FAILED,
+               " reporting end-of-file for gfid=%s", uuid_utoa(locxattr->gfid));
+
+        goto err;
+    }
+
+    if (!size) {
+        op_errno = EINVAL;
+
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_READ_FAILED,
+               " zero size read attempted on gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    if (!(parch->fops.read)) {
+        op_errno = ELIBBAD;
+        goto err;
+    }
+
+    /*
+     * The read request need to be processed. Allocate a request.
+     */
+    req = cvlt_alloc_req(parch);
+
+    if (!req) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, CVLT_NO_MEMORY,
+               " failed to allocated request for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    req->iobuf = iobuf_get_page_aligned(parch->iobuf_pool, size, ALIGN_SIZE);
+    if (!req->iobuf) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /*
+     * Initialize the request object.
+     */
+    req->op_type = CVLT_READ_OP;
+    req->offset = off;
+    req->bytes = size;
+    req->frame = frame;
+    req->szxattr.size = local->stbuf.ia_size;
+    req->szxattr.blocks = local->stbuf.ia_blocks;
+    req->szxattr.blksize = local->stbuf.ia_blksize;
+
+    /*
+     * The file is currently residing inside a data management store.
+     * To read the file contents we need to provide the information
+     * about data management store.
+     */
+    op_ret = cvlt_init_store_info(parch, &(req->store_info));
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract store info for gfid=%s"
+               " offset=%" PRIu64 " size=%" GF_PRI_SIZET
+               ", "
+               " buf=%p",
+               uuid_utoa(locxattr->gfid), off, size, req->iobuf->ptr);
+        goto err;
+    }
+
+    op_ret = cvlt_init_file_info(locxattr, &(req->file_info));
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract file info for gfid=%s"
+               " offset=%" PRIu64 " size=%" GF_PRI_SIZET
+               ", "
+               " buf=%p",
+               uuid_utoa(locxattr->gfid), off, size, req->iobuf->ptr);
+        goto err;
+    }
+
+    /*
+     * Submit the read request.
+     */
+    op_ret = parch->fops.read(&(parch->descinfo), &(req->store_info),
+                              &(req->file_info), off, req->iobuf->ptr, size,
+                              &op_errno, cvlt_readv_complete, req);
+
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " read failed on gfid=%s"
+               " offset=%" PRIu64 " size=%" GF_PRI_SIZET
+               ", "
+               " buf=%p",
+               uuid_utoa(locxattr->gfid), off, size, req->iobuf->ptr);
+        goto err;
+    }
+
+    return 0;
+
+err:
+
+    iobref = iobref_new();
+    gf_msg_debug(plugin, 0, " read unwinding stack op_ret = %d, op_errno = %d",
+                 op_ret, op_errno);
+
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, &iov, 1,
+                        &(local->stbuf), iobref, local->xattr_rsp);
+
+    if (iobref) {
+        iobref_unref(iobref);
+    }
+
+    if (req) {
+        cvlt_free_req(parch, req);
+    }
+
+    return 0;
+}
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.h
new file mode 100644
index 00000000000..c45ac948f6c
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.h
@@ -0,0 +1,84 @@
+/*
+  Copyright (c) 2018 Commvault Systems, Inc. <http://www.commvault.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#ifndef _LIBCVLT_H
+#define _LIBCVLT_H
+
+#include <semaphore.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/compat-errno.h>
+#include "cloudsync-common.h"
+#include "libcvlt-mem-types.h"
+#include "archivestore.h"
+
+enum _cvlt_op {
+    CVLT_READ_OP = 1,
+    CVLT_WRITE_OP = 2,
+    CVLT_RESTORE_OP = 3,
+    CVLT_ARCHIVE_OP = 4,
+    CVLT_LOOKUP_OP = 5,
+    CVLT_XATTR_OP = 6,
+    CVLT_STAT_OP = 7,
+    CVLT_FSTAT_op = 8,
+    CVLT_UNDEF_OP = 127
+};
+typedef enum _cvlt_op cvlt_op_t;
+
+struct _archive;
+struct _cvlt_request {
+    uint64_t offset;
+    uint64_t bytes;
+    struct iobuf *iobuf;
+    struct iobref *iobref;
+    call_frame_t *frame;
+    cvlt_op_t op_type;
+    int32_t op_ret;
+    int32_t op_errno;
+    xlator_t *this;
+    sem_t sem;
+    archstore_info_t store_info;
+    archstore_fileinfo_t file_info;
+    cs_size_xattr_t szxattr;
+};
+typedef struct _cvlt_request cvlt_request_t;
+
+struct _archive {
+    gf_lock_t lock;                /* lock for controlling access   */
+    xlator_t *xl;                  /* xlator                        */
+    void *handle;                  /* handle returned from dlopen   */
+    int32_t nreqs;                 /* num requests active           */
+    struct mem_pool *req_pool;     /* pool for requests             */
+    struct iobuf_pool *iobuf_pool; /* iobuff pool                   */
+    archstore_desc_t descinfo;     /* Archive store descriptor info */
+    archstore_methods_t fops;      /* function pointers             */
+    char *product_id;
+    char *store_id;
+    char *trailer;
+};
+typedef struct _archive archive_t;
+
+void *
+cvlt_init(xlator_t *);
+
+int
+cvlt_reconfigure(xlator_t *, dict_t *);
+
+void
+cvlt_fini(void *);
+
+int
+cvlt_download(call_frame_t *, void *);
+
+int
+cvlt_read(call_frame_t *, void *);
+
+#endif
diff --git a/xlators/features/cloudsync/src/cloudsync.c b/xlators/features/cloudsync/src/cloudsync.c
index 48e27c372b6..7f0b9e563b8 100644
--- a/xlators/features/cloudsync/src/cloudsync.c
+++ b/xlators/features/cloudsync/src/cloudsync.c
@@ -8,1666 +8,2069 @@
  *   cases as published by the Free Software Foundation.
  */
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 #include "cloudsync.h"
 #include "cloudsync-common.h"
-#include "call-stub.h"
+#include <glusterfs/call-stub.h>
 #include "cloudsync-autogen-fops.h"
 
+#include <string.h>
 #include <dlfcn.h>
 
-void
-cs_cleanup_private (cs_private_t *priv)
+static void
+cs_cleanup_private(cs_private_t *priv)
 {
-        if (priv) {
-                if (priv->stores) {
-                        priv->stores->fini (priv->stores->config);
-                        GF_FREE (priv->stores);
-                }
-
-                pthread_spin_destroy (&priv->lock);
-                GF_FREE (priv);
+    if (priv) {
+        if (priv->stores) {
+            priv->stores->fini(priv->stores->config);
+            GF_FREE(priv->stores);
         }
 
-        return;
+        pthread_spin_destroy(&priv->lock);
+        GF_FREE(priv);
+    }
+
+    return;
 }
 
+static struct cs_plugin plugins[] = {
+    {.name = "cloudsyncs3",
+     .library = "cloudsyncs3.so",
+     .description = "cloudsync s3 store."},
+#if defined(__linux__)
+    {.name = "cvlt",
+     .library = "cloudsynccvlt.so",
+     .description = "Commvault content store."},
+#endif
+    {.name = NULL},
+};
+
 int
-cs_init (xlator_t *this)
-{
-        cs_private_t            *priv = NULL;
-        gf_boolean_t            per_vol = _gf_false;
-        int                     ret = 0;
-        char                    *libpath = ("libaws.so");
-        store_methods_t         *store_methods = NULL;
-        void                    *handle = NULL;
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_cs_mt_cs_private_t);
-        if (!priv) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
-                goto out;
+cs_init(xlator_t *this)
+{
+    cs_private_t *priv = NULL;
+    gf_boolean_t per_vol = _gf_false;
+    int ret = 0;
+    char *libpath = NULL;
+    store_methods_t *store_methods = NULL;
+    void *handle = NULL;
+    char *temp_str = NULL;
+    int index = 0;
+    char *libname = NULL;
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_cs_mt_cs_private_t);
+    if (!priv) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+        goto out;
+    }
+
+    priv->this = this;
+
+    this->local_pool = mem_pool_new(cs_local_t, 512);
+    if (!this->local_pool) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM, "initialisation failed.");
+        ret = -1;
+        goto out;
+    }
+
+    this->private = priv;
+
+    GF_OPTION_INIT("cloudsync-remote-read", priv->remote_read, bool, out);
+
+    /* temp workaround. Should be configurable through glusterd*/
+    per_vol = _gf_true;
+
+    if (per_vol) {
+        if (dict_get_str_sizen(this->options, "cloudsync-storetype",
+                               &temp_str) == 0) {
+            for (index = 0; plugins[index].name; index++) {
+                if (!strcmp(temp_str, plugins[index].name)) {
+                    libname = plugins[index].library;
+                    break;
+                }
+            }
+        } else {
+            ret = 0;
         }
 
-        priv->this = this;
+        if (!libname) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0, "no plugin enabled");
+            ret = 0;
+            goto out;
+        }
 
-        this->local_pool = mem_pool_new (cs_local_t, 512);
-        if (!this->local_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, ENOMEM,
-                        "initialisation failed.");
-                ret = -1;
-                goto out;
+        ret = gf_asprintf(&libpath, "%s/%s", CS_PLUGINDIR, libname);
+        if (ret == -1) {
+            goto out;
         }
 
-        this->private = priv;
+        handle = dlopen(libpath, RTLD_NOW);
+        if (!handle) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                   "could not "
+                   "load the required library. %s",
+                   dlerror());
+            ret = 0;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                   "loading library:%s successful", libname);
+        }
 
-        /* temp workaround. Should be configurable through glusterd*/
-        per_vol = _gf_true;
+        priv->stores = GF_CALLOC(1, sizeof(struct cs_remote_stores),
+                                 gf_cs_mt_cs_remote_stores_t);
+        if (!priv->stores) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "Could not "
+                   "allocate memory for priv->stores");
+            ret = -1;
+            goto out;
+        }
 
-        if (per_vol) {
-                /*TODO:Need to make it configurable. This is a temp workaround*/
-                handle = dlopen (libpath, RTLD_NOW);
-                if (!handle) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0, "could not load"
-                                " the required library. %s", dlerror ());
-                        goto out;
-                } else {
-                        gf_msg (this->name, GF_LOG_INFO, 0, 0,
-                                "loading library:%s successful", libpath);
-                }
+        (void)dlerror(); /* clear out previous error string */
 
+        /* load library methods */
+        store_methods = (store_methods_t *)dlsym(handle, "store_ops");
+        if (!store_methods) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "null store_methods %s",
+                   dlerror());
+            ret = -1;
+            goto out;
+        }
 
-                priv->stores = GF_CALLOC (1, sizeof (struct cs_remote_stores),
-                                          gf_cs_mt_cs_remote_stores_t);
-                if (!priv->stores) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0, "Could not "
-                                "allocate memory for priv->stores");
-                        ret = -1;
-                        goto out;
-                }
+        (void)dlerror();
 
-                (void) dlerror (); /* clear out previous error string */
+        if (priv->remote_read) {
+            priv->stores->rdfop = store_methods->fop_remote_read;
+            if (!priv->stores->rdfop) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                       "failed to get"
+                       " read fop %s",
+                       dlerror());
+                ret = -1;
+                goto out;
+            }
+        }
 
-                /* load library methods */
-                store_methods = (store_methods_t *) dlsym (handle, "store_ops");
-                if (!store_methods) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                                "null store_methods %s", dlerror ());
-                        ret = -1;
-                        goto out;
-                }
+        priv->stores->dlfop = store_methods->fop_download;
+        if (!priv->stores->dlfop) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "failed to get"
+                   " download fop %s",
+                   dlerror());
+            ret = -1;
+            goto out;
+        }
 
-                (void) dlerror ();
+        (void)dlerror();
+        priv->stores->init = store_methods->fop_init;
+        if (!priv->stores->init) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "failed to get"
+                   " init fop %s",
+                   dlerror());
+            ret = -1;
+            goto out;
+        }
 
-                priv->stores->dlfop = store_methods->fop_download;
-                if (!priv->stores->dlfop) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0, "failed to get"
-                                " download fop %s", dlerror ());
-                        ret = -1;
-                        goto out;
-                }
+        (void)dlerror();
+        priv->stores->reconfigure = store_methods->fop_reconfigure;
+        if (!priv->stores->reconfigure) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "failed to get"
+                   " reconfigure fop %s",
+                   dlerror());
+            ret = -1;
+            goto out;
+        }
 
-                (void) dlerror ();
-                priv->stores->init = store_methods->fop_init;
-                if (!priv->stores->init) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0, "failed to get"
-                                " init fop %s", dlerror ());
-                        ret = -1;
-                        goto out;
-                }
+        priv->stores->handle = handle;
 
-                (void) dlerror ();
-                priv->stores->reconfigure = store_methods->fop_reconfigure;
-                if (!priv->stores->reconfigure) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0, "failed to get"
-                                " reconfigure fop %s", dlerror ());
-                        ret = -1;
-                        goto out;
-                }
+        priv->stores->config = (void *)((priv->stores->init)(this));
+        if (!priv->stores->config) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "null config");
+            ret = -1;
+            goto out;
+        }
+    }
 
-                priv->stores->handle = handle;
+    ret = 0;
 
-                priv->stores->config = (void *) ((priv->stores->init) (this));
-                if (!priv->stores->config) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0, "null config");
-                        ret = -1;
-                        goto out;
-                }
+out:
+    if (ret == -1) {
+        if (this->local_pool) {
+            mem_pool_destroy(this->local_pool);
+            this->local_pool = NULL;
         }
 
-out:
-        if (ret == -1) {
-                if (this->local_pool)
-                        mem_pool_destroy (this->local_pool);
+        cs_cleanup_private(priv);
 
-                cs_cleanup_private (priv);
+        if (handle) {
+            dlclose(handle);
         }
+    }
 
-        return ret;
-}
+    GF_FREE(libpath);
 
-void
-cs_fini (xlator_t *this)
-{
-        cs_private_t *priv = NULL;
-        priv = this->private;
-
-        cs_cleanup_private (priv);
+    return ret;
 }
 
 int
-cs_reconfigure (xlator_t *this, dict_t *options)
+cs_forget(xlator_t *this, inode_t *inode)
 {
-        cs_private_t    *priv = NULL;
-        int              ret = 0;
+    uint64_t ctx_int = 0;
+    cs_inode_ctx_t *ctx = NULL;
 
-        priv = this->private;
-        if (!priv) {
-                ret = -1;
-                goto out;
-        }
+    inode_ctx_del(inode, this, &ctx_int);
+    if (!ctx_int)
+        return 0;
 
-        /* needed only for per volume configuration*/
-        ret = priv->stores->reconfigure (this, options);
+    ctx = (cs_inode_ctx_t *)(uintptr_t)ctx_int;
 
-out:
-        return ret;
+    GF_FREE(ctx);
+    return 0;
 }
 
-int32_t
-cs_mem_acct_init (xlator_t *this)
+void
+cs_fini(xlator_t *this)
 {
-        int     ret = -1;
+    cs_private_t *priv = NULL;
+    priv = this->private;
 
-        GF_VALIDATE_OR_GOTO ("cloudsync", this, out);
+    cs_cleanup_private(priv);
+}
 
-        ret = xlator_mem_acct_init (this, gf_cs_mt_end + 1);
+int
+cs_reconfigure(xlator_t *this, dict_t *options)
+{
+    cs_private_t *priv = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    if (!priv) {
+        ret = -1;
+        goto out;
+    }
+
+    GF_OPTION_RECONF("cloudsync-remote-read", priv->remote_read, options, bool,
+                     out);
+
+    /* needed only for per volume configuration*/
+    ret = priv->stores->reconfigure(this, options);
 
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                        "Memory accounting init failed");
-                return ret;
-        }
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-cs_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                 dict_t *xdata)
+cs_mem_acct_init(xlator_t *this)
 {
-        gf_dirent_t *tmp = NULL;
-        char        *sxattr = NULL;
-        uint64_t     ia_size = 0;
-        int          ret = 0;
+    int ret = -1;
 
-        list_for_each_entry (tmp, &entries->list, list) {
-                ret = dict_get_str (tmp->dict, GF_CS_OBJECT_SIZE, &sxattr);
-                if (ret) {
-                        gf_msg_trace (this->name, 0, "size xattr found");
-                        continue;
-                }
+    GF_VALIDATE_OR_GOTO("cloudsync", this, out);
 
-                ia_size = atoll (sxattr);
-                tmp->d_stat.ia_size = ia_size;
-        }
+    ret = xlator_mem_acct_init(this, gf_cs_mt_end + 1);
 
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno,
-                             entries, xdata);
-        return 0;
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "Memory accounting init failed");
+        return ret;
+    }
+out:
+    return ret;
 }
 
-
 int32_t
-cs_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t off, dict_t *xdata)
+cs_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *xdata)
 {
-        int ret = 0;
-        int op_errno = ENOMEM;
+    int ret = 0;
+    int op_errno = ENOMEM;
 
+    if (!xdata) {
+        xdata = dict_new();
         if (!xdata) {
-                xdata = dict_new ();
-                if (!xdata) {
-                        goto err;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM,
+                   "failed to create "
+                   "dict");
+            goto err;
         }
-
-        ret = dict_set_int32 (xdata, GF_CS_OBJECT_SIZE, 1);
-        if (ret) {
-                goto err;
-        }
-
-        STACK_WIND (frame, cs_readdirp_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, off, xdata);
-        return 0;
+    }
+
+    ret = dict_set_uint32(xdata, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+    STACK_WIND(frame, default_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno,
-                             NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(readdirp, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-cs_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf, dict_t *xdata)
+cs_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
 {
-        cs_local_t      *local = NULL;
-        int              ret = 0;
-        uint64_t         val = 0;
-
-        local = frame->local;
-
-        /* Do we need lock here? */
-        local->call_cnt++;
-
-        if (op_ret == -1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "truncate failed");
-                ret = dict_get_uint64 (xdata, GF_CS_OBJECT_STATUS, &val);
-                if (ret == 0) {
-                        if (val == GF_CS_ERROR) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                                        "could not get file state, unwinding");
-                                op_ret = -1;
-                                op_errno = EIO;
-                                goto unwind;
-                        } else {
-                                __cs_inode_ctx_update (this, local->loc.inode,
-                                                       val);
-                                gf_msg (this->name, GF_LOG_INFO, 0, 0,
-                                        " state = %ld", val);
-
-                                if (local->call_cnt == 1 &&
-                                    (val == GF_CS_REMOTE ||
-                                     val == GF_CS_DOWNLOADING))  {
-                                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                                0, "will repair and download "
-                                                "the file, current state : %ld",
-                                                val);
-                                        goto repair;
-                                } else {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                                                "second truncate, Unwinding");
-                                        goto unwind;
-                                }
-                        }
+    cs_local_t *local = NULL;
+    int ret = 0;
+    uint64_t val = 0;
+
+    local = frame->local;
+
+    local->call_cnt++;
+
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "truncate failed");
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (ret == 0) {
+            if (val == GF_CS_ERROR) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                       "could not get file state, unwinding");
+                op_ret = -1;
+                op_errno = EIO;
+                goto unwind;
+            } else {
+                __cs_inode_ctx_update(this, local->loc.inode, val);
+                gf_msg(this->name, GF_LOG_INFO, 0, 0, " state = %" PRIu64, val);
+
+                if (local->call_cnt == 1 &&
+                    (val == GF_CS_REMOTE || val == GF_CS_DOWNLOADING)) {
+                    gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                           "will repair and download "
+                           "the file, current state : %" PRIu64,
+                           val);
+                    goto repair;
                 } else {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0, "file state "
-                                "could not be figured, unwinding");
-                        goto unwind;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                           "second truncate, Unwinding");
+                    goto unwind;
                 }
+            }
         } else {
-                /* successful write => file is local */
-                __cs_inode_ctx_update (this, local->loc.inode, GF_CS_LOCAL);
-                gf_msg (this->name, GF_LOG_INFO, 0, 0, "state : GF_CS_LOCAL"
-                        ", truncate successful");
-
-                goto unwind;
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "file state "
+                   "could not be figured, unwinding");
+            goto unwind;
         }
+    } else {
+        /* successful write => file is local */
+        __cs_inode_ctx_update(this, local->loc.inode, GF_CS_LOCAL);
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "state : GF_CS_LOCAL"
+               ", truncate successful");
+
+        goto unwind;
+    }
 
 repair:
-        ret = locate_and_execute (frame);
-        if (ret) {
-                goto unwind;
-        }
+    ret = locate_and_execute(frame);
+    if (ret) {
+        goto unwind;
+    }
 
-        return 0;
+    return 0;
 
 unwind:
-        CS_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf, postbuf,
-                         xdata);
-        return 0;
+    CS_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
 }
 
-
 int32_t
-cs_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-             dict_t *xdata)
+cs_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
 {
-        int                     op_errno        = -1;
-        cs_local_t             *local           = NULL;
-        int                     ret             = 0;
-        cs_inode_ctx_t         *ctx             = NULL;
-        gf_cs_obj_state            state           = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        local = cs_local_init (this, frame, loc, NULL, GF_FOP_TRUNCATE);
-        if (!local) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "local init failed");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        __cs_inode_ctx_get (this, loc->inode, &ctx);
-
-        if (ctx)
-                state = __cs_get_file_state (this, loc->inode, ctx);
-        else
-                state = GF_CS_LOCAL;
-
-        local->xattr_req = xdata ? dict_ref (xdata) : (xdata = dict_new ());
-
-        ret = dict_set_uint32 (local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    cs_local_t *local = NULL;
+    int ret = 0;
+    cs_inode_ctx_t *ctx = NULL;
+    gf_cs_obj_state state = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    local = cs_local_init(this, frame, loc, NULL, GF_FOP_TRUNCATE);
+    if (!local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "local init failed");
+        goto err;
+    }
+
+    __cs_inode_ctx_get(this, loc->inode, &ctx);
+
+    if (ctx)
+        state = __cs_get_file_state(loc->inode, ctx);
+    else
+        state = GF_CS_LOCAL;
+
+    local->xattr_req = xdata ? dict_ref(xdata) : (xdata = dict_new());
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+    local->stub = fop_truncate_stub(frame, cs_resume_truncate, loc, offset,
+                                    xdata);
+    if (!local->stub) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+        goto err;
+    }
+
+    if (state == GF_CS_LOCAL) {
+        STACK_WIND(frame, cs_truncate_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+
+    } else {
+        local->call_cnt++;
+        ret = locate_and_execute(frame);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "dict_set failed key:"
-                        " %s", GF_CS_OBJECT_STATUS);
-                goto err;
-        }
-
-        local->stub = fop_truncate_stub (frame, cs_resume_truncate, loc, offset,
-                                         xdata);
-        if (!local->stub) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        if (state == GF_CS_LOCAL) {
-                STACK_WIND (frame, cs_truncate_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate,
-                            loc, offset, xdata);
-
-        } else {
-                local->call_cnt++;
-                ret = locate_and_execute (frame);
-                if (ret) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
+            goto err;
         }
+    }
 
-        return 0;
+    return 0;
 err:
-        CS_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    CS_STACK_UNWIND(truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    return 0;
 }
 
 int32_t
-cs_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-        int32_t op_ret, int32_t op_errno, struct statvfs *buf,
-        dict_t *xdata)
+cs_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno,
-                             buf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(statfs, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int32_t
-cs_statfs (call_frame_t *frame, xlator_t *this,
-        loc_t *loc,
-        dict_t *xdata)
+cs_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        STACK_WIND (frame, cs_statfs_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->statfs,
-                    loc, xdata);
-        return 0;
+    STACK_WIND(frame, cs_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
 }
 
-
 int32_t
-cs_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-        int32_t op_ret, int32_t op_errno, dict_t *dict,
-        dict_t *xdata)
+cs_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno,
-                             dict, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int32_t
-cs_getxattr (call_frame_t *frame, xlator_t *this,
-        loc_t *loc,
-        const char *name,
-        dict_t *xattr_req)
-{
-        STACK_WIND (frame, cs_getxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->getxattr,
-                    loc, name, xattr_req);
-        return 0;
+cs_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xattr_req)
+{
+    STACK_WIND(frame, cs_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xattr_req);
+    return 0;
 }
 
 int32_t
-cs_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+cs_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        cs_local_t      *local = NULL;
+    cs_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->locked)
-                cs_inodelk_unlock (frame);
+    if (local->locked)
+        cs_inodelk_unlock(frame);
 
-        CS_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+    CS_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-cs_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags, dict_t *xdata)
+cs_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
 {
-        data_t          *tmp = NULL;
-        cs_local_t      *local = NULL;
-        int              ret = 0;
+    data_t *tmp = NULL;
+    cs_local_t *local = NULL;
+    int ret = 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
 
-        local = cs_local_init (this, frame, loc, NULL, GF_FOP_SETXATTR);
-        if (!local) {
-                ret = -1;
-                goto err;
-        }
-
-        local->xattr_req = xdata ? dict_ref (xdata) : (xdata = dict_new ());
+    local = cs_local_init(this, frame, loc, NULL, GF_FOP_SETXATTR);
+    if (!local) {
+        ret = -1;
+        goto err;
+    }
 
-        tmp = dict_get (dict, GF_CS_OBJECT_UPLOAD_COMPLETE);
-        if (tmp) {
-                /* Value of key should be the atime */
-                local->stub = fop_setxattr_stub (frame, cs_resume_setxattr,
-                                                 loc, dict, flags, xdata);
+    local->xattr_req = xdata ? dict_ref(xdata) : (xdata = dict_new());
 
-                if (!local->stub)
-                        goto err;
+    tmp = dict_get_sizen(dict, GF_CS_OBJECT_UPLOAD_COMPLETE);
+    if (tmp) {
+        /* Value of key should be the atime */
+        local->stub = fop_setxattr_stub(frame, cs_resume_setxattr, loc, dict,
+                                        flags, xdata);
 
-                ret = locate_and_execute (frame);
-                if (ret) {
-                        goto err;
-                }
+        if (!local->stub)
+            goto err;
 
-                return 0;
+        ret = locate_and_execute(frame);
+        if (ret) {
+            goto err;
         }
 
-        STACK_WIND (frame, cs_setxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags, xdata);
         return 0;
+    }
+
+    STACK_WIND(frame, cs_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
 err:
-        CS_STACK_UNWIND (setxattr, frame, -1, errno, NULL);
-        return 0;
+    CS_STACK_UNWIND(setxattr, frame, -1, errno, NULL);
+    return 0;
 }
 
-
 int32_t
-cs_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-        int32_t op_ret, int32_t op_errno, dict_t *dict,
-        dict_t *xdata)
+cs_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno,
-                             dict, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int32_t
-cs_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
-              dict_t *xdata)
+cs_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
 {
-        STACK_WIND (frame, cs_fgetxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, cs_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 }
 
-
-
 int32_t
-cs_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+cs_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno,
-                             xdata);
-        return 0;
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int32_t
-cs_fsetxattr (call_frame_t *frame, xlator_t *this,
-        fd_t *fd,
-        dict_t *dict,
-        int32_t flags,
-        dict_t *xdata)
-{
-        STACK_WIND (frame, cs_fsetxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-        return 0;
+cs_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+    STACK_WIND(frame, cs_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
 int32_t
-cs_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-        int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-        struct iatt *postparent,
-        dict_t *xdata)
+cs_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
-
 int32_t
-cs_unlink (call_frame_t *frame, xlator_t *this,
-        loc_t *loc,
-        int32_t flags,
-        dict_t *xattr_req)
+cs_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          dict_t *xattr_req)
 {
-        cs_local_t      *local = NULL;
-        int              ret   = 0;
-
-        local = cs_local_init (this, frame, loc, NULL, GF_FOP_UNLINK);
-        if (!local)
-                goto err;
-
-        local->xattr_req = xattr_req ? dict_ref (xattr_req) : dict_new ();
-
-        ret = dict_set_uint32 (local->xattr_req, GF_CS_OBJECT_STATUS, 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "dict_set failed key:"
-                        " %s", GF_CS_OBJECT_STATUS);
-                goto err;
-        }
-        STACK_WIND (frame, cs_unlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
-                    loc, flags, local->xattr_req);
-        return 0;
+    cs_local_t *local = NULL;
+    int ret = 0;
+
+    local = cs_local_init(this, frame, loc, NULL, GF_FOP_UNLINK);
+    if (!local)
+        goto err;
+
+    local->xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+    STACK_WIND(frame, cs_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, flags, local->xattr_req);
+    return 0;
 err:
-        CS_STACK_UNWIND (unlink, frame, -1, errno, NULL, NULL, NULL);
-        return 0;
+    CS_STACK_UNWIND(unlink, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-cs_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-        int32_t op_ret, int32_t op_errno, fd_t *fd,
-        dict_t *xdata)
-{
-        int              ret = 0;
-        uint64_t         val = 0;
-
-        if (op_ret == 0) {
-                ret = dict_get_uint64 (xdata, GF_CS_OBJECT_STATUS, &val);
-                if (!ret) {
-                        ret = __cs_inode_ctx_update (this, fd->inode, val);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                                        "ctx update failed");
-                        }
-                }
-        } else {
-                cs_inode_ctx_reset (this, fd->inode);
+cs_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    int ret = 0;
+    uint64_t val = 0;
+
+    if (op_ret == 0) {
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (!ret) {
+            ret = __cs_inode_ctx_update(this, fd->inode, val);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx update failed");
+            }
         }
+    } else {
+        cs_inode_ctx_reset(this, fd->inode);
+    }
 
-        CS_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
+    CS_STACK_UNWIND(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
-
 int32_t
-cs_open (call_frame_t *frame, xlator_t *this,
-        loc_t *loc,
-        int32_t flags,
-        fd_t *fd,
-        dict_t *xattr_req)
+cs_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xattr_req)
 {
-        cs_local_t      *local = NULL;
-        int              ret   = 0;
-
-        local = cs_local_init (this, frame, NULL, fd, GF_FOP_OPEN);
-        if (!local)
-                goto err;
-
-        local->xattr_req = xattr_req ? dict_ref (xattr_req) : dict_new ();
-
-        ret = dict_set_uint32 (local->xattr_req, GF_CS_OBJECT_STATUS, 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "dict_set failed key:"
-                        " %s", GF_CS_OBJECT_STATUS);
-                goto err;
-        }
-
-        STACK_WIND (frame, cs_open_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, local->xattr_req);
-        return 0;
+    cs_local_t *local = NULL;
+    int ret = 0;
+
+    local = cs_local_init(this, frame, NULL, fd, GF_FOP_OPEN);
+    if (!local)
+        goto err;
+
+    local->xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+    STACK_WIND(frame, cs_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, local->xattr_req);
+    return 0;
 err:
-        CS_STACK_UNWIND (open, frame, -1, errno, NULL, NULL);
-        return 0;
+    CS_STACK_UNWIND(open, frame, -1, errno, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-cs_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-        int32_t op_ret, int32_t op_errno, struct iatt *buf,
-        dict_t *xdata)
-{
-        int              ret = 0;
-        uint64_t         val = 0;
-        fd_t            *fd  = NULL;
-        cs_local_t      *local = NULL;
-
-        local = frame->local;
-
-        fd = local->fd;
-
-        if (op_ret == 0) {
-                ret = dict_get_uint64 (xdata, GF_CS_OBJECT_STATUS, &val);
-                if (!ret) {
-                        gf_msg_debug (this->name, 0, "state %ld", val);
-                        ret = __cs_inode_ctx_update (this, fd->inode, val);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                                        "ctx update failed");
-                        }
-                }
-        } else {
-                cs_inode_ctx_reset (this, fd->inode);
+cs_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+    int ret = 0;
+    uint64_t val = 0;
+    fd_t *fd = NULL;
+    cs_local_t *local = NULL;
+
+    local = frame->local;
+
+    fd = local->fd;
+
+    if (op_ret == 0) {
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (!ret) {
+            gf_msg_debug(this->name, 0, "state %" PRIu64, val);
+            ret = __cs_inode_ctx_update(this, fd->inode, val);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx update failed");
+            }
         }
+    } else {
+        cs_inode_ctx_reset(this, fd->inode);
+    }
 
-        CS_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
+    CS_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-cs_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr_req)
+cs_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr_req)
 {
-        cs_local_t      *local = NULL;
-        int              ret   = 0;
+    cs_local_t *local = NULL;
+    int ret = 0;
 
-        local = cs_local_init (this, frame, NULL, fd, GF_FOP_FSTAT);
-        if (!local)
-                goto err;
+    local = cs_local_init(this, frame, NULL, fd, GF_FOP_FSTAT);
+    if (!local)
+        goto err;
 
-        if (fd->inode->ia_type == IA_IFDIR)
-                goto wind;
+    if (fd->inode->ia_type == IA_IFDIR)
+        goto wind;
 
-        local->xattr_req = xattr_req ? dict_ref (xattr_req) : dict_new ();
+    local->xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
 
-        ret = dict_set_uint32 (local->xattr_req, GF_CS_OBJECT_STATUS, 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "dict_set failed key:"
-                        " %s", GF_CS_OBJECT_STATUS);
-                goto err;
-        }
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
 
 wind:
-        STACK_WIND (frame, cs_fstat_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat,
-                    fd, local->xattr_req);
-        return 0;
+    STACK_WIND(frame, cs_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, local->xattr_req);
+    return 0;
 err:
-        CS_STACK_UNWIND (fstat, frame, -1, errno, NULL, NULL);
-        return 0;
+    CS_STACK_UNWIND(fstat, frame, -1, errno, NULL, NULL);
+    return 0;
 }
 
 cs_local_t *
-cs_local_init (xlator_t *this, call_frame_t *frame, loc_t *loc, fd_t *fd,
-               glusterfs_fop_t fop)
+cs_local_init(xlator_t *this, call_frame_t *frame, loc_t *loc, fd_t *fd,
+              glusterfs_fop_t fop)
 {
-        cs_local_t      *local = NULL;
-        int              ret   = 0;
-
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto out;
+    cs_local_t *local = NULL;
+    int ret = 0;
 
-        if (loc) {
-                ret = loc_copy (&local->loc, loc);
-                if (ret)
-                        goto out;
-        }
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto out;
 
-        if (fd) {
-                local->fd = fd_ref (fd);
-        }
-
-        local->op_ret   = -1;
-        local->op_errno = EUCLEAN;
-        local->fop      = fop;
-        local->dloffset = 0;
-        frame->local = local;
-        local->locked = _gf_false;
-        local->call_cnt = 0;
+    if (loc) {
+        ret = loc_copy(&local->loc, loc);
+        if (ret)
+            goto out;
+    }
+
+    if (fd) {
+        local->fd = fd_ref(fd);
+    }
+
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+    local->fop = fop;
+    local->dloffset = 0;
+    frame->local = local;
+    local->locked = _gf_false;
+    local->call_cnt = 0;
 out:
-        if (ret) {
-                if (local)
-                        mem_put (local);
-                local = NULL;
-        }
+    if (ret) {
+        if (local)
+            mem_put(local);
+        local = NULL;
+    }
 
-        return local;
+    return local;
 }
 
 call_frame_t *
-cs_lock_frame (call_frame_t *parent_frame)
+cs_lock_frame(call_frame_t *parent_frame)
 {
-        call_frame_t    *lock_frame = NULL;
+    call_frame_t *lock_frame = NULL;
 
-        lock_frame = copy_frame (parent_frame);
+    lock_frame = copy_frame(parent_frame);
 
-        if (lock_frame == NULL)
-                goto out;
+    if (lock_frame == NULL)
+        goto out;
 
-        set_lk_owner_from_ptr (&lock_frame->root->lk_owner, parent_frame->root);
+    set_lk_owner_from_ptr(&lock_frame->root->lk_owner, parent_frame->root);
 
 out:
-        return lock_frame;
-
+    return lock_frame;
 }
 
 void
-cs_lock_wipe (call_frame_t *lock_frame)
+cs_lock_wipe(call_frame_t *lock_frame)
 {
-        CS_STACK_DESTROY (lock_frame);
+    CS_STACK_DESTROY(lock_frame);
 }
 
-
 int32_t
-cs_inodelk_unlock_cbk (call_frame_t *frame, void *cookie,  xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+cs_inodelk_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        cs_lock_wipe (frame);
+    cs_lock_wipe(frame);
 
-        return 0;
+    return 0;
 }
 
 int
-cs_inodelk_unlock (call_frame_t *main_frame)
+cs_inodelk_unlock(call_frame_t *main_frame)
 {
-        xlator_t        *this   = NULL;
-        struct gf_flock  flock  = {0,};
-        call_frame_t    *lock_frame  = NULL;
-        cs_local_t      *lock_local  = NULL;
-        cs_local_t      *main_local  = NULL;
-        int              ret    = 0;
+    xlator_t *this = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    call_frame_t *lock_frame = NULL;
+    cs_local_t *lock_local = NULL;
+    cs_local_t *main_local = NULL;
+    int ret = 0;
 
-        this = main_frame->this;
-        main_local = main_frame->local;
+    this = main_frame->this;
+    main_local = main_frame->local;
 
-        lock_frame = cs_lock_frame (main_frame);
-        if (!lock_frame)
-                goto out;
+    lock_frame = cs_lock_frame(main_frame);
+    if (!lock_frame)
+        goto out;
 
-        lock_local = cs_local_init (this, lock_frame, NULL, NULL, 0);
-        if (!lock_local)
-                goto out;
+    lock_local = cs_local_init(this, lock_frame, NULL, NULL, 0);
+    if (!lock_local)
+        goto out;
 
-        ret = cs_build_loc (&lock_local->loc, main_frame);
-        if (ret) {
-                goto out;
-        }
+    ret = cs_build_loc(&lock_local->loc, main_frame);
+    if (ret) {
+        goto out;
+    }
 
-        flock.l_type = F_UNLCK;
+    flock.l_type = F_UNLCK;
 
-        main_local->locked = _gf_false;
+    main_local->locked = _gf_false;
 
-        STACK_WIND (lock_frame, cs_inodelk_unlock_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->inodelk, CS_LOCK_DOMAIN,
-                    &lock_local->loc, F_SETLKW, &flock, NULL);
+    STACK_WIND(lock_frame, cs_inodelk_unlock_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, CS_LOCK_DOMAIN,
+               &lock_local->loc, F_SETLKW, &flock, NULL);
 
-        return 0;
+    return 0;
 
 out:
-        gf_msg (this->name, GF_LOG_ERROR, 0, 0, "Stale lock would be found on"
-                " server");
+    gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+           "Stale lock would be found on"
+           " server");
 
-        if (lock_frame)
-                cs_lock_wipe (lock_frame);
+    if (lock_frame)
+        cs_lock_wipe(lock_frame);
 
-        return 0;
+    return 0;
 }
 
-void *
-cs_download_task (void *arg)
+int
+cs_download_task(void *arg)
 {
-        call_frame_t    *frame          = NULL;
-        xlator_t        *this           = NULL;
-        cs_private_t    *priv           = NULL;
-        int              ret            = -1;
-        char            *sign_req       = NULL;
-        fd_t            *fd             = NULL;
-        cs_local_t      *local          = NULL;
-        dict_t          *dict           = NULL;
-        int             *retval         = NULL;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    cs_private_t *priv = NULL;
+    int ret = -1;
+    char *sign_req = NULL;
+    fd_t *fd = NULL;
+    cs_local_t *local = NULL;
+    dict_t *dict = NULL;
+
+    frame = (call_frame_t *)arg;
+
+    this = frame->this;
+
+    priv = this->private;
+
+    if (!priv->stores) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "No remote store "
+               "plugins found");
+        ret = -1;
+        goto out;
+    }
+
+    local = frame->local;
+
+    if (local->fd)
+        fd = fd_anonymous(local->fd->inode);
+    else
+        fd = fd_anonymous(local->loc.inode);
+
+    if (!fd) {
+        gf_msg("CS", GF_LOG_ERROR, 0, 0, "fd creation failed");
+        ret = -1;
+        goto out;
+    }
+
+    local->dlfd = fd;
+    local->dloffset = 0;
+
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM,
+               "failed to create "
+               "dict");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_uint32(dict, GF_CS_OBJECT_DOWNLOADING, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "dict_set failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(this, local->fd, dict, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "fsetxattr failed "
+               "key %s",
+               GF_CS_OBJECT_DOWNLOADING);
+        ret = -1;
+        goto out;
+    }
+    /*this calling method is for per volume setting */
+    ret = priv->stores->dlfop(frame, priv->stores->config);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "download failed"
+               ", remotepath: %s",
+               local->remotepath);
+
+        /*using dlfd as it is anonymous and have RDWR flag*/
+        ret = syncop_ftruncate(FIRST_CHILD(this), local->dlfd, 0, NULL, NULL,
+                               NULL, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, -ret, "ftruncate failed");
+        } else {
+            gf_msg_debug(this->name, 0, "ftruncate succeed");
+        }
 
-        frame = (call_frame_t *)arg;
+        ret = -1;
+        goto out;
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "download success, path"
+               " : %s",
+               local->remotepath);
 
-        this = frame->this;
+        ret = syncop_fremovexattr(this, local->fd, GF_CS_OBJECT_REMOTE, NULL,
+                                  NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, -ret,
+                   "removexattr failed, remotexattr");
+            ret = -1;
+            goto out;
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "fremovexattr success, "
+                         "path : %s",
+                         local->remotepath);
+        }
 
-        priv = this->private;
+        ret = syncop_fremovexattr(this, local->fd, GF_CS_OBJECT_DOWNLOADING,
+                                  NULL, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, -ret,
+                   "removexattr failed, downloading xattr, path %s",
+                   local->remotepath);
+            ret = -1;
+            goto out;
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "fremovexattr success"
+                         " path  %s",
+                         local->remotepath);
+        }
+    }
 
-        local = frame->local;
+out:
+    GF_FREE(sign_req);
 
-        retval = GF_CALLOC (1, sizeof(int), gf_common_mt_int);
-        if (!retval) {
-                gf_log (this->name, GF_LOG_ERROR, "insufficient memory");
-                ret = -1;
-                goto out;
-        }
+    if (dict)
+        dict_unref(dict);
 
-        if (local->fd)
-                fd = fd_anonymous (local->fd->inode);
-        else
-                fd = fd_anonymous (local->loc.inode);
+    if (fd) {
+        fd_unref(fd);
+        local->dlfd = NULL;
+    }
 
-        if (!fd) {
-                gf_msg ("CS", GF_LOG_ERROR, 0, 0, "fd creation failed");
-                ret = -1;
-                goto out;
-        }
+    return ret;
+}
 
-        local->dlfd = fd;
-        local->dloffset = 0;
+int
+cs_download(call_frame_t *frame)
+{
+    int ret = 0;
+    cs_local_t *local = NULL;
+    xlator_t *this = NULL;
+
+    local = frame->local;
+    this = frame->this;
+
+    if (!local->remotepath) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "remote path not"
+               " available. Check posix logs to resolve");
+        goto out;
+    }
+
+    ret = cs_download_task((void *)frame);
+out:
+    return ret;
+}
 
-        dict = dict_new ();
-        if (!dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, ENOMEM, "failed to create "
-                        "dict");
-                ret = -1;
-                goto out;
-        }
+int
+cs_set_xattr_req(call_frame_t *frame)
+{
+    cs_local_t *local = NULL;
+    GF_UNUSED int ret = 0;
+
+    local = frame->local;
+
+    /* When remote reads are performed (i.e. reads on remote store),
+     * there needs to be a way to associate a file on gluster volume
+     * with its correspnding file on the remote store. In order to do
+     * that, a unique key can be maintained as an xattr
+     * (GF_CS_XATTR_ARCHIVE_UUID)on the stub file on gluster bricks.
+     * This xattr should be provided to the plugin to
+     * perform the read fop on the correct file. This assumes that the file
+     * hierarchy and name need not be the same on remote store as that of
+     * the gluster volume.
+     */
+    ret = dict_set_sizen_str_sizen(local->xattr_req, GF_CS_XATTR_ARCHIVE_UUID,
+                                   "1");
+
+    return 0;
+}
 
-        ret = dict_set_uint32 (dict, GF_CS_OBJECT_DOWNLOADING, 1);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "dict_set failed");
-                ret = -1;
-                goto out;
+int
+cs_update_xattrs(call_frame_t *frame, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    xlator_t *this = NULL;
+    int size = -1;
+    GF_UNUSED int ret = 0;
+
+    local = frame->local;
+    this = frame->this;
+
+    local->xattrinfo.lxattr = GF_CALLOC(1, sizeof(cs_loc_xattr_t),
+                                        gf_cs_mt_cs_lxattr_t);
+    if (!local->xattrinfo.lxattr) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+
+    gf_uuid_copy(local->xattrinfo.lxattr->gfid, local->loc.gfid);
+
+    if (local->remotepath) {
+        local->xattrinfo.lxattr->file_path = gf_strdup(local->remotepath);
+        if (!local->xattrinfo.lxattr->file_path) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto err;
         }
+    }
+
+    ret = dict_get_gfuuid(xdata, GF_CS_XATTR_ARCHIVE_UUID,
+                          &(local->xattrinfo.lxattr->uuid));
+
+    if (ret) {
+        gf_uuid_clear(local->xattrinfo.lxattr->uuid);
+    }
+    size = strlen(this->name) - strlen("-cloudsync") + 1;
+    local->xattrinfo.lxattr->volname = GF_CALLOC(1, size, gf_common_mt_char);
+    if (!local->xattrinfo.lxattr->volname) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+    strncpy(local->xattrinfo.lxattr->volname, this->name, size - 1);
+    local->xattrinfo.lxattr->volname[size - 1] = '\0';
+
+    return 0;
+err:
+    cs_xattrinfo_wipe(local);
+    return -1;
+}
 
-        ret = syncop_fsetxattr (this, local->fd, dict, 0, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "fsetxattr failed "
-                        "key %s", GF_CS_OBJECT_DOWNLOADING);
-                ret = -1;
-                goto out;
-        }
-        /*this calling method is for per volume setting */
-        ret = priv->stores->dlfop (frame, priv->stores->config);
-        if (ret) {
-                 gf_msg (this->name, GF_LOG_ERROR, 0, 0, "download failed"
-                         ", remotepath: %s", local->remotepath);
-
-                 /*using dlfd as it is anonymous and have RDWR flag*/
-                 ret = syncop_ftruncate (FIRST_CHILD (this), local->dlfd, 0,
-                                         NULL, NULL, NULL, NULL);
-                 if (ret) {
-                         gf_msg (this->name, GF_LOG_ERROR, 0, -ret,
-                                 "ftruncate failed");
-                 } else {
-                         gf_msg_debug (this->name, 0, "ftruncate succeed");
-                 }
-
-                 ret = -1;
-                 goto out;
-        } else {
-                gf_msg (this->name, GF_LOG_INFO, 0, 0, "download success, path"
-                        " : %s", local->remotepath);
+int
+cs_serve_readv(call_frame_t *frame, off_t offset, size_t size, uint32_t flags)
+{
+    xlator_t *this = NULL;
+    cs_private_t *priv = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+    cs_local_t *local = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    priv = this->private;
+
+    if (!local->remotepath) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "remote path not"
+               " available. Check posix logs to resolve");
+        goto out;
+    }
+
+    if (!priv->stores) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "No remote store "
+               "plugins found");
+        ret = -1;
+        goto out;
+    }
+
+    if (local->fd) {
+        fd = fd_anonymous(local->fd->inode);
+    } else {
+        fd = fd_anonymous(local->loc.inode);
+    }
+
+    local->xattrinfo.size = size;
+    local->xattrinfo.offset = offset;
+    local->xattrinfo.flags = flags;
+
+    if (!fd) {
+        gf_msg("CS", GF_LOG_ERROR, 0, 0, "fd creation failed");
+        ret = -1;
+        goto out;
+    }
+
+    local->dlfd = fd;
+    local->dloffset = offset;
+
+    /*this calling method is for per volume setting */
+    ret = priv->stores->rdfop(frame, priv->stores->config);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "read failed"
+               ", remotepath: %s",
+               local->remotepath);
+        ret = -1;
+        goto out;
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "read success, path"
+               " : %s",
+               local->remotepath);
+    }
 
-                ret = syncop_fremovexattr (this, local->fd,
-                                           GF_CS_OBJECT_REMOTE, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, -ret,
-                                "removexattr failed, remotexattr");
-                        ret = -1;
-                        goto out;
-                } else {
-                        gf_msg_debug (this->name, 0, "fremovexattr success, "
-                                      "path : %s", local->remotepath);
-                }
+out:
+    if (fd) {
+        fd_unref(fd);
+        local->dlfd = NULL;
+    }
+    return ret;
+}
 
-                ret = syncop_fremovexattr (this, local->fd,
-                                           GF_CS_OBJECT_DOWNLOADING, NULL,
-                                           NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, -ret,
-                                "removexattr failed, downloading xattr, path %s"
-                                , local->remotepath);
-                        ret = -1;
-                        goto out;
+int32_t
+cs_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iovec *vector, int32_t count,
+             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+    uint64_t val = 0;
+    fd_t *fd = NULL;
+
+    local = frame->local;
+    fd = local->fd;
+
+    local->call_cnt++;
+
+    if (op_ret == -1) {
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (ret == 0) {
+            if (val == GF_CS_ERROR) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                       "could not get file state, unwinding");
+                op_ret = -1;
+                op_errno = EIO;
+                goto unwind;
+            } else {
+                __cs_inode_ctx_update(this, fd->inode, val);
+                gf_msg(this->name, GF_LOG_INFO, 0, 0, " state = %" PRIu64, val);
+
+                if (local->call_cnt == 1 &&
+                    (val == GF_CS_REMOTE || val == GF_CS_DOWNLOADING)) {
+                    gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                           " will read from remote : %" PRIu64, val);
+                    goto repair;
                 } else {
-                        gf_msg_debug (this->name, 0, "fremovexattr success"
-                                      " path  %s", local->remotepath);
+                    gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                           "second readv, Unwinding");
+                    goto unwind;
                 }
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "file state "
+                   "could not be figured, unwinding");
+            goto unwind;
         }
+    } else {
+        /* successful readv => file is local */
+        __cs_inode_ctx_update(this, fd->inode, GF_CS_LOCAL);
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "state : GF_CS_LOCAL"
+               ", readv successful");
 
-out:
-        GF_FREE (sign_req);
+        goto unwind;
+    }
 
-        if (dict)
-                dict_unref (dict);
+repair:
+    ret = locate_and_execute(frame);
+    if (ret) {
+        goto unwind;
+    }
 
-        if (fd) {
-                fd_unref (fd);
-                local->dlfd = NULL;
-        }
+    return 0;
 
-        if (retval) {
-                *retval = ret;
-                pthread_exit (retval);
-        } else {
-                pthread_exit (&ret);
-        }
+unwind:
+    CS_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                    iobref, xdata);
+
+    return 0;
 }
 
-int
-cs_download (call_frame_t *frame)
+int32_t
+cs_resume_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, uint32_t flags, dict_t *xdata)
 {
-        int             *retval = NULL;
-        int             ret = 0;
-        pthread_t       dthread;
-        cs_local_t      *local = NULL;
-        xlator_t        *this = NULL;
+    int ret = 0;
 
-        local = frame->local;
-        this = frame->this;
+    ret = cs_resume_postprocess(this, frame, fd->inode);
+    if (ret) {
+        goto unwind;
+    }
 
-        if (!local->remotepath) {
-                ret = -1;
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "remote path not"
-                        " available. Check posix logs to resolve");
-                goto out;
-        }
+    cs_inodelk_unlock(frame);
 
-        ret = gf_thread_create (&dthread, NULL, &cs_download_task,
-                                (void *)frame, "downloadthread");
+    STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
 
-        pthread_join (dthread, (void **)&retval);
+    return 0;
 
-        ret = *retval;
+unwind:
+    cs_inodelk_unlock(frame);
 
-out:
-        if (retval)
-                GF_FREE (retval);
+    cs_common_cbk(frame);
 
-        return ret;
+    return 0;
 }
 
-int
-cs_stat_check_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
-{
-        cs_local_t      *local = NULL;
-        call_stub_t     *stub  = NULL;
-        char            *filepath = NULL;
-        int              ret = 0;
-        inode_t         *inode = NULL;
-        uint64_t         val = 0;
-
-        local = frame->local;
-
-        if (op_ret == -1) {
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                gf_msg (this->name, GF_LOG_ERROR, 0, op_errno,
-                        "stat check failed");
-                goto err;
-        } else {
-                if (local->fd)
-                        inode = local->fd->inode;
-                else
-                        inode = local->loc.inode;
-
-                if (!inode) {
-                        local->op_ret = -1;
-                        local->op_errno = EINVAL;
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0, "null inode "
-                                "returned");
-                        goto err;
-                }
+int32_t
+cs_resume_remote_readv(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int ret = 0;
+    cs_local_t *local = NULL;
+    gf_cs_obj_state state = -1;
+    cs_inode_ctx_t *ctx = NULL;
+
+    cs_inodelk_unlock(frame);
+
+    local = frame->local;
+    if (!local) {
+        ret = -1;
+        goto unwind;
+    }
+
+    __cs_inode_ctx_get(this, fd->inode, &ctx);
+
+    state = __cs_get_file_state(fd->inode, ctx);
+    if (state == GF_CS_ERROR) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "status is GF_CS_ERROR."
+               " Aborting readv");
+        local->op_ret = -1;
+        local->op_errno = EREMOTE;
+        ret = -1;
+        goto unwind;
+    }
+
+    /* Serve readv from remote store only if it is remote. */
+    gf_msg_debug(this->name, 0, "status of file %s is %d",
+                 local->remotepath ? local->remotepath : "", state);
+
+    /* We will reach this condition if local inode ctx had REMOTE
+     * state when the control was in cs_readv but after stat
+     * we got an updated state saying that the file is LOCAL.
+     */
+    if (state == GF_CS_LOCAL) {
+        STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                   xdata);
+    } else if (state == GF_CS_REMOTE) {
+        ret = cs_resume_remote_readv_postprocess(this, frame, fd->inode, offset,
+                                                 size, flags);
+        /* Failed to submit the remote readv fop to plugin */
+        if (ret) {
+            local->op_ret = -1;
+            local->op_errno = EREMOTE;
+            goto unwind;
+        }
+        /* When the file is in any other intermediate state,
+         * we should not perform remote reads.
+         */
+    } else {
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
 
-                ret = dict_get_uint64 (xdata, GF_CS_OBJECT_STATUS, &val);
-                if (ret == 0) {
-                        if (val == GF_CS_ERROR) {
-                                cs_inode_ctx_reset (this, inode);
-                                local->op_ret = -1;
-                                local->op_errno = EIO;
-                                gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                                        "status = GF_CS_ERROR. failed to get "
-                                        " file state");
-                                goto err;
-                        } else {
-                                ret = __cs_inode_ctx_update (this, inode, val);
-                                gf_msg_debug (this->name, 0, "status : %lu",
-                                              val);
-                                if (ret) {
-                                        gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                                                "ctx update failed");
-                                        local->op_ret = -1;
-                                        local->op_errno = ENOMEM;
-                                        goto err;
-                                }
-                        }
-                } else {
-                        gf_msg_debug (this->name, 0,
-                                      "status not found in dict");
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto err;
-                }
+    return 0;
 
-                ret = dict_get_str (xdata, GF_CS_OBJECT_REMOTE, &filepath);
-                if (filepath) {
-                        gf_msg_debug (this->name, 0, "filepath returned %s",
-                                      filepath);
-                        local->remotepath = gf_strdup (filepath);
-                        if (!local->remotepath) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto err;
-                        }
-                } else {
-                        gf_msg_debug (this->name, 0, "NULL filepath");
-                }
+unwind:
+    cs_common_cbk(frame);
+
+    return 0;
+}
 
-                local->op_ret = 0;
-                local->xattr_rsp = dict_ref (xdata);
-                memcpy (&local->stbuf, stbuf, sizeof (struct iatt));
+int32_t
+cs_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int op_errno = ENOMEM;
+    cs_local_t *local = NULL;
+    int ret = 0;
+    cs_inode_ctx_t *ctx = NULL;
+    gf_cs_obj_state state = -1;
+    cs_private_t *priv = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    priv = this->private;
+
+    local = cs_local_init(this, frame, NULL, fd, GF_FOP_READ);
+    if (!local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "local init failed");
+        goto err;
+    }
+
+    __cs_inode_ctx_get(this, fd->inode, &ctx);
+
+    if (ctx)
+        state = __cs_get_file_state(fd->inode, ctx);
+    else
+        state = GF_CS_LOCAL;
+
+    local->xattr_req = xdata ? dict_ref(xdata) : (xdata = dict_new());
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+    if (priv->remote_read) {
+        local->stub = fop_readv_stub(frame, cs_resume_remote_readv, fd, size,
+                                     offset, flags, xdata);
+    } else {
+        local->stub = fop_readv_stub(frame, cs_resume_readv, fd, size, offset,
+                                     flags, xdata);
+    }
+    if (!local->stub) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+        goto err;
+    }
+
+    if (state == GF_CS_LOCAL) {
+        STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                   xdata);
+    } else {
+        local->call_cnt++;
+        ret = locate_and_execute(frame);
+        if (ret) {
+            goto err;
         }
+    }
 
-        stub = local->stub;
-        local->stub = NULL;
-        call_resume (stub);
+    return 0;
 
-        return 0;
 err:
-        cs_inodelk_unlock (frame);
-
-        cs_common_cbk (frame);
+    CS_STACK_UNWIND(readv, frame, -1, op_errno, NULL, -1, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-cs_do_stat_check (call_frame_t *main_frame)
+cs_resume_remote_readv_postprocess(xlator_t *this, call_frame_t *frame,
+                                   inode_t *inode, off_t offset, size_t size,
+                                   uint32_t flags)
 {
-        cs_local_t       *local  = NULL;
-        xlator_t         *this   = NULL;
-        int               ret   = 0;
+    int ret = 0;
 
-        local = main_frame->local;
-        this = main_frame->this;
+    ret = cs_serve_readv(frame, offset, size, flags);
 
-        ret = dict_set_uint32 (local->xattr_req, GF_CS_OBJECT_REPAIR, 256);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "dict_set failed");
+    return ret;
+}
+
+int
+cs_stat_check_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *stbuf, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    char *filepath = NULL;
+    int ret = 0;
+    inode_t *inode = NULL;
+    uint64_t val = 0;
+
+    local = frame->local;
+
+    if (op_ret == -1) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        gf_msg(this->name, GF_LOG_ERROR, 0, op_errno, "stat check failed");
+        goto err;
+    } else {
+        if (local->fd)
+            inode = local->fd->inode;
+        else
+            inode = local->loc.inode;
+
+        if (!inode) {
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "null inode "
+                   "returned");
+            goto err;
+        }
+
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (ret == 0) {
+            if (val == GF_CS_ERROR) {
+                cs_inode_ctx_reset(this, inode);
+                local->op_ret = -1;
+                local->op_errno = EIO;
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                       "status = GF_CS_ERROR. failed to get "
+                       " file state");
                 goto err;
+            } else {
+                ret = __cs_inode_ctx_update(this, inode, val);
+                gf_msg_debug(this->name, 0, "status : %" PRIu64, val);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx update failed");
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    goto err;
+                }
+            }
+        } else {
+            gf_msg_debug(this->name, 0, "status not found in dict");
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto err;
         }
 
-        if (local->fd) {
-                STACK_WIND (main_frame, cs_stat_check_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fstat, local->fd,
-                            local->xattr_req);
+        ret = dict_get_str_sizen(xdata, GF_CS_OBJECT_REMOTE, &filepath);
+        if (filepath) {
+            gf_msg_debug(this->name, 0, "filepath returned %s", filepath);
+            local->remotepath = gf_strdup(filepath);
+            if (!local->remotepath) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto err;
+            }
         } else {
-                STACK_WIND (main_frame, cs_stat_check_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->stat, &local->loc,
-                            local->xattr_req);
+            gf_msg_debug(this->name, 0, "NULL filepath");
         }
 
-        return 0;
+        ret = cs_update_xattrs(frame, xdata);
+        if (ret)
+            goto err;
 
+        local->op_ret = 0;
+        local->xattr_rsp = dict_ref(xdata);
+        memcpy(&local->stbuf, stbuf, sizeof(struct iatt));
+    }
+
+    stub = local->stub;
+    local->stub = NULL;
+    call_resume(stub);
+
+    return 0;
 err:
-        cs_inodelk_unlock (main_frame);
+    cs_inodelk_unlock(frame);
 
-        cs_common_cbk (main_frame);
+    cs_common_cbk(frame);
 
-        return 0;
+    return 0;
+}
+
+int
+cs_do_stat_check(call_frame_t *main_frame)
+{
+    cs_local_t *local = NULL;
+    xlator_t *this = NULL;
+    int ret = 0;
+
+    local = main_frame->local;
+    this = main_frame->this;
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_REPAIR, 256);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "dict_set failed");
+        goto err;
+    }
+
+    cs_set_xattr_req(main_frame);
+
+    if (local->fd) {
+        STACK_WIND(main_frame, cs_stat_check_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, local->fd, local->xattr_req);
+    } else {
+        STACK_WIND(main_frame, cs_stat_check_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->stat, &local->loc,
+                   local->xattr_req);
+    }
+
+    return 0;
+
+err:
+    cs_inodelk_unlock(main_frame);
+
+    cs_common_cbk(main_frame);
+
+    return 0;
 }
 
 void
-cs_common_cbk (call_frame_t *frame)
+cs_common_cbk(call_frame_t *frame)
 {
-        glusterfs_fop_t fop = -1;
-        cs_local_t      *local = NULL;
+    glusterfs_fop_t fop = -1;
+    cs_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        fop = local->fop;
+    fop = local->fop;
 
-        /*Note: Only the failure case needs to be handled here. Since for
-         * successful stat check the fop will resume anyway. The unwind can
-         * happen from the fop_cbk and each cbk can unlock the inodelk in case
-         * a lock was taken before. The lock status can be stored in frame */
+    /*Note: Only the failure case needs to be handled here. Since for
+     * successful stat check the fop will resume anyway. The unwind can
+     * happen from the fop_cbk and each cbk can unlock the inodelk in case
+     * a lock was taken before. The lock status can be stored in frame */
 
-        /* for failure case  */
+    /* for failure case  */
 
-        /*TODO: add other fops*/
-        switch (fop) {
+    /*TODO: add other fops*/
+    switch (fop) {
         case GF_FOP_WRITE:
-                CS_STACK_UNWIND (writev, frame, local->op_ret,
-                                 local->op_errno, NULL, NULL, NULL);
-                break;
+            CS_STACK_UNWIND(writev, frame, local->op_ret, local->op_errno, NULL,
+                            NULL, NULL);
+            break;
 
         case GF_FOP_SETXATTR:
-                CS_STACK_UNWIND (setxattr, frame, local->op_ret,
-                                 local->op_errno, NULL);
-                break;
+            CS_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                            NULL);
+            break;
         case GF_FOP_READ:
-                CS_STACK_UNWIND (readv, frame, local->op_ret,
-                                 local->op_errno, NULL, 0, NULL, NULL,
-                                 NULL);
-                break;
+            CS_STACK_UNWIND(readv, frame, local->op_ret, local->op_errno, NULL,
+                            0, NULL, NULL, NULL);
+            break;
         case GF_FOP_FTRUNCATE:
-                CS_STACK_UNWIND (ftruncate, frame, local->op_ret,
-                                 local->op_errno, NULL, NULL, NULL);
-                break;
-
-
+            CS_STACK_UNWIND(ftruncate, frame, local->op_ret, local->op_errno,
+                            NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_TRUNCATE:
+            CS_STACK_UNWIND(truncate, frame, local->op_ret, local->op_errno,
+                            NULL, NULL, NULL);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        return;
+    return;
 }
 
 int
-cs_blocking_inodelk_cbk (call_frame_t *lock_frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+cs_blocking_inodelk_cbk(call_frame_t *lock_frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        cs_local_t      *main_local = NULL;
-        call_frame_t    *main_frame = NULL;
-        cs_local_t      *lock_local = NULL;
+    cs_local_t *main_local = NULL;
+    call_frame_t *main_frame = NULL;
+    cs_local_t *lock_local = NULL;
 
-        lock_local = lock_frame->local;
+    lock_local = lock_frame->local;
 
-        main_frame = lock_local->main_frame;
-        main_local = main_frame->local;
+    main_frame = lock_local->main_frame;
+    main_local = main_frame->local;
 
-        if (op_ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "inodelk failed");
-                main_local->op_errno = op_errno;
-                main_local->op_ret = op_ret;
-                goto err;
-        }
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "inodelk failed");
+        main_local->op_errno = op_errno;
+        main_local->op_ret = op_ret;
+        goto err;
+    }
 
-        main_local->locked = _gf_true;
+    main_local->locked = _gf_true;
 
-        cs_lock_wipe (lock_frame);
+    cs_lock_wipe(lock_frame);
 
-        cs_do_stat_check (main_frame);
+    cs_do_stat_check(main_frame);
 
-        return 0;
+    return 0;
 err:
-        cs_common_cbk (main_frame);
+    cs_common_cbk(main_frame);
 
-        cs_lock_wipe (lock_frame);
+    cs_lock_wipe(lock_frame);
 
-        return 0;
+    return 0;
 }
 
 int
-cs_build_loc (loc_t *loc, call_frame_t *frame)
+cs_build_loc(loc_t *loc, call_frame_t *frame)
 {
-        cs_local_t      *local  = NULL;
-        int              ret    = -1;
+    cs_local_t *local = NULL;
+    int ret = -1;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->fd) {
-                loc->inode = inode_ref (local->fd->inode);
-                if (loc->inode) {
-                        gf_uuid_copy (loc->gfid, loc->inode->gfid);
-                        ret = 0;
-                        goto out;
-                } else {
-                        ret = -1;
-                        goto out;
-                }
+    if (local->fd) {
+        loc->inode = inode_ref(local->fd->inode);
+        if (loc->inode) {
+            gf_uuid_copy(loc->gfid, loc->inode->gfid);
+            ret = 0;
+            goto out;
         } else {
-                loc->inode = inode_ref (local->loc.inode);
-                if (loc->inode) {
-                        gf_uuid_copy (loc->gfid, loc->inode->gfid);
-                        ret = 0;
-                        goto out;
-                } else {
-                        ret = -1;
-                        goto out;
-                }
+            ret = -1;
+            goto out;
         }
+    } else {
+        loc->inode = inode_ref(local->loc.inode);
+        if (loc->inode) {
+            gf_uuid_copy(loc->gfid, loc->inode->gfid);
+            ret = 0;
+            goto out;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-cs_blocking_inodelk (call_frame_t *parent_frame)
+cs_blocking_inodelk(call_frame_t *parent_frame)
 {
-        call_frame_t    *lock_frame = NULL;
-        cs_local_t      *lock_local = NULL;
-        xlator_t        *this       = NULL;
-        struct gf_flock  flock      = {0,};
-        int              ret        = 0;
-
-        lock_frame = cs_lock_frame (parent_frame);
-        if (!lock_frame) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "insuffcient memory");
-                goto err;
-        }
-
-        this = parent_frame->this;
-
-        lock_local = cs_local_init (this, lock_frame, NULL, NULL, 0);
-        if (!lock_local) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "local init failed");
-                goto err;
-        }
-
-        lock_local->main_frame = parent_frame;
-
-        flock.l_type = F_WRLCK;
-
-        ret = cs_build_loc (&lock_local->loc, parent_frame);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "build_loc failed");
-                goto err;
-        }
-
-        STACK_WIND (lock_frame, cs_blocking_inodelk_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->inodelk, CS_LOCK_DOMAIN,
-                    &lock_local->loc, F_SETLKW, &flock, NULL);
-
-        return 0;
+    call_frame_t *lock_frame = NULL;
+    cs_local_t *lock_local = NULL;
+    xlator_t *this = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = 0;
+
+    this = parent_frame->this;
+
+    lock_frame = cs_lock_frame(parent_frame);
+    if (!lock_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insuffcient memory");
+        goto err;
+    }
+
+    lock_local = cs_local_init(this, lock_frame, NULL, NULL, 0);
+    if (!lock_local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "local init failed");
+        goto err;
+    }
+
+    lock_local->main_frame = parent_frame;
+
+    flock.l_type = F_WRLCK;
+
+    ret = cs_build_loc(&lock_local->loc, parent_frame);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "build_loc failed");
+        goto err;
+    }
+
+    STACK_WIND(lock_frame, cs_blocking_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, CS_LOCK_DOMAIN,
+               &lock_local->loc, F_SETLKW, &flock, NULL);
+
+    return 0;
 err:
-        if (lock_frame)
-                cs_lock_wipe (lock_frame);
+    if (lock_frame)
+        cs_lock_wipe(lock_frame);
 
-        return -1;
+    return -1;
 }
 
 int
-locate_and_execute (call_frame_t *frame)
+locate_and_execute(call_frame_t *frame)
 {
-        int     ret = 0;
+    int ret = 0;
 
-        ret = cs_blocking_inodelk (frame);
+    ret = cs_blocking_inodelk(frame);
 
-        if (ret)
-                return -1;
-        else
-                return 0;
+    if (ret)
+        return -1;
+    else
+        return 0;
 }
 
 int32_t
-cs_resume_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    off_t offset, dict_t *xattr_req)
+cs_resume_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   off_t offset, dict_t *xattr_req)
 {
-        cs_local_t      *local  = NULL;
-        int              ret    = 0;
+    cs_local_t *local = NULL;
+    int ret = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        ret = cs_resume_postprocess (this, frame, loc->inode);
-        if (ret) {
-                goto unwind;
-        }
+    ret = cs_resume_postprocess(this, frame, loc->inode);
+    if (ret) {
+        goto unwind;
+    }
 
-        cs_inodelk_unlock (frame);
+    cs_inodelk_unlock(frame);
 
-        STACK_WIND (frame, cs_truncate_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
-                    loc, offset, local->xattr_req);
+    STACK_WIND(frame, cs_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset,
+               local->xattr_req);
 
-        return 0;
+    return 0;
 
 unwind:
-        cs_inodelk_unlock (frame);
+    cs_inodelk_unlock(frame);
 
-        cs_common_cbk (frame);
+    cs_common_cbk(frame);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-cs_resume_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    dict_t *dict, int32_t flags, dict_t *xdata)
+cs_resume_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *dict, int32_t flags, dict_t *xdata)
 {
-        cs_local_t      *local = NULL;
-        cs_inode_ctx_t  *ctx   = NULL;
-        gf_cs_obj_state  state = GF_CS_ERROR;
-
-        local = frame->local;
-
-        __cs_inode_ctx_get (this, loc->inode, &ctx);
-
-        state = __cs_get_file_state (this, loc->inode, ctx);
-
-        if (state == GF_CS_ERROR) {
-                /* file is already remote */
-                local->op_ret = -1;
-                local->op_errno = EINVAL;
-                gf_msg (this->name, GF_LOG_WARNING, 0, 0,
-                        "file %s , could not figure file state", loc->path);
-                goto unwind;
-        }
-
-
-        if (state == GF_CS_REMOTE) {
-                /* file is already remote */
-                local->op_ret = -1;
-                local->op_errno = EINVAL;
-                gf_msg (this->name, GF_LOG_WARNING, 0, EINVAL,
-                        "file %s is already remote", loc->path);
-                goto unwind;
-        }
-
-        if (state == GF_CS_DOWNLOADING) {
-                gf_msg (this->name, GF_LOG_WARNING, 0, 0,
-                        " file is in downloading state.");
-                local->op_ret = -1;
-                local->op_errno = EINVAL;
-                goto unwind;
-        }
-
-        STACK_WIND (frame, cs_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
-                    local->xattr_req);
-
-        return 0;
+    cs_local_t *local = NULL;
+    cs_inode_ctx_t *ctx = NULL;
+    gf_cs_obj_state state = GF_CS_ERROR;
+
+    local = frame->local;
+
+    __cs_inode_ctx_get(this, loc->inode, &ctx);
+
+    state = __cs_get_file_state(loc->inode, ctx);
+
+    if (state == GF_CS_ERROR) {
+        /* file is already remote */
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+               "file %s , could not figure file state", loc->path);
+        goto unwind;
+    }
+
+    if (state == GF_CS_REMOTE) {
+        /* file is already remote */
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, 0, EINVAL,
+               "file %s is already remote", loc->path);
+        goto unwind;
+    }
+
+    if (state == GF_CS_DOWNLOADING) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+               " file is in downloading state.");
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
+
+    STACK_WIND(frame, cs_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+               local->xattr_req);
+
+    return 0;
 unwind:
-        cs_inodelk_unlock (frame);
+    cs_inodelk_unlock(frame);
 
-        cs_common_cbk (frame);
+    cs_common_cbk(frame);
 
-        return 0;
+    return 0;
 }
 
-
-
-
 gf_cs_obj_state
-__cs_get_file_state (xlator_t *this, inode_t *inode, cs_inode_ctx_t *ctx)
+__cs_get_file_state(inode_t *inode, cs_inode_ctx_t *ctx)
 {
-        gf_cs_obj_state    state = -1;
+    gf_cs_obj_state state = -1;
 
-        if (!ctx)
-                return GF_CS_ERROR;
+    if (!ctx)
+        return GF_CS_ERROR;
 
-        LOCK (&inode->lock);
-        {
-                state = ctx->state;
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        state = ctx->state;
+    }
+    UNLOCK(&inode->lock);
 
-        return state;
+    return state;
 }
 
 void
-__cs_inode_ctx_get (xlator_t *this, inode_t *inode, cs_inode_ctx_t **ctx)
+__cs_inode_ctx_get(xlator_t *this, inode_t *inode, cs_inode_ctx_t **ctx)
 {
-        uint64_t        ctxint = 0;
-        int             ret = 0;
+    uint64_t ctxint = 0;
+    int ret = 0;
 
-        LOCK (&inode->lock);
-        {
-        ret = __inode_ctx_get (inode, this, &ctxint);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctxint);
+    }
+    UNLOCK(&inode->lock);
 
-        if (ret)
-                *ctx = NULL;
-        else
-                *ctx = (cs_inode_ctx_t *)ctxint;
+    if (ret)
+        *ctx = NULL;
+    else
+        *ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
 
-        return;
+    return;
 }
 
 int
-__cs_inode_ctx_update (xlator_t *this, inode_t *inode, uint64_t val)
+__cs_inode_ctx_update(xlator_t *this, inode_t *inode, uint64_t val)
 {
-        cs_inode_ctx_t  *ctx = NULL;
-        uint64_t         ctxint = 0;
-        int              ret = 0;
+    cs_inode_ctx_t *ctx = NULL;
+    uint64_t ctxint = 0;
+    int ret = 0;
 
-        LOCK (&inode->lock);
-        {
-        ret = __inode_ctx_get (inode, this, &ctxint);
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctxint);
         if (ret) {
-                ctx = GF_CALLOC (1, sizeof (*ctx), gf_cs_mt_cs_inode_ctx_t);
-                if (!ctx) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                                "ctx allocation failed");
-                        ret = -1;
-                        goto out;
-                }
+            ctx = GF_CALLOC(1, sizeof(*ctx), gf_cs_mt_cs_inode_ctx_t);
+            if (!ctx) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx allocation failed");
+                ret = -1;
+                goto out;
+            }
 
-                ctx->state = val;
+            ctx->state = val;
 
-                ctxint = (uint64_t) ctx;
+            ctxint = (uint64_t)(uintptr_t)ctx;
 
-                ret = __inode_ctx_set (inode, this, &ctxint);
-                if (ret) {
-                        GF_FREE (ctx);
-                        goto out;
-                }
+            ret = __inode_ctx_set(inode, this, &ctxint);
+            if (ret) {
+                GF_FREE(ctx);
+                goto out;
+            }
         } else {
-                ctx = (cs_inode_ctx_t *) ctxint;
-
-                ctx->state = val;
-        }
+            ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
 
+            ctx->state = val;
         }
+    }
 
 out:
-        UNLOCK (&inode->lock);
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
 int
-cs_inode_ctx_reset (xlator_t *this, inode_t *inode)
+cs_inode_ctx_reset(xlator_t *this, inode_t *inode)
 {
-        cs_inode_ctx_t  *ctx = NULL;
-        uint64_t         ctxint = 0;
+    cs_inode_ctx_t *ctx = NULL;
+    uint64_t ctxint = 0;
 
-        inode_ctx_del (inode, this, &ctxint);
-        if (!ctxint) {
-                return 0;
-        }
+    inode_ctx_del(inode, this, &ctxint);
+    if (!ctxint) {
+        return 0;
+    }
 
-        ctx = (cs_inode_ctx_t *)ctxint;
+    ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
 
-        GF_FREE (ctx);
-        return 0;
+    GF_FREE(ctx);
+    return 0;
 }
 
 int
-cs_resume_postprocess (xlator_t *this, call_frame_t *frame, inode_t *inode)
+cs_resume_postprocess(xlator_t *this, call_frame_t *frame, inode_t *inode)
 {
-        cs_local_t      *local = NULL;
-        gf_cs_obj_state  state = -1;
-        cs_inode_ctx_t  *ctx   = NULL;
-        int              ret   = 0;
-
-        local = frame->local;
-        if (!local) {
-                ret = -1;
-                goto out;
-        }
-
-        __cs_inode_ctx_get (this, inode, &ctx);
-
-        state = __cs_get_file_state (this, inode, ctx);
-        if (state == GF_CS_ERROR) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "status is GF_CS_ERROR."
-                        " Aborting write");
-                local->op_ret = -1;
-                local->op_errno = EREMOTE;
-                ret = -1;
-                goto out;
-        }
-
-        if (state == GF_CS_REMOTE || state == GF_CS_DOWNLOADING) {
-                gf_msg_debug (this->name, 0, "status is %d", state);
-                ret = cs_download (frame);
-                if (ret == 0) {
-                        gf_msg_debug (this->name, 0, "Winding for Final Write");
-                } else {
-                        gf_msg (this->name, GF_LOG_ERROR, 0, 0,
-                                " download failed, unwinding writev");
-                        local->op_ret = -1;
-                        local->op_errno = EREMOTE;
-                        ret = -1;
-                }
+    cs_local_t *local = NULL;
+    gf_cs_obj_state state = -1;
+    cs_inode_ctx_t *ctx = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    if (!local) {
+        ret = -1;
+        goto out;
+    }
+
+    __cs_inode_ctx_get(this, inode, &ctx);
+
+    state = __cs_get_file_state(inode, ctx);
+    if (state == GF_CS_ERROR) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "status is GF_CS_ERROR."
+               " Aborting write");
+        local->op_ret = -1;
+        local->op_errno = EREMOTE;
+        ret = -1;
+        goto out;
+    }
+
+    if (state == GF_CS_REMOTE || state == GF_CS_DOWNLOADING) {
+        gf_msg_debug(this->name, 0, "status is %d", state);
+        ret = cs_download(frame);
+        if (ret == 0) {
+            gf_msg_debug(this->name, 0, "Winding for Final Write");
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   " download failed, unwinding writev");
+            local->op_ret = -1;
+            local->op_errno = EREMOTE;
+            ret = -1;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
+
 int32_t
-cs_fdctx_to_dict (xlator_t *this,
-        fd_t *fd,
-        dict_t *dict)
+cs_fdctx_to_dict(xlator_t *this, fd_t *fd, dict_t *dict)
 {
-        return 0;
+    return 0;
 }
 
-
 int32_t
-cs_inode (xlator_t *this)
+cs_inode(xlator_t *this)
 {
-        return 0;
+    return 0;
 }
 
-
 int32_t
-cs_inode_to_dict (xlator_t *this,
-        dict_t *dict)
+cs_inode_to_dict(xlator_t *this, dict_t *dict)
 {
-        return 0;
+    return 0;
 }
 
-
 int32_t
-cs_history (xlator_t *this)
+cs_history(xlator_t *this)
 {
-        return 0;
+    return 0;
 }
 
-
 int32_t
-cs_fd (xlator_t *this)
+cs_fd(xlator_t *this)
 {
-        return 0;
+    return 0;
 }
 
-
 int32_t
-cs_fd_to_dict (xlator_t *this,
-        dict_t *dict)
+cs_fd_to_dict(xlator_t *this, dict_t *dict)
 {
-        return 0;
+    return 0;
 }
 
-
 int32_t
-cs_fdctx (xlator_t *this,
-        fd_t *fd)
+cs_fdctx(xlator_t *this, fd_t *fd)
 {
-        return 0;
+    return 0;
 }
 
-
 int32_t
-cs_inodectx (xlator_t *this,
-        inode_t *ino)
+cs_inodectx(xlator_t *this, inode_t *ino)
 {
-        return 0;
+    return 0;
 }
 
-
 int32_t
-cs_inodectx_to_dict (xlator_t *this,
-        inode_t *ino,
-        dict_t *dict)
+cs_inodectx_to_dict(xlator_t *this, inode_t *ino, dict_t *dict)
 {
-        return 0;
+    return 0;
 }
 
-
 int32_t
-cs_priv_to_dict (xlator_t *this,
-        dict_t *dict, char *brickname)
+cs_priv_to_dict(xlator_t *this, dict_t *dict, char *brickname)
 {
-        return 0;
+    return 0;
 }
 
-
 int32_t
-cs_priv (xlator_t *this)
+cs_priv(xlator_t *this)
 {
-        return 0;
+    return 0;
 }
 
 int
-cs_notify (xlator_t *this, int event, void *data, ...)
+cs_notify(xlator_t *this, int event, void *data, ...)
 {
-        return default_notify (this, event, data);
+    return default_notify(this, event, data);
 }
 
-
 struct xlator_fops cs_fops = {
-        .stat                 = cs_stat,
-        .readdirp             = cs_readdirp,
-        .truncate             = cs_truncate,
-        .seek                 = cs_seek,
-        .statfs               = cs_statfs,
-        .fallocate            = cs_fallocate,
-        .discard              = cs_discard,
-        .getxattr             = cs_getxattr,
-        .writev               = cs_writev,
-        .setxattr             = cs_setxattr,
-        .fgetxattr            = cs_fgetxattr,
-        .lookup               = cs_lookup,
-        .fsetxattr            = cs_fsetxattr,
-        .readv                = cs_readv,
-        .ftruncate            = cs_ftruncate,
-        .rchecksum            = cs_rchecksum,
-        .unlink               = cs_unlink,
-        .open                 = cs_open,
-        .fstat                = cs_fstat,
-        .zerofill             = cs_zerofill,
+    .stat = cs_stat,
+    .readdirp = cs_readdirp,
+    .truncate = cs_truncate,
+    .seek = cs_seek,
+    .statfs = cs_statfs,
+    .fallocate = cs_fallocate,
+    .discard = cs_discard,
+    .getxattr = cs_getxattr,
+    .writev = cs_writev,
+    .setxattr = cs_setxattr,
+    .fgetxattr = cs_fgetxattr,
+    .lookup = cs_lookup,
+    .fsetxattr = cs_fsetxattr,
+    .readv = cs_readv,
+    .ftruncate = cs_ftruncate,
+    .rchecksum = cs_rchecksum,
+    .unlink = cs_unlink,
+    .open = cs_open,
+    .fstat = cs_fstat,
+    .zerofill = cs_zerofill,
 };
 
 struct xlator_cbks cs_cbks = {
+    .forget = cs_forget,
 };
 
 struct xlator_dumpops cs_dumpops = {
-        .fdctx_to_dict        = cs_fdctx_to_dict,
-        .inode                = cs_inode,
-        .inode_to_dict        = cs_inode_to_dict,
-        .history              = cs_history,
-        .fd                   = cs_fd,
-        .fd_to_dict           = cs_fd_to_dict,
-        .fdctx                = cs_fdctx,
-        .inodectx             = cs_inodectx,
-        .inodectx_to_dict     = cs_inodectx_to_dict,
-        .priv_to_dict         = cs_priv_to_dict,
-        .priv                 = cs_priv,
+    .fdctx_to_dict = cs_fdctx_to_dict,
+    .inode = cs_inode,
+    .inode_to_dict = cs_inode_to_dict,
+    .history = cs_history,
+    .fd = cs_fd,
+    .fd_to_dict = cs_fd_to_dict,
+    .fdctx = cs_fdctx,
+    .inodectx = cs_inodectx,
+    .inodectx_to_dict = cs_inodectx_to_dict,
+    .priv_to_dict = cs_priv_to_dict,
+    .priv = cs_priv,
 };
 
 struct volume_options cs_options[] = {
-       { .key  = {NULL} },
+    {.key = {"cloudsync-storetype"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Defines which remote store is enabled"},
+    {.key = {"cloudsync-remote-read"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .description = "Defines a remote read fop when on"},
+    {.key = {"cloudsync-store-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Defines a volume wide store id"},
+    {.key = {"cloudsync-product-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Defines a volume wide product id"},
+    {.key = {NULL}},
 };
 
 xlator_api_t xlator_api = {
-        .init           = cs_init,
-        .fini           = cs_fini,
-        .notify         = cs_notify,
-        .reconfigure    = cs_reconfigure,
-        .mem_acct_init  = cs_mem_acct_init,
-        .dumpops        = &cs_dumpops,
-        .fops           = &cs_fops,
-        .cbks           = &cs_cbks,
-        .options        = cs_options,
-        .identifier     = "cloudsync",
+    .init = cs_init,
+    .fini = cs_fini,
+    .notify = cs_notify,
+    .reconfigure = cs_reconfigure,
+    .mem_acct_init = cs_mem_acct_init,
+    .dumpops = &cs_dumpops,
+    .fops = &cs_fops,
+    .cbks = &cs_cbks,
+    .options = cs_options,
+    .identifier = "cloudsync",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/features/cloudsync/src/cloudsync.h b/xlators/features/cloudsync/src/cloudsync.h
index bd54d760864..d24141978d6 100644
--- a/xlators/features/cloudsync/src/cloudsync.h
+++ b/xlators/features/cloudsync/src/cloudsync.h
@@ -8,93 +8,116 @@
  *   cases as published by the Free Software Foundation.
  */
 
-
 #ifndef __CLOUDSYNC_H__
 #define __CLOUDSYNC_H__
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "defaults.h"
-#include "syncop.h"
-#include "call-stub.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
 #include "cloudsync-common.h"
 #include "cloudsync-autogen-fops.h"
 
-
+#define ALIGN_SIZE 4096
 #define CS_LOCK_DOMAIN "cs.protect.file.stat"
 typedef struct cs_dlstore {
-        off_t           off;
-        struct iovec   *vector;
-        int32_t         count;
-        struct iobref  *iobref;
-        uint32_t        flags;
+    off_t off;
+    struct iovec *vector;
+    int32_t count;
+    struct iobref *iobref;
+    uint32_t flags;
 } cs_dlstore;
 
 typedef struct cs_inode_ctx {
-        gf_cs_obj_state state;
+    cs_loc_xattr_t locxattr;
+    gf_cs_obj_state state;
 } cs_inode_ctx_t;
 
+struct cs_plugin {
+    char *name;        /* store name */
+    char *library;     /* library to load for the given store */
+    char *description; /* description about the store */
+};
+
 cs_local_t *
-cs_local_init (xlator_t *this, call_frame_t *frame, loc_t *loc, fd_t *fd,
-               glusterfs_fop_t fop);
+cs_local_init(xlator_t *this, call_frame_t *frame, loc_t *loc, fd_t *fd,
+              glusterfs_fop_t fop);
 
 int
-locate_and_execute (call_frame_t *frame);
-
+locate_and_execute(call_frame_t *frame);
 
 int32_t
-cs_resume_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    dict_t *dict, int32_t flags, dict_t *xdata);
+cs_resume_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *dict, int32_t flags, dict_t *xdata);
 
 int32_t
-cs_inodelk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-                       int32_t op_errno, dict_t *xdata);
+cs_inodelk_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 size_t
-cs_write_callback (void *lcurlbuf, size_t size, size_t nitems, void *frame);
+cs_write_callback(void *lcurlbuf, size_t size, size_t nitems, void *frame);
 
 void
-cs_common_cbk (call_frame_t *frame);
+cs_common_cbk(call_frame_t *frame);
 
 gf_boolean_t
-cs_is_file_remote (struct iatt *stbuf, dict_t *xattr);
+cs_is_file_remote(struct iatt *stbuf, dict_t *xattr);
 
 int32_t
-cs_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *xdata);
+cs_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata);
 int
-cs_build_loc (loc_t *loc, call_frame_t *frame);
+cs_build_loc(loc_t *loc, call_frame_t *frame);
 
 int
-cs_blocking_inodelk_cbk (call_frame_t *lock_frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata);
+cs_blocking_inodelk_cbk(call_frame_t *lock_frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
-int cs_read_authinfo(xlator_t *this);
+int
+cs_read_authinfo(xlator_t *this);
 
 int
-__cs_inode_ctx_update (xlator_t *this, inode_t *inode, uint64_t val);
+__cs_inode_ctx_update(xlator_t *this, inode_t *inode, uint64_t val);
 
 int
-cs_inode_ctx_reset (xlator_t *this, inode_t *inode);
+cs_inode_ctx_reset(xlator_t *this, inode_t *inode);
 
 void
-__cs_inode_ctx_get (xlator_t *this, inode_t *inode, cs_inode_ctx_t **ctx);
+__cs_inode_ctx_get(xlator_t *this, inode_t *inode, cs_inode_ctx_t **ctx);
 
 gf_cs_obj_state
-__cs_get_file_state (xlator_t *this, inode_t *inode, cs_inode_ctx_t *ctx);
+__cs_get_file_state(inode_t *inode, cs_inode_ctx_t *ctx);
 
 int
-cs_inodelk_unlock (call_frame_t *main_frame);
+cs_inodelk_unlock(call_frame_t *main_frame);
 
 int
-cs_resume_postprocess (xlator_t *this, call_frame_t *frame, inode_t *inode);
+cs_resume_postprocess(xlator_t *this, call_frame_t *frame, inode_t *inode);
 
 int32_t
-cs_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf, dict_t *xdata);
+cs_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata);
 int32_t
-cs_resume_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    off_t offset, dict_t *xattr_req);
-#endif /* __CLOUDSYNC_H__ */
+cs_resume_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   off_t offset, dict_t *xattr_req);
 
+int32_t
+cs_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iovec *vector, int32_t count,
+             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata);
+int32_t
+cs_resume_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, uint32_t flags, dict_t *xdata);
+int32_t
+cs_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata);
+
+int
+cs_resume_remote_readv_postprocess(xlator_t *this, call_frame_t *frame,
+                                   inode_t *inode, off_t offset, size_t size,
+                                   uint32_t flags);
+int
+cs_serve_readv(call_frame_t *frame, off_t offset, size_t size, uint32_t flags);
+#endif /* __CLOUDSYNC_H__ */
diff --git a/xlators/features/compress/src/cdc-helper.c b/xlators/features/compress/src/cdc-helper.c
index 0a9a0e3d29c..f973ff56cf5 100644
--- a/xlators/features/compress/src/cdc-helper.c
+++ b/xlators/features/compress/src/cdc-helper.c
@@ -8,9 +8,9 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "syscall.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/syscall.h>
 
 #include "cdc.h"
 #include "cdc-mem-types.h"
@@ -34,118 +34,110 @@
  * gzip_header is added only during debugging.
  * Refer to the function cdc_dump_iovec_to_disk
  */
-static const char gzip_header[10] =
-        {
-                '\037', '\213', Z_DEFLATED, 0,
-                0,      0,      0,          0,
-                0,      GF_CDC_OS_ID
-        };
+static const char gzip_header[10] = {'\037', '\213', Z_DEFLATED,  0, 0, 0, 0,
+                                     0,      0,      GF_CDC_OS_ID};
 
 static int32_t
-cdc_next_iovec (xlator_t *this, cdc_info_t *ci)
+cdc_next_iovec(xlator_t *this, cdc_info_t *ci)
 {
-        int ret = -1;
-
-        ci->ncount++;
-        /* check for iovec overflow -- should not happen */
-        if (ci->ncount == MAX_IOVEC) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Zlib output buffer overflow"
-                        " ->ncount (%d) | ->MAX_IOVEC (%d)",
-                        ci->ncount, MAX_IOVEC);
-                goto out;
-        }
-
-        ret = 0;
-
- out:
-        return ret;
+    int ret = -1;
+
+    ci->ncount++;
+    /* check for iovec overflow -- should not happen */
+    if (ci->ncount == MAX_IOVEC) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Zlib output buffer overflow"
+               " ->ncount (%d) | ->MAX_IOVEC (%d)",
+               ci->ncount, MAX_IOVEC);
+        goto out;
+    }
+
+    ret = 0;
+
+out:
+    return ret;
 }
 
 static void
-cdc_put_long (unsigned char *string, unsigned long x)
+cdc_put_long(unsigned char *string, unsigned long x)
 {
-        string[0] = (unsigned char) (x & 0xff);
-        string[1] = (unsigned char) ((x & 0xff00) >> 8);
-        string[2] = (unsigned char) ((x & 0xff0000) >> 16);
-        string[3] = (unsigned char) ((x & 0xff000000) >> 24);
+    string[0] = (unsigned char)(x & 0xff);
+    string[1] = (unsigned char)((x & 0xff00) >> 8);
+    string[2] = (unsigned char)((x & 0xff0000) >> 16);
+    string[3] = (unsigned char)((x & 0xff000000) >> 24);
 }
 
 static unsigned long
-cdc_get_long (unsigned char *buf)
+cdc_get_long(unsigned char *buf)
 {
-        return ((unsigned long) buf[0])
-                | (((unsigned long) buf[1]) << 8)
-                | (((unsigned long) buf[2]) << 16)
-                | (((unsigned long) buf[3]) << 24);
+    return ((unsigned long)buf[0]) | (((unsigned long)buf[1]) << 8) |
+           (((unsigned long)buf[2]) << 16) | (((unsigned long)buf[3]) << 24);
 }
 
 static int32_t
-cdc_init_gzip_trailer (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+cdc_init_gzip_trailer(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
 {
-        int   ret = -1;
-        char *buf = NULL;
+    int ret = -1;
+    char *buf = NULL;
 
-        ret = cdc_next_iovec (this, ci);
-        if (ret)
-                goto out;
+    ret = cdc_next_iovec(this, ci);
+    if (ret)
+        goto out;
 
-        buf = CURR_VEC(ci).iov_base =
-                (char *) GF_CALLOC (1, GF_CDC_VALIDATION_SIZE,
-                                    gf_cdc_mt_gzip_trailer_t);
+    buf = CURR_VEC(ci).iov_base = (char *)GF_CALLOC(1, GF_CDC_VALIDATION_SIZE,
+                                                    gf_cdc_mt_gzip_trailer_t);
 
-        if (!CURR_VEC(ci).iov_base)
-                goto out;
+    if (!CURR_VEC(ci).iov_base)
+        goto out;
 
-        CURR_VEC(ci).iov_len = GF_CDC_VALIDATION_SIZE;
+    CURR_VEC(ci).iov_len = GF_CDC_VALIDATION_SIZE;
 
-        cdc_put_long ((unsigned char *)&buf[0], ci->crc);
-        cdc_put_long ((unsigned char *)&buf[4], ci->stream.total_in);
+    cdc_put_long((unsigned char *)&buf[0], ci->crc);
+    cdc_put_long((unsigned char *)&buf[4], ci->stream.total_in);
 
-        ret = 0;
+    ret = 0;
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 static int32_t
-cdc_alloc_iobuf_and_init_vec (xlator_t *this,
-                              cdc_priv_t *priv, cdc_info_t *ci,
-                              int size)
+cdc_alloc_iobuf_and_init_vec(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
+                             int size)
 {
-        int           ret       = -1;
-        int           alloc_len = 0;
-        struct iobuf *iobuf     = NULL;
+    int ret = -1;
+    int alloc_len = 0;
+    struct iobuf *iobuf = NULL;
 
-        ret = cdc_next_iovec (this, ci);
-        if (ret)
-                goto out;
+    ret = cdc_next_iovec(this, ci);
+    if (ret)
+        goto out;
 
-        alloc_len = size ? size : ci->buffer_size;
+    alloc_len = size ? size : ci->buffer_size;
 
-        iobuf = iobuf_get2 (this->ctx->iobuf_pool, alloc_len);
-        if (!iobuf)
-                goto out;
+    iobuf = iobuf_get2(this->ctx->iobuf_pool, alloc_len);
+    if (!iobuf)
+        goto out;
 
-        ret = iobref_add (ci->iobref, iobuf);
-        if (ret)
-                goto out;
+    ret = iobref_add(ci->iobref, iobuf);
+    if (ret)
+        goto out;
 
-        /* Initialize this iovec */
-        CURR_VEC(ci).iov_base = iobuf->ptr;
-        CURR_VEC(ci).iov_len  = alloc_len;
+    /* Initialize this iovec */
+    CURR_VEC(ci).iov_base = iobuf->ptr;
+    CURR_VEC(ci).iov_len = alloc_len;
 
-        ret = 0;
+    ret = 0;
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 static void
-cdc_init_zlib_output_stream (cdc_priv_t *priv, cdc_info_t *ci, int size)
+cdc_init_zlib_output_stream(cdc_priv_t *priv, cdc_info_t *ci, int size)
 {
-        ci->stream.next_out  = (unsigned char *) CURR_VEC(ci).iov_base;
-        ci->stream.avail_out = size ? size : ci->buffer_size;
+    ci->stream.next_out = (unsigned char *)CURR_VEC(ci).iov_base;
+    ci->stream.avail_out = size ? size : ci->buffer_size;
 }
 
 /* This routine is for testing and debugging only.
@@ -153,391 +145,383 @@ cdc_init_zlib_output_stream (cdc_priv_t *priv, cdc_info_t *ci, int size)
  * So each gzip dump file is at least 18 bytes in size.
  */
 void
-cdc_dump_iovec_to_disk (xlator_t *this, cdc_info_t *ci, const char *file)
+cdc_dump_iovec_to_disk(xlator_t *this, cdc_info_t *ci, const char *file)
 {
-        int    i      = 0;
-        int    fd     = 0;
-        size_t written = 0;
-        size_t total_written = 0;
-
-        fd = open (file, O_WRONLY|O_CREAT|O_TRUNC, 0777 );
-        if (fd < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Cannot open file: %s", file);
-                return;
-        }
-
-        written = sys_write (fd, (char *) gzip_header, 10);
+    int i = 0;
+    int fd = 0;
+    size_t written = 0;
+    size_t total_written = 0;
+
+    fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0777);
+    if (fd < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "Cannot open file: %s", file);
+        return;
+    }
+
+    written = sys_write(fd, (char *)gzip_header, 10);
+    total_written += written;
+    for (i = 0; i < ci->ncount; i++) {
+        written = sys_write(fd, (char *)ci->vec[i].iov_base,
+                            ci->vec[i].iov_len);
         total_written += written;
-        for (i = 0; i < ci->ncount; i++) {
-                written = sys_write (fd, (char *) ci->vec[i].iov_base, ci->vec[i].iov_len);
-                total_written += written;
-        }
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                        "dump'd %zu bytes to %s", total_written, GF_CDC_DEBUG_DUMP_FILE );
+    gf_log(this->name, GF_LOG_DEBUG, "dump'd %zu bytes to %s", total_written,
+           GF_CDC_DEBUG_DUMP_FILE);
 
-        sys_close (fd);
+    sys_close(fd);
 }
 
 static int32_t
-cdc_flush_libz_buffer (cdc_priv_t *priv, xlator_t *this, cdc_info_t *ci,
-                       int (*libz_func)(z_streamp, int),
-                       int flush)
+cdc_flush_libz_buffer(cdc_priv_t *priv, xlator_t *this, cdc_info_t *ci,
+                      int (*libz_func)(z_streamp, int), int flush)
 {
-        int32_t ret = Z_OK;
-        int done = 0;
-        unsigned int deflate_len = 0;
+    int32_t ret = Z_OK;
+    int done = 0;
+    unsigned int deflate_len = 0;
 
-        for (;;) {
-                deflate_len = ci->buffer_size - ci->stream.avail_out;
+    for (;;) {
+        deflate_len = ci->buffer_size - ci->stream.avail_out;
 
-                if (deflate_len != 0) {
-                        CURR_VEC(ci).iov_len = deflate_len;
+        if (deflate_len != 0) {
+            CURR_VEC(ci).iov_len = deflate_len;
 
-                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-                        if (ret) {
-                                ret = Z_MEM_ERROR;
-                                break;
-                        }
+            ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+            if (ret) {
+                ret = Z_MEM_ERROR;
+                break;
+            }
 
-                        /* Re-position Zlib output buffer */
-                        cdc_init_zlib_output_stream (priv, ci, 0);
-                }
+            /* Re-position Zlib output buffer */
+            cdc_init_zlib_output_stream(priv, ci, 0);
+        }
 
-                if (done) {
-                        ci->ncount--;
-                        break;
-                }
+        if (done) {
+            ci->ncount--;
+            break;
+        }
 
-                ret = libz_func (&ci->stream, flush);
+        ret = libz_func(&ci->stream, flush);
 
-                if (ret == Z_BUF_ERROR) {
-                        ret = Z_OK;
-                        ci->ncount--;
-                        break;
-                }
+        if (ret == Z_BUF_ERROR) {
+            ret = Z_OK;
+            ci->ncount--;
+            break;
+        }
 
-                done = (ci->stream.avail_out != 0 || ret == Z_STREAM_END);
+        done = (ci->stream.avail_out != 0 || ret == Z_STREAM_END);
 
-                if (ret != Z_OK && ret != Z_STREAM_END)
-                        break;
-        }
+        if (ret != Z_OK && ret != Z_STREAM_END)
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
 static int32_t
-do_cdc_compress (struct iovec *vec, xlator_t *this, cdc_priv_t *priv,
-                 cdc_info_t *ci)
+do_cdc_compress(struct iovec *vec, xlator_t *this, cdc_priv_t *priv,
+                cdc_info_t *ci)
 {
-        int ret = -1;
+    int ret = -1;
 
-        /* Initialize defalte */
-        ret = deflateInit2 (&ci->stream, priv->cdc_level, Z_DEFLATED,
-                            priv->window_size, priv->mem_level,
-                            Z_DEFAULT_STRATEGY);
+    /* Initialize defalte */
+    ret = deflateInit2(&ci->stream, priv->cdc_level, Z_DEFLATED,
+                       priv->window_size, priv->mem_level, Z_DEFAULT_STRATEGY);
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "unable to init Zlib (retval: %d)", ret);
-                goto out;
-        }
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "unable to init Zlib (retval: %d)",
+               ret);
+        goto out;
+    }
 
-        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-        if (ret)
-                goto out;
+    ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+    if (ret)
+        goto out;
 
-        /* setup output buffer */
-        cdc_init_zlib_output_stream (priv, ci, 0);
+    /* setup output buffer */
+    cdc_init_zlib_output_stream(priv, ci, 0);
 
-        /* setup input buffer */
-        ci->stream.next_in  = (unsigned char *) vec->iov_base;
-        ci->stream.avail_in = vec->iov_len;
+    /* setup input buffer */
+    ci->stream.next_in = (unsigned char *)vec->iov_base;
+    ci->stream.avail_in = vec->iov_len;
 
-        ci->crc = crc32 (ci->crc, (const Bytef *) vec->iov_base, vec->iov_len);
+    ci->crc = crc32(ci->crc, (const Bytef *)vec->iov_base, vec->iov_len);
 
-        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%d buffer_size=%d",
-                ci->crc, ci->stream.avail_in, ci->buffer_size);
+    gf_log(this->name, GF_LOG_DEBUG, "crc=%lu len=%d buffer_size=%d", ci->crc,
+           ci->stream.avail_in, ci->buffer_size);
 
-        /* compress !! */
-        while (ci->stream.avail_in != 0) {
-                if (ci->stream.avail_out == 0) {
+    /* compress !! */
+    while (ci->stream.avail_in != 0) {
+        if (ci->stream.avail_out == 0) {
+            CURR_VEC(ci).iov_len = ci->buffer_size;
 
-                        CURR_VEC(ci).iov_len = ci->buffer_size;
+            ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+            if (ret)
+                break;
 
-                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-                        if (ret)
-                                break;
-
-                        /* Re-position Zlib output buffer */
-                        cdc_init_zlib_output_stream (priv, ci, 0);
-                }
-
-                ret = deflate (&ci->stream, Z_NO_FLUSH);
-                if (ret != Z_OK)
-                        break;
+            /* Re-position Zlib output buffer */
+            cdc_init_zlib_output_stream(priv, ci, 0);
         }
 
- out:
-        return ret;
+        ret = deflate(&ci->stream, Z_NO_FLUSH);
+        if (ret != Z_OK)
+            break;
+    }
+
+out:
+    return ret;
 }
 
 int32_t
-cdc_compress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
-              dict_t **xdata)
+cdc_compress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci, dict_t **xdata)
 {
-        int ret = -1;
-        int i   = 0;
+    int ret = -1;
+    int i = 0;
 
-        ci->iobref = iobref_new ();
-        if (!ci->iobref)
-                goto out;
+    ci->iobref = iobref_new();
+    if (!ci->iobref)
+        goto out;
 
+    if (!*xdata) {
+        *xdata = dict_new();
         if (!*xdata) {
-                *xdata = dict_new ();
-                if (!*xdata) {
-                        gf_log (this->name, GF_LOG_ERROR, "Cannot allocate xdata"
-                                " dict");
-                        goto out;
-                }
-        }
-
-        /* data */
-        for (i = 0; i < ci->count; i++) {
-                ret = do_cdc_compress (&ci->vector[i], this, priv, ci);
-                if (ret != Z_OK)
-                        goto deflate_cleanup_out;
-        }
-
-        /* flush zlib buffer */
-        ret = cdc_flush_libz_buffer (priv, this, ci, deflate, Z_FINISH);
-        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Compression Error: ret (%d)", ret);
-                ret = -1;
-                goto deflate_cleanup_out;
-        }
-
-        /* trailer */
-        ret = cdc_init_gzip_trailer (this, priv, ci);
-        if (ret)
-                goto deflate_cleanup_out;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Compressed %ld to %ld bytes",
-                ci->stream.total_in, ci->stream.total_out);
-
-        ci->nbytes = ci->stream.total_out + GF_CDC_VALIDATION_SIZE;
-
-        /* set deflated canary value for identification */
-        ret = dict_set_int32 (*xdata, GF_CDC_DEFLATE_CANARY_VAL, 1);
-        if (ret) {
-                /* Send uncompressed data if we can't _tell_ the client
-                 * that deflated data is on it's way. So, we just log
-                 * the faliure and continue as usual.
-                 */
-                 gf_log (this->name, GF_LOG_ERROR,
-                 "Data deflated, but could not set canary"
-                 " value in dict for identification");
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Cannot allocate xdata"
+                   " dict");
+            goto out;
         }
+    }
+
+    /* data */
+    for (i = 0; i < ci->count; i++) {
+        ret = do_cdc_compress(&ci->vector[i], this, priv, ci);
+        if (ret != Z_OK)
+            goto deflate_cleanup_out;
+    }
+
+    /* flush zlib buffer */
+    ret = cdc_flush_libz_buffer(priv, this, ci, deflate, Z_FINISH);
+    if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+        gf_log(this->name, GF_LOG_ERROR, "Compression Error: ret (%d)", ret);
+        ret = -1;
+        goto deflate_cleanup_out;
+    }
+
+    /* trailer */
+    ret = cdc_init_gzip_trailer(this, priv, ci);
+    if (ret)
+        goto deflate_cleanup_out;
+
+    gf_log(this->name, GF_LOG_DEBUG, "Compressed %ld to %ld bytes",
+           ci->stream.total_in, ci->stream.total_out);
+
+    ci->nbytes = ci->stream.total_out + GF_CDC_VALIDATION_SIZE;
+
+    /* set deflated canary value for identification */
+    ret = dict_set_int32(*xdata, GF_CDC_DEFLATE_CANARY_VAL, 1);
+    if (ret) {
+        /* Send uncompressed data if we can't _tell_ the client
+         * that deflated data is on it's way. So, we just log
+         * the failure and continue as usual.
+         */
+        gf_log(this->name, GF_LOG_ERROR,
+               "Data deflated, but could not set canary"
+               " value in dict for identification");
+    }
 
-        /* This is to be used in testing */
-        if ( priv->debug ) {
-                cdc_dump_iovec_to_disk (this, ci, GF_CDC_DEBUG_DUMP_FILE );
-        }
+    /* This is to be used in testing */
+    if (priv->debug) {
+        cdc_dump_iovec_to_disk(this, ci, GF_CDC_DEBUG_DUMP_FILE);
+    }
 
- deflate_cleanup_out:
-        (void) deflateEnd(&ci->stream);
+deflate_cleanup_out:
+    (void)deflateEnd(&ci->stream);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
-
 /* deflate content is checked by the presence of a canary
  * value in the dict as the key
  */
 static int32_t
-cdc_check_content_for_deflate (dict_t *xdata)
+cdc_check_content_for_deflate(dict_t *xdata)
 {
-        return dict_get (xdata, GF_CDC_DEFLATE_CANARY_VAL) ? -1 : 0;
+    return dict_get(xdata, GF_CDC_DEFLATE_CANARY_VAL) ? -1 : 0;
 }
 
 static unsigned long
-cdc_extract_crc (char *trailer)
+cdc_extract_crc(char *trailer)
 {
-        return cdc_get_long ((unsigned char *) &trailer[0]);
+    return cdc_get_long((unsigned char *)&trailer[0]);
 }
 
 static unsigned long
-cdc_extract_size (char *trailer)
+cdc_extract_size(char *trailer)
 {
-        return cdc_get_long ((unsigned char *) &trailer[4]);
+    return cdc_get_long((unsigned char *)&trailer[4]);
 }
 
 static int32_t
-cdc_validate_inflate (cdc_info_t *ci, unsigned long crc,
-                      unsigned long len)
+cdc_validate_inflate(cdc_info_t *ci, unsigned long crc, unsigned long len)
 {
-        return !((crc == ci->crc)
-                 /* inflated length is hidden inside
-                  * Zlib stream struct */
-                 && (len == ci->stream.total_out));
+    return !((crc == ci->crc)
+             /* inflated length is hidden inside
+              * Zlib stream struct */
+             && (len == ci->stream.total_out));
 }
 
 static int32_t
-do_cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+do_cdc_decompress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
 {
-        int            ret          = -1;
-        int            i            = 0;
-        int            len          = 0;
-        char          *inflte       = NULL;
-        char          *trailer      = NULL;
-        struct iovec   vec          = {0,};
-        unsigned long  computed_crc = 0;
-        unsigned long  computed_len = 0;
-
-        ret = inflateInit2 (&ci->stream, priv->window_size);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Zlib: Unable to initialize inflate");
-                goto out;
+    int ret = -1;
+    int i = 0;
+    int len = 0;
+    char *inflte = NULL;
+    char *trailer = NULL;
+    struct iovec vec = {
+        0,
+    };
+    unsigned long computed_crc = 0;
+    unsigned long computed_len = 0;
+
+    ret = inflateInit2(&ci->stream, priv->window_size);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Zlib: Unable to initialize inflate");
+        goto out;
+    }
+
+    vec = THIS_VEC(ci, 0);
+
+    trailer = (char *)(((char *)vec.iov_base) + vec.iov_len -
+                       GF_CDC_VALIDATION_SIZE);
+
+    /* CRC of uncompressed data */
+    computed_crc = cdc_extract_crc(trailer);
+
+    /* size of uncomrpessed data */
+    computed_len = cdc_extract_size(trailer);
+
+    gf_log(this->name, GF_LOG_DEBUG, "crc=%lu len=%lu buffer_size=%d",
+           computed_crc, computed_len, ci->buffer_size);
+
+    inflte = vec.iov_base;
+    len = vec.iov_len - GF_CDC_VALIDATION_SIZE;
+
+    /* allocate buffer of the original length of the data */
+    ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+    if (ret)
+        goto out;
+
+    /* setup output buffer */
+    cdc_init_zlib_output_stream(priv, ci, 0);
+
+    /* setup input buffer */
+    ci->stream.next_in = (unsigned char *)inflte;
+    ci->stream.avail_in = len;
+
+    while (ci->stream.avail_in != 0) {
+        if (ci->stream.avail_out == 0) {
+            CURR_VEC(ci).iov_len = ci->buffer_size;
+
+            ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+            if (ret)
+                break;
+
+            /* Re-position Zlib output buffer */
+            cdc_init_zlib_output_stream(priv, ci, 0);
         }
 
-        vec = THIS_VEC(ci, 0);
-
-        trailer = (char *) (((char *) vec.iov_base) + vec.iov_len
-                - GF_CDC_VALIDATION_SIZE);
-
-        /* CRC of uncompressed data */
-        computed_crc = cdc_extract_crc (trailer);
-
-        /* size of uncomrpessed data */
-        computed_len = cdc_extract_size (trailer);
-
-        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%lu buffer_size=%d",
-                computed_crc, computed_len, ci->buffer_size);
-
-        inflte = vec.iov_base ;
-        len = vec.iov_len - GF_CDC_VALIDATION_SIZE;
-
-        /* allocate buffer of the original length of the data */
-        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-        if (ret)
-                goto out;
-
-        /* setup output buffer */
-        cdc_init_zlib_output_stream (priv, ci, 0);
-
-        /* setup input buffer */
-        ci->stream.next_in = (unsigned char *) inflte;
-        ci->stream.avail_in = len;
-
-        while (ci->stream.avail_in != 0) {
-                if (ci->stream.avail_out == 0) {
-                        CURR_VEC(ci).iov_len = ci->buffer_size;
-
-                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-                        if (ret)
-                                break;
-
-                        /* Re-position Zlib output buffer */
-                        cdc_init_zlib_output_stream (priv, ci, 0);
-                }
-
-                ret = inflate (&ci->stream, Z_NO_FLUSH);
-                if (ret == Z_STREAM_ERROR)
-                        break;
-        }
-
-        /* flush zlib buffer */
-        ret = cdc_flush_libz_buffer (priv, this, ci, inflate, Z_SYNC_FLUSH);
-        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Decompression Error: ret (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-
-        /* compute CRC of the uncompresses data to check for
-         * correctness */
-
-        for (i = 0; i < ci->ncount; i++) {
-                ci->crc = crc32 (ci->crc,
-                                 (const Bytef *) ci->vec[i].iov_base,
-                                 ci->vec[i].iov_len);
-        }
-
-        /* validate inflated data */
-        ret = cdc_validate_inflate (ci, computed_crc, computed_len);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Checksum or length mismatched in inflated data");
-        }
-
- out:
-        return ret;
+        ret = inflate(&ci->stream, Z_NO_FLUSH);
+        if (ret == Z_STREAM_ERROR)
+            break;
+    }
+
+    /* flush zlib buffer */
+    ret = cdc_flush_libz_buffer(priv, this, ci, inflate, Z_SYNC_FLUSH);
+    if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+        gf_log(this->name, GF_LOG_ERROR, "Decompression Error: ret (%d)", ret);
+        ret = -1;
+        goto out;
+    }
+
+    /* compute CRC of the uncompresses data to check for
+     * correctness */
+
+    for (i = 0; i < ci->ncount; i++) {
+        ci->crc = crc32(ci->crc, (const Bytef *)ci->vec[i].iov_base,
+                        ci->vec[i].iov_len);
+    }
+
+    /* validate inflated data */
+    ret = cdc_validate_inflate(ci, computed_crc, computed_len);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Checksum or length mismatched in inflated data");
+    }
+
+out:
+    return ret;
 }
 
 int32_t
-cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
-                dict_t *xdata)
+cdc_decompress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci, dict_t *xdata)
 {
-        int32_t ret = -1;
-
-        /* check for deflate content */
-        if (!cdc_check_content_for_deflate (xdata)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Content not deflated, passing through ...");
-                goto passthrough_out;
-        }
-
-        ci->iobref = iobref_new ();
-        if (!ci->iobref)
-                goto passthrough_out;
-
-        /* do we need to do this? can we assume that one iovec
-         * will hold per request data every time?
-         *
-         * server/client protocol seems to deal with a single
-         * iovec even if op_ret > 1M. So, it looks ok to
-         * assume that a single iovec will contain all the
-         * data (This saves us a lot from finding the trailer
-         * and the data since it could have been split-up onto
-         * two adjacent iovec's.
-         *
-         * But, in case this translator is loaded above quick-read
-         * for some reason, then it's entirely possible that we get
-         * multiple iovec's...
-         *
-         * This case (handled below) is not tested. (by loading the
-         * xlator below quick-read)
-         */
-
-        /* @@ I_HOPE_THIS_IS_NEVER_HIT */
-        if (ci->count > 1) {
-                gf_log (this->name, GF_LOG_WARNING, "unable to handle"
-                        " multiple iovecs (%d in number)", ci->count);
-                goto inflate_cleanup_out;
-                /* TODO: coallate all iovecs in one */
-        }
-
-        ret = do_cdc_decompress (this, priv, ci);
-        if (ret)
-                goto inflate_cleanup_out;
-
-        ci->nbytes = ci->stream.total_out;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Inflated %ld to %ld bytes",
-                ci->stream.total_in, ci->stream.total_out);
-
- inflate_cleanup_out:
-        (void) inflateEnd (&ci->stream);
-
- passthrough_out:
-        return ret;
+    int32_t ret = -1;
+
+    /* check for deflate content */
+    if (!cdc_check_content_for_deflate(xdata)) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "Content not deflated, passing through ...");
+        goto passthrough_out;
+    }
+
+    ci->iobref = iobref_new();
+    if (!ci->iobref)
+        goto passthrough_out;
+
+    /* do we need to do this? can we assume that one iovec
+     * will hold per request data every time?
+     *
+     * server/client protocol seems to deal with a single
+     * iovec even if op_ret > 1M. So, it looks ok to
+     * assume that a single iovec will contain all the
+     * data (This saves us a lot from finding the trailer
+     * and the data since it could have been split-up onto
+     * two adjacent iovec's.
+     *
+     * But, in case this translator is loaded above quick-read
+     * for some reason, then it's entirely possible that we get
+     * multiple iovec's...
+     *
+     * This case (handled below) is not tested. (by loading the
+     * xlator below quick-read)
+     */
+
+    /* @@ I_HOPE_THIS_IS_NEVER_HIT */
+    if (ci->count > 1) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "unable to handle"
+               " multiple iovecs (%d in number)",
+               ci->count);
+        goto inflate_cleanup_out;
+        /* TODO: coallate all iovecs in one */
+    }
+
+    ret = do_cdc_decompress(this, priv, ci);
+    if (ret)
+        goto inflate_cleanup_out;
+
+    ci->nbytes = ci->stream.total_out;
+
+    gf_log(this->name, GF_LOG_DEBUG, "Inflated %ld to %ld bytes",
+           ci->stream.total_in, ci->stream.total_out);
+
+inflate_cleanup_out:
+    (void)inflateEnd(&ci->stream);
+
+passthrough_out:
+    return ret;
 }
 
 #endif
diff --git a/xlators/features/compress/src/cdc-mem-types.h b/xlators/features/compress/src/cdc-mem-types.h
index ead2c70ba6e..928afdd2efe 100644
--- a/xlators/features/compress/src/cdc-mem-types.h
+++ b/xlators/features/compress/src/cdc-mem-types.h
@@ -11,13 +11,13 @@
 #ifndef __CDC_MEM_TYPES_H
 #define __CDC_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_cdc_mem_types {
-        gf_cdc_mt_priv_t         = gf_common_mt_end + 1,
-        gf_cdc_mt_vec_t          = gf_common_mt_end + 2,
-        gf_cdc_mt_gzip_trailer_t = gf_common_mt_end + 3,
-        gf_cdc_mt_end            = gf_common_mt_end + 4,
+    gf_cdc_mt_priv_t = gf_common_mt_end + 1,
+    gf_cdc_mt_vec_t = gf_common_mt_end + 2,
+    gf_cdc_mt_gzip_trailer_t = gf_common_mt_end + 3,
+    gf_cdc_mt_end = gf_common_mt_end + 4,
 };
 
 #endif
diff --git a/xlators/features/compress/src/cdc.c b/xlators/features/compress/src/cdc.c
index e33d4efc1a1..b0b51e914ed 100644
--- a/xlators/features/compress/src/cdc.c
+++ b/xlators/features/compress/src/cdc.c
@@ -10,347 +10,339 @@
 
 #include <sys/uio.h>
 
-#include "xlator.h"
-#include "defaults.h"
-#include "logging.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/logging.h>
 
 #include "cdc.h"
 #include "cdc-mem-types.h"
 
 static void
-cdc_cleanup_iobref (cdc_info_t *ci)
+cdc_cleanup_iobref(cdc_info_t *ci)
 {
-        assert(ci->iobref != NULL);
-        iobref_clear (ci->iobref);
+    assert(ci->iobref != NULL);
+    iobref_clear(ci->iobref);
 }
 
 int32_t
-cdc_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno,
-               struct iovec *vector, int32_t count,
-               struct iatt *stbuf, struct iobref *iobref,
-               dict_t *xdata)
+cdc_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iovec *vector, int32_t count,
+              struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-        int         ret   = -1;
-        cdc_priv_t *priv  = NULL;
-        cdc_info_t  ci    = {0,};
-
-        GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
-
-        priv = this->private;
-
-        if (op_ret <= 0)
-                goto default_out;
-
-        if ( (priv->min_file_size != 0)
-             && (op_ret < priv->min_file_size) )
-                goto default_out;
-
-        ci.count       = count;
-        ci.ibytes      = op_ret;
-        ci.vector      = vector;
-        ci.buf         = NULL;
-        ci.iobref      = NULL;
-        ci.ncount      = 0;
-        ci.crc         = 0;
-        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
-
-/* A readv compresses on the server side and decompresses on the client side
- */
-        if (priv->op_mode == GF_CDC_MODE_SERVER) {
-                ret = cdc_compress (this, priv, &ci, &xdata);
-        } else if (priv->op_mode == GF_CDC_MODE_CLIENT) {
-                ret = cdc_decompress (this, priv, &ci, xdata);
-        } else {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Invalid operation mode (%d)", priv->op_mode);
-        }
-
-        if (ret)
-                goto default_out;
-
-        STACK_UNWIND_STRICT (readv, frame, ci.nbytes, op_errno,
-                             ci.vec, ci.ncount, stbuf, iobref,
-                             xdata);
-        cdc_cleanup_iobref (&ci);
-        return 0;
-
- default_out:
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, stbuf, iobref, xdata);
-        return 0;
+    int ret = -1;
+    cdc_priv_t *priv = NULL;
+    cdc_info_t ci = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("cdc", this, default_out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, default_out);
+
+    priv = this->private;
+
+    if (op_ret <= 0)
+        goto default_out;
+
+    if ((priv->min_file_size != 0) && (op_ret < priv->min_file_size))
+        goto default_out;
+
+    ci.count = count;
+    ci.ibytes = op_ret;
+    ci.vector = vector;
+    ci.buf = NULL;
+    ci.iobref = NULL;
+    ci.ncount = 0;
+    ci.crc = 0;
+    ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+    /* A readv compresses on the server side and decompresses on the client side
+     */
+    if (priv->op_mode == GF_CDC_MODE_SERVER) {
+        ret = cdc_compress(this, priv, &ci, &xdata);
+    } else if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+        ret = cdc_decompress(this, priv, &ci, xdata);
+    } else {
+        gf_log(this->name, GF_LOG_ERROR, "Invalid operation mode (%d)",
+               priv->op_mode);
+    }
+
+    if (ret)
+        goto default_out;
+
+    STACK_UNWIND_STRICT(readv, frame, ci.nbytes, op_errno, ci.vec, ci.ncount,
+                        stbuf, iobref, xdata);
+    cdc_cleanup_iobref(&ci);
+    return 0;
+
+default_out:
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                        iobref, xdata);
+    return 0;
 }
 
 int32_t
-cdc_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t offset, uint32_t flags,
-           dict_t *xdata)
+cdc_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
-        fop_readv_cbk_t cbk = NULL;
+    fop_readv_cbk_t cbk = NULL;
 
 #ifdef HAVE_LIB_Z
-        cbk = cdc_readv_cbk;
+    cbk = cdc_readv_cbk;
 #else
-        cbk = default_readv_cbk;
+    cbk = default_readv_cbk;
 #endif
-        STACK_WIND (frame, cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags, xdata);
-        return 0;
+    STACK_WIND(frame, cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+               fd, size, offset, flags, xdata);
+    return 0;
 }
 
 int32_t
-cdc_writev_cbk (call_frame_t *frame,
-                void *cookie,
-                xlator_t *this,
-                int32_t op_ret,
-                int32_t op_errno,
-                struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+cdc_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
 {
-
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int32_t
-cdc_writev (call_frame_t *frame,
-            xlator_t *this,
-            fd_t *fd,
-            struct iovec *vector,
-            int32_t count,
-            off_t offset,
-            uint32_t flags,
-            struct iobref *iobref, dict_t *xdata)
+cdc_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
-        int          ret   = -1;
-        cdc_priv_t  *priv  = NULL;
-        cdc_info_t   ci    = {0,};
-        size_t       isize = 0;
-
-        GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
-
-        priv = this->private;
-
-        isize =  iov_length(vector, count);
-
-        if (isize <= 0)
-            goto default_out;
-
-        if ( (priv->min_file_size != 0) && (isize < priv->min_file_size) )
-            goto default_out;
-
-        ci.count       = count;
-        ci.ibytes      = isize;
-        ci.vector      = vector;
-        ci.buf         = NULL;
-        ci.iobref      = NULL;
-        ci.ncount      = 0;
-        ci.crc         = 0;
-        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
-
-/* A writev compresses on the client side and decompresses on the server side
- */
-	    if (priv->op_mode == GF_CDC_MODE_CLIENT) {
-		    ret = cdc_compress (this, priv, &ci, &xdata);
-	    } else if (priv->op_mode == GF_CDC_MODE_SERVER) {
-		    ret = cdc_decompress (this, priv, &ci, xdata);
-	    } else {
-		    gf_log (this->name, GF_LOG_ERROR, "Invalid operation mode (%d) ", priv->op_mode);
-	    }
-
-	    if (ret)
-		    goto default_out;
-
-	    STACK_WIND (frame,
-                    cdc_writev_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->writev,
-                    fd, ci.vec, ci.ncount, offset, flags,
-                    iobref, xdata);
-
-        cdc_cleanup_iobref (&ci);
-        return 0;
-
- default_out:
-        STACK_WIND (frame,
-                    cdc_writev_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->writev,
-                    fd, vector, count, offset, flags,
-                    iobref, xdata);
-        return 0;
+    int ret = -1;
+    cdc_priv_t *priv = NULL;
+    cdc_info_t ci = {
+        0,
+    };
+    size_t isize = 0;
+
+    GF_VALIDATE_OR_GOTO("cdc", this, err);
+    GF_VALIDATE_OR_GOTO(this->name, frame, err);
+
+    priv = this->private;
+
+    isize = iov_length(vector, count);
+
+    if (isize <= 0)
+        goto default_out;
+
+    if ((priv->min_file_size != 0) && (isize < priv->min_file_size))
+        goto default_out;
+
+    ci.count = count;
+    ci.ibytes = isize;
+    ci.vector = vector;
+    ci.buf = NULL;
+    ci.iobref = NULL;
+    ci.ncount = 0;
+    ci.crc = 0;
+    ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+    /* A writev compresses on the client side and decompresses on the server
+     * side
+     */
+    if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+        ret = cdc_compress(this, priv, &ci, &xdata);
+    } else if (priv->op_mode == GF_CDC_MODE_SERVER) {
+        ret = cdc_decompress(this, priv, &ci, xdata);
+    } else {
+        gf_log(this->name, GF_LOG_ERROR, "Invalid operation mode (%d) ",
+               priv->op_mode);
+    }
+
+    if (ret)
+        goto default_out;
+
+    STACK_WIND(frame, cdc_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, ci.vec, ci.ncount, offset,
+               flags, iobref, xdata);
+
+    cdc_cleanup_iobref(&ci);
+    return 0;
+
+default_out:
+    STACK_WIND(frame, cdc_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(writev, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_cdc_mt_end);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       "failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_cdc_mt_end);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               "failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int         ret      = -1;
-        char       *temp_str = NULL;
-        cdc_priv_t *priv     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("cdc", this, err);
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Need subvolume == 1");
-                goto err;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Dangling volume. Check volfile");
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_cdc_mt_priv_t);
-        if (!priv) {
-                goto err;
-        }
-
-        /* Check if debug mode is turned on */
-        GF_OPTION_INIT ("debug", priv->debug, bool, err);
-        if( priv->debug ) {
-                gf_log (this->name, GF_LOG_DEBUG, "CDC debug option turned on");
-        }
-
-        /* Set Gzip Window Size */
-        GF_OPTION_INIT ("window-size", priv->window_size, int32, err);
-        if ( (priv->window_size > GF_CDC_MAX_WINDOWSIZE)
-             || (priv->window_size < GF_CDC_DEF_WINDOWSIZE) ) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Invalid gzip window size (%d), using default",
-                        priv->window_size);
-                priv->window_size = GF_CDC_DEF_WINDOWSIZE;
-        }
-
-        /* Set Gzip (De)Compression Level */
-        GF_OPTION_INIT ("compression-level", priv->cdc_level, int32, err);
-        if ( ((priv->cdc_level < 1) || (priv->cdc_level > 9))
-             && (priv->cdc_level != GF_CDC_DEF_COMPRESSION) ) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Invalid gzip (de)compression level (%d),"
-                        " using default", priv->cdc_level);
-                priv->cdc_level = GF_CDC_DEF_COMPRESSION;
-        }
-
-        /* Set Gzip Memory Level */
-        GF_OPTION_INIT ("mem-level", priv->mem_level, int32, err);
-        if ( (priv->mem_level < 1) || (priv->mem_level > 9) ) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Invalid gzip memory level, using the default");
-                priv->mem_level = GF_CDC_DEF_MEMLEVEL;
-        }
-
-        /* Set min file size to enable compression */
-        GF_OPTION_INIT ("min-size", priv->min_file_size, int32, err);
-
-        /* Mode of operation - Server/Client */
-        ret = dict_get_str (this->options, "mode", &temp_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Operation mode not specified !!");
-                goto err;
-        }
-
-        if (GF_CDC_MODE_IS_CLIENT (temp_str)) {
-                priv->op_mode = GF_CDC_MODE_CLIENT;
-        } else if (GF_CDC_MODE_IS_SERVER (temp_str)) {
-                priv->op_mode = GF_CDC_MODE_SERVER;
-        } else {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Bogus operation mode (%s) specified", temp_str);
-                goto err;
-        }
-
-        this->private = priv;
-        gf_log (this->name, GF_LOG_DEBUG, "CDC xlator loaded in (%s) mode",temp_str);
-        return 0;
-
- err:
-        if (priv)
-                GF_FREE (priv);
-
-        return -1;
+    int ret = -1;
+    char *temp_str = NULL;
+    cdc_priv_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("cdc", this, err);
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR, "Need subvolume == 1");
+        goto err;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "Dangling volume. Check volfile");
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_cdc_mt_priv_t);
+    if (!priv) {
+        goto err;
+    }
+
+    /* Check if debug mode is turned on */
+    GF_OPTION_INIT("debug", priv->debug, bool, err);
+    if (priv->debug) {
+        gf_log(this->name, GF_LOG_DEBUG, "CDC debug option turned on");
+    }
+
+    /* Set Gzip Window Size */
+    GF_OPTION_INIT("window-size", priv->window_size, int32, err);
+    if ((priv->window_size > GF_CDC_MAX_WINDOWSIZE) ||
+        (priv->window_size < GF_CDC_DEF_WINDOWSIZE)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Invalid gzip window size (%d), using default",
+               priv->window_size);
+        priv->window_size = GF_CDC_DEF_WINDOWSIZE;
+    }
+
+    /* Set Gzip (De)Compression Level */
+    GF_OPTION_INIT("compression-level", priv->cdc_level, int32, err);
+    if (((priv->cdc_level < 1) || (priv->cdc_level > 9)) &&
+        (priv->cdc_level != GF_CDC_DEF_COMPRESSION)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Invalid gzip (de)compression level (%d),"
+               " using default",
+               priv->cdc_level);
+        priv->cdc_level = GF_CDC_DEF_COMPRESSION;
+    }
+
+    /* Set Gzip Memory Level */
+    GF_OPTION_INIT("mem-level", priv->mem_level, int32, err);
+    if ((priv->mem_level < 1) || (priv->mem_level > 9)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Invalid gzip memory level, using the default");
+        priv->mem_level = GF_CDC_DEF_MEMLEVEL;
+    }
+
+    /* Set min file size to enable compression */
+    GF_OPTION_INIT("min-size", priv->min_file_size, int32, err);
+
+    /* Mode of operation - Server/Client */
+    ret = dict_get_str(this->options, "mode", &temp_str);
+    if (ret) {
+        gf_log(this->name, GF_LOG_CRITICAL, "Operation mode not specified !!");
+        goto err;
+    }
+
+    if (GF_CDC_MODE_IS_CLIENT(temp_str)) {
+        priv->op_mode = GF_CDC_MODE_CLIENT;
+    } else if (GF_CDC_MODE_IS_SERVER(temp_str)) {
+        priv->op_mode = GF_CDC_MODE_SERVER;
+    } else {
+        gf_log(this->name, GF_LOG_CRITICAL,
+               "Bogus operation mode (%s) specified", temp_str);
+        goto err;
+    }
+
+    this->private = priv;
+    gf_log(this->name, GF_LOG_DEBUG, "CDC xlator loaded in (%s) mode",
+           temp_str);
+    return 0;
+
+err:
+    if (priv)
+        GF_FREE(priv);
+
+    return -1;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        cdc_priv_t *priv = this->private;
+    cdc_priv_t *priv = this->private;
 
-        if (priv)
-                GF_FREE (priv);
-        this->private = NULL;
-        return;
+    if (priv)
+        GF_FREE(priv);
+    this->private = NULL;
+    return;
 }
 
 struct xlator_fops fops = {
-        .readv  = cdc_readv,
-        .writev = cdc_writev,
+    .readv = cdc_readv,
+    .writev = cdc_writev,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks = {};
 
 struct volume_options options[] = {
-        { .key  = {"window-size"},
-          .default_value = "-15",
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Size of the zlib history buffer."
-        },
-        { .key  = {"mem-level"},
-          .default_value = "8",
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Memory allocated for internal compression state. "
-                         "1 uses minimum memory but is slow and reduces "
-                         "compression ratio; memLevel=9 uses maximum memory "
-                         "for optimal speed. The default value is 8."
-        },
-        { .key  = {"compression-level"},
-          .default_value = "-1",
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Compression levels \n"
-                         "0 : no compression, 1 : best speed, \n"
-                         "9 : best compression, -1 : default compression "
-        },
-        { .key  = {"min-size"},
-          .default_value = "0",
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Data is compressed only when its size exceeds this."
-        },
-        { .key  = {"mode"},
-          .value = {"server", "client"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Set on the basis of where the xlator is loaded. "
-                         "This option should NOT be configured by user."
-        },
-        { .key = {"debug"},
-          .default_value = "false",
-          .type = GF_OPTION_TYPE_BOOL,
-          .description = "This is used in testing. Will dump compressed data "
-                         "to disk as a gzip file."
-        },
-        { .key  = {NULL}
-        },
+    {.key = {"window-size"},
+     .default_value = "-15",
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Size of the zlib history buffer."},
+    {.key = {"mem-level"},
+     .default_value = "8",
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Memory allocated for internal compression state. "
+                    "1 uses minimum memory but is slow and reduces "
+                    "compression ratio; memLevel=9 uses maximum memory "
+                    "for optimal speed. The default value is 8."},
+    {.key = {"compression-level"},
+     .default_value = "-1",
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Compression levels \n"
+                    "0 : no compression, 1 : best speed, \n"
+                    "9 : best compression, -1 : default compression "},
+    {.key = {"min-size"},
+     .default_value = "0",
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Data is compressed only when its size exceeds this."},
+    {.key = {"mode"},
+     .value = {"server", "client"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Set on the basis of where the xlator is loaded. "
+                    "This option should NOT be configured by user."},
+    {.key = {"debug"},
+     .default_value = "false",
+     .type = GF_OPTION_TYPE_BOOL,
+     .description = "This is used in testing. Will dump compressed data "
+                    "to disk as a gzip file."},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {GD_OP_VERSION_3_9_0},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "cdc",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/features/compress/src/cdc.h b/xlators/features/compress/src/cdc.h
index 71f4d2317bb..cb87b06a989 100644
--- a/xlators/features/compress/src/cdc.h
+++ b/xlators/features/compress/src/cdc.h
@@ -15,41 +15,41 @@
 #include "zlib.h"
 #endif
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
 #ifndef MAX_IOVEC
 #define MAX_IOVEC 16
 #endif
 
 typedef struct cdc_priv {
-        int window_size;
-        int mem_level;
-        int cdc_level;
-        int min_file_size;
-        int op_mode;
-        gf_boolean_t debug;
-        gf_lock_t lock;
+    int window_size;
+    int mem_level;
+    int cdc_level;
+    int min_file_size;
+    int op_mode;
+    gf_boolean_t debug;
+    gf_lock_t lock;
 } cdc_priv_t;
 
 typedef struct cdc_info {
-        /* input bits */
-        int            count;
-        int32_t        ibytes;
-        struct iovec  *vector;
-        struct iatt   *buf;
-
-        /* output bits */
-        int            ncount;
-        int            nbytes;
-        int            buffer_size;
-        struct iovec   vec[MAX_IOVEC];
-        struct iobref *iobref;
-
-        /* zlib bits */
+    /* input bits */
+    int count;
+    int32_t ibytes;
+    struct iovec *vector;
+    struct iatt *buf;
+
+    /* output bits */
+    int ncount;
+    int nbytes;
+    int buffer_size;
+    struct iovec vec[MAX_IOVEC];
+    struct iobref *iobref;
+
+    /* zlib bits */
 #ifdef HAVE_LIB_Z
-        z_stream      stream;
+    z_stream stream;
 #endif
-        unsigned long crc;
+    unsigned long crc;
 } cdc_info_t;
 
 #define NVEC(ci) (ci->ncount - 1)
@@ -57,8 +57,8 @@ typedef struct cdc_info {
 #define THIS_VEC(ci, i) ci->vector[i]
 
 /* Gzip defaults */
-#define GF_CDC_DEF_WINDOWSIZE  -15 /* default value */
-#define GF_CDC_MAX_WINDOWSIZE  -8  /* max value     */
+#define GF_CDC_DEF_WINDOWSIZE -15 /* default value */
+#define GF_CDC_MAX_WINDOWSIZE -8  /* max value     */
 
 #ifdef HAVE_LIB_Z
 #define GF_CDC_DEF_COMPRESSION Z_DEFAULT_COMPRESSION
@@ -66,15 +66,15 @@ typedef struct cdc_info {
 #define GF_CDC_DEF_COMPRESSION -1
 #endif
 
-#define GF_CDC_DEF_MEMLEVEL    8
-#define GF_CDC_DEF_BUFFERSIZE  262144 // 256K - default compression buffer size
+#define GF_CDC_DEF_MEMLEVEL 8
+#define GF_CDC_DEF_BUFFERSIZE 262144  // 256K - default compression buffer size
 
 /* Operation mode
  * If xlator is loaded on client, readv decompresses and writev compresses
  * If xlator is loaded on server, readv compresses and writev decompresses
  */
-#define GF_CDC_MODE_CLIENT   0
-#define GF_CDC_MODE_SERVER   1
+#define GF_CDC_MODE_CLIENT 0
+#define GF_CDC_MODE_SERVER 1
 
 /* min size of data to do cmpression
  * 0 == compress even 1byte
@@ -87,21 +87,13 @@ typedef struct cdc_info {
 #define GF_CDC_DEFLATE_CANARY_VAL "deflate"
 #define GF_CDC_DEBUG_DUMP_FILE "/tmp/cdcdump.gz"
 
-#define GF_CDC_MODE_IS_CLIENT(m) \
-        (strcmp (m, "client") == 0)
+#define GF_CDC_MODE_IS_CLIENT(m) (strcmp(m, "client") == 0)
 
-#define GF_CDC_MODE_IS_SERVER(m) \
-        (strcmp (m, "server") == 0)
+#define GF_CDC_MODE_IS_SERVER(m) (strcmp(m, "server") == 0)
 
 int32_t
-cdc_compress (xlator_t *this,
-              cdc_priv_t *priv,
-              cdc_info_t *ci,
-              dict_t **xdata);
+cdc_compress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci, dict_t **xdata);
 int32_t
-cdc_decompress (xlator_t *this,
-                cdc_priv_t *priv,
-                cdc_info_t *ci,
-                dict_t *xdata);
+cdc_decompress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci, dict_t *xdata);
 
 #endif
diff --git a/xlators/features/gfid-access/src/gfid-access-mem-types.h b/xlators/features/gfid-access/src/gfid-access-mem-types.h
index 168d67b431f..1c4d0b93de2 100644
--- a/xlators/features/gfid-access/src/gfid-access-mem-types.h
+++ b/xlators/features/gfid-access/src/gfid-access-mem-types.h
@@ -11,13 +11,12 @@
 #ifndef _GFID_ACCESS_MEM_TYPES_H
 #define _GFID_ACCESS_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_changelog_mem_types {
-        gf_gfid_access_mt_priv_t = gf_common_mt_end + 1,
-        gf_gfid_access_mt_gfid_t,
-        gf_gfid_access_mt_end
+    gf_gfid_access_mt_priv_t = gf_common_mt_end + 1,
+    gf_gfid_access_mt_gfid_t,
+    gf_gfid_access_mt_end
 };
 
 #endif
-
diff --git a/xlators/features/gfid-access/src/gfid-access.c b/xlators/features/gfid-access/src/gfid-access.c
index 49d781c59ba..3fea5672a21 100644
--- a/xlators/features/gfid-access/src/gfid-access.c
+++ b/xlators/features/gfid-access/src/gfid-access.c
@@ -8,1421 +8,1413 @@
    cases as published by the Free Software Foundation.
 */
 #include "gfid-access.h"
-#include "inode.h"
-#include "byte-order.h"
-#include "statedump.h"
-
+#include <glusterfs/inode.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/statedump.h>
 
 int
-ga_valid_inode_loc_copy (loc_t *dst, loc_t *src, xlator_t *this)
+ga_valid_inode_loc_copy(loc_t *dst, loc_t *src, xlator_t *this)
 {
-        int      ret        = 0;
-        uint64_t value      = 0;
-
-        /* if its an entry operation, on the virtual */
-        /* directory inode as parent, we need to handle */
-        /* it properly */
-        ret = loc_copy (dst, src);
-        if (ret < 0)
-                goto out;
-
-        /*
-         * Change ALL virtual inodes with real-inodes in loc
-         */
-        if (dst->parent) {
-                ret = inode_ctx_get (dst->parent, this, &value);
-                if (ret < 0) {
-                        ret = 0; //real-inode
-                        goto out;
-                }
-                inode_unref (dst->parent);
-                dst->parent = inode_ref ((inode_t*)value);
-                gf_uuid_copy (dst->pargfid, dst->parent->gfid);
+    int ret = 0;
+    uint64_t value = 0;
+
+    /* if its an entry operation, on the virtual */
+    /* directory inode as parent, we need to handle */
+    /* it properly */
+    ret = loc_copy(dst, src);
+    if (ret < 0)
+        goto out;
+
+    /*
+     * Change ALL virtual inodes with real-inodes in loc
+     */
+    if (dst->parent) {
+        ret = inode_ctx_get(dst->parent, this, &value);
+        if (ret < 0) {
+            ret = 0;  // real-inode
+            goto out;
         }
+        inode_unref(dst->parent);
+        dst->parent = inode_ref((inode_t *)(uintptr_t)value);
+        gf_uuid_copy(dst->pargfid, dst->parent->gfid);
+    }
 
-        if (dst->inode) {
-                ret = inode_ctx_get (dst->inode, this, &value);
-                if (ret < 0) {
-                        ret = 0; //real-inode
-                        goto out;
-                }
-                inode_unref (dst->inode);
-                dst->inode = inode_ref ((inode_t*)value);
-                gf_uuid_copy (dst->gfid, dst->inode->gfid);
+    if (dst->inode) {
+        ret = inode_ctx_get(dst->inode, this, &value);
+        if (ret < 0) {
+            ret = 0;  // real-inode
+            goto out;
         }
+        inode_unref(dst->inode);
+        dst->inode = inode_ref((inode_t *)(uintptr_t)value);
+        gf_uuid_copy(dst->gfid, dst->inode->gfid);
+    }
 out:
 
-        return ret;
+    return ret;
 }
 
 void
-ga_newfile_args_free (ga_newfile_args_t *args)
+ga_newfile_args_free(ga_newfile_args_t *args)
 {
-        if (!args)
-                goto out;
+    if (!args)
+        goto out;
 
-        GF_FREE (args->bname);
+    GF_FREE(args->bname);
 
-        if (S_ISLNK (args->st_mode) && args->args.symlink.linkpath) {
-                GF_FREE (args->args.symlink.linkpath);
-                args->args.symlink.linkpath = NULL;
-        }
+    if (S_ISLNK(args->st_mode) && args->args.symlink.linkpath) {
+        GF_FREE(args->args.symlink.linkpath);
+        args->args.symlink.linkpath = NULL;
+    }
 
-        mem_put (args);
+    mem_put(args);
 out:
-        return;
+    return;
 }
 
-
 void
-ga_heal_args_free (ga_heal_args_t *args)
+ga_heal_args_free(ga_heal_args_t *args)
 {
-        if (!args)
-                goto out;
+    if (!args)
+        goto out;
 
-        GF_FREE (args->bname);
+    GF_FREE(args->bname);
 
-        mem_put (args);
+    mem_put(args);
 out:
-        return;
+    return;
 }
 
-
 ga_newfile_args_t *
-ga_newfile_parse_args (xlator_t *this, data_t *data)
+ga_newfile_parse_args(xlator_t *this, data_t *data)
 {
-        ga_newfile_args_t *args     = NULL;
-        ga_private_t      *priv     = NULL;
-        int                len      = 0;
-        int                blob_len = 0;
-        int                min_len  = 0;
-        void              *blob     = NULL;
-
-        priv = this->private;
-
-        blob = data->data;
-        blob_len = data->len;
-
-        min_len = sizeof (args->uid) + sizeof (args->gid) + sizeof (args->gfid)
-                + sizeof (args->st_mode) + 2 + 2;
-        if (blob_len < min_len) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Invalid length: Total length is less "
-                        "than minimum length.");
-                goto err;
+    ga_newfile_args_t *args = NULL;
+    ga_private_t *priv = NULL;
+    int len = 0;
+    int blob_len = 0;
+    int min_len = 0;
+    void *blob = NULL;
+
+    priv = this->private;
+
+    blob = data->data;
+    blob_len = data->len;
+
+    min_len = sizeof(args->uid) + sizeof(args->gid) + sizeof(args->gfid) +
+              sizeof(args->st_mode) + 2 + 2;
+    if (blob_len < min_len) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Invalid length: Total length is less "
+               "than minimum length.");
+        goto err;
+    }
+
+    args = mem_get0(priv->newfile_args_pool);
+    if (args == NULL)
+        goto err;
+
+    args->uid = ntoh32(*(uint32_t *)blob);
+    blob += sizeof(uint32_t);
+    blob_len -= sizeof(uint32_t);
+
+    args->gid = ntoh32(*(uint32_t *)blob);
+    blob += sizeof(uint32_t);
+    blob_len -= sizeof(uint32_t);
+
+    memcpy(args->gfid, blob, sizeof(args->gfid));
+    blob += sizeof(args->gfid);
+    blob_len -= sizeof(args->gfid);
+
+    args->st_mode = ntoh32(*(uint32_t *)blob);
+    blob += sizeof(uint32_t);
+    blob_len -= sizeof(uint32_t);
+
+    len = strnlen(blob, blob_len);
+    if (len == blob_len) {
+        gf_log(this->name, GF_LOG_ERROR, "gfid: %s. No null byte present.",
+               args->gfid);
+        goto err;
+    }
+
+    args->bname = GF_MALLOC(len + 1, gf_common_mt_char);
+    if (args->bname == NULL)
+        goto err;
+
+    memcpy(args->bname, blob, (len + 1));
+    blob += (len + 1);
+    blob_len -= (len + 1);
+
+    if (S_ISDIR(args->st_mode)) {
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
         }
-
-        args = mem_get0 (priv->newfile_args_pool);
-        if (args == NULL)
-                goto err;
-
-        args->uid = ntoh32 (*(uint32_t *)blob);
-        blob += sizeof (uint32_t);
-        blob_len -= sizeof (uint32_t);
-
-        args->gid = ntoh32 (*(uint32_t *)blob);
-        blob += sizeof (uint32_t);
-        blob_len -= sizeof (uint32_t);
-
-        memcpy (args->gfid, blob, sizeof (args->gfid));
-        blob += sizeof (args->gfid);
-        blob_len -= sizeof (args->gfid);
-
-        args->st_mode = ntoh32 (*(uint32_t *)blob);
-        blob += sizeof (uint32_t);
-        blob_len -= sizeof (uint32_t);
-
-        len = strnlen (blob, blob_len);
+        args->args.mkdir.mode = ntoh32(*(uint32_t *)blob);
+        blob += sizeof(uint32_t);
+        blob_len -= sizeof(uint32_t);
+
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
+        }
+        args->args.mkdir.umask = ntoh32(*(uint32_t *)blob);
+        blob_len -= sizeof(uint32_t);
+        if (blob_len < 0) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
+        }
+    } else if (S_ISLNK(args->st_mode)) {
+        len = strnlen(blob, blob_len);
         if (len == blob_len) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "gfid: %s. No null byte present.",
-                        args->gfid);
-                goto err;
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
         }
+        args->args.symlink.linkpath = GF_MALLOC(len + 1, gf_common_mt_char);
+        if (args->args.symlink.linkpath == NULL)
+            goto err;
 
-        args->bname = GF_CALLOC (1, (len + 1), gf_common_mt_char);
-        if (args->bname == NULL)
-                goto err;
-
-        memcpy (args->bname, blob, (len + 1));
-        blob += (len + 1);
+        memcpy(args->args.symlink.linkpath, blob, (len + 1));
         blob_len -= (len + 1);
-
-        if (S_ISDIR (args->st_mode)) {
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mkdir.mode = ntoh32 (*(uint32_t *)blob);
-                blob += sizeof (uint32_t);
-                blob_len -= sizeof (uint32_t);
-
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mkdir.umask = ntoh32 (*(uint32_t *)blob);
-                blob_len -= sizeof (uint32_t);
-                if (blob_len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-        } else if (S_ISLNK (args->st_mode)) {
-                len = strnlen (blob, blob_len);
-                if (len == blob_len) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.symlink.linkpath = GF_CALLOC (1, len + 1,
-                                                         gf_common_mt_char);
-                if (args->args.symlink.linkpath == NULL)
-                        goto err;
-
-                memcpy (args->args.symlink.linkpath, blob, (len + 1));
-                blob_len -= (len + 1);
-        } else {
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mknod.mode = ntoh32 (*(uint32_t *)blob);
-                blob += sizeof (uint32_t);
-                blob_len -= sizeof (uint32_t);
-
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mknod.rdev = ntoh32 (*(uint32_t *)blob);
-                blob += sizeof (uint32_t);
-                blob_len -= sizeof (uint32_t);
-
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mknod.umask = ntoh32 (*(uint32_t *)blob);
-                blob_len -= sizeof (uint32_t);
+    } else {
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
         }
-
-        if (blob_len) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "gfid: %s. Invalid length",
-                        args->gfid);
-                goto err;
+        args->args.mknod.mode = ntoh32(*(uint32_t *)blob);
+        blob += sizeof(uint32_t);
+        blob_len -= sizeof(uint32_t);
+
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
         }
+        args->args.mknod.rdev = ntoh32(*(uint32_t *)blob);
+        blob += sizeof(uint32_t);
+        blob_len -= sizeof(uint32_t);
+
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
+        }
+        args->args.mknod.umask = ntoh32(*(uint32_t *)blob);
+        blob_len -= sizeof(uint32_t);
+    }
+
+    if (blob_len) {
+        gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+               args->gfid);
+        goto err;
+    }
 
-        return args;
+    return args;
 
 err:
-        if (args)
-                ga_newfile_args_free (args);
+    if (args)
+        ga_newfile_args_free(args);
 
-        return NULL;
+    return NULL;
 }
 
 ga_heal_args_t *
-ga_heal_parse_args (xlator_t *this, data_t *data)
+ga_heal_parse_args(xlator_t *this, data_t *data)
 {
-        ga_heal_args_t *args     = NULL;
-        ga_private_t   *priv     = NULL;
-        void           *blob     = NULL;
-        int             len      = 0;
-        int             blob_len = 0;
+    ga_heal_args_t *args = NULL;
+    ga_private_t *priv = NULL;
+    void *blob = NULL;
+    int len = 0;
+    int blob_len = 0;
 
-        blob = data->data;
-        blob_len = data->len;
+    blob = data->data;
+    blob_len = data->len;
 
-        priv = this->private;
+    priv = this->private;
 
-        /* bname should at least contain a character */
-        if (blob_len < (sizeof (args->gfid) + 2))
-                goto err;
+    /* bname should at least contain a character */
+    if (blob_len < (sizeof(args->gfid) + 2))
+        goto err;
 
-        args = mem_get0 (priv->heal_args_pool);
-        if (!args)
-                goto err;
+    args = mem_get0(priv->heal_args_pool);
+    if (!args)
+        goto err;
 
-        memcpy (args->gfid, blob, sizeof (args->gfid));
-        blob += sizeof (args->gfid);
-        blob_len -= sizeof (args->gfid);
+    memcpy(args->gfid, blob, sizeof(args->gfid));
+    blob += sizeof(args->gfid);
+    blob_len -= sizeof(args->gfid);
 
-        len = strnlen (blob, blob_len);
-        if (len == blob_len)
-                goto err;
+    len = strnlen(blob, blob_len);
+    if (len == blob_len)
+        goto err;
 
-        args->bname = GF_CALLOC (1, len + 1, gf_common_mt_char);
-        if (!args->bname)
-                goto err;
+    args->bname = GF_MALLOC(len + 1, gf_common_mt_char);
+    if (!args->bname)
+        goto err;
 
-        memcpy (args->bname, blob, len);
-        blob_len -= (len + 1);
+    memcpy(args->bname, blob, len);
+    args->bname[len] = '\0';
+    blob_len -= (len + 1);
 
-        if (blob_len)
-                goto err;
+    if (blob_len)
+        goto err;
 
-        return args;
+    return args;
 
 err:
-        if (args)
-                ga_heal_args_free (args);
+    if (args)
+        ga_heal_args_free(args);
 
-        return NULL;
+    return NULL;
 }
 
 static int32_t
-ga_fill_tmp_loc (loc_t *loc, xlator_t *this, uuid_t gfid,
-                 char *bname, dict_t *xdata, loc_t *new_loc)
+ga_fill_tmp_loc(loc_t *loc, xlator_t *this, uuid_t gfid, char *bname,
+                dict_t *xdata, loc_t *new_loc)
 {
-        int       ret    = -1;
-        uint64_t  value  = 0;
-        inode_t  *parent = NULL;
-        unsigned char *gfid_ptr = NULL;
-
-        parent = loc->inode;
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (!ret) {
-                parent = (void *)value;
-                if (gf_uuid_is_null (parent->gfid))
-                        parent = loc->inode;
-        }
-
-        /* parent itself should be looked up */
-        gf_uuid_copy (new_loc->pargfid, parent->gfid);
-        new_loc->parent = inode_ref (parent);
-
-        new_loc->inode = inode_grep (parent->table, parent, bname);
-        if (!new_loc->inode) {
-                new_loc->inode = inode_new (parent->table);
-                gf_uuid_copy (new_loc->inode->gfid, gfid);
-        }
-
-        loc_path (new_loc, bname);
-        if (new_loc->path) {
-                new_loc->name = strrchr (new_loc->path, '/');
-                if (new_loc->name)
-                        new_loc->name++;
-        }
-
-        gfid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!gfid_ptr) {
-                ret = -1;
-                goto out;
-        }
-        gf_uuid_copy (gfid_ptr, gfid);
-        ret = dict_set_gfuuid (xdata, "gfid-req", gfid_ptr, false);
-        if (ret < 0)
-                goto out;
-
-        ret = 0;
+    int ret = -1;
+    uint64_t value = 0;
+    inode_t *parent = NULL;
+    unsigned char *gfid_ptr = NULL;
+
+    parent = loc->inode;
+    ret = inode_ctx_get(loc->inode, this, &value);
+    if (!ret) {
+        parent = (void *)(uintptr_t)value;
+        if (gf_uuid_is_null(parent->gfid))
+            parent = loc->inode;
+    }
+
+    /* parent itself should be looked up */
+    gf_uuid_copy(new_loc->pargfid, parent->gfid);
+    new_loc->parent = inode_ref(parent);
+
+    new_loc->inode = inode_grep(parent->table, parent, bname);
+    if (!new_loc->inode) {
+        new_loc->inode = inode_new(parent->table);
+        gf_uuid_copy(new_loc->inode->gfid, gfid);
+    }
+
+    loc_path(new_loc, bname);
+    if (new_loc->path) {
+        new_loc->name = strrchr(new_loc->path, '/');
+        if (new_loc->name)
+            new_loc->name++;
+    }
+
+    gfid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!gfid_ptr) {
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_copy(gfid_ptr, gfid);
+    ret = dict_set_gfuuid(xdata, "gfid-req", gfid_ptr, false);
+    if (ret < 0)
+        goto out;
+
+    ret = 0;
 
 out:
-        if (ret && gfid_ptr)
-                GF_FREE (gfid_ptr);
-        return ret;
+    if (ret && gfid_ptr)
+        GF_FREE(gfid_ptr);
+    return ret;
 }
 
-
-
 static gf_boolean_t
-__is_gfid_access_dir (uuid_t gfid)
+__is_gfid_access_dir(uuid_t gfid)
 {
-        uuid_t  aux_gfid;
-
-        memset (aux_gfid, 0, 16);
-        aux_gfid[15] = GF_AUX_GFID;
+    static uuid_t aux_gfid = {0, 0, 0, 0, 0, 0, 0, 0,
+                              0, 0, 0, 0, 0, 0, 0, GF_AUX_GFID};
 
-        if (gf_uuid_compare (gfid, aux_gfid) == 0)
-                return _gf_true;
+    if (gf_uuid_compare(gfid, aux_gfid) == 0)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 int32_t
-ga_forget (xlator_t *this, inode_t *inode)
+ga_forget(xlator_t *this, inode_t *inode)
 {
-        int       ret = -1;
-        uint64_t  value = 0;
-        inode_t  *tmp_inode = NULL;
+    int ret = -1;
+    uint64_t value = 0;
+    inode_t *tmp_inode = NULL;
 
-        ret = inode_ctx_del (inode, this, &value);
-        if (ret)
-                goto out;
+    ret = inode_ctx_del(inode, this, &value);
+    if (ret)
+        goto out;
 
-        tmp_inode = (void *)value;
-        inode_unref (tmp_inode);
+    tmp_inode = (void *)(uintptr_t)value;
+    inode_unref(tmp_inode);
 
 out:
-        return 0;
+    return 0;
 }
 
-
 static int
-ga_heal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno,
-             inode_t *inode, struct iatt *stat, dict_t *dict,
-             struct iatt *postparent)
+ga_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, inode_t *inode, struct iatt *stat, dict_t *dict,
+            struct iatt *postparent)
 {
-        call_frame_t *orig_frame = NULL;
+    call_frame_t *orig_frame = NULL;
 
-        orig_frame = frame->local;
-        frame->local = NULL;
+    orig_frame = frame->local;
+    frame->local = NULL;
 
-        /* don't worry about inode linking and other stuff. They'll happen on
-         * the next lookup.
-         */
-        STACK_DESTROY (frame->root);
+    /* don't worry about inode linking and other stuff. They'll happen on
+     * the next lookup.
+     */
+    STACK_DESTROY(frame->root);
 
-        STACK_UNWIND_STRICT (setxattr, orig_frame, op_ret, op_errno, dict);
+    STACK_UNWIND_STRICT(setxattr, orig_frame, op_ret, op_errno, dict);
 
-        return 0;
+    return 0;
 }
 
 static int
-ga_newentry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent,
-                 dict_t *xdata)
+ga_newentry_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-        ga_local_t *local = NULL;
+    ga_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        /* don't worry about inode linking and other stuff. They'll happen on
-         * the next lookup.
-         */
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
+    /* don't worry about inode linking and other stuff. They'll happen on
+     * the next lookup.
+     */
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
 
-        STACK_UNWIND_STRICT (setxattr, local->orig_frame, op_ret,
-                             op_errno, xdata);
+    STACK_UNWIND_STRICT(setxattr, local->orig_frame, op_ret, op_errno, xdata);
 
-        if (local->xdata)
-                dict_unref (local->xdata);
-        loc_wipe (&local->loc);
-        mem_put (local);
+    if (local->xdata)
+        dict_unref(local->xdata);
+    loc_wipe(&local->loc);
+    mem_put(local);
 
-        return 0;
+    return 0;
 }
 
 static int
-ga_newentry_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *stat, dict_t *xdata,
-                        struct iatt *postparent)
+ga_newentry_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *stat, dict_t *xdata,
+                       struct iatt *postparent)
 
 {
-        ga_local_t *local = NULL;
+    ga_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if ((op_ret < 0) && ((op_errno != ENOENT) && (op_errno != ESTALE)))
-                goto err;
+    if ((op_ret < 0) && ((op_errno != ENOENT) && (op_errno != ESTALE)))
+        goto err;
 
-        STACK_WIND (frame, ga_newentry_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, &local->loc, local->mode,
-                    local->rdev, local->umask, local->xdata);
-        return 0;
+    STACK_WIND(frame, ga_newentry_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, &local->loc, local->mode,
+               local->rdev, local->umask, local->xdata);
+    return 0;
 
 err:
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
-        STACK_UNWIND_STRICT (setxattr, local->orig_frame, op_ret, op_errno,
-                             xdata);
-        if (local->xdata)
-                dict_unref (local->xdata);
-        loc_wipe (&local->loc);
-        mem_put (local);
-
-        return 0;
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    STACK_UNWIND_STRICT(setxattr, local->orig_frame, op_ret, op_errno, xdata);
+    if (local->xdata)
+        dict_unref(local->xdata);
+    loc_wipe(&local->loc);
+    mem_put(local);
+
+    return 0;
 }
 
 int32_t
-ga_new_entry (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
-              dict_t *xdata)
+ga_new_entry(call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
+             dict_t *xdata)
 {
-        int                ret       = -1;
-        ga_newfile_args_t *args      = NULL;
-        loc_t              tmp_loc   = {0,};
-        call_frame_t      *new_frame = NULL;
-        ga_local_t        *local     = NULL;
-        uuid_t             gfid      = {0,};
-
-        args = ga_newfile_parse_args (this, data);
-        if (!args)
-                goto out;
-
-        ret = gf_uuid_parse (args->gfid, gfid);
-        if (ret)
-                goto out;
-
-        if (!xdata) {
-                xdata = dict_new ();
-        } else {
-                xdata = dict_ref (xdata);
-        }
-
-        if (!xdata) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = ga_fill_tmp_loc (loc, this, gfid,
-                               args->bname, xdata, &tmp_loc);
-        if (ret)
-                goto out;
-
-        new_frame = copy_frame (frame);
-        if (!new_frame)
-                goto out;
-
-        local = mem_get0 (this->local_pool);
-        local->orig_frame = frame;
-
-        loc_copy (&local->loc, &tmp_loc);
-
-        new_frame->local = local;
-        new_frame->root->uid = args->uid;
-        new_frame->root->gid = args->gid;
-
-        if (S_ISDIR (args->st_mode)) {
-                STACK_WIND (new_frame, ga_newentry_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
-                            &tmp_loc, args->args.mkdir.mode,
-                            args->args.mkdir.umask, xdata);
-        } else if (S_ISLNK (args->st_mode)) {
-                STACK_WIND (new_frame, ga_newentry_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
-                            args->args.symlink.linkpath,
-                            &tmp_loc, 0, xdata);
-        } else {
-                /* use 07777 (4 7s) for considering the Sticky bits etc) */
-                ((ga_local_t *)new_frame->local)->mode =
-                     (S_IFMT & args->st_mode) | (07777 & args->args.mknod.mode);
-
-                ((ga_local_t *)new_frame->local)->umask =
-                                                         args->args.mknod.umask;
-                ((ga_local_t *)new_frame->local)->rdev  = args->args.mknod.rdev;
-                ((ga_local_t *)new_frame->local)->xdata = dict_ref (xdata);
-
-                /* send a named lookup, so that dht can cleanup up stale linkto
-                 * files etc.
-                 */
-                STACK_WIND (new_frame, ga_newentry_lookup_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup,
-                            &tmp_loc, NULL);
-        }
+    int ret = -1;
+    ga_newfile_args_t *args = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    call_frame_t *new_frame = NULL;
+    ga_local_t *local = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    if (!xdata) {
+        xdata = dict_new();
+    } else {
+        xdata = dict_ref(xdata);
+    }
+
+    if (!xdata) {
+        ret = -1;
+        goto out;
+    }
+
+    args = ga_newfile_parse_args(this, data);
+    if (!args)
+        goto out;
+
+    ret = gf_uuid_parse(args->gfid, gfid);
+    if (ret)
+        goto out;
+
+    ret = ga_fill_tmp_loc(loc, this, gfid, args->bname, xdata, &tmp_loc);
+    if (ret)
+        goto out;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame)
+        goto out;
+
+    local = mem_get0(this->local_pool);
+    local->orig_frame = frame;
+
+    loc_copy(&local->loc, &tmp_loc);
+
+    new_frame->local = local;
+    new_frame->root->uid = args->uid;
+    new_frame->root->gid = args->gid;
+
+    if (S_ISDIR(args->st_mode)) {
+        STACK_WIND(new_frame, ga_newentry_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mkdir, &tmp_loc,
+                   args->args.mkdir.mode, args->args.mkdir.umask, xdata);
+    } else if (S_ISLNK(args->st_mode)) {
+        STACK_WIND(new_frame, ga_newentry_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->symlink,
+                   args->args.symlink.linkpath, &tmp_loc, 0, xdata);
+    } else {
+        /* use 07777 (4 7s) for considering the Sticky bits etc) */
+        ((ga_local_t *)new_frame->local)->mode = (S_IFMT & args->st_mode) |
+                                                 (07777 &
+                                                  args->args.mknod.mode);
+
+        ((ga_local_t *)new_frame->local)->umask = args->args.mknod.umask;
+        ((ga_local_t *)new_frame->local)->rdev = args->args.mknod.rdev;
+        ((ga_local_t *)new_frame->local)->xdata = dict_ref(xdata);
+
+        /* send a named lookup, so that dht can cleanup up stale linkto
+         * files etc.
+         */
+        STACK_WIND(new_frame, ga_newentry_lookup_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, &tmp_loc, NULL);
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        ga_newfile_args_free (args);
+    ga_newfile_args_free(args);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        loc_wipe (&tmp_loc);
+    loc_wipe(&tmp_loc);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-ga_heal_entry (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
-               dict_t *xdata)
+ga_heal_entry(call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
+              dict_t *xdata)
 {
-        int             ret       = -1;
-        ga_heal_args_t *args      = NULL;
-        loc_t           tmp_loc   = {0,};
-        call_frame_t   *new_frame = NULL;
-        uuid_t          gfid      = {0,};
-
-        args = ga_heal_parse_args (this, data);
-        if (!args)
-                goto out;
-
-        ret = gf_uuid_parse (args->gfid, gfid);
-        if (ret)
-                goto out;
-
-        if (!xdata)
-                xdata = dict_new ();
-        else
-                xdata = dict_ref (xdata);
-
-        if (!xdata) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = ga_fill_tmp_loc (loc, this, gfid, args->bname,
-                               xdata, &tmp_loc);
-        if (ret)
-                goto out;
-
-        new_frame = copy_frame (frame);
-        if (!new_frame)
-                goto out;
-
-        new_frame->local = (void *)frame;
-
-        STACK_WIND (new_frame, ga_heal_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->lookup,
-                    &tmp_loc, xdata);
-
-        ret = 0;
+    int ret = -1;
+    ga_heal_args_t *args = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    call_frame_t *new_frame = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    args = ga_heal_parse_args(this, data);
+    if (!args)
+        goto out;
+
+    ret = gf_uuid_parse(args->gfid, gfid);
+    if (ret)
+        goto out;
+
+    if (!xdata)
+        xdata = dict_new();
+    else
+        xdata = dict_ref(xdata);
+
+    if (!xdata) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = ga_fill_tmp_loc(loc, this, gfid, args->bname, xdata, &tmp_loc);
+    if (ret)
+        goto out;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame)
+        goto out;
+
+    new_frame->local = (void *)frame;
+
+    STACK_WIND(new_frame, ga_heal_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
+
+    ret = 0;
 out:
-        if (args)
-                ga_heal_args_free (args);
+    if (args)
+        ga_heal_args_free(args);
 
-        loc_wipe (&tmp_loc);
+    loc_wipe(&tmp_loc);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-ga_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 dict_t *xdata)
+ga_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-ga_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags, dict_t *xdata)
+ga_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
 {
-        data_t  *data     = NULL;
-        int      op_errno = ENOMEM;
-        int      ret      = 0;
-        loc_t   ga_loc    = {0, };
-
-        GFID_ACCESS_INODE_OP_CHECK (loc, op_errno, err);
-
-        data = dict_get (dict, GF_FUSE_AUX_GFID_NEWFILE);
-        if (data) {
-                ret = ga_new_entry (frame, this, loc, data, xdata);
-                if (ret)
-                        goto err;
-                return 0;
-        }
+    data_t *data = NULL;
+    int op_errno = ENOMEM;
+    int ret = 0;
+    loc_t ga_loc = {
+        0,
+    };
+
+    GFID_ACCESS_INODE_OP_CHECK(loc, op_errno, err);
+
+    data = dict_get(dict, GF_FUSE_AUX_GFID_NEWFILE);
+    if (data) {
+        ret = ga_new_entry(frame, this, loc, data, xdata);
+        if (ret)
+            goto err;
+        return 0;
+    }
 
-        data = dict_get (dict, GF_FUSE_AUX_GFID_HEAL);
-        if (data) {
-                ret = ga_heal_entry (frame, this, loc, data, xdata);
-                if (ret)
-                        goto err;
-                return 0;
-        }
+    data = dict_get(dict, GF_FUSE_AUX_GFID_HEAL);
+    if (data) {
+        ret = ga_heal_entry(frame, this, loc, data, xdata);
+        if (ret)
+            goto err;
+        return 0;
+    }
 
-        //If the inode is a virtual inode change the inode otherwise perform
-        //the operation on same inode
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
+    // If the inode is a virtual inode change the inode otherwise perform
+    // the operation on same inode
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
 
-        STACK_WIND (frame, ga_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, &ga_loc, dict, flags,
-                    xdata);
+    STACK_WIND(frame, ga_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, &ga_loc, dict, flags, xdata);
 
-        loc_wipe (&ga_loc);
-        return 0;
+    loc_wipe(&ga_loc);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(setxattr, frame, -1, op_errno, xdata);
+    return 0;
 }
 
-
 int32_t
-ga_virtual_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, inode_t *inode,
-                       struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+ga_virtual_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
+                      struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        int             ret                     = 0;
-        inode_t         *cbk_inode              = NULL;
-        inode_t         *true_inode             = NULL;
-        uuid_t          random_gfid             = {0,};
-        inode_t         *linked_inode           = NULL;
-
-        if (frame->local)
-                cbk_inode = frame->local;
-        else
-                cbk_inode = inode_ref (inode);
-
-        frame->local = NULL;
-        if (op_ret)
-                goto unwind;
-
-        if (!IA_ISDIR (buf->ia_type))
+    int ret = 0;
+    inode_t *cbk_inode = NULL;
+    inode_t *true_inode = NULL;
+    uuid_t random_gfid = {
+        0,
+    };
+    inode_t *linked_inode = NULL;
+
+    if (frame->local)
+        cbk_inode = frame->local;
+    else
+        cbk_inode = inode_ref(inode);
+
+    frame->local = NULL;
+    if (op_ret)
+        goto unwind;
+
+    if (!IA_ISDIR(buf->ia_type))
+        goto unwind;
+
+    /* need to send back a different inode for linking in itable */
+    if (cbk_inode == inode) {
+        /* check if the inode is in the 'itable' or
+           if its just previously discover()'d inode */
+        true_inode = inode_find(inode->table, buf->ia_gfid);
+        if (!true_inode) {
+            /* This unref is for 'inode_ref()' done in beginning.
+               This is needed as cbk_inode is allocated new inode
+               whose unref is taken at the end*/
+            inode_unref(cbk_inode);
+            cbk_inode = inode_new(inode->table);
+
+            if (!cbk_inode) {
+                op_ret = -1;
+                op_errno = ENOMEM;
                 goto unwind;
+            }
+            /* the inode is not present in itable, ie, the actual
+               path is not yet looked up. Use the current inode
+               itself for now */
 
-        /* need to send back a different inode for linking in itable */
-        if (cbk_inode == inode) {
-                /* check if the inode is in the 'itable' or
-                   if its just previously discover()'d inode */
-                true_inode = inode_find (inode->table, buf->ia_gfid);
-                if (!true_inode) {
-                        /* This unref is for 'inode_ref()' done in beginning.
-                           This is needed as cbk_inode is allocated new inode
-                           whose unref is taken at the end*/
-                        inode_unref (cbk_inode);
-                        cbk_inode = inode_new (inode->table);
-
-                        if (!cbk_inode) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                goto unwind;
-                        }
-                        /* the inode is not present in itable, ie, the actual
-                           path is not yet looked up. Use the current inode
-                           itself for now */
-
-                        linked_inode = inode_link (inode, NULL, NULL, buf);
-                        inode = linked_inode;
-                } else {
-                        /* 'inode_ref()' has been done in inode_find() */
-                        inode = true_inode;
-                }
-
-                ret = inode_ctx_put (cbk_inode, this, (uint64_t)inode);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set the inode ctx with"
-                                "the actual inode");
-                        if (inode)
-                                inode_unref (inode);
-                }
-                inode = NULL;
+            linked_inode = inode_link(inode, NULL, NULL, buf);
+            inode = linked_inode;
+        } else {
+            /* 'inode_ref()' has been done in inode_find() */
+            inode = true_inode;
         }
 
-        if (!gf_uuid_is_null (cbk_inode->gfid)) {
-                /* if the previous linked inode is used, use the
-                   same gfid */
-                gf_uuid_copy (random_gfid, cbk_inode->gfid);
-        } else {
-                /* replace the buf->ia_gfid to a random gfid
-                   for directory, for files, what we received is fine */
-                gf_uuid_generate (random_gfid);
+        ret = inode_ctx_put(cbk_inode, this, (uint64_t)(uintptr_t)inode);
+        if (ret) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "failed to set the inode ctx with"
+                   "the actual inode");
+            if (inode)
+                inode_unref(inode);
         }
+        inode = NULL;
+    }
+
+    if (!gf_uuid_is_null(cbk_inode->gfid)) {
+        /* if the previous linked inode is used, use the
+           same gfid */
+        gf_uuid_copy(random_gfid, cbk_inode->gfid);
+    } else {
+        /* replace the buf->ia_gfid to a random gfid
+           for directory, for files, what we received is fine */
+        gf_uuid_generate(random_gfid);
+    }
 
-        gf_uuid_copy (buf->ia_gfid, random_gfid);
+    gf_uuid_copy(buf->ia_gfid, random_gfid);
 
-        buf->ia_ino = gfid_to_ino (buf->ia_gfid);
+    buf->ia_ino = gfid_to_ino(buf->ia_gfid);
 
 unwind:
-        /* Lookup on non-existing gfid returns ESTALE.
-           Convert into ENOENT for virtual lookup*/
-        if (op_errno == ESTALE)
-               op_errno = ENOENT;
+    /* Lookup on non-existing gfid returns ESTALE.
+       Convert into ENOENT for virtual lookup*/
+    if (op_errno == ESTALE)
+        op_errno = ENOENT;
 
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, cbk_inode, buf,
-                             xdata, postparent);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, cbk_inode, buf, xdata,
+                        postparent);
 
-        /* Also handles inode_unref of frame->local if done in ga_lookup */
-        if (cbk_inode)
-               inode_unref (cbk_inode);
+    /* Also handles inode_unref of frame->local if done in ga_lookup */
+    if (cbk_inode)
+        inode_unref(cbk_inode);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ga_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+ga_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+              struct iatt *postparent)
 {
-        ga_private_t *priv = NULL;
+    ga_private_t *priv = NULL;
 
-        /* if the entry in question is not 'root',
-           then follow the normal path */
-        if (op_ret || !__is_root_gfid(buf->ia_gfid))
-                goto unwind;
+    /* if the entry in question is not 'root',
+       then follow the normal path */
+    if (op_ret || !__is_root_gfid(buf->ia_gfid))
+        goto unwind;
 
-        priv = this->private;
+    priv = this->private;
 
-        /* do we need to copy root stbuf everytime? */
-        /* mostly yes, as we want to have the 'stat' info show latest
-           in every _cbk() */
+    /* do we need to copy root stbuf every time? */
+    /* mostly yes, as we want to have the 'stat' info show latest
+       in every _cbk() */
 
-        /* keep the reference for root stat buf */
-        priv->root_stbuf = *buf;
-        priv->gfiddir_stbuf = priv->root_stbuf;
-        priv->gfiddir_stbuf.ia_gfid[15] = GF_AUX_GFID;
-        priv->gfiddir_stbuf.ia_ino = GF_AUX_GFID;
+    /* keep the reference for root stat buf */
+    priv->root_stbuf = *buf;
+    priv->gfiddir_stbuf = priv->root_stbuf;
+    priv->gfiddir_stbuf.ia_gfid[15] = GF_AUX_GFID;
+    priv->gfiddir_stbuf.ia_ino = GF_AUX_GFID;
 
 unwind:
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xdata, postparent);
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    return 0;
 }
 
 int32_t
-ga_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+ga_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        ga_private_t *priv     = NULL;
-        int           ret      = -1;
-        uuid_t        tmp_gfid = {0,};
-        loc_t         tmp_loc  = {0,};
-        uint64_t      value    = 0;
-        inode_t      *inode    = NULL;
-        inode_t      *true_inode    = NULL;
-        int32_t       op_errno = ENOENT;
-
-        priv = this->private;
-
-        /* Handle nameless lookup on ".gfid" */
-        if (!loc->parent && __is_gfid_access_dir(loc->gfid)) {
-                STACK_UNWIND_STRICT (lookup, frame, 0, 0, loc->inode,
-                                     &priv->gfiddir_stbuf, xdata,
-                                     &priv->root_stbuf);
-                return 0;
-        }
-
-        /* if its discover(), no need for any action here */
-        if (!loc->name)
-                goto wind;
-
-        /* if its revalidate, and inode is not of type directory,
-           proceed with 'wind' */
-        if (loc->inode && loc->inode->ia_type &&
-            !IA_ISDIR (loc->inode->ia_type)) {
-
-                /* a revalidate on ".gfid/<dentry>" is possible, check for it */
-                if (((loc->parent &&
-                      __is_gfid_access_dir (loc->parent->gfid)) ||
-                     __is_gfid_access_dir (loc->pargfid))) {
-
-                        /* here, just send 'loc->gfid' and 'loc->inode' */
-                        tmp_loc.inode = inode_ref (loc->inode);
-                        gf_uuid_copy (tmp_loc.gfid, loc->inode->gfid);
+    ga_private_t *priv = NULL;
+    int ret = -1;
+    uuid_t tmp_gfid = {
+        0,
+    };
+    loc_t tmp_loc = {
+        0,
+    };
+    uint64_t value = 0;
+    inode_t *inode = NULL;
+    inode_t *true_inode = NULL;
+    int32_t op_errno = ENOENT;
+
+    priv = this->private;
+
+    /* Handle nameless lookup on ".gfid" */
+    if (!loc->parent && __is_gfid_access_dir(loc->gfid)) {
+        STACK_UNWIND_STRICT(lookup, frame, 0, 0, loc->inode,
+                            &priv->gfiddir_stbuf, xdata, &priv->root_stbuf);
+        return 0;
+    }
 
-                        STACK_WIND (frame, default_lookup_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->lookup,
-                                    &tmp_loc, xdata);
+    /* if its discover(), no need for any action here */
+    if (!loc->name)
+        goto wind;
 
-                        inode_unref (tmp_loc.inode);
+    /* if its revalidate, and inode is not of type directory,
+       proceed with 'wind' */
+    if (loc->inode && loc->inode->ia_type && !IA_ISDIR(loc->inode->ia_type)) {
+        /* a revalidate on ".gfid/<dentry>" is possible, check for it */
+        if (((loc->parent && __is_gfid_access_dir(loc->parent->gfid)) ||
+             __is_gfid_access_dir(loc->pargfid))) {
+            /* here, just send 'loc->gfid' and 'loc->inode' */
+            tmp_loc.inode = inode_ref(loc->inode);
+            gf_uuid_copy(tmp_loc.gfid, loc->inode->gfid);
 
-                        return 0;
-                }
+            STACK_WIND(frame, default_lookup_cbk, FIRST_CHILD(this),
+                       FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
 
-                /* not something to bother, continue the flow */
-                goto wind;
-        }
+            inode_unref(tmp_loc.inode);
 
-        /* need to check if the lookup is on virtual dir */
-        if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&
-            ((loc->parent && __is_root_gfid (loc->parent->gfid)) ||
-             __is_root_gfid (loc->pargfid))) {
-                /* this means, the query is on '/.gfid', return the fake stat,
-                   and say success */
-
-                STACK_UNWIND_STRICT (lookup, frame, 0, 0, loc->inode,
-                                     &priv->gfiddir_stbuf, xdata,
-                                     &priv->root_stbuf);
-                return 0;
+            return 0;
         }
 
-        /* now, check if the lookup() is on an existing entry,
-           but on gfid-path */
-        if (!((loc->parent && __is_gfid_access_dir (loc->parent->gfid)) ||
-              __is_gfid_access_dir (loc->pargfid))) {
-                if (!loc->parent)
-                        goto wind;
-
-                ret = inode_ctx_get (loc->parent, this, &value);
-                if (ret)
-                        goto wind;
+        /* not something to bother, continue the flow */
+        goto wind;
+    }
 
-                inode = (inode_t *) value;
+    /* need to check if the lookup is on virtual dir */
+    if ((loc->name && !strcmp(GF_GFID_DIR, loc->name)) &&
+        ((loc->parent && __is_root_gfid(loc->parent->gfid)) ||
+         __is_root_gfid(loc->pargfid))) {
+        /* this means, the query is on '/.gfid', return the fake stat,
+           and say success */
 
-                ret = loc_copy_overload_parent (&tmp_loc, loc, inode);
-                if (ret)
-                        goto err;
+        STACK_UNWIND_STRICT(lookup, frame, 0, 0, loc->inode,
+                            &priv->gfiddir_stbuf, xdata, &priv->root_stbuf);
+        return 0;
+    }
 
-                STACK_WIND (frame, ga_lookup_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->lookup, &tmp_loc, xdata);
+    /* now, check if the lookup() is on an existing entry,
+       but on gfid-path */
+    if (!((loc->parent && __is_gfid_access_dir(loc->parent->gfid)) ||
+          __is_gfid_access_dir(loc->pargfid))) {
+        if (!loc->parent)
+            goto wind;
 
-                loc_wipe (&tmp_loc);
-                return 0;
-        }
-
-        /* make sure the 'basename' is actually a 'canonical-gfid',
-           otherwise, return error */
-        ret = gf_uuid_parse (loc->name, tmp_gfid);
+        ret = inode_ctx_get(loc->parent, this, &value);
         if (ret)
-                goto err;
+            goto wind;
 
-        /* if its fresh lookup, go ahead and send it down, if not,
-           for directory, we need indirection to actual dir inode */
-        if (!(loc->inode && loc->inode->ia_type))
-                goto discover;
+        inode = (inode_t *)(uintptr_t)value;
 
-        /* revalidate on directory */
-        ret = inode_ctx_get (loc->inode, this, &value);
+        ret = loc_copy_overload_parent(&tmp_loc, loc, inode);
         if (ret)
-                goto err;
-
-        inode = (void *)value;
+            goto err;
 
-        /* valid inode, already looked up, work on that */
-        if (inode->ia_type)
-                goto discover;
+        STACK_WIND(frame, ga_lookup_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
 
-        /* check if the inode is in the 'itable' or
-           if its just previously discover()'d inode */
-        true_inode = inode_find (loc->inode->table, tmp_gfid);
-        if (true_inode) {
-                /* time do another lookup and update the context
-                   with proper inode */
-                op_errno = ESTALE;
-                /* 'inode_ref()' done in inode_find */
-                inode_unref (true_inode);
-                goto err;
-        }
+        loc_wipe(&tmp_loc);
+        return 0;
+    }
+
+    /* make sure the 'basename' is actually a 'canonical-gfid',
+       otherwise, return error */
+    ret = gf_uuid_parse(loc->name, tmp_gfid);
+    if (ret)
+        goto err;
+
+    /* if its fresh lookup, go ahead and send it down, if not,
+       for directory, we need indirection to actual dir inode */
+    if (!(loc->inode && loc->inode->ia_type))
+        goto discover;
+
+    /* revalidate on directory */
+    ret = inode_ctx_get(loc->inode, this, &value);
+    if (ret)
+        goto err;
+
+    inode = (void *)(uintptr_t)value;
+
+    /* valid inode, already looked up, work on that */
+    if (inode->ia_type)
+        goto discover;
+
+    /* check if the inode is in the 'itable' or
+       if its just previously discover()'d inode */
+    true_inode = inode_find(loc->inode->table, tmp_gfid);
+    if (true_inode) {
+        /* time do another lookup and update the context
+           with proper inode */
+        op_errno = ESTALE;
+        /* 'inode_ref()' done in inode_find */
+        inode_unref(true_inode);
+        goto err;
+    }
 
 discover:
-        /* for the virtual entries, we don't need to send 'gfid-req' key, as
-           for these entries, we don't want to 'set' a new gfid */
-        if (xdata)
-                dict_del (xdata, "gfid-req");
+    /* for the virtual entries, we don't need to send 'gfid-req' key, as
+       for these entries, we don't want to 'set' a new gfid */
+    if (xdata)
+        dict_del(xdata, "gfid-req");
 
-        gf_uuid_copy (tmp_loc.gfid, tmp_gfid);
+    gf_uuid_copy(tmp_loc.gfid, tmp_gfid);
 
-        /* if revalidate, then we need to have the proper reference */
-        if (inode) {
-                tmp_loc.inode = inode_ref (inode);
-                frame->local = inode_ref (loc->inode);
-        } else {
-                tmp_loc.inode = inode_ref (loc->inode);
-        }
+    /* if revalidate, then we need to have the proper reference */
+    if (inode) {
+        tmp_loc.inode = inode_ref(inode);
+        frame->local = inode_ref(loc->inode);
+    } else {
+        tmp_loc.inode = inode_ref(loc->inode);
+    }
 
-        STACK_WIND (frame, ga_virtual_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
+    STACK_WIND(frame, ga_virtual_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
 
-        inode_unref (tmp_loc.inode);
+    inode_unref(tmp_loc.inode);
 
-        return 0;
+    return 0;
 
 wind:
-        /* used for all the normal lookup path */
-        STACK_WIND (frame, ga_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    /* used for all the normal lookup path */
+    STACK_WIND(frame, ga_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, loc->inode,
-                             &priv->gfiddir_stbuf, xdata,
-                             &priv->root_stbuf);
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, loc->inode,
+                        &priv->gfiddir_stbuf, xdata, &priv->root_stbuf);
+    return 0;
 }
 
 int
-ga_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          mode_t umask, dict_t *xdata)
+ga_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata)
 {
-        int op_errno = ENOMEM;
+    int op_errno = ENOMEM;
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        STACK_WIND (frame, default_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
-                    xdata);
+    STACK_WIND(frame, default_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, loc->inode,
-                             NULL, NULL, NULL, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(mkdir, frame, -1, op_errno, loc->inode, NULL, NULL,
+                        NULL, xdata);
+    return 0;
 }
 
-
 int
-ga_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+ga_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        int op_errno = ENOMEM;
+    int op_errno = ENOMEM;
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        STACK_WIND (frame, default_create_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND(frame, default_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL, xdata);
-
-        return 0;
+    STACK_UNWIND_STRICT(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, xdata);
 
+    return 0;
 }
 
 int
-ga_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
-            loc_t *loc, mode_t umask, dict_t *xdata)
+ga_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+           loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        int op_errno = ENOMEM;
+    int op_errno = ENOMEM;
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        STACK_WIND (frame, default_symlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
-                    linkname, loc, umask, xdata);
-        return 0;
+    STACK_WIND(frame, default_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, xdata);
+    STACK_UNWIND_STRICT(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-ga_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dev_t rdev, mode_t umask, dict_t *xdata)
+ga_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        int op_errno = ENOMEM;
+    int op_errno = ENOMEM;
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        STACK_WIND (frame, default_mknod_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev,
-                    umask, xdata);
+    STACK_WIND(frame, default_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
 
-        return 0;
+    return 0;
 err:
-        STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, xdata);
+    STACK_UNWIND_STRICT(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-ga_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
-          dict_t *xdata)
+ga_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+         dict_t *xdata)
 {
-        int   op_errno = ENOMEM;
-        int   ret      = -1;
-        loc_t ga_loc   = {0, };
+    int op_errno = ENOMEM;
+    int ret = -1;
+    loc_t ga_loc = {
+        0,
+    };
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
 
-        STACK_WIND (frame, default_rmdir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
-                    &ga_loc, flag, xdata);
+    STACK_WIND(frame, default_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, &ga_loc, flag, xdata);
 
-        loc_wipe (&ga_loc);
-        return 0;
+    loc_wipe(&ga_loc);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL,
-                             NULL, xdata);
+    STACK_UNWIND_STRICT(rmdir, frame, -1, op_errno, NULL, NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-ga_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
-           dict_t *xdata)
+ga_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
+          dict_t *xdata)
 {
-        int   op_errno = ENOMEM;
-        int   ret      = -1;
-        loc_t ga_loc   = {0, };
+    int op_errno = ENOMEM;
+    int ret = -1;
+    loc_t ga_loc = {
+        0,
+    };
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
 
-        STACK_WIND (frame, default_unlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
-                    &ga_loc, xflag, xdata);
+    STACK_WIND(frame, default_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, &ga_loc, xflag, xdata);
 
-        loc_wipe (&ga_loc);
-        return 0;
+    loc_wipe(&ga_loc);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL,
-                             NULL, xdata);
+    STACK_UNWIND_STRICT(unlink, frame, -1, op_errno, NULL, NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-ga_rename (call_frame_t *frame, xlator_t *this,
-           loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+ga_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
 {
-        int   op_errno  = ENOMEM;
-        int   ret       = 0;
-        loc_t ga_oldloc = {0, };
-        loc_t ga_newloc = {0, };
-
-        GFID_ACCESS_ENTRY_OP_CHECK (oldloc, op_errno, err);
-        GFID_ACCESS_ENTRY_OP_CHECK (newloc, op_errno, err);
-
-        ret = ga_valid_inode_loc_copy (&ga_oldloc, oldloc, this);
-        if (ret < 0)
-                goto err;
-
-        ret = ga_valid_inode_loc_copy (&ga_newloc, newloc, this);
-        if (ret < 0) {
-                loc_wipe (&ga_oldloc);
-                goto err;
-        }
-
-        STACK_WIND (frame, default_rename_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
-                    &ga_oldloc, &ga_newloc, xdata);
-
-        loc_wipe (&ga_newloc);
-        loc_wipe (&ga_oldloc);
-        return 0;
+    int op_errno = ENOMEM;
+    int ret = 0;
+    loc_t ga_oldloc = {
+        0,
+    };
+    loc_t ga_newloc = {
+        0,
+    };
+
+    GFID_ACCESS_ENTRY_OP_CHECK(oldloc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(newloc, op_errno, err);
+
+    ret = ga_valid_inode_loc_copy(&ga_oldloc, oldloc, this);
+    if (ret < 0)
+        goto err;
+
+    ret = ga_valid_inode_loc_copy(&ga_newloc, newloc, this);
+    if (ret < 0) {
+        loc_wipe(&ga_oldloc);
+        goto err;
+    }
+
+    STACK_WIND(frame, default_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, &ga_oldloc, &ga_newloc, xdata);
+
+    loc_wipe(&ga_newloc);
+    loc_wipe(&ga_oldloc);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL, xdata);
+    STACK_UNWIND_STRICT(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-ga_link (call_frame_t *frame, xlator_t *this,
-         loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+ga_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
 {
-        int   op_errno  = ENOMEM;
-        int   ret       = 0;
-        loc_t ga_oldloc = {0, };
-        loc_t ga_newloc = {0, };
-
-        GFID_ACCESS_ENTRY_OP_CHECK (oldloc, op_errno, err);
-        GFID_ACCESS_ENTRY_OP_CHECK (newloc, op_errno, err);
-
-        ret = ga_valid_inode_loc_copy (&ga_oldloc, oldloc, this);
-        if (ret < 0)
-                goto err;
-
-        ret = ga_valid_inode_loc_copy (&ga_newloc, newloc, this);
-        if (ret < 0) {
-                loc_wipe (&ga_oldloc);
-                goto err;
-        }
-
-        STACK_WIND (frame, default_link_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
-                    &ga_oldloc, &ga_newloc, xdata);
-
-        loc_wipe (&ga_newloc);
-        loc_wipe (&ga_oldloc);
-        return 0;
+    int op_errno = ENOMEM;
+    int ret = 0;
+    loc_t ga_oldloc = {
+        0,
+    };
+    loc_t ga_newloc = {
+        0,
+    };
+
+    GFID_ACCESS_ENTRY_OP_CHECK(oldloc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(newloc, op_errno, err);
+
+    ret = ga_valid_inode_loc_copy(&ga_oldloc, oldloc, this);
+    if (ret < 0)
+        goto err;
author	Kaushal M <kaushal@redhat.com>	2012-12-13 12:51:03 +0530
committer	Anand Avati <avati@redhat.com>	2012-12-13 14:46:42 -0800
commit	1b681f27e04ae3b74a461ef2601ac8c48ad6b1c3 (patch)
tree	7607160f0b0fdc434d8864b6072ee8f860ddfd5f
parent	ef95b1ec0981e5f9859f5308e15ae33608bd6c29 (diff)